github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 17:28:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

buildEsQuery allow ignore_unmapped for nested fields queries (#134580)

Browse source
This commit is contained in:
Anton Dosov 2022-06-21 16:47:41 +02:00 • committed by GitHub
parent 851c4ebe60
commit 12d04a9e55
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 281 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

55
packages/kbn-es-query/src/es_query/build_es_query.test.ts
View file

@ -202,5 +202,60 @@ describe('build query', () => {
expect(result).toEqual(expectedResult);
});
it('should allow to use ignore_unmapped for nested fields', () => {
const queries = [
{ query: 'nestedField: { child: "something" }', language: 'kuery' },
] as Query[];
const filters = [
{
query: { exists: { field: 'nestedField.child' } },
meta: { type: 'exists', alias: '', disabled: false, negate: false },
},
];
const result = buildEsQuery(indexPattern, queries, filters, { nestedIgnoreUnmapped: true });
const expected = {
bool: {
must: [],
filter: [
{
nested: {
ignore_unmapped: true,
path: 'nestedField',
query: {
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
'nestedField.child': 'something',
},
},
],
},
},
score_mode: 'none',
},
},
{
nested: {
path: 'nestedField',
query: {
exists: {
field: 'nestedField.child',
},
},
ignore_unmapped: true,
},
},
],
should: [],
must_not: [],
},
};
expect(result).toEqual(expected);
});
});
});

32
packages/kbn-es-query/src/es_query/build_es_query.ts
View file

@ -13,17 +13,18 @@ import { buildQueryFromFilters } from './from_filters';
import { buildQueryFromLucene } from './from_lucene';
import { Filter, Query } from '../filters';
import { BoolQuery, DataViewBase } from './types';
import { KueryQueryOptions } from '../kuery';
import type { KueryQueryOptions } from '../kuery';
import type { EsQueryFiltersConfig } from './from_filters';
/**
* Configurations to be used while constructing an ES query.
* @public
*/
export type EsQueryConfig = KueryQueryOptions & {
allowLeadingWildcards: boolean;
queryStringOptions: SerializableRecord;
ignoreFilterIfFieldNotInIndex: boolean;
};
export type EsQueryConfig = KueryQueryOptions &
EsQueryFiltersConfig & {
allowLeadingWildcards?: boolean;
queryStringOptions?: SerializableRecord;
};
function removeMatchAll<T>(filters: T[]) {
return filters.filter(
@ -59,20 +60,23 @@ export function buildEsQuery(
const kueryQuery = buildQueryFromKuery(
indexPattern,
queriesByLanguage.kuery,
config.allowLeadingWildcards,
config.dateFormatTZ,
config.filtersInMustClause
{ allowLeadingWildcards: config.allowLeadingWildcards },
{
dateFormatTZ: config.dateFormatTZ,
filtersInMustClause: config.filtersInMustClause,
nestedIgnoreUnmapped: config.nestedIgnoreUnmapped,
}
);
const luceneQuery = buildQueryFromLucene(
queriesByLanguage.lucene,
config.queryStringOptions,
config.dateFormatTZ
);
const filterQuery = buildQueryFromFilters(
filters,
indexPattern,
config.ignoreFilterIfFieldNotInIndex
);
const filterQuery = buildQueryFromFilters(filters, indexPattern, {
ignoreFilterIfFieldNotInIndex: config.ignoreFilterIfFieldNotInIndex,
nestedIgnoreUnmapped: config.nestedIgnoreUnmapped,
});
return {
bool: {

58
packages/kbn-es-query/src/es_query/from_filters.test.ts
View file

@ -19,7 +19,9 @@ describe('build query', () => {
describe('buildQueryFromFilters', () => {
test('should return the parameters of an Elasticsearch bool query', () => {
const result = buildQueryFromFilters([], indexPattern, false);
const result = buildQueryFromFilters([], indexPattern, {
ignoreFilterIfFieldNotInIndex: false,
});
const expected = {
must: [],
filter: [],
@ -43,7 +45,9 @@ describe('build query', () => {
const expectedESQueries = [{ match_all: {} }, { exists: { field: 'foo' } }];
const result = buildQueryFromFilters(filters, indexPattern, false);
const result = buildQueryFromFilters(filters, indexPattern, {
ignoreFilterIfFieldNotInIndex: false,
});
expect(result.filter).toEqual(expectedESQueries);
});
@ -55,14 +59,18 @@ describe('build query', () => {
meta: { type: 'match_all', negate: true, disabled: true },
} as MatchAllFilter,
] as Filter[];
const result = buildQueryFromFilters(filters, indexPattern, false);
const result = buildQueryFromFilters(filters, indexPattern, {
ignoreFilterIfFieldNotInIndex: false,
});
expect(result.must_not).toEqual([]);
});
test('should remove falsy filters', () => {
const filters = [null, undefined] as unknown as Filter[];
const result = buildQueryFromFilters(filters, indexPattern, false);
const result = buildQueryFromFilters(filters, indexPattern, {
ignoreFilterIfFieldNotInIndex: false,
});
expect(result.must_not).toEqual([]);
expect(result.must).toEqual([]);
@ -78,7 +86,9 @@ describe('build query', () => {
const expectedESQueries = [{ match_all: {} }];
const result = buildQueryFromFilters(filters, indexPattern, false);
const result = buildQueryFromFilters(filters, indexPattern, {
ignoreFilterIfFieldNotInIndex: false,
});
expect(result.must_not).toEqual(expectedESQueries);
});
@ -97,7 +107,9 @@ describe('build query', () => {
},
];
const result = buildQueryFromFilters(filters, indexPattern, false);
const result = buildQueryFromFilters(filters, indexPattern, {
ignoreFilterIfFieldNotInIndex: false,
});
expect(result.filter).toEqual(expectedESQueries);
});
@ -116,7 +128,9 @@ describe('build query', () => {
},
];
const result = buildQueryFromFilters(filters, indexPattern, false);
const result = buildQueryFromFilters(filters, indexPattern, {
ignoreFilterIfFieldNotInIndex: false,
});
expect(result.filter).toEqual(expectedESQueries);
});
@ -130,7 +144,9 @@ describe('build query', () => {
] as Filter[];
const expectedESQueries = [{ query_string: { query: 'foo' } }];
const result = buildQueryFromFilters(filters, indexPattern, false);
const result = buildQueryFromFilters(filters, indexPattern, {
ignoreFilterIfFieldNotInIndex: false,
});
expect(result.filter).toEqual(expectedESQueries);
});
@ -159,5 +175,31 @@ describe('build query', () => {
const result = buildQueryFromFilters(filters, indexPattern);
expect(result.filter).toEqual(expectedESQueries);
});
test('should allow to configure ignore_unmapped for filters targeting nested fields in a nested query', () => {
const filters = [
{
query: { exists: { field: 'nestedField.child' } },
meta: { type: 'exists', alias: '', disabled: false, negate: false },
},
];
const expectedESQueries = [
{
nested: {
path: 'nestedField',
query: {
exists: {
field: 'nestedField.child',
},
},
ignore_unmapped: true,
},
},
];
const result = buildQueryFromFilters(filters, indexPattern, { nestedIgnoreUnmapped: true });
expect(result.filter).toEqual(expectedESQueries);
});
});
});

25
packages/kbn-es-query/src/es_query/from_filters.ts
View file

@ -38,6 +38,23 @@ const translateToQuery = (filter: Partial<Filter>): estypes.QueryDslQueryContain
return filter.query || filter;
};
/**
* Options for building query for filters
*/
export interface EsQueryFiltersConfig {
/**
* by default filters that use fields that can't be found in the specified index pattern are not applied. Set this to true if you want to apply them anyway.
*/
ignoreFilterIfFieldNotInIndex?: boolean;
/**
* the nested field type requires a special query syntax, which includes an optional ignore_unmapped parameter that indicates whether to ignore an unmapped path and not return any documents instead of an error.
* The optional ignore_unmapped parameter defaults to false.
* This `nestedIgnoreUnmapped` param allows creating queries with "ignore_unmapped": true
*/
nestedIgnoreUnmapped?: boolean;
}
/**
* @param filters
* @param indexPattern
@ -49,7 +66,9 @@ const translateToQuery = (filter: Partial<Filter>): estypes.QueryDslQueryContain
export const buildQueryFromFilters = (
filters: Filter[] = [],
indexPattern: DataViewBase | undefined,
ignoreFilterIfFieldNotInIndex: boolean = false
{ ignoreFilterIfFieldNotInIndex = false, nestedIgnoreUnmapped }: EsQueryFiltersConfig = {
ignoreFilterIfFieldNotInIndex: false,
}
): BoolQuery => {
filters = filters.filter((filter) => filter && !isFilterDisabled(filter));
@ -63,7 +82,9 @@ export const buildQueryFromFilters = (
.map((filter) => {
return migrateFilter(filter, indexPattern);
})
.map((filter) => handleNestedFilter(filter, indexPattern))
.map((filter) =>
handleNestedFilter(filter, indexPattern, { ignoreUnmapped: nestedIgnoreUnmapped })
)
.map(cleanFilter)
.map(translateToQuery);
};

15
packages/kbn-es-query/src/es_query/from_kuery.test.ts
View file

@ -22,7 +22,7 @@ describe('build query', () => {
describe('buildQueryFromKuery', () => {
test('should return the parameters of an Elasticsearch bool query', () => {
const result = buildQueryFromKuery(undefined, [], true);
const result = buildQueryFromKuery(undefined, [], { allowLeadingWildcards: true });
const expected = {
must: [],
filter: [],
@ -42,7 +42,7 @@ describe('build query', () => {
return toElasticsearchQuery(fromKueryExpression(query.query), indexPattern);
});
const result = buildQueryFromKuery(indexPattern, queries, true);
const result = buildQueryFromKuery(indexPattern, queries, { allowLeadingWildcards: true });
expect(result.filter).toEqual(expectedESQueries);
});
@ -55,7 +55,14 @@ describe('build query', () => {
});
});
const result = buildQueryFromKuery(indexPattern, queries, true, 'America/Phoenix');
const result = buildQueryFromKuery(
indexPattern,
queries,
{ allowLeadingWildcards: true },
{
dateFormatTZ: 'America/Phoenix',
}
);
expect(result.filter).toEqual(expectedESQueries);
});
@ -68,7 +75,7 @@ describe('build query', () => {
return toElasticsearchQuery(fromKueryExpression(query.query), indexPattern);
});
const result = buildQueryFromKuery(indexPattern, queries, true);
const result = buildQueryFromKuery(indexPattern, queries, { allowLeadingWildcards: true });
expect(result.filter).toEqual(expectedESQueries);
});

26
packages/kbn-es-query/src/es_query/from_kuery.ts
View file

@ -6,30 +6,42 @@
* Side Public License, v 1.
*/
import { SerializableRecord } from '@kbn/utility-types';
import { Query } from '../filters';
import { fromKueryExpression, toElasticsearchQuery, nodeTypes, KueryNode } from '../kuery';
import {
fromKueryExpression,
toElasticsearchQuery,
nodeTypes,
KueryNode,
KueryQueryOptions,
} from '../kuery';
import { BoolQuery, DataViewBase } from './types';
/** @internal */
export function buildQueryFromKuery(
indexPattern: DataViewBase | undefined,
queries: Query[] = [],
allowLeadingWildcards: boolean = false,
dateFormatTZ?: string,
filtersInMustClause: boolean = false
{ allowLeadingWildcards = false }: { allowLeadingWildcards?: boolean } = {
allowLeadingWildcards: false,
},
{ filtersInMustClause = false, dateFormatTZ, nestedIgnoreUnmapped }: KueryQueryOptions = {
filtersInMustClause: false,
}
): BoolQuery {
const queryASTs = queries.map((query) => {
return fromKueryExpression(query.query, { allowLeadingWildcards });
});
return buildQuery(indexPattern, queryASTs, { dateFormatTZ, filtersInMustClause });
return buildQuery(indexPattern, queryASTs, {
filtersInMustClause,
dateFormatTZ,
nestedIgnoreUnmapped,
});
}
function buildQuery(
indexPattern: DataViewBase | undefined,
queryASTs: KueryNode[],
config: SerializableRecord = {}
config: KueryQueryOptions = {}
): BoolQuery {
const compoundQueryAST = nodeTypes.function.buildNode('and', queryASTs);
const kueryQuery = toElasticsearchQuery(compoundQueryAST, indexPattern, config);

2
packages/kbn-es-query/src/es_query/from_lucene.ts
View file

@ -15,7 +15,7 @@ import { BoolQuery } from './types';
/** @internal */
export function buildQueryFromLucene(
queries: Query[],
queryStringOptions: SerializableRecord,
queryStringOptions: SerializableRecord = {},
dateFormatTZ?: string
): BoolQuery {
const combinedQueries = (queries || []).map((query) => {

22
packages/kbn-es-query/src/es_query/handle_nested_filter.test.ts
View file

@ -39,6 +39,28 @@ describe('handleNestedFilter', function () {
});
});
it('should allow to configure ignore_unmapped', () => {
const field = getField('nestedField.child');
const filter = buildPhraseFilter(field!, 'foo', indexPattern);
const result = handleNestedFilter(filter, indexPattern, { ignoreUnmapped: true });
expect(result).toEqual({
meta: {
index: 'logstash-*',
},
query: {
nested: {
path: 'nestedField',
query: {
match_phrase: {
'nestedField.child': 'foo',
},
},
ignore_unmapped: true,
},
},
});
});
it('should return filter untouched if it does not target a nested field', () => {
const field = getField('extension');
const filter = buildPhraseFilter(field!, 'jpg', indexPattern);

9
packages/kbn-es-query/src/es_query/handle_nested_filter.ts
View file

@ -11,7 +11,11 @@ import { DataViewBase } from './types';
import { getDataViewFieldSubtypeNested } from '../utils';
/** @internal */
export const handleNestedFilter = (filter: Filter, indexPattern?: DataViewBase) => {
export const handleNestedFilter = (
filter: Filter,
indexPattern?: DataViewBase,
config: { ignoreUnmapped?: boolean } = {}
) => {
if (!indexPattern) return filter;
const fieldName = getFilterField(filter);
@ -36,6 +40,9 @@ export const handleNestedFilter = (filter: Filter, indexPattern?: DataViewBase)
nested: {
path: subTypeNested.nested.path,
query: query.query || query,
...(typeof config.ignoreUnmapped === 'boolean' && {
ignore_unmapped: config.ignoreUnmapped,
}),
},
},
};

1
packages/kbn-es-query/src/es_query/index.ts
View file

@ -7,6 +7,7 @@
*/
export { migrateFilter } from './migrate_filter';
export type { EsQueryFiltersConfig } from './from_filters';
export type { EsQueryConfig } from './build_es_query';
export { buildEsQuery } from './build_es_query';
export { buildQueryFromFilters } from './from_filters';

1
packages/kbn-es-query/src/index.ts
View file

@ -11,6 +11,7 @@ export type {
DataViewBase,
DataViewFieldBase,
EsQueryConfig,
EsQueryFiltersConfig,
IFieldSubType,
IFieldSubTypeMulti,
IFieldSubTypeNested,

26
packages/kbn-es-query/src/kuery/functions/is.test.ts
View file

@ -314,6 +314,32 @@ describe('kuery functions', () => {
expect(result).toEqual(expected);
});
test('should allow to configure ignore_unmapped for a nested query', () => {
const expected = {
bool: {
should: [
{
nested: {
path: 'nestedField.nestedChild',
query: {
match: {
'nestedField.nestedChild.doublyNestedChild': 'foo',
},
},
score_mode: 'none',
ignore_unmapped: true,
},
},
],
minimum_should_match: 1,
},
};
const node = nodeTypes.function.buildNode('is', '*doublyNested*', 'foo');
const result = is.toElasticsearchQuery(node, indexPattern, { nestedIgnoreUnmapped: true });
expect(result).toEqual(expected);
});
});
});
});

3
packages/kbn-es-query/src/kuery/functions/is.ts
View file

@ -114,6 +114,9 @@ export function toElasticsearchQuery(
path: subTypeNested.nested.path,
query,
score_mode: 'none',
...(typeof config.nestedIgnoreUnmapped === 'boolean' && {
ignore_unmapped: config.nestedIgnoreUnmapped,
}),
},
};
}

3
packages/kbn-es-query/src/kuery/functions/nested.ts
View file

@ -37,6 +37,9 @@ export function toElasticsearchQuery(
nested: { path: fullPath },
}) as estypes.QueryDslQueryContainer,
score_mode: 'none',
...(typeof config.nestedIgnoreUnmapped === 'boolean' && {
ignore_unmapped: config.nestedIgnoreUnmapped,
}),
},
};
}

30
packages/kbn-es-query/src/kuery/functions/range.test.ts
View file

@ -229,6 +229,36 @@ describe('kuery functions', () => {
expect(result).toEqual(expected);
});
test('should allow to configure ignore_unmapped for a nested query', () => {
const expected = {
bool: {
should: [
{
nested: {
path: 'nestedField.nestedChild',
query: {
range: {
'nestedField.nestedChild.doublyNestedChild': {
lt: 8000,
},
},
},
score_mode: 'none',
ignore_unmapped: true,
},
},
],
minimum_should_match: 1,
},
};
const node = nodeTypes.function.buildNode('range', '*doublyNested*', 'lt', 8000);
const result = range.toElasticsearchQuery(node, indexPattern, {
nestedIgnoreUnmapped: true,
});
expect(result).toEqual(expected);
});
});
});
});

3
packages/kbn-es-query/src/kuery/functions/range.ts
View file

@ -66,6 +66,9 @@ export function toElasticsearchQuery(
path: subTypeNested.nested.path,
query,
score_mode: 'none',
...(typeof config.nestedIgnoreUnmapped === 'boolean' && {
ignore_unmapped: config.nestedIgnoreUnmapped,
}),
},
};
}

7
packages/kbn-es-query/src/kuery/types.ts
View file

@ -39,4 +39,11 @@ export { nodeTypes } from './node_types';
export interface KueryQueryOptions {
filtersInMustClause?: boolean;
dateFormatTZ?: string;
/**
* the Nested field type requires a special query syntax, which includes an optional ignore_unmapped parameter that indicates whether to ignore an unmapped path and not return any documents instead of an error.
* The optional ignore_unmapped parameter defaults to false.
* The `nestedIgnoreUnmapped` param allows creating queries with "ignore_unmapped": true
*/
nestedIgnoreUnmapped?: boolean;
}