github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-23 09:19:04 -04:00
Code Issues Projects Releases Packages Wiki Activity

[8.x] [ML] Fix query for pattern analysis and change point analysis (#194742) (#195246)

Browse source 
# Backport

This will backport the following commits from `main` to `8.x`:
- [[ML] Fix query for pattern analysis and change point analysis
(#194742)](https://github.com/elastic/kibana/pull/194742)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"James
Gowdy","email":"jgowdy@elastic.co"},"sourceCommit":{"committedDate":"2024-10-07T12:41:50Z","message":"[ML]
Fix query for pattern analysis and change point analysis
(#194742)\n\nFixes
https://github.com/elastic/kibana/issues/190710\r\n\r\nAdds an
additional check for `query_string` to the query creating\r\nfunction to
adjust the query if only a single `query_string` condition\r\nis being
used.\r\nThis function was originally only used for pattern analysis,
but has\r\nbeen renamed and moved to a common location so change point
analysis can\r\nalso use
it.","sha":"02f277efa7470035671f67d4e86e74ea5f52838e","branchLabelMapping":{"^v9.0.0$":"main","^v8.16.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix",":ml","v9.0.0","v8.16.0","backport:version"],"title":"[ML]
Fix query for pattern analysis and change point
analysis","number":194742,"url":"https://github.com/elastic/kibana/pull/194742","mergeCommit":{"message":"[ML]
Fix query for pattern analysis and change point analysis
(#194742)\n\nFixes
https://github.com/elastic/kibana/issues/190710\r\n\r\nAdds an
additional check for `query_string` to the query creating\r\nfunction to
adjust the query if only a single `query_string` condition\r\nis being
used.\r\nThis function was originally only used for pattern analysis,
but has\r\nbeen renamed and moved to a common location so change point
analysis can\r\nalso use
it.","sha":"02f277efa7470035671f67d4e86e74ea5f52838e"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/194742","number":194742,"mergeCommit":{"message":"[ML]
Fix query for pattern analysis and change point analysis
(#194742)\n\nFixes
https://github.com/elastic/kibana/issues/190710\r\n\r\nAdds an
additional check for `query_string` to the query creating\r\nfunction to
adjust the query if only a single `query_string` condition\r\nis being
used.\r\nThis function was originally only used for pattern analysis,
but has\r\nbeen renamed and moved to a common location so change point
analysis can\r\nalso use
it.","sha":"02f277efa7470035671f67d4e86e74ea5f52838e"}},{"branch":"8.x","label":"v8.16.0","branchLabelMappingKey":"^v8.16.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: James Gowdy <jgowdy@elastic.co>
This commit is contained in:
Kibana Machine 2024-10-08 02:18:53 +11:00 • committed by GitHub
parent c099f33d32
commit 19631389a4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 177 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

156
x-pack/packages/ml/aiops_common/create_default_query.test.ts Normal file
View file

@ -0,0 +1,156 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { createDefaultQuery } from './create_default_query';
describe('createDefaultQuery', () => {
it('should create a default match_all query when no input query is provided', () => {
const result = createDefaultQuery(undefined, 'timestamp', undefined);
expect(result).toEqual({
bool: {
must: [{ match_all: {} }],
},
});
});
it('should wrap an existing match_all query in a bool must clause', () => {
const inputQuery = { match_all: {} };
const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
expect(result).toEqual({
bool: {
must: [{ match_all: {} }],
},
});
});
it('should wrap an existing query_string query in a bool must clause', () => {
const inputQuery = { query_string: { query: '*' } };
const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
expect(result).toEqual({
bool: {
must: [{ query_string: { query: '*' } }],
},
});
});
it('should wrap an existing multi_match query in a bool should clause', () => {
const inputQuery = { multi_match: { query: 'test', fields: ['field1', 'field2'] } };
const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
expect(result).toEqual({
bool: {
must: [],
should: { multi_match: { query: 'test', fields: ['field1', 'field2'] } },
},
});
});
it('should add a time range filter to the query', () => {
const timeRange = { from: 1609459200000, to: 1609545600000 };
const result = createDefaultQuery(undefined, 'timestamp', timeRange);
expect(result).toEqual({
bool: {
must: [
{ match_all: {} },
{
range: {
timestamp: {
gte: 1609459200000,
lte: 1609545600000,
format: 'epoch_millis',
},
},
},
],
},
});
});
it('should merge existing bool query with new time range filter', () => {
const inputQuery = { bool: { must: [{ term: { field: 'value' } }] } };
const timeRange = { from: 1609459200000, to: 1609545600000 };
const result = createDefaultQuery(inputQuery, 'timestamp', timeRange);
expect(result).toEqual({
bool: {
must: [
{ term: { field: 'value' } },
{
range: {
timestamp: {
gte: 1609459200000,
lte: 1609545600000,
format: 'epoch_millis',
},
},
},
],
},
});
});
it('should handle an existing bool query with must clause', () => {
const inputQuery = { bool: { must: [{ term: { field: 'value' } }] } };
const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
expect(result).toEqual({
bool: {
must: [{ term: { field: 'value' } }],
},
});
});
it('should handle an existing bool query with should clause', () => {
const inputQuery = { bool: { should: [{ term: { field: 'value' } }] } };
const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
expect(result).toEqual({
bool: {
must: [],
should: [{ term: { field: 'value' } }],
},
});
});
it('should handle an existing bool query with must_not clause', () => {
const inputQuery = { bool: { must_not: [{ term: { field: 'value' } }] } };
const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
expect(result).toEqual({
bool: {
must: [],
must_not: [{ term: { field: 'value' } }],
},
});
});
it('should handle an existing bool query with filter clause', () => {
const inputQuery = { bool: { filter: [{ term: { field: 'value' } }] } };
const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
expect(result).toEqual({
bool: {
must: [],
filter: [{ term: { field: 'value' } }],
},
});
});
it('should handle an input query with multiple clauses', () => {
const inputQuery = {
bool: {
must: [{ term: { field1: 'value1' } }],
should: [{ term: { field2: 'value2' } }],
must_not: [{ term: { field3: 'value3' } }],
filter: [{ term: { field4: 'value4' } }],
},
};
const result = createDefaultQuery(inputQuery, 'timestamp', undefined);
expect(result).toEqual({
bool: {
must: [{ term: { field1: 'value1' } }],
should: [{ term: { field2: 'value2' } }],
must_not: [{ term: { field3: 'value3' } }],
filter: [{ term: { field4: 'value4' } }],
},
});
});
});

14
x-pack/packages/ml/aiops_log_pattern_analysis/create_categorize_query.ts → x-pack/packages/ml/aiops_common/create_default_query.ts
View file

@ -7,9 +7,12 @@
import { cloneDeep } from 'lodash';
import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
import type {
QueryDslBoolQuery,
QueryDslQueryContainer,
} from '@elastic/elasticsearch/lib/api/types';
export function createCategorizeQuery(
export function createDefaultQuery(
queryIn: QueryDslQueryContainer | undefined,
timeField: string,
timeRange: { from: number; to: number } | undefined
@ -17,14 +20,19 @@ export function createCategorizeQuery(
const query = cloneDeep(queryIn ?? { match_all: {} });
if (query.bool === undefined) {
query.bool = {};
query.bool = Object.create(null) as QueryDslBoolQuery;
}
if (query.bool.must === undefined) {
query.bool.must = [];
if (query.match_all !== undefined) {
query.bool.must.push({ match_all: query.match_all });
delete query.match_all;
}
if (query.query_string !== undefined) {
query.bool.must.push({ query_string: query.query_string });
delete query.query_string;
}
}
if (query.multi_match !== undefined) {
query.bool.should = {

4
x-pack/packages/ml/aiops_log_pattern_analysis/create_category_request.ts
View file

@ -14,7 +14,7 @@ import { isPopulatedObject } from '@kbn/ml-is-populated-object/src/is_populated_
import type { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils';
import { createCategorizeQuery } from './create_categorize_query';
import { createDefaultQuery } from '@kbn/aiops-common/create_default_query';
const CATEGORY_LIMIT = 1000;
const EXAMPLE_LIMIT = 4;
@ -38,7 +38,7 @@ export function createCategoryRequest(
useStandardTokenizer: boolean = true,
includeSparkline: boolean = true
) {
const query = createCategorizeQuery(queryIn, timeField, timeRange);
const query = createDefaultQuery(queryIn, timeField, timeRange);
const aggs = {
categories: {
categorize_text: {

1
x-pack/packages/ml/aiops_log_pattern_analysis/tsconfig.json
View file

@ -23,5 +23,6 @@
"@kbn/saved-search-plugin",
"@kbn/data-views-plugin",
"@kbn/ml-is-populated-object",
"@kbn/aiops-common",
]
}

22
x-pack/plugins/aiops/public/components/change_point_detection/change_point_detection_context.tsx
View file

@ -16,6 +16,7 @@ import { ES_FIELD_TYPES } from '@kbn/field-types';
import { type QueryDslQueryContainer } from '@kbn/data-views-plugin/common/types';
import type { TimeBuckets, TimeBucketsInterval } from '@kbn/ml-time-buckets';
import { useTimeBuckets } from '@kbn/ml-time-buckets';
import { createDefaultQuery } from '@kbn/aiops-common/create_default_query';
import { useFilterQueryUpdates } from '../../hooks/use_filters_query';
import { type ChangePointType, DEFAULT_AGG_FUNCTION } from './constants';
import {
@ -261,23 +262,10 @@ export const ChangePointDetectionContextProvider: FC<PropsWithChildren<unknown>>
const combinedQuery = useMemo(() => {
const mergedQuery = createMergedEsQuery(resultQuery, resultFilters, dataView, uiSettings);
if (!Array.isArray(mergedQuery.bool?.filter)) {
if (!mergedQuery.bool) {
mergedQuery.bool = {};
}
mergedQuery.bool.filter = [];
}
mergedQuery.bool!.filter.push({
range: {
[dataView.timeFieldName!]: {
from: searchBounds.min?.valueOf(),
to: searchBounds.max?.valueOf(),
},
},
});
return mergedQuery;
const to = searchBounds.max?.valueOf();
const from = searchBounds.min?.valueOf();
const timeRange = to !== undefined && from !== undefined ? { from, to } : undefined;
return createDefaultQuery(mergedQuery, dataView.timeFieldName!, timeRange);
}, [resultFilters, resultQuery, uiSettings, dataView, searchBounds]);
if (!bucketInterval) return null;

4
x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts
View file

@ -14,7 +14,7 @@ import type { FieldValidationResults } from '@kbn/ml-category-validator';
import type { HttpFetchOptions } from '@kbn/core/public';
import { AIOPS_API_ENDPOINT } from '@kbn/aiops-common/constants';
import { createCategorizeQuery } from '@kbn/aiops-log-pattern-analysis/create_categorize_query';
import { createDefaultQuery } from '@kbn/aiops-common/create_default_query';
import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
@ -32,7 +32,7 @@ export function useValidateFieldRequest() {
runtimeMappings: MappingRuntimeFields | undefined,
headers?: HttpFetchOptions['headers']
) => {
const query = createCategorizeQuery(queryIn, timeField, timeRange);
const query = createDefaultQuery(queryIn, timeField, timeRange);
const resp = await http.post<FieldValidationResults>(
AIOPS_API_ENDPOINT.CATEGORIZATION_FIELD_VALIDATION,
{