github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-06-27 18:51:07 -04:00
Code Issues Projects Releases Packages Wiki Activity

ES|QL pattern formatting (#222871)

Browse source 
Adds a recommended query for the `CATEGORIZE` function in ES|QL.
Adds keyword highlighting for the patterns and the ability to open a new
Discover tab to filter for docs which match the selected pattern.


https://github.com/user-attachments/assets/9ed8c5b0-7e92-4cc8-88dd-cb7749b5ffd3

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>
This commit is contained in:
James Gowdy 2025-06-25 11:33:30 +01:00 • committed by GitHub
parent 831004deac
commit 0d2930b3d0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
49 changed files with 1112 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

12
.github/CODEOWNERS vendored
View file

@ -410,6 +410,7 @@ src/platform/packages/shared/home/sample_data_card @elastic/appex-sharedux
src/platform/packages/shared/home/sample_data_tab @elastic/appex-sharedux
src/platform/packages/shared/home/sample_data_types @elastic/appex-sharedux
src/platform/packages/shared/kbn-actions-types @elastic/response-ops
src/platform/packages/shared/kbn-aiops-utils @elastic/ml-ui
src/platform/packages/shared/kbn-alerting-types @elastic/response-ops
src/platform/packages/shared/kbn-alerts-as-data-utils @elastic/response-ops
src/platform/packages/shared/kbn-alerts-ui-shared @elastic/response-ops
@ -1220,6 +1221,7 @@ x-pack/test_serverless/api_integration/test_suites/common/platform_security @ela
/x-pack/test_serverless/functional/test_suites/common/examples/search_examples @elastic/kibana-data-discovery
/x-pack/test_serverless/functional/test_suites/common/examples/unified_field_list_examples @elastic/kibana-data-discovery
/x-pack/test_serverless/functional/test_suites/common/management/data_views @elastic/kibana-data-discovery
src/platform/plugins/shared/discover/public/context_awareness/profile_providers/common/patterns @elastic/ml-ui
src/platform/plugins/shared/discover/public/context_awareness/profile_providers/security @elastic/kibana-data-discovery @elastic/security-threat-hunting-investigations
# TODO: this deprecation_logs folder should be owned by kibana management team after 9.0
src/platform/plugins/shared/discover/public/context_awareness/profile_providers/common/deprecation_logs @elastic/kibana-data-discovery @elastic/kibana-core
@ -2836,11 +2838,11 @@ src/platform/testfunctional/page_objects/solution_navigation.ts @elastic/appex-s
/x-pack/test/api_integration/deployment_agnostic/apis/intercepts/*.ts @elastic/appex-sharedux
# OpenAPI spec files
oas_docs/linters @elastic/core-docs @elastic/experience-docs
oas_docs/overlays @elastic/core-docs @elastic/experience-docs
oas_docs/kibana.info.serverless.yaml @elastic/core-docs @elastic/experience-docs
oas_docs/kibana.info.yaml @elastic/core-docs @elastic/experience-docs
oas_docs/output @elastic/core-docs @elastic/experience-docs
oas_docs/linters @elastic/core-docs @elastic/experience-docs
oas_docs/overlays @elastic/core-docs @elastic/experience-docs
oas_docs/kibana.info.serverless.yaml @elastic/core-docs @elastic/experience-docs
oas_docs/kibana.info.yaml @elastic/core-docs @elastic/experience-docs
oas_docs/output @elastic/core-docs @elastic/experience-docs
# Documentation settings files
docs/settings-gen @elastic/platform-docs @elastic/experience-docs

1
package.json
View file

@ -183,6 +183,7 @@
"@kbn/aiops-log-rate-analysis": "link:x-pack/platform/packages/shared/ml/aiops_log_rate_analysis",
"@kbn/aiops-plugin": "link:x-pack/platform/plugins/shared/aiops",
"@kbn/aiops-test-utils": "link:x-pack/platform/packages/private/ml/aiops_test_utils",
"@kbn/aiops-utils": "link:src/platform/packages/shared/kbn-aiops-utils",
"@kbn/alerting-api-integration-test-plugin": "link:x-pack/platform/test/alerting_api_integration/common/plugins/alerts",
"@kbn/alerting-comparators": "link:x-pack/platform/packages/shared/kbn-alerting-comparators",
"@kbn/alerting-example-plugin": "link:x-pack/examples/alerting_example",

3
src/platform/packages/shared/kbn-aiops-utils/README.md Normal file
View file

@ -0,0 +1,3 @@
# @kbn/aiops-utils
AIOps utils for plugins which are not in x-pack

13
src/platform/packages/shared/kbn-aiops-utils/index.ts Normal file
View file

@ -0,0 +1,13 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export {
getCategorizationDataViewField,
getCategorizationField,
} from './src/get_categorization_field';

14
src/platform/packages/shared/kbn-aiops-utils/jest.config.js Normal file
View file

@ -0,0 +1,14 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
module.exports = {
preset: '@kbn/test',
rootDir: '../../../../..',
roots: ['<rootDir>/src/platform/packages/shared/kbn-aiops-utils'],
};

7
src/platform/packages/shared/kbn-aiops-utils/kibana.jsonc Normal file
View file

@ -0,0 +1,7 @@
{
"type": "shared-common",
"id": "@kbn/aiops-utils",
"owner": "@elastic/ml-ui",
"group": "platform",
"visibility": "shared"
}

6
src/platform/packages/shared/kbn-aiops-utils/package.json Normal file
View file

@ -0,0 +1,6 @@
{
"name": "@kbn/aiops-utils",
"private": true,
"version": "1.0.0",
"license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0"
}

226
src/platform/packages/shared/kbn-aiops-utils/src/get_categorization_field.test.ts Normal file
View file

@ -0,0 +1,226 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { createStubDataView } from '@kbn/data-views-plugin/common/stubs';
import { getCategorizationField } from './get_categorization_field';
import { getCategorizationDataViewField } from './get_categorization_field';
describe('get_categorization_field utils', () => {
describe('getCategorizationField', () => {
it('returns "message" if present', () => {
const fields = ['foo', 'bar', 'message', 'baz'];
expect(getCategorizationField(fields)).toBe('message');
});
it('returns "error.message" if present and "message" is not', () => {
const fields = ['foo', 'error.message', 'baz'];
expect(getCategorizationField(fields)).toBe('error.message');
});
it('returns "event.original" if present and others are not', () => {
const fields = ['event.original', 'foo', 'bar'];
expect(getCategorizationField(fields)).toBe('event.original');
});
it('returns first field if none of the priority fields are present', () => {
const fields = ['foo', 'bar', 'baz'];
expect(getCategorizationField(fields)).toBe('foo');
});
it('returns first field, skipping meta data fields, if none of the priority fields are present', () => {
const fields = ['_id', 'foo', 'bar', 'baz'];
expect(getCategorizationField(fields)).toBe('foo');
});
it('returns the first matching field according to priority', () => {
const fields = ['event.original', 'error.message', 'message'];
expect(getCategorizationField(fields)).toBe('message');
});
it('handles empty array', () => {
expect(getCategorizationField([])).toBeUndefined();
});
});
describe('getCategorizationDataViewField', () => {
it('returns messageField as "message" if present', () => {
const dataView = createStubDataView({
spec: {
id: 'test',
fields: {
foo: {
searchable: false,
aggregatable: false,
name: 'foo',
type: 'type',
esTypes: ['keyword'],
},
message: {
searchable: true,
aggregatable: true,
name: 'message',
type: 'text',
esTypes: ['text'],
},
'error.message': {
searchable: true,
aggregatable: true,
name: 'error.message',
type: 'text',
esTypes: ['text'],
},
},
},
});
const result = getCategorizationDataViewField(dataView);
expect(result.messageField?.name).toBe('message');
expect(result.dataViewFields.map((f) => f.name)).toContain('message');
expect(result.dataViewFields.length).toBe(2);
});
it('returns messageField as "error.message" if "message" is not present', () => {
const dataView = createStubDataView({
spec: {
id: 'test2',
fields: {
foo: {
searchable: false,
aggregatable: false,
name: 'foo',
type: 'type',
esTypes: ['keyword'],
},
'error.message': {
searchable: true,
aggregatable: true,
name: 'error.message',
type: 'text',
esTypes: ['text'],
},
bar: {
searchable: true,
aggregatable: true,
name: 'bar',
type: 'text',
esTypes: ['text'],
},
},
},
});
const result = getCategorizationDataViewField(dataView);
expect(result.messageField?.name).toBe('error.message');
expect(result.dataViewFields.map((f) => f.name)).toContain('error.message');
expect(result.dataViewFields.length).toBe(2);
});
it('handles empty fields array', () => {
const dataView = createStubDataView({
spec: {
id: 'test6',
fields: {},
},
});
const result = getCategorizationDataViewField(dataView);
expect(result.messageField).toBeNull();
expect(result.dataViewFields.length).toBe(0);
});
it('returns the first matching field according to priority', () => {
const dataView = createStubDataView({
spec: {
id: 'test7',
fields: {
'event.original': {
searchable: true,
aggregatable: true,
name: 'event.original',
type: 'text',
esTypes: ['text'],
},
'error.message': {
searchable: true,
aggregatable: true,
name: 'error.message',
type: 'text',
esTypes: ['text'],
},
message: {
searchable: true,
aggregatable: true,
name: 'message',
type: 'text',
esTypes: ['text'],
},
},
},
});
const result = getCategorizationDataViewField(dataView);
expect(result.messageField?.name).toBe('message');
expect(result.dataViewFields.map((f) => f.name)).toEqual(
expect.arrayContaining(['message', 'error.message', 'event.original'])
);
expect(result.dataViewFields.length).toBe(3);
});
it('returns the first field if no priority fields are present', () => {
const dataView = createStubDataView({
spec: {
id: 'test8',
fields: {
foo: {
searchable: true,
aggregatable: true,
name: 'foo',
type: 'text',
esTypes: ['text'],
},
bar: {
searchable: true,
aggregatable: true,
name: 'bar',
type: 'text',
esTypes: ['text'],
},
},
},
});
const result = getCategorizationDataViewField(dataView);
expect(result.messageField?.name).toBe('foo');
expect(result.dataViewFields.map((f) => f.name)).toEqual(
expect.arrayContaining(['foo', 'bar'])
);
expect(result.dataViewFields.length).toBe(2);
});
it('returns null messageField if no text fields are present', () => {
const dataView = createStubDataView({
spec: {
id: 'test9',
fields: {
foo: {
searchable: true,
aggregatable: true,
name: 'foo',
type: 'keyword',
esTypes: ['keyword'],
},
bar: {
searchable: true,
aggregatable: true,
name: 'bar',
type: 'keyword',
esTypes: ['keyword'],
},
},
},
});
const result = getCategorizationDataViewField(dataView);
expect(result.messageField).toBeNull();
expect(result.dataViewFields.map((f) => f.name)).toEqual(expect.arrayContaining([]));
expect(result.dataViewFields.length).toBe(0);
});
});
});

72
src/platform/packages/shared/kbn-aiops-utils/src/get_categorization_field.ts Normal file
View file

@ -0,0 +1,72 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
import { ES_FIELD_TYPES } from '@kbn/field-types';
const FIELD_PRIORITY = ['message', 'error.message', 'event.original'];
const METADATA_FIELDS = [
'_version',
'_id',
'_index',
'_source',
'_ignored',
'_index_mode',
'_score',
];
/**
* This function returns the categorization field from the list of fields.
* It checks for the presence of 'message', 'error.message', or 'event.original' in that order.
* If none of these fields are present, it returns the first field from the list,
* Assumes text fields have been passed in the `fields` array.
*
* @param fields, the list of fields to check
* @returns string | undefined, the categorization field if found, otherwise undefined
*/
export function getCategorizationField(fields: string[]): string | undefined {
const fieldSet = new Set(fields);
for (const field of FIELD_PRIORITY) {
if (fieldSet.has(field)) {
return field;
}
}
// Filter out metadata fields
const filteredFields = fields.filter((field) => !METADATA_FIELDS.includes(field));
return filteredFields[0] ?? undefined;
}
/**
* This function returns the categorization field from the DataView.
* It checks for the presence of 'message', 'error.message', or 'event.original' in that order.
* If none of these fields are present, it returns the first text field from the DataView.
*
* @param dataView, the DataView to check
* @returns an object containing the message field DataViewField and dataViewFields
*/
export function getCategorizationDataViewField(dataView: DataView): {
messageField: DataViewField | null;
dataViewFields: DataViewField[];
} {
const dataViewFields = dataView.fields.filter((f) => f.esTypes?.includes(ES_FIELD_TYPES.TEXT));
const categorizationFieldName = getCategorizationField(dataViewFields.map((f) => f.name));
if (categorizationFieldName) {
return {
messageField: dataView.fields.getByName(categorizationFieldName) ?? null,
dataViewFields,
};
}
return {
messageField: null,
dataViewFields,
};
}

22
src/platform/packages/shared/kbn-aiops-utils/tsconfig.json Normal file
View file

@ -0,0 +1,22 @@
{
"extends": "../../../../../tsconfig.base.json",
"compilerOptions": {
"outDir": "target/types",
"types": [
"jest",
"node",
"react"
]
},
"include": [
"**/*.ts",
"**/*.tsx",
],
"exclude": [
"target/**/*"
],
"kbn_references": [
"@kbn/data-views-plugin",
"@kbn/field-types",
]
}

1
src/platform/packages/shared/kbn-esql-utils/index.ts
View file

@ -44,6 +44,7 @@ export {
getCategorizeColumns,
extractCategorizeTokens,
getArgsFromRenameFunction,
getCategorizeField,
} from './src';
export { ENABLE_ESQL, FEEDBACK_LINK } from './constants';

1
src/platform/packages/shared/kbn-esql-utils/src/index.ts
View file

@ -26,6 +26,7 @@ export {
fixESQLQueryWithVariables,
getCategorizeColumns,
getArgsFromRenameFunction,
getCategorizeField,
} from './utils/query_parsing_helpers';
export { queryCannotBeSampled } from './utils/query_cannot_be_sampled';
export {

1
src/platform/packages/shared/kbn-esql-utils/src/utils/extract_categorize_tokens.test.ts
View file

@ -48,5 +48,6 @@ describe('extractCategorizeTokens()', () => {
expect(extractCategorizeTokens(regexString4)).toStrictEqual(['someString']);
const regexString5 = 'justAString'; // No '.*?' or '.+?'
expect(extractCategorizeTokens(regexString5)).toStrictEqual(['justAString']);
expect(extractCategorizeTokens('.*?foo\\.bar.*?')).toEqual(['foo.bar']);
});
});

3
src/platform/packages/shared/kbn-esql-utils/src/utils/extract_categorize_tokens.ts
View file

@ -17,6 +17,9 @@
export function extractCategorizeTokens(regexString: string) {
let cleanedString = regexString;
// Remove backslashes
cleanedString = cleanedString.replace(/\\/g, '');
// Strip leading '.*?'
if (cleanedString.startsWith('.*?')) {
cleanedString = cleanedString.substring('.*?'.length);

40
src/platform/packages/shared/kbn-esql-utils/src/utils/query_parsing_helpers.test.ts
View file

@ -23,6 +23,7 @@ import {
fixESQLQueryWithVariables,
getCategorizeColumns,
getArgsFromRenameFunction,
getCategorizeField,
} from './query_parsing_helpers';
import { monaco } from '@kbn/monaco';
import { parse, walk } from '@kbn/esql-ast';
@ -815,4 +816,43 @@ describe('esql query helpers', () => {
});
});
});
describe('getCategorizeField', () => {
it('should return the field used in categorize', () => {
const esql = 'FROM index | STATS COUNT() BY categorize(field1)';
const expected = ['field1'];
expect(getCategorizeField(esql)).toEqual(expected);
});
it('should return the field used in categorize for multiple breakdowns', () => {
const esql = 'FROM index | STATS COUNT() BY categorize(field1), field2';
const expected = ['field1'];
expect(getCategorizeField(esql)).toEqual(expected);
});
it('should return the field used in categorize for multiple breakdowns with BUCKET', () => {
const esql =
'FROM index | STATS count_per_day = COUNT() BY Pattern=CATEGORIZE(message), @timestamp=BUCKET(@timestamp, 1 day)';
const expected = ['message'];
expect(getCategorizeField(esql)).toEqual(expected);
});
it('should return the field used in categorize if the result is stored in a new column', () => {
const esql = 'FROM index | STATS COUNT() BY pattern = categorize(field1)';
const expected = ['field1'];
expect(getCategorizeField(esql)).toEqual(expected);
});
it('should return the field used in categorize for a complex query', () => {
const esql =
'FROM index | STATS count_per_day = COUNT() BY Pattern=CATEGORIZE(message), @timestamp=BUCKET(@timestamp, 1 day) | STATS Count=SUM(count_per_day), Trend=VALUES(count_per_day) BY Pattern';
const expected = ['message'];
expect(getCategorizeField(esql)).toEqual(expected);
});
it('should return an empty array if no categorize is present', () => {
const esql = 'FROM index | STATS COUNT() BY field1';
const expected: string[] = [];
expect(getCategorizeField(esql)).toEqual(expected);
});
});
});

52
src/platform/packages/shared/kbn-esql-utils/src/utils/query_parsing_helpers.ts
View file

@ -362,3 +362,55 @@ export const getArgsFromRenameFunction = (
renamed: renameFunction.args[0] as ESQLColumn,
};
};
/**
* Extracts the fields used in the CATEGORIZE function from an ESQL query.
* @param esql: string - The ESQL query string
*/
export const getCategorizeField = (esql: string): string[] => {
const { root } = parse(esql);
const statsCommand = root.commands.find(({ name }) => name === 'stats');
if (!statsCommand) {
return [];
}
const options: ESQLCommandOption[] = [];
const columns: string[] = [];
walk(statsCommand, {
visitCommandOption: (node) => options.push(node),
});
const statsByOptions = options.find(({ name }) => name === 'by');
// categorize is part of the stats by command
if (!statsByOptions) {
return [];
}
const categorizeOptions = statsByOptions.args.filter((arg) => {
return (arg as ESQLFunction).text.toLowerCase().indexOf('categorize') !== -1;
}) as ESQLFunction[];
if (categorizeOptions.length) {
categorizeOptions.forEach((arg) => {
// ... STATS ... BY CATEGORIZE(field)
if (isFunctionExpression(arg) && arg.name === 'categorize') {
walk(arg, {
visitColumn: (node) => columns.push(node.name),
});
} else {
// ... STATS ... BY pattern = CATEGORIZE(field)
walk(arg, {
visitFunction: (node) => {
if (node.name === 'categorize') {
const columnArgs = node.args.filter((a) => isColumn(a));
columnArgs.forEach((c) => columns.push((c as ESQLColumn).name));
}
},
});
}
});
}
return columns;
};

1
src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.from.test.ts
View file

@ -110,6 +110,7 @@ describe('autocomplete.suggest', () => {
const recommendedQueries = getRecommendedQueries({
fromCommand: '',
timeField: 'dateField',
categorizationField: 'textField',
});
const { assertSuggestions } = await setup();
const expected = [

40
src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts
View file

@ -38,10 +38,15 @@ const commandDefinitions = unmodifiedCommandDefinitions.filter(
({ name, hidden }) => !hidden && name !== 'rrf'
);
const getRecommendedQueriesSuggestionsFromTemplates = (fromCommand: string, timeField?: string) =>
const getRecommendedQueriesSuggestionsFromTemplates = (
fromCommand: string,
timeField?: string,
categorizationField?: string
) =>
getRecommendedQueries({
fromCommand,
timeField,
categorizationField,
});
describe('autocomplete', () => {
@ -91,7 +96,8 @@ describe('autocomplete', () => {
describe('New command', () => {
const recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
'FROM logs*',
'dateField'
'dateField',
'textField'
);
testSuggestions('/', [
...sourceCommands.map((name) => name.toUpperCase() + ' '),
@ -251,7 +257,8 @@ describe('autocomplete', () => {
// source command
let recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
'FROM logs*',
'dateField'
'dateField',
'textField'
);
testSuggestions('f/', [
...sourceCommands.map((cmd) => `${cmd.toUpperCase()} `),
@ -477,7 +484,8 @@ describe('autocomplete', () => {
});
let recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
'FROM logs*',
'dateField'
'dateField',
'textField'
);
// Source command
testSuggestions('F/', [
@ -569,7 +577,11 @@ describe('autocomplete', () => {
);
});
recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates('', 'dateField');
recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
'',
'dateField',
'textField'
);
// PIPE (|)
testSuggestions('FROM a /', [
@ -620,7 +632,8 @@ describe('autocomplete', () => {
);
recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
'index1',
'dateField'
'dateField',
'textField'
);
testSuggestions(
@ -643,7 +656,8 @@ describe('autocomplete', () => {
recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
'index2',
'dateField'
'dateField',
'textField'
);
testSuggestions(
'FROM index1, index2/',
@ -669,7 +683,8 @@ describe('autocomplete', () => {
// we're setting it ourselves.
recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
'foo$bar',
'dateField'
'dateField',
'textField'
);
testSuggestions(
'FROM foo$bar/',
@ -701,7 +716,8 @@ describe('autocomplete', () => {
// This is an identifier that matches multiple sources
recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
'i*',
'dateField'
'dateField',
'textField'
);
testSuggestions(
'FROM i*/',
@ -722,7 +738,11 @@ describe('autocomplete', () => {
);
});
recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates('', 'dateField');
recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
'',
'dateField',
'textField'
);
// FROM source METADATA
testSuggestions('FROM index1 M/', [attachTriggerCommand('METADATA ')]);

51
src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/recommended_queries/suggestions.ts
View file

@ -9,22 +9,36 @@
import type { RecommendedQuery } from '@kbn/esql-types';
import type { SuggestionRawDefinition, GetColumnsByTypeFn } from '../types';
import { getRecommendedQueries } from './templates';
import { METADATA_FIELDS } from '../../shared/constants';
export const getRecommendedQueriesSuggestionsFromStaticTemplates = async (
getFieldsByType: GetColumnsByTypeFn,
fromCommand: string = ''
): Promise<SuggestionRawDefinition[]> => {
const fieldSuggestions = await getFieldsByType('date', [], {
openSuggestions: true,
});
const [fieldSuggestions, textFieldSuggestions] = await Promise.all([
getFieldsByType(['date'], [], { openSuggestions: true }),
// get text fields separately to avoid mixing them with date fields
getFieldsByType(['text'], [], { openSuggestions: true }),
]);
let timeField = '';
let categorizationField: string | undefined = '';
if (fieldSuggestions.length) {
timeField =
fieldSuggestions?.find((field) => field.label === '@timestamp')?.label ||
fieldSuggestions[0].label;
}
const recommendedQueries = getRecommendedQueries({ fromCommand, timeField });
if (textFieldSuggestions.length) {
categorizationField = getCategorizationField(textFieldSuggestions.map((field) => field.label));
}
const recommendedQueries = getRecommendedQueries({
fromCommand,
timeField,
categorizationField,
});
const suggestions: SuggestionRawDefinition[] = recommendedQueries.map((query) => {
return {
@ -95,3 +109,32 @@ export const getRecommendedQueriesTemplatesFromExtensions = (
return recommendedQueriesTemplates;
};
/**
* This function returns the categorization field from the list of fields.
* It checks for the presence of 'message', 'error.message', or 'event.original' in that order.
* If none of these fields are present, it returns the first field from the list,
* Assumes text fields have been passed in the `fields` array.
*
* This function is a duplicate of the one in src/platform/packages/shared/kbn-aiops-utils.
* It is included here to avoid build errors due to bazel
*
* TODO: Remove this function once the bazel issue is resolved.
*
* @param fields, the list of fields to check
* @returns string | undefined, the categorization field if found, otherwise undefined
*/
export function getCategorizationField(fields: string[]): string | undefined {
const fieldPriority = ['message', 'error.message', 'event.original'];
const fieldSet = new Set(fields);
for (const field of fieldPriority) {
if (fieldSet.has(field)) {
return field;
}
}
// Filter out metadata fields
const filteredFields = fields.filter((field) => !METADATA_FIELDS.includes(field));
return filteredFields[0] ?? undefined;
}

25
src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/recommended_queries/templates.ts
View file

@ -14,9 +14,11 @@ import { i18n } from '@kbn/i18n';
export const getRecommendedQueries = ({
fromCommand,
timeField,
categorizationField,
}: {
fromCommand: string;
timeField?: string;
categorizationField?: string;
}) => {
const queries = [
{
@ -135,6 +137,8 @@ export const getRecommendedQueries = ({
label: i18n.translate(
'kbn-esql-validation-autocomplete.recommendedQueries.categorize.label',
{
// TODO this item should be hidden if AIOps is disabled or we're not running with a platinum license
// the capability aiops.enabled can be used to check both of these conditions
defaultMessage: 'Detect change points',
}
),
@ -170,6 +174,27 @@ export const getRecommendedQueries = ({
},
]
: []),
...(categorizationField
? // TODO this item should be hidden if AIOps is disabled or we're not running with a platinum license
// the capability aiops.enabled can be used to check both of these conditions
[
{
label: i18n.translate(
'kbn-esql-validation-autocomplete.recommendedQueries.patternAnalysis.label',
{
defaultMessage: 'Identify patterns',
}
),
description: i18n.translate(
'kbn-esql-validation-autocomplete.recommendedQueries.patternAnalysis.description',
{
defaultMessage: 'Use the CATEGORIZE function to identify patterns in your logs',
}
),
queryString: `${fromCommand}\n | STATS Count=COUNT(*) BY Pattern=CATEGORIZE(${categorizationField})\n | SORT Count DESC`,
},
]
: []),
];
return queries;
};

27
src/platform/plugins/shared/discover/public/application/main/state_management/discover_app_state_container.test.ts
View file

@ -297,12 +297,14 @@ describe('Test discover app state container', () => {
const state = getStateContainer();
expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
columns: false,
hideChart: false,
rowHeight: false,
breakdownField: false,
});
state.initAndSync();
expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
columns: true,
hideChart: true,
rowHeight: true,
breakdownField: true,
});
@ -314,12 +316,14 @@ describe('Test discover app state container', () => {
const state = getStateContainer();
expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
columns: false,
hideChart: false,
rowHeight: false,
breakdownField: false,
});
state.initAndSync();
expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
columns: false,
hideChart: true,
rowHeight: true,
breakdownField: true,
});
@ -331,17 +335,38 @@ describe('Test discover app state container', () => {
const state = getStateContainer();
expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
columns: false,
hideChart: false,
rowHeight: false,
breakdownField: false,
});
state.initAndSync();
expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
columns: true,
hideChart: true,
rowHeight: false,
breakdownField: true,
});
});
it('should call setResetDefaultProfileState correctly with initial hide chart', () => {
const stateStorageGetSpy = jest.spyOn(stateStorage, 'get');
stateStorageGetSpy.mockReturnValue({ hideChart: true });
const state = getStateContainer();
expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
columns: false,
hideChart: false,
rowHeight: false,
breakdownField: false,
});
state.initAndSync();
expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
columns: true,
hideChart: false,
rowHeight: true,
breakdownField: true,
});
});
it('should call setResetDefaultProfileState correctly with saved search', () => {
const stateStorageGetSpy = jest.spyOn(stateStorage, 'get');
stateStorageGetSpy.mockReturnValue({ columns: ['test'], rowHeight: 5 });
@ -354,12 +379,14 @@ describe('Test discover app state container', () => {
const state = getStateContainer();
expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
columns: false,
hideChart: false,
rowHeight: false,
breakdownField: false,
});
state.initAndSync();
expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
columns: false,
hideChart: false,
rowHeight: false,
breakdownField: false,
});

6
src/platform/plugins/shared/discover/public/application/main/state_management/discover_app_state_container.ts
View file

@ -271,7 +271,10 @@ export const getDiscoverAppStateContainer = ({
// Set the default profile state only if not loading a saved search,
// to avoid overwriting saved search state
if (!currentSavedSearch.id) {
const { breakdownField, columns, rowHeight } = getCurrentUrlState(stateStorage, services);
const { breakdownField, columns, rowHeight, hideChart } = getCurrentUrlState(
stateStorage,
services
);
// Only set default state which is not already set in the URL
internalState.dispatch(
@ -280,6 +283,7 @@ export const getDiscoverAppStateContainer = ({
columns: columns === undefined,
rowHeight: rowHeight === undefined,
breakdownField: breakdownField === undefined,
hideChart: hideChart === undefined,
},
})
);

4
src/platform/plugins/shared/discover/public/application/main/state_management/discover_data_state_container.test.ts
View file

@ -183,6 +183,7 @@ describe('test getDataStateContainer', () => {
columns: true,
rowHeight: true,
breakdownField: true,
hideChart: false,
},
})
);
@ -200,6 +201,7 @@ describe('test getDataStateContainer', () => {
columns: false,
rowHeight: false,
breakdownField: false,
hideChart: false,
});
expect(stateContainer.appState.get().columns).toEqual(['message', 'extension']);
expect(stateContainer.appState.get().rowHeight).toEqual(3);
@ -225,6 +227,7 @@ describe('test getDataStateContainer', () => {
columns: false,
rowHeight: false,
breakdownField: false,
hideChart: false,
},
})
);
@ -240,6 +243,7 @@ describe('test getDataStateContainer', () => {
columns: false,
rowHeight: false,
breakdownField: false,
hideChart: false,
});
expect(stateContainer.appState.get().columns).toEqual(['default_column']);
expect(stateContainer.appState.get().rowHeight).toBeUndefined();

1
src/platform/plugins/shared/discover/public/application/main/state_management/discover_data_state_container.ts
View file

@ -359,6 +359,7 @@ export function getDataStateContainer({
columns: false,
rowHeight: false,
breakdownField: false,
hideChart: false,
},
})
);

1
src/platform/plugins/shared/discover/public/application/main/state_management/redux/internal_state.ts
View file

@ -55,6 +55,7 @@ export const defaultTabState: Omit<TabState, keyof TabItem> = {
columns: false,
rowHeight: false,
breakdownField: false,
hideChart: false,
},
documentsRequest: {
loadingStatus: LoadingStatus.Uninitialized,

1
src/platform/plugins/shared/discover/public/application/main/state_management/redux/types.ts
View file

@ -70,6 +70,7 @@ export interface TabState extends TabItem {
columns: boolean;
rowHeight: boolean;
breakdownField: boolean;
hideChart: boolean;
};
documentsRequest: DocumentsRequest;
totalHitsRequest: TotalHitsRequest;

8
src/platform/plugins/shared/discover/public/application/main/state_management/utils/build_esql_fetch_subscribe.test.ts
View file

@ -492,6 +492,7 @@ describe('buildEsqlFetchSubscribe', () => {
const documents$ = stateContainer.dataState.data$.documents$;
expect(omit(stateContainer.getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
columns: false,
hideChart: false,
rowHeight: false,
breakdownField: false,
});
@ -507,6 +508,7 @@ describe('buildEsqlFetchSubscribe', () => {
await waitFor(() =>
expect(omit(stateContainer.getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
columns: true,
hideChart: true,
rowHeight: true,
breakdownField: true,
})
@ -521,6 +523,7 @@ describe('buildEsqlFetchSubscribe', () => {
columns: false,
rowHeight: false,
breakdownField: false,
hideChart: false,
},
})
);
@ -534,6 +537,7 @@ describe('buildEsqlFetchSubscribe', () => {
columns: false,
rowHeight: false,
breakdownField: false,
hideChart: false,
})
);
documents$.next({
@ -550,6 +554,7 @@ describe('buildEsqlFetchSubscribe', () => {
columns: true,
rowHeight: true,
breakdownField: true,
hideChart: true,
})
);
documents$.next({
@ -567,6 +572,7 @@ describe('buildEsqlFetchSubscribe', () => {
columns: false,
rowHeight: false,
breakdownField: false,
hideChart: false,
});
documents$.next({
fetchStatus: FetchStatus.PARTIAL,
@ -578,6 +584,7 @@ describe('buildEsqlFetchSubscribe', () => {
columns: false,
rowHeight: false,
breakdownField: false,
hideChart: false,
})
);
documents$.next({
@ -590,6 +597,7 @@ describe('buildEsqlFetchSubscribe', () => {
columns: true,
rowHeight: false,
breakdownField: false,
hideChart: false,
})
);
});

2
src/platform/plugins/shared/discover/public/application/main/state_management/utils/build_esql_fetch_subscribe.ts
View file

@ -98,6 +98,7 @@ export const buildEsqlFetchSubscribe = ({
columns: true,
rowHeight: true,
breakdownField: true,
hideChart: true,
},
})
);
@ -158,6 +159,7 @@ export const buildEsqlFetchSubscribe = ({
columns: true,
rowHeight: false,
breakdownField: false,
hideChart: false,
},
})
);

2
src/platform/plugins/shared/discover/public/application/main/state_management/utils/change_data_view.test.ts
View file

@ -84,6 +84,7 @@ describe('changeDataView', () => {
columns: false,
rowHeight: false,
breakdownField: false,
hideChart: false,
})
);
await changeDataView({ dataViewId: dataViewComplexMock.id!, ...params });
@ -92,6 +93,7 @@ describe('changeDataView', () => {
columns: true,
rowHeight: true,
breakdownField: true,
hideChart: true,
})
);
});

1
src/platform/plugins/shared/discover/public/application/main/state_management/utils/change_data_view.ts
View file

@ -82,6 +82,7 @@ export async function changeDataView({
columns: true,
rowHeight: true,
breakdownField: true,
hideChart: true,
},
})
);

6
src/platform/plugins/shared/discover/public/application/main/state_management/utils/get_default_profile_state.test.ts
View file

@ -32,6 +32,7 @@ describe('getDefaultProfileState', () => {
columns: false,
rowHeight: false,
breakdownField: true,
hideChart: false,
},
dataView: dataViewWithTimefieldMock,
}).getPreFetchState();
@ -45,6 +46,7 @@ describe('getDefaultProfileState', () => {
columns: false,
rowHeight: false,
breakdownField: true,
hideChart: false,
},
dataView: emptyDataView,
}).getPreFetchState();
@ -61,6 +63,7 @@ describe('getDefaultProfileState', () => {
columns: true,
rowHeight: false,
breakdownField: false,
hideChart: false,
},
dataView: dataViewWithTimefieldMock,
}).getPostFetchState({
@ -87,6 +90,7 @@ describe('getDefaultProfileState', () => {
columns: true,
rowHeight: false,
breakdownField: false,
hideChart: false,
},
dataView: emptyDataView,
}).getPostFetchState({
@ -116,6 +120,7 @@ describe('getDefaultProfileState', () => {
columns: false,
rowHeight: true,
breakdownField: false,
hideChart: false,
},
dataView: dataViewWithTimefieldMock,
}).getPostFetchState({
@ -135,6 +140,7 @@ describe('getDefaultProfileState', () => {
columns: false,
rowHeight: false,
breakdownField: false,
hideChart: false,
},
dataView: dataViewWithTimefieldMock,
}).getPostFetchState({

4
src/platform/plugins/shared/discover/public/application/main/state_management/utils/get_default_profile_state.ts
View file

@ -44,6 +44,10 @@ export const getDefaultProfileState = ({
stateUpdate.breakdownField = defaultState.breakdownField;
}
if (defaultState.hideChart !== undefined) {
stateUpdate.hideChart = defaultState.hideChart;
}
return Object.keys(stateUpdate).length ? stateUpdate : undefined;
},

10
src/platform/plugins/shared/discover/public/context_awareness/profile_providers/common/patterns/index.ts Normal file
View file

@ -0,0 +1,10 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
export { createPatternDataSourceProfileProvider } from './profile';

135
src/platform/plugins/shared/discover/public/context_awareness/profile_providers/common/patterns/pattern_cell_renderer.tsx Normal file
View file

@ -0,0 +1,135 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import React, { useMemo } from 'react';
import type { FC } from 'react';
import type { UseEuiTheme } from '@elastic/eui';
import { EuiCode, EuiSpacer, EuiText } from '@elastic/eui';
import { css } from '@emotion/react';
import type { DataTableRecord } from '@kbn/discover-utils';
import { extractCategorizeTokens } from '@kbn/esql-utils';
import { FormattedMessage } from '@kbn/i18n-react';
import { useMemoCss } from '@kbn/css-utils/public/use_memo_css';
interface Props {
pattern: string;
isDetails: boolean;
defaultRowHeight?: number;
}
export const PatternCellRenderer: FC<Props> = ({ pattern, isDetails, defaultRowHeight }) => {
const styles = useMemoCss(componentStyles);
const keywords = useMemo(() => extractCategorizeTokens(pattern), [pattern]);
const containerStyle = useMemo(() => getContainerStyle(defaultRowHeight), [defaultRowHeight]);
const formattedTokens = useMemo(
() =>
keywords.map((keyword, index) => {
return (
<EuiCode key={index} css={styles.keyword}>
{keyword}
</EuiCode>
);
}),
[styles, keywords]
);
if (isDetails) {
return (
<div css={styles.detailsContainer}>
<EuiText size="s">
<strong>
<FormattedMessage
id="discover.contextAwareness.patternCellRenderer.tokensLabel"
defaultMessage="Tokens"
/>
</strong>
<EuiSpacer size="s" />
{formattedTokens}
</EuiText>
<EuiSpacer size="m" />
<EuiText size="s">
<strong>
<FormattedMessage
id="discover.contextAwareness.patternCellRenderer.regexLabel"
defaultMessage="Regex"
/>
</strong>
<EuiSpacer size="s" />
<span data-test-subj="euiDataGridExpansionPopover-patternRegex">{pattern}</span>
</EuiText>
<EuiSpacer size="s" />
</div>
);
}
return <div css={containerStyle}>{formattedTokens}</div>;
};
const componentStyles = {
keyword: ({ euiTheme }: UseEuiTheme) =>
css({
marginRight: euiTheme.size.xs,
marginBottom: `calc(${euiTheme.size.m} / 2)`,
display: 'inline-block',
padding: `${euiTheme.size.xxs} ${euiTheme.size.s}`,
backgroundColor: euiTheme.colors.lightestShade,
borderRadius: euiTheme.border.radius.small,
color: euiTheme.colors.textPrimary,
fontSize: euiTheme.size.m,
}),
detailsContainer: ({ euiTheme }: UseEuiTheme) =>
css({
maxWidth: '600px',
}),
};
export function getPatternCellRenderer(
row: DataTableRecord,
columnId: string,
isDetails: boolean,
defaultRowHeight?: number
) {
const pattern = row.flattened[columnId];
if (pattern === undefined) {
return '-';
}
return (
<PatternCellRenderer
pattern={String(pattern)}
isDetails={isDetails}
defaultRowHeight={defaultRowHeight}
/>
);
}
function getContainerStyle(defaultRowHeight?: number) {
// the keywords are slightly larger than the default text height,
// so they need to be adjusted to fit within the row height while
// not truncating the bottom of the text
let rowHeight = 2;
if (defaultRowHeight === undefined) {
rowHeight = 2;
} else if (defaultRowHeight < 2) {
rowHeight = 1;
} else {
rowHeight = Math.floor(defaultRowHeight / 1.5);
}
return {
display: '-webkit-box',
WebkitBoxOrient: 'vertical' as const,
WebkitLineClamp: rowHeight,
overflow: 'hidden',
textOverflow: 'ellipsis',
};
}

138
src/platform/plugins/shared/discover/public/context_awareness/profile_providers/common/patterns/profile.ts Normal file
View file

@ -0,0 +1,138 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { isOfAggregateQueryType } from '@kbn/es-query';
import { extractCategorizeTokens, getCategorizeColumns, getCategorizeField } from '@kbn/esql-utils';
import { i18n } from '@kbn/i18n';
import type { DataGridCellValueElementProps } from '@kbn/unified-data-table';
import { DataSourceType, isDataSourceType } from '../../../../../common/data_sources';
import type { DataSourceProfileProvider } from '../../../profiles';
import { DataSourceCategory } from '../../../profiles';
import { getPatternCellRenderer } from './pattern_cell_renderer';
import type { ProfileProviderServices } from '../../profile_provider_services';
const DOC_LIMIT = 10000;
export const createPatternDataSourceProfileProvider = (
services: ProfileProviderServices
): DataSourceProfileProvider<{
patternColumns: string[];
}> => ({
profileId: 'patterns-data-source-profile',
profile: {
getCellRenderers:
(prev, { context }) =>
(params) => {
const { rowHeight } = params;
const { patternColumns } = context;
if (!patternColumns || patternColumns.length === 0) {
return prev(params);
}
const patternRenderers = patternColumns.reduce(
(acc, column) =>
Object.assign(acc, {
[column]: (props: DataGridCellValueElementProps) =>
getPatternCellRenderer(props.row, props.columnId, props.isDetails, rowHeight),
}),
{}
);
return {
...prev(params),
...patternRenderers,
};
},
getAdditionalCellActions:
(prev, { context }) =>
() => {
return [
...prev(),
{
id: 'patterns-action-view-docs-in-discover',
getDisplayName: () =>
i18n.translate('discover.docViews.patterns.cellAction.viewResults', {
defaultMessage: 'View matching results',
}),
getIconType: () => 'discoverApp',
isCompatible: (compatibleContext) => {
const { query, field } = compatibleContext;
const { patternColumns } = context;
if (!isOfAggregateQueryType(query) || field === undefined) {
return false;
}
return patternColumns.includes(field.name);
},
execute: (executeContext) => {
const index = executeContext.dataView?.getIndexPattern();
if (
!isOfAggregateQueryType(executeContext.query) ||
!executeContext.value ||
!index
) {
return;
}
const pattern = extractCategorizeTokens(executeContext.value as string).join(' ');
const categoryField = getCategorizeField(executeContext.query.esql);
if (!categoryField || !pattern) {
return;
}
const query = {
...executeContext.query,
esql: `FROM ${index}\n | WHERE MATCH(${categoryField}, "${pattern}", {"auto_generate_synonyms_phrase_query": false, "fuzziness": 0, "operator": "AND"})\n | LIMIT ${DOC_LIMIT}`,
};
const discoverLink = services.locator.getRedirectUrl({
query,
timeRange: executeContext.timeRange,
hideChart: false,
});
window.open(discoverLink, '_blank');
},
},
];
},
getDefaultAppState: (prev) => (params) => {
return {
...prev(params),
hideChart: true,
columns: [
{ name: 'Count', width: 150 },
{ name: 'Pattern', width: undefined },
],
};
},
},
resolve: (params) => {
if (!isDataSourceType(params.dataSource, DataSourceType.Esql)) {
return { isMatch: false };
}
const query = params.query;
if (!isOfAggregateQueryType(query)) {
return { isMatch: false };
}
const patternColumns = getCategorizeColumns(query.esql);
if (patternColumns.length === 0) {
return { isMatch: false };
}
return {
isMatch: true,
context: {
category: DataSourceCategory.Default,
patternColumns,
},
};
},
});

2
src/platform/plugins/shared/discover/public/context_awareness/profile_providers/register_profile_providers.ts
View file

@ -29,6 +29,7 @@ import { createTracesDataSourceProfileProvider } from './observability/traces_da
import { createDeprecationLogsDataSourceProfileProvider } from './common/deprecation_logs';
import { createClassicNavRootProfileProvider } from './common/classic_nav_root_profile';
import { createObservabilityDocumentProfileProviders } from './observability/observability_profile_providers';
import { createPatternDataSourceProfileProvider } from './common/patterns';
import { createSecurityDocumentProfileProvider } from './security/security_document_profile';
/**
@ -147,6 +148,7 @@ const createRootProfileProviders = (providerServices: ProfileProviderServices) =
*/
const createDataSourceProfileProviders = (providerServices: ProfileProviderServices) => [
createExampleDataSourceProfileProvider(),
createPatternDataSourceProfileProvider(providerServices),
createDeprecationLogsDataSourceProfileProvider(),
createTracesDataSourceProfileProvider(providerServices),
...createObservabilityLogsDataSourceProfileProviders(providerServices),

4
src/platform/plugins/shared/discover/public/context_awareness/types.ts
View file

@ -140,6 +140,10 @@ export interface DefaultAppStateExtension {
* The field to apply for the histogram breakdown
*/
breakdownField?: string;
/**
* The state for chart visibility toggle
*/
hideChart?: boolean;
}
/**

38
src/platform/plugins/shared/unified_search/public/query_string_input/esql_menu_popover.test.tsx
View file

@ -12,6 +12,7 @@ import { BehaviorSubject } from 'rxjs';
import { screen, render, waitFor } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
import { createStubDataView, stubLogstashFieldSpecMap } from '@kbn/data-views-plugin/common/stubs';
import { stubIndexPattern } from '@kbn/data-plugin/public/stubs';
import { coreMock } from '@kbn/core/public/mocks';
import type { DataView } from '@kbn/data-views-plugin/common';
@ -95,6 +96,7 @@ describe('ESQLMenuPopover', () => {
await waitFor(() => {
expect(screen.getByText('Count of logs')).toBeInTheDocument();
expect(screen.getByText('Average bytes')).toBeInTheDocument();
expect(screen.getByText('Identify patterns')).toBeInTheDocument();
});
});
@ -113,4 +115,40 @@ describe('ESQLMenuPopover', () => {
await userEvent.click(screen.getByRole('button'));
expect(screen.queryByTestId('esql-recommended-queries')).toBeInTheDocument();
});
it('should show identify patterns recommended query', async () => {
const stubLogstashDataView = createStubDataView({
spec: {
id: 'logstash-*',
title: 'logstash-*',
timeFieldName: 'time',
fields: {
...stubLogstashFieldSpecMap,
message: {
name: 'message',
type: 'string',
esTypes: ['text'],
aggregatable: true,
searchable: true,
count: 0,
readFromDocValues: true,
scripted: false,
isMapped: true,
},
},
},
});
renderESQLPopover(stubLogstashDataView);
// open the popover and check for recommended queries
await userEvent.click(screen.getByRole('button'));
expect(screen.queryByTestId('esql-recommended-queries')).toBeInTheDocument();
// Open the nested section to see the recommended queries
await waitFor(() => userEvent.click(screen.getByTestId('esql-recommended-queries')));
await waitFor(() => {
expect(screen.getByText('Identify patterns')).toBeInTheDocument();
});
});
});

14
src/platform/plugins/shared/unified_search/public/query_string_input/esql_menu_popover.tsx
View file

@ -28,6 +28,7 @@ import {
getRecommendedQueriesTemplatesFromExtensions,
} from '@kbn/esql-validation-autocomplete';
import { LanguageDocumentationFlyout } from '@kbn/language-documentation';
import { getCategorizationField } from '@kbn/aiops-utils';
import type { IUnifiedSearchPluginServices } from '../types';
export interface ESQLMenuPopoverProps {
@ -52,17 +53,25 @@ export const ESQLMenuPopover: React.FC<ESQLMenuPopoverProps> = ({
RecommendedQuery[]
>([]);
const { queryForRecommendedQueries, timeFieldName } = useMemo(() => {
const { queryForRecommendedQueries, timeFieldName, categorizationField } = useMemo(() => {
if (adHocDataview && typeof adHocDataview !== 'string') {
const textFields = adHocDataview.fields?.getByType('string') ?? [];
let tempCategorizationField;
if (textFields.length) {
tempCategorizationField = getCategorizationField(textFields.map((field) => field.name));
}
return {
queryForRecommendedQueries: `FROM ${adHocDataview.name}`,
timeFieldName:
adHocDataview.timeFieldName ?? adHocDataview.fields?.getByType('date')?.[0]?.name,
categorizationField: tempCategorizationField,
};
}
return {
queryForRecommendedQueries: '',
timeFieldName: undefined,
categorizationField: undefined,
};
}, [adHocDataview]);
@ -130,11 +139,13 @@ export const ESQLMenuPopover: React.FC<ESQLMenuPopoverProps> = ({
}))
);
}
// Handle the static recommended queries, no solutions specific
if (queryForRecommendedQueries && timeFieldName) {
const recommendedQueriesFromStaticTemplates = getRecommendedQueries({
fromCommand: queryForRecommendedQueries,
timeField: timeFieldName,
categorizationField,
});
recommendedQueries.push(...recommendedQueriesFromStaticTemplates);
@ -239,6 +250,7 @@ export const ESQLMenuPopover: React.FC<ESQLMenuPopoverProps> = ({
timeFieldName,
toggleLanguageComponent,
solutionsRecommendedQueries, // This dependency is fine here, as it *uses* the state
categorizationField,
]);
const esqlMenuPopoverStyles = css`

3
src/platform/plugins/shared/unified_search/tsconfig.json
View file

@ -50,7 +50,8 @@
"@kbn/react-kibana-mount",
"@kbn/field-utils",
"@kbn/language-documentation",
"@kbn/esql-types"
"@kbn/esql-types",
"@kbn/aiops-utils",
],
"exclude": [
"target/**/*",

2
tsconfig.base.json
View file

@ -40,6 +40,8 @@
"@kbn/aiops-plugin/*": ["x-pack/platform/plugins/shared/aiops/*"],
"@kbn/aiops-test-utils": ["x-pack/platform/packages/private/ml/aiops_test_utils"],
"@kbn/aiops-test-utils/*": ["x-pack/platform/packages/private/ml/aiops_test_utils/*"],
"@kbn/aiops-utils": ["src/platform/packages/shared/kbn-aiops-utils"],
"@kbn/aiops-utils/*": ["src/platform/packages/shared/kbn-aiops-utils/*"],
"@kbn/alerting-api-integration-helpers": ["x-pack/platform/test/alerting_api_integration/packages/helpers"],
"@kbn/alerting-api-integration-helpers/*": ["x-pack/platform/test/alerting_api_integration/packages/helpers/*"],
"@kbn/alerting-api-integration-test-plugin": ["x-pack/platform/test/alerting_api_integration/common/plugins/alerts"],

5
x-pack/platform/plugins/shared/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_discover.tsx
View file

@ -25,6 +25,7 @@ import { css } from '@emotion/react';
import { useTableState } from '@kbn/ml-in-memory-table/hooks/use_table_state';
import useMountedState from 'react-use/lib/useMountedState';
import { getEsQueryConfig } from '@kbn/data-service';
import { getCategorizationDataViewField } from '@kbn/aiops-utils';
import {
type LogCategorizationPageUrlState,
getDefaultLogCategorizationAppState,
@ -42,7 +43,7 @@ import { useValidateFieldRequest } from '../use_validate_category_field';
import { FieldValidationCallout } from '../category_validation_callout';
import { useMinimumTimeRange } from './use_minimum_time_range';
import { createAdditionalConfigHash, createDocumentStatsHash, getMessageField } from '../utils';
import { createAdditionalConfigHash, createDocumentStatsHash } from '../utils';
import { DiscoverTabs } from './discover_tabs';
import { useRandomSamplerStorage } from '../sampling_menu';
import { useActions } from '../category_table/use_actions';
@ -126,7 +127,7 @@ export const LogCategorizationDiscover: FC<LogCategorizationEmbeddableProps> = (
setCurrentDocumentStatsHash(null);
setSelectedField(null);
setLoading(null);
const { dataViewFields, messageField } = getMessageField(dataView);
const { dataViewFields, messageField } = getCategorizationDataViewField(dataView);
setFields(dataViewFields);
setSelectedField(messageField);
},

34
x-pack/platform/plugins/shared/aiops/public/components/log_categorization/utils.ts
View file

@ -6,8 +6,6 @@
*/
import { stringHash } from '@kbn/ml-string-hash';
import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
import { ES_FIELD_TYPES } from '@kbn/field-types';
import type { DocumentStats } from '../../hooks/use_document_count_stats';
/**
@ -26,35 +24,3 @@ export function createDocumentStatsHash(documentStats: DocumentStats) {
export function createAdditionalConfigHash(additionalStrings: string[] = []) {
return stringHash(`${additionalStrings.join('')}`);
}
/**
* Retrieves the message field from a DataView object.
* If the message field is not found, it falls back to error.message or event.original or the first text field in the DataView.
*
* @param dataView - The DataView object containing the fields.
* @returns An object containing the message field and all the fields in the DataView.
*/
export function getMessageField(dataView: DataView): {
messageField: DataViewField | null;
dataViewFields: DataViewField[];
} {
const dataViewFields = dataView.fields.filter((f) => f.esTypes?.includes(ES_FIELD_TYPES.TEXT));
let messageField: DataViewField | null | undefined = dataViewFields.find(
(f) => f.name === 'message'
);
if (messageField === undefined) {
messageField = dataViewFields.find((f) => f.name === 'error.message');
}
if (messageField === undefined) {
messageField = dataViewFields.find((f) => f.name === 'event.original');
}
if (messageField === undefined) {
if (dataViewFields.length > 0) {
messageField = dataViewFields[0];
} else {
messageField = null;
}
}
return { messageField, dataViewFields };
}

5
x-pack/platform/plugins/shared/aiops/public/embeddables/pattern_analysis/pattern_analysis_initializer.tsx
View file

@ -29,6 +29,7 @@ import { isPopulatedObject } from '@kbn/ml-is-populated-object';
import useObservable from 'react-use/lib/useObservable';
import type { DataViewField } from '@kbn/data-views-plugin/public';
import useMountedState from 'react-use/lib/useMountedState';
import { getCategorizationDataViewField } from '@kbn/aiops-utils';
import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
import { DataSourceContextProvider } from '../../hooks/use_data_source';
import type { PatternAnalysisEmbeddableRuntimeState } from './types';
@ -43,7 +44,7 @@ import {
DEFAULT_MINIMUM_TIME_RANGE_OPTION,
type MinimumTimeRangeOption,
} from '../../components/log_categorization/log_categorization_for_embeddable/minimum_time_range';
import { getMessageField } from '../../components/log_categorization/utils';
import { FieldSelector } from '../../components/log_categorization/log_categorization_for_embeddable/field_selector';
import { SamplingPanel } from '../../components/log_categorization/sampling_menu/sampling_panel';
@ -272,7 +273,7 @@ export const FormControls: FC<{
setSelectedField(null);
return;
}
const { dataViewFields, messageField } = getMessageField(dataView);
const { dataViewFields, messageField } = getCategorizationDataViewField(dataView);
setFields(dataViewFields);
const field = dataViewFields.find((f) => f.name === formInput.fieldName);
if (formInput.fieldName === undefined) {

1
x-pack/platform/plugins/shared/aiops/tsconfig.json
View file

@ -77,6 +77,7 @@
"@kbn/apm-utils",
"@kbn/ml-field-stats-flyout",
"@kbn/config-schema",
"@kbn/aiops-utils",
],
"exclude": [
"target/**/*",

1
x-pack/test/functional/apps/aiops/index.ts
View file

@ -38,5 +38,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
loadTestFile(require.resolve('./change_point_detection_cases'));
loadTestFile(require.resolve('./log_pattern_analysis'));
loadTestFile(require.resolve('./log_pattern_analysis_in_discover'));
loadTestFile(require.resolve('./log_pattern_analysis_esql_in_discover'));
});
}

121
x-pack/test/functional/apps/aiops/log_pattern_analysis_esql_in_discover.ts Normal file
View file

@ -0,0 +1,121 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
export default function ({ getPageObjects, getService }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const kibanaServer = getService('kibanaServer');
const security = getService('security');
const dataGrid = getService('dataGrid');
const monacoEditor = getService('monacoEditor');
const aiops = getService('aiops');
const browser = getService('browser');
const retry = getService('retry');
const testSubjects = getService('testSubjects');
const { common, discover, header, timePicker, unifiedFieldList } = getPageObjects([
'common',
'discover',
'header',
'timePicker',
'unifiedFieldList',
]);
const defaultSettings = {
defaultIndex: 'logstash-*',
};
async function retrySwitchTab(tabIndex: number, seconds: number) {
await retry.tryForTime(seconds * 1000, async () => {
await browser.switchTab(tabIndex);
});
}
describe('log pattern analysis ES|QL in discover', function () {
before(async () => {
await kibanaServer.savedObjects.cleanStandardList();
await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']);
await kibanaServer.importExport.load(
'src/platform/test/functional/fixtures/kbn_archiver/discover'
);
await esArchiver.loadIfNeeded(
'src/platform/test/functional/fixtures/es_archiver/logstash_functional'
);
await kibanaServer.uiSettings.replace(defaultSettings);
await common.navigateToApp('discover');
await timePicker.setDefaultAbsoluteRange();
await discover.waitUntilSearchingHasFinished();
await discover.selectTextBaseLang();
await discover.waitUntilSearchingHasFinished();
});
beforeEach(async () => {
await discover.clickNewSearchButton();
await header.waitUntilLoadingHasFinished();
await discover.waitUntilSearchingHasFinished();
});
let tabsCount = 1;
afterEach(async () => {
if (tabsCount > 1) {
await browser.closeCurrentWindow();
await retrySwitchTab(0, 10);
tabsCount--;
}
});
it('should render categorize fields correctly', async () => {
// set query to be categorize command
// ensure the columns are correct
await monacoEditor.setCodeEditorValue(
'from logstash-*\n | STATS Count=COUNT(*) BY Pattern=CATEGORIZE(@message)\n | SORT Count DESC'
);
await testSubjects.click('querySubmitButton');
const columns = ['Count', 'Pattern'];
await unifiedFieldList.clickFieldListItemAdd('Count');
await unifiedFieldList.clickFieldListItemAdd('Pattern');
await header.waitUntilLoadingHasFinished();
await discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
// refresh the table and ensure the columns are still correct
await browser.refresh();
await header.waitUntilLoadingHasFinished();
await discover.waitUntilSearchingHasFinished();
expect(await dataGrid.getHeaderFields()).to.eql(columns);
// get the first pattern cell and ensure it has the correct number of tokens
const cell = await dataGrid.getCellElement(0, 3);
const tokens = await cell.findAllByCssSelector('.unifiedDataTable__cellValue code');
expect(tokens.length).to.be.greaterThan(0);
// click the expand action button in the cell
// ensure the popover is displayed and the regex is correct
await dataGrid.clickCellExpandButton(0, { columnIndex: 3, columnName: 'Pattern' });
const regexElement = await testSubjects.find('euiDataGridExpansionPopover-patternRegex');
const regexText = await regexElement.getVisibleText();
// The pattern might change based on the data, so rather than checking for an exact match,
// we check that the string contains part of a regex
expect(regexText).to.contain('.*?');
// Click the view docs in Discover button
// ensure it opens a new tab and the discover doc count is greater than 0
// We cannot look for an exact count as the count may vary based on sampling (when that is added)
await dataGrid.clickCellExpandPopoverAction('patterns-action-view-docs-in-discover');
await retrySwitchTab(1, 10);
tabsCount++;
await aiops.logPatternAnalysisPage.assertDiscoverDocCountExists();
// ensure the discover doc count is greater than 0
await aiops.logPatternAnalysisPage.assertDiscoverDocCountGreaterThan(0);
});
});
}

2
x-pack/test/functional/apps/aiops/log_pattern_analysis_in_discover.ts
View file

@ -22,7 +22,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
});
}
describe('log pattern analysis', function () {
describe('log pattern analysis in discover', function () {
let tabsCount = 1;
afterEach(async () => {

4
yarn.lock
View file

@ -3849,6 +3849,10 @@
version "0.0.0"
uid ""
"@kbn/aiops-utils@link:src/platform/packages/shared/kbn-aiops-utils":
version "0.0.0"
uid ""
"@kbn/alerting-api-integration-helpers@link:x-pack/platform/test/alerting_api_integration/packages/helpers":
version "0.0.0"
uid ""