ES|QL pattern formatting (#222871)

Adds a recommended query for the `CATEGORIZE` function in ES|QL.
Adds keyword highlighting for the patterns and the ability to open a new
Discover tab to filter for docs which match the selected pattern.


https://github.com/user-attachments/assets/9ed8c5b0-7e92-4cc8-88dd-cb7749b5ffd3

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
Co-authored-by: Stratoula Kalafateli <efstratia.kalafateli@elastic.co>

.github/CODEOWNERS

@@ -410,6 +410,7 @@ src/platform/packages/shared/home/sample_data_card @elastic/appex-sharedux
 src/platform/packages/shared/home/sample_data_tab @elastic/appex-sharedux
 src/platform/packages/shared/home/sample_data_types @elastic/appex-sharedux
 src/platform/packages/shared/kbn-actions-types @elastic/response-ops
+src/platform/packages/shared/kbn-aiops-utils @elastic/ml-ui
 src/platform/packages/shared/kbn-alerting-types @elastic/response-ops
 src/platform/packages/shared/kbn-alerts-as-data-utils @elastic/response-ops
 src/platform/packages/shared/kbn-alerts-ui-shared @elastic/response-ops
@@ -1220,6 +1221,7 @@ x-pack/test_serverless/api_integration/test_suites/common/platform_security @ela
 /x-pack/test_serverless/functional/test_suites/common/examples/search_examples @elastic/kibana-data-discovery
 /x-pack/test_serverless/functional/test_suites/common/examples/unified_field_list_examples @elastic/kibana-data-discovery
 /x-pack/test_serverless/functional/test_suites/common/management/data_views @elastic/kibana-data-discovery
+src/platform/plugins/shared/discover/public/context_awareness/profile_providers/common/patterns @elastic/ml-ui
 src/platform/plugins/shared/discover/public/context_awareness/profile_providers/security @elastic/kibana-data-discovery @elastic/security-threat-hunting-investigations
 # TODO: this deprecation_logs folder should be owned by kibana management team after 9.0
 src/platform/plugins/shared/discover/public/context_awareness/profile_providers/common/deprecation_logs @elastic/kibana-data-discovery @elastic/kibana-core
@@ -2836,11 +2838,11 @@ src/platform/testfunctional/page_objects/solution_navigation.ts @elastic/appex-s
 /x-pack/test/api_integration/deployment_agnostic/apis/intercepts/*.ts @elastic/appex-sharedux
 
 # OpenAPI spec files
-oas_docs/linters                              @elastic/core-docs @elastic/experience-docs 
+oas_docs/linters                              @elastic/core-docs @elastic/experience-docs
-oas_docs/overlays                             @elastic/core-docs @elastic/experience-docs 
+oas_docs/overlays                             @elastic/core-docs @elastic/experience-docs
-oas_docs/kibana.info.serverless.yaml          @elastic/core-docs @elastic/experience-docs 
+oas_docs/kibana.info.serverless.yaml          @elastic/core-docs @elastic/experience-docs
-oas_docs/kibana.info.yaml                     @elastic/core-docs @elastic/experience-docs 
+oas_docs/kibana.info.yaml                     @elastic/core-docs @elastic/experience-docs
-oas_docs/output                               @elastic/core-docs @elastic/experience-docs 
+oas_docs/output                               @elastic/core-docs @elastic/experience-docs
 
 # Documentation settings files
 docs/settings-gen                             @elastic/platform-docs @elastic/experience-docs

package.json

@@ -183,6 +183,7 @@
     "@kbn/aiops-log-rate-analysis": "link:x-pack/platform/packages/shared/ml/aiops_log_rate_analysis",
     "@kbn/aiops-plugin": "link:x-pack/platform/plugins/shared/aiops",
     "@kbn/aiops-test-utils": "link:x-pack/platform/packages/private/ml/aiops_test_utils",
+    "@kbn/aiops-utils": "link:src/platform/packages/shared/kbn-aiops-utils",
     "@kbn/alerting-api-integration-test-plugin": "link:x-pack/platform/test/alerting_api_integration/common/plugins/alerts",
     "@kbn/alerting-comparators": "link:x-pack/platform/packages/shared/kbn-alerting-comparators",
     "@kbn/alerting-example-plugin": "link:x-pack/examples/alerting_example",

src/platform/packages/shared/kbn-aiops-utils/README.md

@@ -0,0 +1,3 @@
+# @kbn/aiops-utils
+
+AIOps utils for plugins which are not in x-pack

src/platform/packages/shared/kbn-aiops-utils/index.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export {
+  getCategorizationDataViewField,
+  getCategorizationField,
+} from './src/get_categorization_field';

src/platform/packages/shared/kbn-aiops-utils/jest.config.js

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../../..',
+  roots: ['<rootDir>/src/platform/packages/shared/kbn-aiops-utils'],
+};

src/platform/packages/shared/kbn-aiops-utils/kibana.jsonc

@@ -0,0 +1,7 @@
+{
+  "type": "shared-common",
+  "id": "@kbn/aiops-utils",
+  "owner": "@elastic/ml-ui",
+  "group": "platform",
+  "visibility": "shared"
+}

src/platform/packages/shared/kbn-aiops-utils/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "@kbn/aiops-utils",
+  "private": true,
+  "version": "1.0.0",
+  "license": "Elastic License 2.0 OR AGPL-3.0-only OR SSPL-1.0"
+}

src/platform/packages/shared/kbn-aiops-utils/src/get_categorization_field.test.ts

@@ -0,0 +1,226 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { createStubDataView } from '@kbn/data-views-plugin/common/stubs';
+import { getCategorizationField } from './get_categorization_field';
+import { getCategorizationDataViewField } from './get_categorization_field';
+
+describe('get_categorization_field utils', () => {
+  describe('getCategorizationField', () => {
+    it('returns "message" if present', () => {
+      const fields = ['foo', 'bar', 'message', 'baz'];
+      expect(getCategorizationField(fields)).toBe('message');
+    });
+
+    it('returns "error.message" if present and "message" is not', () => {
+      const fields = ['foo', 'error.message', 'baz'];
+      expect(getCategorizationField(fields)).toBe('error.message');
+    });
+
+    it('returns "event.original" if present and others are not', () => {
+      const fields = ['event.original', 'foo', 'bar'];
+      expect(getCategorizationField(fields)).toBe('event.original');
+    });
+
+    it('returns first field if none of the priority fields are present', () => {
+      const fields = ['foo', 'bar', 'baz'];
+      expect(getCategorizationField(fields)).toBe('foo');
+    });
+
+    it('returns first field, skipping meta data fields, if none of the priority fields are present', () => {
+      const fields = ['_id', 'foo', 'bar', 'baz'];
+      expect(getCategorizationField(fields)).toBe('foo');
+    });
+
+    it('returns the first matching field according to priority', () => {
+      const fields = ['event.original', 'error.message', 'message'];
+      expect(getCategorizationField(fields)).toBe('message');
+    });
+
+    it('handles empty array', () => {
+      expect(getCategorizationField([])).toBeUndefined();
+    });
+  });
+
+  describe('getCategorizationDataViewField', () => {
+    it('returns messageField as "message" if present', () => {
+      const dataView = createStubDataView({
+        spec: {
+          id: 'test',
+          fields: {
+            foo: {
+              searchable: false,
+              aggregatable: false,
+              name: 'foo',
+              type: 'type',
+              esTypes: ['keyword'],
+            },
+            message: {
+              searchable: true,
+              aggregatable: true,
+              name: 'message',
+              type: 'text',
+              esTypes: ['text'],
+            },
+            'error.message': {
+              searchable: true,
+              aggregatable: true,
+              name: 'error.message',
+              type: 'text',
+              esTypes: ['text'],
+            },
+          },
+        },
+      });
+      const result = getCategorizationDataViewField(dataView);
+      expect(result.messageField?.name).toBe('message');
+      expect(result.dataViewFields.map((f) => f.name)).toContain('message');
+      expect(result.dataViewFields.length).toBe(2);
+    });
+    it('returns messageField as "error.message" if "message" is not present', () => {
+      const dataView = createStubDataView({
+        spec: {
+          id: 'test2',
+          fields: {
+            foo: {
+              searchable: false,
+              aggregatable: false,
+              name: 'foo',
+              type: 'type',
+              esTypes: ['keyword'],
+            },
+            'error.message': {
+              searchable: true,
+              aggregatable: true,
+              name: 'error.message',
+              type: 'text',
+              esTypes: ['text'],
+            },
+            bar: {
+              searchable: true,
+              aggregatable: true,
+              name: 'bar',
+              type: 'text',
+              esTypes: ['text'],
+            },
+          },
+        },
+      });
+      const result = getCategorizationDataViewField(dataView);
+      expect(result.messageField?.name).toBe('error.message');
+      expect(result.dataViewFields.map((f) => f.name)).toContain('error.message');
+      expect(result.dataViewFields.length).toBe(2);
+    });
+
+    it('handles empty fields array', () => {
+      const dataView = createStubDataView({
+        spec: {
+          id: 'test6',
+          fields: {},
+        },
+      });
+      const result = getCategorizationDataViewField(dataView);
+      expect(result.messageField).toBeNull();
+      expect(result.dataViewFields.length).toBe(0);
+    });
+
+    it('returns the first matching field according to priority', () => {
+      const dataView = createStubDataView({
+        spec: {
+          id: 'test7',
+          fields: {
+            'event.original': {
+              searchable: true,
+              aggregatable: true,
+              name: 'event.original',
+              type: 'text',
+              esTypes: ['text'],
+            },
+            'error.message': {
+              searchable: true,
+              aggregatable: true,
+              name: 'error.message',
+              type: 'text',
+              esTypes: ['text'],
+            },
+            message: {
+              searchable: true,
+              aggregatable: true,
+              name: 'message',
+              type: 'text',
+              esTypes: ['text'],
+            },
+          },
+        },
+      });
+      const result = getCategorizationDataViewField(dataView);
+      expect(result.messageField?.name).toBe('message');
+      expect(result.dataViewFields.map((f) => f.name)).toEqual(
+        expect.arrayContaining(['message', 'error.message', 'event.original'])
+      );
+      expect(result.dataViewFields.length).toBe(3);
+    });
+    it('returns the first field if no priority fields are present', () => {
+      const dataView = createStubDataView({
+        spec: {
+          id: 'test8',
+          fields: {
+            foo: {
+              searchable: true,
+              aggregatable: true,
+              name: 'foo',
+              type: 'text',
+              esTypes: ['text'],
+            },
+            bar: {
+              searchable: true,
+              aggregatable: true,
+              name: 'bar',
+              type: 'text',
+              esTypes: ['text'],
+            },
+          },
+        },
+      });
+      const result = getCategorizationDataViewField(dataView);
+      expect(result.messageField?.name).toBe('foo');
+      expect(result.dataViewFields.map((f) => f.name)).toEqual(
+        expect.arrayContaining(['foo', 'bar'])
+      );
+      expect(result.dataViewFields.length).toBe(2);
+    });
+    it('returns null messageField if no text fields are present', () => {
+      const dataView = createStubDataView({
+        spec: {
+          id: 'test9',
+          fields: {
+            foo: {
+              searchable: true,
+              aggregatable: true,
+              name: 'foo',
+              type: 'keyword',
+              esTypes: ['keyword'],
+            },
+            bar: {
+              searchable: true,
+              aggregatable: true,
+              name: 'bar',
+              type: 'keyword',
+              esTypes: ['keyword'],
+            },
+          },
+        },
+      });
+      const result = getCategorizationDataViewField(dataView);
+      expect(result.messageField).toBeNull();
+      expect(result.dataViewFields.map((f) => f.name)).toEqual(expect.arrayContaining([]));
+      expect(result.dataViewFields.length).toBe(0);
+    });
+  });
+});

src/platform/packages/shared/kbn-aiops-utils/src/get_categorization_field.ts

@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
+import { ES_FIELD_TYPES } from '@kbn/field-types';
+
+const FIELD_PRIORITY = ['message', 'error.message', 'event.original'];
+
+const METADATA_FIELDS = [
+  '_version',
+  '_id',
+  '_index',
+  '_source',
+  '_ignored',
+  '_index_mode',
+  '_score',
+];
+
+/**
+ * This function returns the categorization field from the list of fields.
+ * It checks for the presence of 'message', 'error.message', or 'event.original' in that order.
+ * If none of these fields are present, it returns the first field from the list,
+ * Assumes text fields have been passed in the `fields` array.
+ *
+ * @param fields, the list of fields to check
+ * @returns string | undefined, the categorization field if found, otherwise undefined
+ */
+export function getCategorizationField(fields: string[]): string | undefined {
+  const fieldSet = new Set(fields);
+  for (const field of FIELD_PRIORITY) {
+    if (fieldSet.has(field)) {
+      return field;
+    }
+  }
+
+  // Filter out metadata fields
+  const filteredFields = fields.filter((field) => !METADATA_FIELDS.includes(field));
+  return filteredFields[0] ?? undefined;
+}
+
+/**
+ * This function returns the categorization field from the DataView.
+ * It checks for the presence of 'message', 'error.message', or 'event.original' in that order.
+ * If none of these fields are present, it returns the first text field from the DataView.
+ *
+ * @param dataView, the DataView to check
+ * @returns an object containing the message field DataViewField and dataViewFields
+ */
+export function getCategorizationDataViewField(dataView: DataView): {
+  messageField: DataViewField | null;
+  dataViewFields: DataViewField[];
+} {
+  const dataViewFields = dataView.fields.filter((f) => f.esTypes?.includes(ES_FIELD_TYPES.TEXT));
+  const categorizationFieldName = getCategorizationField(dataViewFields.map((f) => f.name));
+  if (categorizationFieldName) {
+    return {
+      messageField: dataView.fields.getByName(categorizationFieldName) ?? null,
+      dataViewFields,
+    };
+  }
+
+  return {
+    messageField: null,
+    dataViewFields,
+  };
+}

src/platform/packages/shared/kbn-aiops-utils/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "extends": "../../../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "target/types",
+    "types": [
+      "jest",
+      "node",
+      "react"
+    ]
+  },
+  "include": [
+    "**/*.ts",
+    "**/*.tsx",
+  ],
+  "exclude": [
+    "target/**/*"
+  ],
+  "kbn_references": [
+    "@kbn/data-views-plugin",
+    "@kbn/field-types",
+  ]
+}

src/platform/packages/shared/kbn-esql-utils/index.ts

@@ -44,6 +44,7 @@ export {
   getCategorizeColumns,
   extractCategorizeTokens,
   getArgsFromRenameFunction,
+  getCategorizeField,
 } from './src';
 
 export { ENABLE_ESQL, FEEDBACK_LINK } from './constants';

src/platform/packages/shared/kbn-esql-utils/src/index.ts

@@ -26,6 +26,7 @@ export {
   fixESQLQueryWithVariables,
   getCategorizeColumns,
   getArgsFromRenameFunction,
+  getCategorizeField,
 } from './utils/query_parsing_helpers';
 export { queryCannotBeSampled } from './utils/query_cannot_be_sampled';
 export {

src/platform/packages/shared/kbn-esql-utils/src/utils/extract_categorize_tokens.test.ts

@@ -48,5 +48,6 @@ describe('extractCategorizeTokens()', () => {
     expect(extractCategorizeTokens(regexString4)).toStrictEqual(['someString']);
     const regexString5 = 'justAString'; // No '.*?' or '.+?'
     expect(extractCategorizeTokens(regexString5)).toStrictEqual(['justAString']);
+    expect(extractCategorizeTokens('.*?foo\\.bar.*?')).toEqual(['foo.bar']);
   });
 });

src/platform/packages/shared/kbn-esql-utils/src/utils/extract_categorize_tokens.ts

@@ -17,6 +17,9 @@
 export function extractCategorizeTokens(regexString: string) {
   let cleanedString = regexString;
 
+  // Remove backslashes
+  cleanedString = cleanedString.replace(/\\/g, '');
+
   // Strip leading '.*?'
   if (cleanedString.startsWith('.*?')) {
     cleanedString = cleanedString.substring('.*?'.length);

src/platform/packages/shared/kbn-esql-utils/src/utils/query_parsing_helpers.test.ts

@@ -23,6 +23,7 @@ import {
   fixESQLQueryWithVariables,
   getCategorizeColumns,
   getArgsFromRenameFunction,
+  getCategorizeField,
 } from './query_parsing_helpers';
 import { monaco } from '@kbn/monaco';
 import { parse, walk } from '@kbn/esql-ast';
@@ -815,4 +816,43 @@ describe('esql query helpers', () => {
       });
     });
   });
+
+  describe('getCategorizeField', () => {
+    it('should return the field used in categorize', () => {
+      const esql = 'FROM index | STATS COUNT() BY categorize(field1)';
+      const expected = ['field1'];
+      expect(getCategorizeField(esql)).toEqual(expected);
+    });
+
+    it('should return the field used in categorize for multiple breakdowns', () => {
+      const esql = 'FROM index | STATS COUNT() BY categorize(field1), field2';
+      const expected = ['field1'];
+      expect(getCategorizeField(esql)).toEqual(expected);
+    });
+
+    it('should return the field used in categorize for multiple breakdowns with BUCKET', () => {
+      const esql =
+        'FROM index | STATS count_per_day = COUNT() BY Pattern=CATEGORIZE(message), @timestamp=BUCKET(@timestamp, 1 day)';
+      const expected = ['message'];
+      expect(getCategorizeField(esql)).toEqual(expected);
+    });
+
+    it('should return the field used in categorize if the result is stored in a new column', () => {
+      const esql = 'FROM index | STATS COUNT() BY pattern = categorize(field1)';
+      const expected = ['field1'];
+      expect(getCategorizeField(esql)).toEqual(expected);
+    });
+
+    it('should return the field used in categorize for a complex query', () => {
+      const esql =
+        'FROM index | STATS count_per_day = COUNT() BY Pattern=CATEGORIZE(message), @timestamp=BUCKET(@timestamp, 1 day)  | STATS Count=SUM(count_per_day), Trend=VALUES(count_per_day) BY Pattern';
+      const expected = ['message'];
+      expect(getCategorizeField(esql)).toEqual(expected);
+    });
+    it('should return an empty array if no categorize is present', () => {
+      const esql = 'FROM index | STATS COUNT() BY field1';
+      const expected: string[] = [];
+      expect(getCategorizeField(esql)).toEqual(expected);
+    });
+  });
 });

src/platform/packages/shared/kbn-esql-utils/src/utils/query_parsing_helpers.ts

@@ -362,3 +362,55 @@ export const getArgsFromRenameFunction = (
     renamed: renameFunction.args[0] as ESQLColumn,
   };
 };
+
+/**
+ * Extracts the fields used in the CATEGORIZE function from an ESQL query.
+ * @param esql: string - The ESQL query string
+ */
+export const getCategorizeField = (esql: string): string[] => {
+  const { root } = parse(esql);
+  const statsCommand = root.commands.find(({ name }) => name === 'stats');
+  if (!statsCommand) {
+    return [];
+  }
+  const options: ESQLCommandOption[] = [];
+  const columns: string[] = [];
+
+  walk(statsCommand, {
+    visitCommandOption: (node) => options.push(node),
+  });
+
+  const statsByOptions = options.find(({ name }) => name === 'by');
+
+  // categorize is part of the stats by command
+  if (!statsByOptions) {
+    return [];
+  }
+
+  const categorizeOptions = statsByOptions.args.filter((arg) => {
+    return (arg as ESQLFunction).text.toLowerCase().indexOf('categorize') !== -1;
+  }) as ESQLFunction[];
+
+  if (categorizeOptions.length) {
+    categorizeOptions.forEach((arg) => {
+      // ... STATS ... BY CATEGORIZE(field)
+      if (isFunctionExpression(arg) && arg.name === 'categorize') {
+        walk(arg, {
+          visitColumn: (node) => columns.push(node.name),
+        });
+      } else {
+        // ... STATS ... BY pattern = CATEGORIZE(field)
+        walk(arg, {
+          visitFunction: (node) => {
+            if (node.name === 'categorize') {
+              const columnArgs = node.args.filter((a) => isColumn(a));
+              columnArgs.forEach((c) => columns.push((c as ESQLColumn).name));
+            }
+          },
+        });
+      }
+    });
+  }
+
+  return columns;
+};

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/__tests__/autocomplete.command.from.test.ts

@@ -110,6 +110,7 @@ describe('autocomplete.suggest', () => {
         const recommendedQueries = getRecommendedQueries({
           fromCommand: '',
           timeField: 'dateField',
+          categorizationField: 'textField',
         });
         const { assertSuggestions } = await setup();
         const expected = [

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/autocomplete.test.ts

@@ -38,10 +38,15 @@ const commandDefinitions = unmodifiedCommandDefinitions.filter(
   ({ name, hidden }) => !hidden && name !== 'rrf'
 );
 
-const getRecommendedQueriesSuggestionsFromTemplates = (fromCommand: string, timeField?: string) =>
+const getRecommendedQueriesSuggestionsFromTemplates = (
+  fromCommand: string,
+  timeField?: string,
+  categorizationField?: string
+) =>
   getRecommendedQueries({
     fromCommand,
     timeField,
+    categorizationField,
   });
 
 describe('autocomplete', () => {
@@ -91,7 +96,8 @@ describe('autocomplete', () => {
   describe('New command', () => {
     const recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
       'FROM logs*',
-      'dateField'
+      'dateField',
+      'textField'
     );
     testSuggestions('/', [
       ...sourceCommands.map((name) => name.toUpperCase() + ' '),
@@ -251,7 +257,8 @@ describe('autocomplete', () => {
     // source command
     let recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
       'FROM logs*',
-      'dateField'
+      'dateField',
+      'textField'
     );
     testSuggestions('f/', [
       ...sourceCommands.map((cmd) => `${cmd.toUpperCase()} `),
@@ -477,7 +484,8 @@ describe('autocomplete', () => {
     });
     let recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
       'FROM logs*',
-      'dateField'
+      'dateField',
+      'textField'
     );
     // Source command
     testSuggestions('F/', [
@@ -569,7 +577,11 @@ describe('autocomplete', () => {
       );
     });
 
-    recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates('', 'dateField');
+    recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
+      '',
+      'dateField',
+      'textField'
+    );
 
     // PIPE (|)
     testSuggestions('FROM a /', [
@@ -620,7 +632,8 @@ describe('autocomplete', () => {
       );
       recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
         'index1',
-        'dateField'
+        'dateField',
+        'textField'
       );
 
       testSuggestions(
@@ -643,7 +656,8 @@ describe('autocomplete', () => {
 
       recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
         'index2',
-        'dateField'
+        'dateField',
+        'textField'
       );
       testSuggestions(
         'FROM index1, index2/',
@@ -669,7 +683,8 @@ describe('autocomplete', () => {
       // we're setting it ourselves.
       recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
         'foo$bar',
-        'dateField'
+        'dateField',
+        'textField'
       );
       testSuggestions(
         'FROM foo$bar/',
@@ -701,7 +716,8 @@ describe('autocomplete', () => {
       // This is an identifier that matches multiple sources
       recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
         'i*',
-        'dateField'
+        'dateField',
+        'textField'
       );
       testSuggestions(
         'FROM i*/',
@@ -722,7 +738,11 @@ describe('autocomplete', () => {
       );
     });
 
-    recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates('', 'dateField');
+    recommendedQuerySuggestions = getRecommendedQueriesSuggestionsFromTemplates(
+      '',
+      'dateField',
+      'textField'
+    );
     // FROM source METADATA
     testSuggestions('FROM index1 M/', [attachTriggerCommand('METADATA ')]);
 

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/recommended_queries/suggestions.ts

@@ -9,22 +9,36 @@
 import type { RecommendedQuery } from '@kbn/esql-types';
 import type { SuggestionRawDefinition, GetColumnsByTypeFn } from '../types';
 import { getRecommendedQueries } from './templates';
+import { METADATA_FIELDS } from '../../shared/constants';
 
 export const getRecommendedQueriesSuggestionsFromStaticTemplates = async (
   getFieldsByType: GetColumnsByTypeFn,
   fromCommand: string = ''
 ): Promise<SuggestionRawDefinition[]> => {
-  const fieldSuggestions = await getFieldsByType('date', [], {
+  const [fieldSuggestions, textFieldSuggestions] = await Promise.all([
-    openSuggestions: true,
+    getFieldsByType(['date'], [], { openSuggestions: true }),
-  });
+    // get text fields separately to avoid mixing them with date fields
+    getFieldsByType(['text'], [], { openSuggestions: true }),
+  ]);
+
   let timeField = '';
+  let categorizationField: string | undefined = '';
+
   if (fieldSuggestions.length) {
     timeField =
       fieldSuggestions?.find((field) => field.label === '@timestamp')?.label ||
       fieldSuggestions[0].label;
   }
 
-  const recommendedQueries = getRecommendedQueries({ fromCommand, timeField });
+  if (textFieldSuggestions.length) {
+    categorizationField = getCategorizationField(textFieldSuggestions.map((field) => field.label));
+  }
+
+  const recommendedQueries = getRecommendedQueries({
+    fromCommand,
+    timeField,
+    categorizationField,
+  });
 
   const suggestions: SuggestionRawDefinition[] = recommendedQueries.map((query) => {
     return {
@@ -95,3 +109,32 @@ export const getRecommendedQueriesTemplatesFromExtensions = (
 
   return recommendedQueriesTemplates;
 };
+
+/**
+ * This function returns the categorization field from the list of fields.
+ * It checks for the presence of 'message', 'error.message', or 'event.original' in that order.
+ * If none of these fields are present, it returns the first field from the list,
+ * Assumes text fields have been passed in the `fields` array.
+ *
+ * This function is a duplicate of the one in src/platform/packages/shared/kbn-aiops-utils.
+ * It is included here to avoid build errors due to bazel
+ *
+ * TODO: Remove this function once the bazel issue is resolved.
+ *
+ * @param fields, the list of fields to check
+ * @returns string | undefined, the categorization field if found, otherwise undefined
+ */
+
+export function getCategorizationField(fields: string[]): string | undefined {
+  const fieldPriority = ['message', 'error.message', 'event.original'];
+  const fieldSet = new Set(fields);
+  for (const field of fieldPriority) {
+    if (fieldSet.has(field)) {
+      return field;
+    }
+  }
+
+  // Filter out metadata fields
+  const filteredFields = fields.filter((field) => !METADATA_FIELDS.includes(field));
+  return filteredFields[0] ?? undefined;
+}

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/recommended_queries/templates.ts

@@ -14,9 +14,11 @@ import { i18n } from '@kbn/i18n';
 export const getRecommendedQueries = ({
   fromCommand,
   timeField,
+  categorizationField,
 }: {
   fromCommand: string;
   timeField?: string;
+  categorizationField?: string;
 }) => {
   const queries = [
     {
@@ -135,6 +137,8 @@ export const getRecommendedQueries = ({
             label: i18n.translate(
               'kbn-esql-validation-autocomplete.recommendedQueries.categorize.label',
               {
+                // TODO this item should be hidden if AIOps is disabled or we're not running with a platinum license
+                // the capability aiops.enabled can be used to check both of these conditions
                 defaultMessage: 'Detect change points',
               }
             ),
@@ -170,6 +174,27 @@ export const getRecommendedQueries = ({
           },
         ]
       : []),
+    ...(categorizationField
+      ? // TODO this item should be hidden if AIOps is disabled or we're not running with a platinum license
+        // the capability aiops.enabled can be used to check both of these conditions
+        [
+          {
+            label: i18n.translate(
+              'kbn-esql-validation-autocomplete.recommendedQueries.patternAnalysis.label',
+              {
+                defaultMessage: 'Identify patterns',
+              }
+            ),
+            description: i18n.translate(
+              'kbn-esql-validation-autocomplete.recommendedQueries.patternAnalysis.description',
+              {
+                defaultMessage: 'Use the CATEGORIZE function to identify patterns in your logs',
+              }
+            ),
+            queryString: `${fromCommand}\n  | STATS Count=COUNT(*) BY Pattern=CATEGORIZE(${categorizationField})\n  | SORT Count DESC`,
+          },
+        ]
+      : []),
   ];
   return queries;
 };

src/platform/plugins/shared/discover/public/application/main/state_management/discover_app_state_container.test.ts

@@ -297,12 +297,14 @@ describe('Test discover app state container', () => {
       const state = getStateContainer();
       expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
         columns: false,
+        hideChart: false,
         rowHeight: false,
         breakdownField: false,
       });
       state.initAndSync();
       expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
         columns: true,
+        hideChart: true,
         rowHeight: true,
         breakdownField: true,
       });
@@ -314,12 +316,14 @@ describe('Test discover app state container', () => {
       const state = getStateContainer();
       expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
         columns: false,
+        hideChart: false,
         rowHeight: false,
         breakdownField: false,
       });
       state.initAndSync();
       expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
         columns: false,
+        hideChart: true,
         rowHeight: true,
         breakdownField: true,
       });
@@ -331,17 +335,38 @@ describe('Test discover app state container', () => {
       const state = getStateContainer();
       expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
         columns: false,
+        hideChart: false,
         rowHeight: false,
         breakdownField: false,
       });
       state.initAndSync();
       expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
         columns: true,
+        hideChart: true,
         rowHeight: false,
         breakdownField: true,
       });
     });
 
+    it('should call setResetDefaultProfileState correctly with initial hide chart', () => {
+      const stateStorageGetSpy = jest.spyOn(stateStorage, 'get');
+      stateStorageGetSpy.mockReturnValue({ hideChart: true });
+      const state = getStateContainer();
+      expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
+        columns: false,
+        hideChart: false,
+        rowHeight: false,
+        breakdownField: false,
+      });
+      state.initAndSync();
+      expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
+        columns: true,
+        hideChart: false,
+        rowHeight: true,
+        breakdownField: true,
+      });
+    });
+
     it('should call setResetDefaultProfileState correctly with saved search', () => {
       const stateStorageGetSpy = jest.spyOn(stateStorage, 'get');
       stateStorageGetSpy.mockReturnValue({ columns: ['test'], rowHeight: 5 });
@@ -354,12 +379,14 @@ describe('Test discover app state container', () => {
       const state = getStateContainer();
       expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
         columns: false,
+        hideChart: false,
         rowHeight: false,
         breakdownField: false,
       });
       state.initAndSync();
       expect(omit(getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
         columns: false,
+        hideChart: false,
         rowHeight: false,
         breakdownField: false,
       });

src/platform/plugins/shared/discover/public/application/main/state_management/discover_app_state_container.ts

@@ -271,7 +271,10 @@ export const getDiscoverAppStateContainer = ({
     // Set the default profile state only if not loading a saved search,
     // to avoid overwriting saved search state
     if (!currentSavedSearch.id) {
-      const { breakdownField, columns, rowHeight } = getCurrentUrlState(stateStorage, services);
+      const { breakdownField, columns, rowHeight, hideChart } = getCurrentUrlState(
+        stateStorage,
+        services
+      );
 
       // Only set default state which is not already set in the URL
       internalState.dispatch(
@@ -280,6 +283,7 @@ export const getDiscoverAppStateContainer = ({
             columns: columns === undefined,
             rowHeight: rowHeight === undefined,
             breakdownField: breakdownField === undefined,
+            hideChart: hideChart === undefined,
           },
         })
       );

src/platform/plugins/shared/discover/public/application/main/state_management/discover_data_state_container.test.ts

@@ -183,6 +183,7 @@ describe('test getDataStateContainer', () => {
           columns: true,
           rowHeight: true,
           breakdownField: true,
+          hideChart: false,
         },
       })
     );
@@ -200,6 +201,7 @@ describe('test getDataStateContainer', () => {
       columns: false,
       rowHeight: false,
       breakdownField: false,
+      hideChart: false,
     });
     expect(stateContainer.appState.get().columns).toEqual(['message', 'extension']);
     expect(stateContainer.appState.get().rowHeight).toEqual(3);
@@ -225,6 +227,7 @@ describe('test getDataStateContainer', () => {
           columns: false,
           rowHeight: false,
           breakdownField: false,
+          hideChart: false,
         },
       })
     );
@@ -240,6 +243,7 @@ describe('test getDataStateContainer', () => {
       columns: false,
       rowHeight: false,
       breakdownField: false,
+      hideChart: false,
     });
     expect(stateContainer.appState.get().columns).toEqual(['default_column']);
     expect(stateContainer.appState.get().rowHeight).toBeUndefined();

src/platform/plugins/shared/discover/public/application/main/state_management/discover_data_state_container.ts

@@ -359,6 +359,7 @@ export function getDataStateContainer({
                     columns: false,
                     rowHeight: false,
                     breakdownField: false,
+                    hideChart: false,
                   },
                 })
               );

src/platform/plugins/shared/discover/public/application/main/state_management/redux/internal_state.ts

@@ -55,6 +55,7 @@ export const defaultTabState: Omit<TabState, keyof TabItem> = {
     columns: false,
     rowHeight: false,
     breakdownField: false,
+    hideChart: false,
   },
   documentsRequest: {
     loadingStatus: LoadingStatus.Uninitialized,

src/platform/plugins/shared/discover/public/application/main/state_management/redux/types.ts

@@ -70,6 +70,7 @@ export interface TabState extends TabItem {
     columns: boolean;
     rowHeight: boolean;
     breakdownField: boolean;
+    hideChart: boolean;
   };
   documentsRequest: DocumentsRequest;
   totalHitsRequest: TotalHitsRequest;

src/platform/plugins/shared/discover/public/application/main/state_management/utils/build_esql_fetch_subscribe.test.ts

@@ -492,6 +492,7 @@ describe('buildEsqlFetchSubscribe', () => {
     const documents$ = stateContainer.dataState.data$.documents$;
     expect(omit(stateContainer.getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
       columns: false,
+      hideChart: false,
       rowHeight: false,
       breakdownField: false,
     });
@@ -507,6 +508,7 @@ describe('buildEsqlFetchSubscribe', () => {
     await waitFor(() =>
       expect(omit(stateContainer.getCurrentTab().resetDefaultProfileState, 'resetId')).toEqual({
         columns: true,
+        hideChart: true,
         rowHeight: true,
         breakdownField: true,
       })
@@ -521,6 +523,7 @@ describe('buildEsqlFetchSubscribe', () => {
           columns: false,
           rowHeight: false,
           breakdownField: false,
+          hideChart: false,
         },
       })
     );
@@ -534,6 +537,7 @@ describe('buildEsqlFetchSubscribe', () => {
         columns: false,
         rowHeight: false,
         breakdownField: false,
+        hideChart: false,
       })
     );
     documents$.next({
@@ -550,6 +554,7 @@ describe('buildEsqlFetchSubscribe', () => {
         columns: true,
         rowHeight: true,
         breakdownField: true,
+        hideChart: true,
       })
     );
     documents$.next({
@@ -567,6 +572,7 @@ describe('buildEsqlFetchSubscribe', () => {
       columns: false,
       rowHeight: false,
       breakdownField: false,
+      hideChart: false,
     });
     documents$.next({
       fetchStatus: FetchStatus.PARTIAL,
@@ -578,6 +584,7 @@ describe('buildEsqlFetchSubscribe', () => {
         columns: false,
         rowHeight: false,
         breakdownField: false,
+        hideChart: false,
       })
     );
     documents$.next({
@@ -590,6 +597,7 @@ describe('buildEsqlFetchSubscribe', () => {
         columns: true,
         rowHeight: false,
         breakdownField: false,
+        hideChart: false,
       })
     );
   });

src/platform/plugins/shared/discover/public/application/main/state_management/utils/build_esql_fetch_subscribe.ts

@@ -98,6 +98,7 @@ export const buildEsqlFetchSubscribe = ({
                 columns: true,
                 rowHeight: true,
                 breakdownField: true,
+                hideChart: true,
               },
             })
           );
@@ -158,6 +159,7 @@ export const buildEsqlFetchSubscribe = ({
             columns: true,
             rowHeight: false,
             breakdownField: false,
+            hideChart: false,
           },
         })
       );

src/platform/plugins/shared/discover/public/application/main/state_management/utils/change_data_view.test.ts

@@ -84,6 +84,7 @@ describe('changeDataView', () => {
         columns: false,
         rowHeight: false,
         breakdownField: false,
+        hideChart: false,
       })
     );
     await changeDataView({ dataViewId: dataViewComplexMock.id!, ...params });
@@ -92,6 +93,7 @@ describe('changeDataView', () => {
         columns: true,
         rowHeight: true,
         breakdownField: true,
+        hideChart: true,
       })
     );
   });

src/platform/plugins/shared/discover/public/application/main/state_management/utils/change_data_view.ts

@@ -82,6 +82,7 @@ export async function changeDataView({
           columns: true,
           rowHeight: true,
           breakdownField: true,
+          hideChart: true,
         },
       })
     );

src/platform/plugins/shared/discover/public/application/main/state_management/utils/get_default_profile_state.test.ts

@@ -32,6 +32,7 @@ describe('getDefaultProfileState', () => {
           columns: false,
           rowHeight: false,
           breakdownField: true,
+          hideChart: false,
         },
         dataView: dataViewWithTimefieldMock,
       }).getPreFetchState();
@@ -45,6 +46,7 @@ describe('getDefaultProfileState', () => {
           columns: false,
           rowHeight: false,
           breakdownField: true,
+          hideChart: false,
         },
         dataView: emptyDataView,
       }).getPreFetchState();
@@ -61,6 +63,7 @@ describe('getDefaultProfileState', () => {
           columns: true,
           rowHeight: false,
           breakdownField: false,
+          hideChart: false,
         },
         dataView: dataViewWithTimefieldMock,
       }).getPostFetchState({
@@ -87,6 +90,7 @@ describe('getDefaultProfileState', () => {
           columns: true,
           rowHeight: false,
           breakdownField: false,
+          hideChart: false,
         },
         dataView: emptyDataView,
       }).getPostFetchState({
@@ -116,6 +120,7 @@ describe('getDefaultProfileState', () => {
           columns: false,
           rowHeight: true,
           breakdownField: false,
+          hideChart: false,
         },
         dataView: dataViewWithTimefieldMock,
       }).getPostFetchState({
@@ -135,6 +140,7 @@ describe('getDefaultProfileState', () => {
           columns: false,
           rowHeight: false,
           breakdownField: false,
+          hideChart: false,
         },
         dataView: dataViewWithTimefieldMock,
       }).getPostFetchState({

src/platform/plugins/shared/discover/public/application/main/state_management/utils/get_default_profile_state.ts

@@ -44,6 +44,10 @@ export const getDefaultProfileState = ({
         stateUpdate.breakdownField = defaultState.breakdownField;
       }
 
+      if (defaultState.hideChart !== undefined) {
+        stateUpdate.hideChart = defaultState.hideChart;
+      }
+
       return Object.keys(stateUpdate).length ? stateUpdate : undefined;
     },
 

src/platform/plugins/shared/discover/public/context_awareness/profile_providers/common/patterns/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export { createPatternDataSourceProfileProvider } from './profile';

src/platform/plugins/shared/discover/public/context_awareness/profile_providers/common/patterns/pattern_cell_renderer.tsx

@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import React, { useMemo } from 'react';
+import type { FC } from 'react';
+import type { UseEuiTheme } from '@elastic/eui';
+import { EuiCode, EuiSpacer, EuiText } from '@elastic/eui';
+import { css } from '@emotion/react';
+import type { DataTableRecord } from '@kbn/discover-utils';
+import { extractCategorizeTokens } from '@kbn/esql-utils';
+import { FormattedMessage } from '@kbn/i18n-react';
+import { useMemoCss } from '@kbn/css-utils/public/use_memo_css';
+
+interface Props {
+  pattern: string;
+  isDetails: boolean;
+  defaultRowHeight?: number;
+}
+
+export const PatternCellRenderer: FC<Props> = ({ pattern, isDetails, defaultRowHeight }) => {
+  const styles = useMemoCss(componentStyles);
+
+  const keywords = useMemo(() => extractCategorizeTokens(pattern), [pattern]);
+  const containerStyle = useMemo(() => getContainerStyle(defaultRowHeight), [defaultRowHeight]);
+
+  const formattedTokens = useMemo(
+    () =>
+      keywords.map((keyword, index) => {
+        return (
+          <EuiCode key={index} css={styles.keyword}>
+            {keyword}
+          </EuiCode>
+        );
+      }),
+    [styles, keywords]
+  );
+
+  if (isDetails) {
+    return (
+      <div css={styles.detailsContainer}>
+        <EuiText size="s">
+          <strong>
+            <FormattedMessage
+              id="discover.contextAwareness.patternCellRenderer.tokensLabel"
+              defaultMessage="Tokens"
+            />
+          </strong>
+          <EuiSpacer size="s" />
+          {formattedTokens}
+        </EuiText>
+
+        <EuiSpacer size="m" />
+
+        <EuiText size="s">
+          <strong>
+            <FormattedMessage
+              id="discover.contextAwareness.patternCellRenderer.regexLabel"
+              defaultMessage="Regex"
+            />
+          </strong>
+          <EuiSpacer size="s" />
+          <span data-test-subj="euiDataGridExpansionPopover-patternRegex">{pattern}</span>
+        </EuiText>
+        <EuiSpacer size="s" />
+      </div>
+    );
+  }
+
+  return <div css={containerStyle}>{formattedTokens}</div>;
+};
+
+const componentStyles = {
+  keyword: ({ euiTheme }: UseEuiTheme) =>
+    css({
+      marginRight: euiTheme.size.xs,
+      marginBottom: `calc(${euiTheme.size.m} / 2)`,
+      display: 'inline-block',
+      padding: `${euiTheme.size.xxs} ${euiTheme.size.s}`,
+      backgroundColor: euiTheme.colors.lightestShade,
+      borderRadius: euiTheme.border.radius.small,
+      color: euiTheme.colors.textPrimary,
+      fontSize: euiTheme.size.m,
+    }),
+  detailsContainer: ({ euiTheme }: UseEuiTheme) =>
+    css({
+      maxWidth: '600px',
+    }),
+};
+
+export function getPatternCellRenderer(
+  row: DataTableRecord,
+  columnId: string,
+  isDetails: boolean,
+  defaultRowHeight?: number
+) {
+  const pattern = row.flattened[columnId];
+  if (pattern === undefined) {
+    return '-';
+  }
+  return (
+    <PatternCellRenderer
+      pattern={String(pattern)}
+      isDetails={isDetails}
+      defaultRowHeight={defaultRowHeight}
+    />
+  );
+}
+
+function getContainerStyle(defaultRowHeight?: number) {
+  // the keywords are slightly larger than the default text height,
+  // so they need to be adjusted to fit within the row height while
+  // not truncating the bottom of the text
+  let rowHeight = 2;
+  if (defaultRowHeight === undefined) {
+    rowHeight = 2;
+  } else if (defaultRowHeight < 2) {
+    rowHeight = 1;
+  } else {
+    rowHeight = Math.floor(defaultRowHeight / 1.5);
+  }
+
+  return {
+    display: '-webkit-box',
+    WebkitBoxOrient: 'vertical' as const,
+    WebkitLineClamp: rowHeight,
+    overflow: 'hidden',
+    textOverflow: 'ellipsis',
+  };
+}

src/platform/plugins/shared/discover/public/context_awareness/profile_providers/common/patterns/profile.ts

@@ -0,0 +1,138 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { isOfAggregateQueryType } from '@kbn/es-query';
+import { extractCategorizeTokens, getCategorizeColumns, getCategorizeField } from '@kbn/esql-utils';
+import { i18n } from '@kbn/i18n';
+import type { DataGridCellValueElementProps } from '@kbn/unified-data-table';
+import { DataSourceType, isDataSourceType } from '../../../../../common/data_sources';
+import type { DataSourceProfileProvider } from '../../../profiles';
+import { DataSourceCategory } from '../../../profiles';
+import { getPatternCellRenderer } from './pattern_cell_renderer';
+import type { ProfileProviderServices } from '../../profile_provider_services';
+
+const DOC_LIMIT = 10000;
+
+export const createPatternDataSourceProfileProvider = (
+  services: ProfileProviderServices
+): DataSourceProfileProvider<{
+  patternColumns: string[];
+}> => ({
+  profileId: 'patterns-data-source-profile',
+  profile: {
+    getCellRenderers:
+      (prev, { context }) =>
+      (params) => {
+        const { rowHeight } = params;
+        const { patternColumns } = context;
+        if (!patternColumns || patternColumns.length === 0) {
+          return prev(params);
+        }
+        const patternRenderers = patternColumns.reduce(
+          (acc, column) =>
+            Object.assign(acc, {
+              [column]: (props: DataGridCellValueElementProps) =>
+                getPatternCellRenderer(props.row, props.columnId, props.isDetails, rowHeight),
+            }),
+          {}
+        );
+
+        return {
+          ...prev(params),
+          ...patternRenderers,
+        };
+      },
+    getAdditionalCellActions:
+      (prev, { context }) =>
+      () => {
+        return [
+          ...prev(),
+          {
+            id: 'patterns-action-view-docs-in-discover',
+            getDisplayName: () =>
+              i18n.translate('discover.docViews.patterns.cellAction.viewResults', {
+                defaultMessage: 'View matching results',
+              }),
+            getIconType: () => 'discoverApp',
+            isCompatible: (compatibleContext) => {
+              const { query, field } = compatibleContext;
+              const { patternColumns } = context;
+              if (!isOfAggregateQueryType(query) || field === undefined) {
+                return false;
+              }
+              return patternColumns.includes(field.name);
+            },
+            execute: (executeContext) => {
+              const index = executeContext.dataView?.getIndexPattern();
+              if (
+                !isOfAggregateQueryType(executeContext.query) ||
+                !executeContext.value ||
+                !index
+              ) {
+                return;
+              }
+
+              const pattern = extractCategorizeTokens(executeContext.value as string).join(' ');
+              const categoryField = getCategorizeField(executeContext.query.esql);
+
+              if (!categoryField || !pattern) {
+                return;
+              }
+
+              const query = {
+                ...executeContext.query,
+                esql: `FROM ${index}\n  | WHERE MATCH(${categoryField}, "${pattern}", {"auto_generate_synonyms_phrase_query": false, "fuzziness": 0, "operator": "AND"})\n  | LIMIT ${DOC_LIMIT}`,
+              };
+
+              const discoverLink = services.locator.getRedirectUrl({
+                query,
+                timeRange: executeContext.timeRange,
+                hideChart: false,
+              });
+              window.open(discoverLink, '_blank');
+            },
+          },
+        ];
+      },
+    getDefaultAppState: (prev) => (params) => {
+      return {
+        ...prev(params),
+        hideChart: true,
+        columns: [
+          { name: 'Count', width: 150 },
+          { name: 'Pattern', width: undefined },
+        ],
+      };
+    },
+  },
+  resolve: (params) => {
+    if (!isDataSourceType(params.dataSource, DataSourceType.Esql)) {
+      return { isMatch: false };
+    }
+
+    const query = params.query;
+
+    if (!isOfAggregateQueryType(query)) {
+      return { isMatch: false };
+    }
+
+    const patternColumns = getCategorizeColumns(query.esql);
+    if (patternColumns.length === 0) {
+      return { isMatch: false };
+    }
+
+    return {
+      isMatch: true,
+      context: {
+        category: DataSourceCategory.Default,
+        patternColumns,
+      },
+    };
+  },
+});

src/platform/plugins/shared/discover/public/context_awareness/profile_providers/register_profile_providers.ts

@@ -29,6 +29,7 @@ import { createTracesDataSourceProfileProvider } from './observability/traces_da
 import { createDeprecationLogsDataSourceProfileProvider } from './common/deprecation_logs';
 import { createClassicNavRootProfileProvider } from './common/classic_nav_root_profile';
 import { createObservabilityDocumentProfileProviders } from './observability/observability_profile_providers';
+import { createPatternDataSourceProfileProvider } from './common/patterns';
 import { createSecurityDocumentProfileProvider } from './security/security_document_profile';
 
 /**
@@ -147,6 +148,7 @@ const createRootProfileProviders = (providerServices: ProfileProviderServices) =
  */
 const createDataSourceProfileProviders = (providerServices: ProfileProviderServices) => [
   createExampleDataSourceProfileProvider(),
+  createPatternDataSourceProfileProvider(providerServices),
   createDeprecationLogsDataSourceProfileProvider(),
   createTracesDataSourceProfileProvider(providerServices),
   ...createObservabilityLogsDataSourceProfileProviders(providerServices),

src/platform/plugins/shared/discover/public/context_awareness/types.ts

@@ -140,6 +140,10 @@ export interface DefaultAppStateExtension {
    * The field to apply for the histogram breakdown
    */
   breakdownField?: string;
+  /**
+   * The state for chart visibility toggle
+   */
+  hideChart?: boolean;
 }
 
 /**

src/platform/plugins/shared/unified_search/public/query_string_input/esql_menu_popover.test.tsx

@@ -12,6 +12,7 @@ import { BehaviorSubject } from 'rxjs';
 import { screen, render, waitFor } from '@testing-library/react';
 import userEvent from '@testing-library/user-event';
 import { KibanaContextProvider } from '@kbn/kibana-react-plugin/public';
+import { createStubDataView, stubLogstashFieldSpecMap } from '@kbn/data-views-plugin/common/stubs';
 import { stubIndexPattern } from '@kbn/data-plugin/public/stubs';
 import { coreMock } from '@kbn/core/public/mocks';
 import type { DataView } from '@kbn/data-views-plugin/common';
@@ -95,6 +96,7 @@ describe('ESQLMenuPopover', () => {
     await waitFor(() => {
       expect(screen.getByText('Count of logs')).toBeInTheDocument();
       expect(screen.getByText('Average bytes')).toBeInTheDocument();
+      expect(screen.getByText('Identify patterns')).toBeInTheDocument();
     });
   });
 
@@ -113,4 +115,40 @@ describe('ESQLMenuPopover', () => {
     await userEvent.click(screen.getByRole('button'));
     expect(screen.queryByTestId('esql-recommended-queries')).toBeInTheDocument();
   });
+
+  it('should show identify patterns recommended query', async () => {
+    const stubLogstashDataView = createStubDataView({
+      spec: {
+        id: 'logstash-*',
+        title: 'logstash-*',
+        timeFieldName: 'time',
+        fields: {
+          ...stubLogstashFieldSpecMap,
+          message: {
+            name: 'message',
+            type: 'string',
+            esTypes: ['text'],
+            aggregatable: true,
+            searchable: true,
+            count: 0,
+            readFromDocValues: true,
+            scripted: false,
+            isMapped: true,
+          },
+        },
+      },
+    });
+
+    renderESQLPopover(stubLogstashDataView);
+
+    // open the popover and check for recommended queries
+    await userEvent.click(screen.getByRole('button'));
+    expect(screen.queryByTestId('esql-recommended-queries')).toBeInTheDocument();
+    // Open the nested section to see the recommended queries
+    await waitFor(() => userEvent.click(screen.getByTestId('esql-recommended-queries')));
+
+    await waitFor(() => {
+      expect(screen.getByText('Identify patterns')).toBeInTheDocument();
+    });
+  });
 });

src/platform/plugins/shared/unified_search/public/query_string_input/esql_menu_popover.tsx

@@ -28,6 +28,7 @@ import {
   getRecommendedQueriesTemplatesFromExtensions,
 } from '@kbn/esql-validation-autocomplete';
 import { LanguageDocumentationFlyout } from '@kbn/language-documentation';
+import { getCategorizationField } from '@kbn/aiops-utils';
 import type { IUnifiedSearchPluginServices } from '../types';
 
 export interface ESQLMenuPopoverProps {
@@ -52,17 +53,25 @@ export const ESQLMenuPopover: React.FC<ESQLMenuPopoverProps> = ({
     RecommendedQuery[]
   >([]);
 
-  const { queryForRecommendedQueries, timeFieldName } = useMemo(() => {
+  const { queryForRecommendedQueries, timeFieldName, categorizationField } = useMemo(() => {
     if (adHocDataview && typeof adHocDataview !== 'string') {
+      const textFields = adHocDataview.fields?.getByType('string') ?? [];
+      let tempCategorizationField;
+      if (textFields.length) {
+        tempCategorizationField = getCategorizationField(textFields.map((field) => field.name));
+      }
+
       return {
         queryForRecommendedQueries: `FROM ${adHocDataview.name}`,
         timeFieldName:
           adHocDataview.timeFieldName ?? adHocDataview.fields?.getByType('date')?.[0]?.name,
+        categorizationField: tempCategorizationField,
       };
     }
     return {
       queryForRecommendedQueries: '',
       timeFieldName: undefined,
+      categorizationField: undefined,
     };
   }, [adHocDataview]);
 
@@ -130,11 +139,13 @@ export const ESQLMenuPopover: React.FC<ESQLMenuPopoverProps> = ({
         }))
       );
     }
+
     // Handle the static recommended queries, no solutions specific
     if (queryForRecommendedQueries && timeFieldName) {
       const recommendedQueriesFromStaticTemplates = getRecommendedQueries({
         fromCommand: queryForRecommendedQueries,
         timeField: timeFieldName,
+        categorizationField,
       });
 
       recommendedQueries.push(...recommendedQueriesFromStaticTemplates);
@@ -239,6 +250,7 @@ export const ESQLMenuPopover: React.FC<ESQLMenuPopoverProps> = ({
     timeFieldName,
     toggleLanguageComponent,
     solutionsRecommendedQueries, // This dependency is fine here, as it *uses* the state
+    categorizationField,
   ]);
 
   const esqlMenuPopoverStyles = css`

src/platform/plugins/shared/unified_search/tsconfig.json

@@ -50,7 +50,8 @@
     "@kbn/react-kibana-mount",
     "@kbn/field-utils",
     "@kbn/language-documentation",
-    "@kbn/esql-types"
+    "@kbn/esql-types",
+    "@kbn/aiops-utils",
   ],
   "exclude": [
     "target/**/*",

tsconfig.base.json

@@ -40,6 +40,8 @@
       "@kbn/aiops-plugin/*": ["x-pack/platform/plugins/shared/aiops/*"],
       "@kbn/aiops-test-utils": ["x-pack/platform/packages/private/ml/aiops_test_utils"],
       "@kbn/aiops-test-utils/*": ["x-pack/platform/packages/private/ml/aiops_test_utils/*"],
+      "@kbn/aiops-utils": ["src/platform/packages/shared/kbn-aiops-utils"],
+      "@kbn/aiops-utils/*": ["src/platform/packages/shared/kbn-aiops-utils/*"],
       "@kbn/alerting-api-integration-helpers": ["x-pack/platform/test/alerting_api_integration/packages/helpers"],
       "@kbn/alerting-api-integration-helpers/*": ["x-pack/platform/test/alerting_api_integration/packages/helpers/*"],
       "@kbn/alerting-api-integration-test-plugin": ["x-pack/platform/test/alerting_api_integration/common/plugins/alerts"],

x-pack/platform/plugins/shared/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_discover.tsx

@@ -25,6 +25,7 @@ import { css } from '@emotion/react';
 import { useTableState } from '@kbn/ml-in-memory-table/hooks/use_table_state';
 import useMountedState from 'react-use/lib/useMountedState';
 import { getEsQueryConfig } from '@kbn/data-service';
+import { getCategorizationDataViewField } from '@kbn/aiops-utils';
 import {
   type LogCategorizationPageUrlState,
   getDefaultLogCategorizationAppState,
@@ -42,7 +43,7 @@ import { useValidateFieldRequest } from '../use_validate_category_field';
 import { FieldValidationCallout } from '../category_validation_callout';
 import { useMinimumTimeRange } from './use_minimum_time_range';
 
-import { createAdditionalConfigHash, createDocumentStatsHash, getMessageField } from '../utils';
+import { createAdditionalConfigHash, createDocumentStatsHash } from '../utils';
 import { DiscoverTabs } from './discover_tabs';
 import { useRandomSamplerStorage } from '../sampling_menu';
 import { useActions } from '../category_table/use_actions';
@@ -126,7 +127,7 @@ export const LogCategorizationDiscover: FC<LogCategorizationEmbeddableProps> = (
       setCurrentDocumentStatsHash(null);
       setSelectedField(null);
       setLoading(null);
-      const { dataViewFields, messageField } = getMessageField(dataView);
+      const { dataViewFields, messageField } = getCategorizationDataViewField(dataView);
       setFields(dataViewFields);
       setSelectedField(messageField);
     },

x-pack/platform/plugins/shared/aiops/public/components/log_categorization/utils.ts

@@ -6,8 +6,6 @@
  */
 
 import { stringHash } from '@kbn/ml-string-hash';
-import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
-import { ES_FIELD_TYPES } from '@kbn/field-types';
 import type { DocumentStats } from '../../hooks/use_document_count_stats';
 
 /**
@@ -26,35 +24,3 @@ export function createDocumentStatsHash(documentStats: DocumentStats) {
 export function createAdditionalConfigHash(additionalStrings: string[] = []) {
   return stringHash(`${additionalStrings.join('')}`);
 }
-
-/**
- * Retrieves the message field from a DataView object.
- * If the message field is not found, it falls back to error.message or event.original or the first text field in the DataView.
- *
- * @param dataView - The DataView object containing the fields.
- * @returns An object containing the message field and all the fields in the DataView.
- */
-export function getMessageField(dataView: DataView): {
-  messageField: DataViewField | null;
-  dataViewFields: DataViewField[];
-} {
-  const dataViewFields = dataView.fields.filter((f) => f.esTypes?.includes(ES_FIELD_TYPES.TEXT));
-
-  let messageField: DataViewField | null | undefined = dataViewFields.find(
-    (f) => f.name === 'message'
-  );
-  if (messageField === undefined) {
-    messageField = dataViewFields.find((f) => f.name === 'error.message');
-  }
-  if (messageField === undefined) {
-    messageField = dataViewFields.find((f) => f.name === 'event.original');
-  }
-  if (messageField === undefined) {
-    if (dataViewFields.length > 0) {
-      messageField = dataViewFields[0];
-    } else {
-      messageField = null;
-    }
-  }
-  return { messageField, dataViewFields };
-}

x-pack/platform/plugins/shared/aiops/public/embeddables/pattern_analysis/pattern_analysis_initializer.tsx

@@ -29,6 +29,7 @@ import { isPopulatedObject } from '@kbn/ml-is-populated-object';
 import useObservable from 'react-use/lib/useObservable';
 import type { DataViewField } from '@kbn/data-views-plugin/public';
 import useMountedState from 'react-use/lib/useMountedState';
+import { getCategorizationDataViewField } from '@kbn/aiops-utils';
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 import { DataSourceContextProvider } from '../../hooks/use_data_source';
 import type { PatternAnalysisEmbeddableRuntimeState } from './types';
@@ -43,7 +44,7 @@ import {
   DEFAULT_MINIMUM_TIME_RANGE_OPTION,
   type MinimumTimeRangeOption,
 } from '../../components/log_categorization/log_categorization_for_embeddable/minimum_time_range';
-import { getMessageField } from '../../components/log_categorization/utils';
+
 import { FieldSelector } from '../../components/log_categorization/log_categorization_for_embeddable/field_selector';
 import { SamplingPanel } from '../../components/log_categorization/sampling_menu/sampling_panel';
 
@@ -272,7 +273,7 @@ export const FormControls: FC<{
             setSelectedField(null);
             return;
           }
-          const { dataViewFields, messageField } = getMessageField(dataView);
+          const { dataViewFields, messageField } = getCategorizationDataViewField(dataView);
           setFields(dataViewFields);
           const field = dataViewFields.find((f) => f.name === formInput.fieldName);
           if (formInput.fieldName === undefined) {

x-pack/platform/plugins/shared/aiops/tsconfig.json

@@ -77,6 +77,7 @@
     "@kbn/apm-utils",
     "@kbn/ml-field-stats-flyout",
     "@kbn/config-schema",
+    "@kbn/aiops-utils",
   ],
   "exclude": [
     "target/**/*",

x-pack/test/functional/apps/aiops/index.ts

@@ -38,5 +38,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./change_point_detection_cases'));
     loadTestFile(require.resolve('./log_pattern_analysis'));
     loadTestFile(require.resolve('./log_pattern_analysis_in_discover'));
+    loadTestFile(require.resolve('./log_pattern_analysis_esql_in_discover'));
   });
 }

x-pack/test/functional/apps/aiops/log_pattern_analysis_esql_in_discover.ts

@@ -0,0 +1,121 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ getPageObjects, getService }: FtrProviderContext) {
+  const esArchiver = getService('esArchiver');
+  const kibanaServer = getService('kibanaServer');
+  const security = getService('security');
+  const dataGrid = getService('dataGrid');
+  const monacoEditor = getService('monacoEditor');
+  const aiops = getService('aiops');
+  const browser = getService('browser');
+  const retry = getService('retry');
+  const testSubjects = getService('testSubjects');
+  const { common, discover, header, timePicker, unifiedFieldList } = getPageObjects([
+    'common',
+    'discover',
+    'header',
+    'timePicker',
+    'unifiedFieldList',
+  ]);
+
+  const defaultSettings = {
+    defaultIndex: 'logstash-*',
+  };
+
+  async function retrySwitchTab(tabIndex: number, seconds: number) {
+    await retry.tryForTime(seconds * 1000, async () => {
+      await browser.switchTab(tabIndex);
+    });
+  }
+
+  describe('log pattern analysis ES|QL in discover', function () {
+    before(async () => {
+      await kibanaServer.savedObjects.cleanStandardList();
+      await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']);
+      await kibanaServer.importExport.load(
+        'src/platform/test/functional/fixtures/kbn_archiver/discover'
+      );
+      await esArchiver.loadIfNeeded(
+        'src/platform/test/functional/fixtures/es_archiver/logstash_functional'
+      );
+      await kibanaServer.uiSettings.replace(defaultSettings);
+      await common.navigateToApp('discover');
+      await timePicker.setDefaultAbsoluteRange();
+      await discover.waitUntilSearchingHasFinished();
+      await discover.selectTextBaseLang();
+      await discover.waitUntilSearchingHasFinished();
+    });
+
+    beforeEach(async () => {
+      await discover.clickNewSearchButton();
+      await header.waitUntilLoadingHasFinished();
+      await discover.waitUntilSearchingHasFinished();
+    });
+
+    let tabsCount = 1;
+
+    afterEach(async () => {
+      if (tabsCount > 1) {
+        await browser.closeCurrentWindow();
+        await retrySwitchTab(0, 10);
+        tabsCount--;
+      }
+    });
+
+    it('should render categorize fields correctly', async () => {
+      // set query to be categorize command
+      // ensure the columns are correct
+      await monacoEditor.setCodeEditorValue(
+        'from logstash-*\n  | STATS Count=COUNT(*) BY Pattern=CATEGORIZE(@message)\n | SORT Count DESC'
+      );
+      await testSubjects.click('querySubmitButton');
+      const columns = ['Count', 'Pattern'];
+      await unifiedFieldList.clickFieldListItemAdd('Count');
+      await unifiedFieldList.clickFieldListItemAdd('Pattern');
+      await header.waitUntilLoadingHasFinished();
+      await discover.waitUntilSearchingHasFinished();
+      expect(await dataGrid.getHeaderFields()).to.eql(columns);
+
+      // refresh the table and ensure the columns are still correct
+      await browser.refresh();
+      await header.waitUntilLoadingHasFinished();
+      await discover.waitUntilSearchingHasFinished();
+      expect(await dataGrid.getHeaderFields()).to.eql(columns);
+
+      // get the first pattern cell and ensure it has the correct number of tokens
+      const cell = await dataGrid.getCellElement(0, 3);
+      const tokens = await cell.findAllByCssSelector('.unifiedDataTable__cellValue code');
+
+      expect(tokens.length).to.be.greaterThan(0);
+
+      // click the expand action button in the cell
+      // ensure the popover is displayed and the regex is correct
+      await dataGrid.clickCellExpandButton(0, { columnIndex: 3, columnName: 'Pattern' });
+
+      const regexElement = await testSubjects.find('euiDataGridExpansionPopover-patternRegex');
+      const regexText = await regexElement.getVisibleText();
+      // The pattern might change based on the data, so rather than checking for an exact match,
+      // we check that the string contains part of a regex
+      expect(regexText).to.contain('.*?');
+
+      // Click the view docs in Discover button
+      // ensure it opens a new tab and the discover doc count is greater than 0
+      // We cannot look for an exact count as the count may vary based on sampling (when that is added)
+      await dataGrid.clickCellExpandPopoverAction('patterns-action-view-docs-in-discover');
+      await retrySwitchTab(1, 10);
+      tabsCount++;
+
+      await aiops.logPatternAnalysisPage.assertDiscoverDocCountExists();
+      // ensure the discover doc count is greater than 0
+      await aiops.logPatternAnalysisPage.assertDiscoverDocCountGreaterThan(0);
+    });
+  });
+}

x-pack/test/functional/apps/aiops/log_pattern_analysis_in_discover.ts

@@ -22,7 +22,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
     });
   }
 
-  describe('log pattern analysis', function () {
+  describe('log pattern analysis in discover', function () {
     let tabsCount = 1;
 
     afterEach(async () => {

yarn.lock

@@ -3849,6 +3849,10 @@
   version "0.0.0"
   uid ""
 
+"@kbn/aiops-utils@link:src/platform/packages/shared/kbn-aiops-utils":
+  version "0.0.0"
+  uid ""
+
 "@kbn/alerting-api-integration-helpers@link:x-pack/platform/test/alerting_api_integration/packages/helpers":
   version "0.0.0"
   uid ""