[Security Solution] Move ES|QL parsing functionality into @kbn/securitysolution-utils package (#202772)

## Summary

With this PR we move existing `parseEsqlQuery` method into a shared
security solution utils package. We need to the same functionality in
"SIEM migrations" feature. Previously we duplicated the code in [this
PR](https://github.com/elastic/kibana/pull/202331/files#diff-b5f1a952a5e5a9685a4fef5d1f5a4c3b53ce338333e569bb6f92ccf2681100b7R54)
and these are the follow-up changes to make parsing functionality shared
for easier re-use within security solution.

### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-securitysolution-utils/src/esql/index.ts

@@ -9,3 +9,4 @@
 
 export * from './compute_if_esql_query_aggregating';
 export * from './get_index_list_from_esql_query';
+export * from './parse_esql_query';

packages/kbn-securitysolution-utils/src/esql/parse_esql_query.test.ts

@@ -0,0 +1,119 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { parseEsqlQuery } from './parse_esql_query';
+
+describe('parseEsqlQuery', () => {
+  describe('ES|QL query syntax', () => {
+    it.each([['incorrect syntax'], ['from test* metadata']])(
+      'detects incorrect syntax in "%s"',
+      (esqlQuery) => {
+        const result = parseEsqlQuery(esqlQuery);
+        expect(result.errors.length).toEqual(1);
+        expect(result.errors[0].message.startsWith('SyntaxError:')).toBeTruthy();
+        expect(parseEsqlQuery(esqlQuery)).toMatchObject({
+          hasMetadataOperator: false,
+          isEsqlQueryAggregating: false,
+        });
+      }
+    );
+
+    it.each([
+      ['from test* metadata _id'],
+      [
+        'FROM kibana_sample_data_logs | STATS total_bytes = SUM(bytes) BY host | WHERE total_bytes > 200000 | SORT total_bytes DESC | LIMIT 10',
+      ],
+      [
+        `from packetbeat* metadata
+        _id
+        | limit 100`,
+      ],
+      [
+        `FROM kibana_sample_data_logs |
+         STATS total_bytes = SUM(bytes) BY host |
+         WHERE total_bytes > 200000 |
+         SORT total_bytes DESC |
+         LIMIT 10`,
+      ],
+    ])('parses correctly valid syntax in "%s"', (esqlQuery) => {
+      const result = parseEsqlQuery(esqlQuery);
+      expect(result.errors.length).toEqual(0);
+      expect(result).toMatchObject({ errors: [] });
+    });
+  });
+
+  describe('METADATA operator', () => {
+    it.each([
+      ['from test*'],
+      ['from metadata*'],
+      ['from test* | keep metadata'],
+      ['from test* | eval x="metadata _id"'],
+    ])('detects when METADATA operator is missing in a NON aggregating query "%s"', (esqlQuery) => {
+      expect(parseEsqlQuery(esqlQuery)).toEqual({
+        errors: [],
+        hasMetadataOperator: false,
+        isEsqlQueryAggregating: false,
+      });
+    });
+
+    it.each([
+      ['from test* metadata _id'],
+      ['from test* metadata _id, _index'],
+      ['from test* metadata _index, _id'],
+      ['from test*  metadata _id '],
+      ['from test*    metadata _id '],
+      ['from test*  metadata _id | limit 10'],
+      [
+        `from packetbeat* metadata
+
+        _id
+        | limit 100`,
+      ],
+    ])('detects existin METADATA operator in a NON aggregating query "%s"', (esqlQuery) =>
+      expect(parseEsqlQuery(esqlQuery)).toEqual({
+        errors: [],
+        hasMetadataOperator: true,
+        isEsqlQueryAggregating: false,
+      })
+    );
+
+    it('detects missing METADATA operator in an aggregating query "from test* | stats c = count(*) by fieldA"', () =>
+      expect(parseEsqlQuery('from test* | stats c = count(*) by fieldA')).toEqual({
+        errors: [],
+        hasMetadataOperator: false,
+        isEsqlQueryAggregating: true,
+      }));
+  });
+
+  describe('METADATA _id field for NON aggregating queries', () => {
+    it('detects missing METADATA "_id" field', () => {
+      expect(parseEsqlQuery('from test*')).toEqual({
+        errors: [],
+        hasMetadataOperator: false,
+        isEsqlQueryAggregating: false,
+      });
+    });
+
+    it('detects existing METADATA "_id" field', async () => {
+      expect(parseEsqlQuery('from test* metadata _id')).toEqual({
+        errors: [],
+        hasMetadataOperator: true,
+        isEsqlQueryAggregating: false,
+      });
+    });
+  });
+
+  describe('METADATA _id field for aggregating queries', () => {
+    it('detects existing METADATA operator with missing "_id" field', () => {
+      expect(
+        parseEsqlQuery('from test* metadata someField | stats c = count(*) by fieldA')
+      ).toEqual({ errors: [], hasMetadataOperator: false, isEsqlQueryAggregating: true });
+    });
+  });
+});

packages/kbn-securitysolution-utils/src/esql/parse_esql_query.ts

@@ -1,21 +1,40 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import type { ESQLAstQueryExpression, ESQLCommandOption, EditorError } from '@kbn/esql-ast';
-import { parse } from '@kbn/esql-ast';
+import { type ESQLAstQueryExpression, parse, ESQLCommandOption, EditorError } from '@kbn/esql-ast';
 import { isColumnItem, isOptionItem } from '@kbn/esql-validation-autocomplete';
-import { isAggregatingQuery } from '@kbn/securitysolution-utils';
+import { isAggregatingQuery } from './compute_if_esql_query_aggregating';
 
-interface ParseEsqlQueryResult {
+export interface ParseEsqlQueryResult {
   errors: EditorError[];
   isEsqlQueryAggregating: boolean;
   hasMetadataOperator: boolean;
 }
 
+/**
+ * check if esql query valid for Security rule:
+ * - if it's non aggregation query it must have metadata operator
+ */
+export const parseEsqlQuery = (query: string): ParseEsqlQueryResult => {
+  const { root, errors } = parse(query);
+  const isEsqlQueryAggregating = isAggregatingQuery(root);
+
+  return {
+    errors,
+    isEsqlQueryAggregating,
+    hasMetadataOperator: computeHasMetadataOperator(root),
+  };
+};
+
+/**
+ * checks whether query has metadata _id operator
+ */
 function computeHasMetadataOperator(astExpression: ESQLAstQueryExpression): boolean {
   // Check whether the `from` command has `metadata` operator
   const metadataOption = getMetadataOption(astExpression);
@@ -50,13 +69,3 @@ function getMetadataOption(astExpression: ESQLAstQueryExpression): ESQLCommandOp
 
   return undefined;
 }
-
-export const parseEsqlQuery = (query: string): ParseEsqlQueryResult => {
-  const { root, errors } = parse(query);
-  const isEsqlQueryAggregating = isAggregatingQuery(root);
-  return {
-    errors,
-    isEsqlQueryAggregating,
-    hasMetadataOperator: computeHasMetadataOperator(root),
-  };
-};

packages/kbn-securitysolution-utils/tsconfig.json

@@ -13,7 +13,8 @@
   "kbn_references": [
     "@kbn/i18n",
     "@kbn/esql-utils",
-    "@kbn/esql-ast"
+    "@kbn/esql-ast",
+    "@kbn/esql-validation-autocomplete"
   ],
   "exclude": [
     "target/**/*",

x-pack/plugins/security_solution/public/detection_engine/rule_creation/components/esql_query_edit/validators/esql_query_validator_factory.ts

@@ -7,10 +7,7 @@
 
 import type { QueryClient } from '@tanstack/react-query';
 import type { DatatableColumn } from '@kbn/expressions-plugin/common';
-import type { ESQLAstQueryExpression, ESQLCommandOption } from '@kbn/esql-ast';
-import { parse } from '@kbn/esql-ast';
-import { isAggregatingQuery } from '@kbn/securitysolution-utils';
-import { isColumnItem, isOptionItem } from '@kbn/esql-validation-autocomplete';
+import { parseEsqlQuery } from '@kbn/securitysolution-utils';
 import type { FormData, ValidationError, ValidationFunc } from '../../../../../shared_imports';
 import type { FieldValueQueryBar } from '../../../../rule_creation_ui/components/query_bar_field';
 import { fetchEsqlQueryColumns } from '../../../logic/esql_query_columns';
@@ -79,59 +76,6 @@ function hasIdColumn(columns: DatatableColumn[]): boolean {
   return columns.some(({ id }) => '_id' === id);
 }
 
-/**
- * check if esql query valid for Security rule:
- * - if it's non aggregation query it must have metadata operator
- */
-function parseEsqlQuery(query: string) {
-  const { root, errors } = parse(query);
-  const isEsqlQueryAggregating = isAggregatingQuery(root);
-
-  return {
-    errors,
-    isEsqlQueryAggregating,
-    hasMetadataOperator: computeHasMetadataOperator(root),
-  };
-}
-
-/**
- * checks whether query has metadata _id operator
- */
-function computeHasMetadataOperator(astExpression: ESQLAstQueryExpression): boolean {
-  // Check whether the `from` command has `metadata` operator
-  const metadataOption = getMetadataOption(astExpression);
-  if (!metadataOption) {
-    return false;
-  }
-
-  // Check whether the `metadata` operator has `_id` argument
-  const idColumnItem = metadataOption.args.find(
-    (fromArg) => isColumnItem(fromArg) && fromArg.name === '_id'
-  );
-  if (!idColumnItem) {
-    return false;
-  }
-
-  return true;
-}
-
-function getMetadataOption(astExpression: ESQLAstQueryExpression): ESQLCommandOption | undefined {
-  const fromCommand = astExpression.commands.find((x) => x.name === 'from');
-
-  if (!fromCommand?.args) {
-    return undefined;
-  }
-
-  // Check whether the `from` command has `metadata` operator
-  for (const fromArg of fromCommand.args) {
-    if (isOptionItem(fromArg) && fromArg.name === 'metadata') {
-      return fromArg;
-    }
-  }
-
-  return undefined;
-}
-
 function constructSyntaxError(error: Error): ValidationError {
   return {
     code: ESQL_ERROR_CODES.INVALID_SYNTAX,

x-pack/plugins/security_solution/server/lib/siem_migrations/rules/task/agent/sub_graphs/translate_rule/nodes/validation/validation.ts

@@ -7,8 +7,8 @@
 
 import type { Logger } from '@kbn/core/server';
 import { isEmpty } from 'lodash/fp';
+import { parseEsqlQuery } from '@kbn/securitysolution-utils';
 import type { GraphNode } from '../../types';
-import { parseEsqlQuery } from './esql_query';
 
 interface GetValidationNodeParams {
   logger: Logger;

x-pack/plugins/security_solution/tsconfig.json

@@ -209,8 +209,6 @@
     "@kbn/core-theme-browser",
     "@kbn/integration-assistant-plugin",
     "@kbn/avc-banner",
-    "@kbn/esql-ast",
-    "@kbn/esql-validation-autocomplete",
     "@kbn/config",
     "@kbn/openapi-common",
     "@kbn/securitysolution-lists-common",