[8.x] [ES|QL] Validation and autocomplete support for the CHANGE_POINT command (#216043) (#216657)

# Backport This will backport the following commits from `main` to `8.x`: - [[ES|QL] Validation and autocomplete support for the `CHANGE_POINT` command (#216043)](https://github.com/elastic/kibana/pull/216043)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)  Co-authored-by: Dima Arnautov <dmitrii.arnautov@elastic.co>
2025-04-24 01:38:56 -04:00 · 2025-04-01 18:01:01 +02:00 · 2025-04-01 18:01:01 +02:00 · 189a1eb14a
commit 189a1eb14a
parent 2d7bbd76f6
5 changed files with 419 additions and 7 deletions
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/tests/autocomplete.command.change_point.test.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/tests/autocomplete.command.change_point.test.ts
@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { attachTriggerCommand, getFieldNamesByType, setup } from './helpers';
+import { ESQL_NUMBER_TYPES } from '../../shared/esql_types';
+
+describe('autocomplete.suggest', () => {
+  describe('CHANGE_POINT', () => {
+    let assertSuggestions: Awaited<ReturnType<typeof setup>>['assertSuggestions'];
+
+    beforeEach(async () => {
+      const setupResult = await setup();
+      assertSuggestions = setupResult.assertSuggestions;
+    });
+
+    it('suggests value columns of numeric types', async () => {
+      await assertSuggestions(
+        `from a | change_point /`,
+        getFieldNamesByType(ESQL_NUMBER_TYPES).map((v) => `${v} `)
+      );
+    });
+
+    it('suggests ON after value column', async () => {
+      await assertSuggestions(
+        `from a | change_point value /`,
+        ['ON ', 'AS ', '| '].map(attachTriggerCommand)
+      );
+      await assertSuggestions(
+        `from a | change_point value O/`,
+        ['ON ', 'AS ', '| '].map(attachTriggerCommand)
+      );
+    });
+
+    it('suggests fields after ON', async () => {
+      await assertSuggestions(
+        `from a | change_point value on /`,
+        getFieldNamesByType('any').map((v) => `${v} `)
+      );
+      await assertSuggestions(
+        `from a | change_point value on fi/`,
+        getFieldNamesByType('any').map((v) => `${v} `)
+      );
+    });
+
+    describe('AS', () => {
+      it('suggests AS after ON <field>', async () => {
+        await assertSuggestions(
+          `from a | change_point value on field /`,
+          ['AS ', '| '].map(attachTriggerCommand)
+        );
+      });
+
+      it('suggests default field name for AS clauses with an empty ON', async () => {
+        await assertSuggestions(`from a | change_point value as / `, ['changePointType, ']);
+
+        await assertSuggestions(`from a | change_point value on field as changePointType,/`, [
+          'pValue',
+        ]);
+      });
+
+      it('suggests default field name for AS clauses', async () => {
+        await assertSuggestions(`from a | change_point value on field as / `, [
+          'changePointType, ',
+        ]);
+
+        await assertSuggestions(`from a | change_point value on field as changePointType,/`, [
+          'pValue',
+        ]);
+      });
+
+      it('suggests a default pValue column name', async () => {
+        await assertSuggestions(`from a | change_point value on field as changePointType,pValu/`, [
+          'pValue',
+        ]);
+      });
+
+      it('suggests pipe after complete command', async () => {
+        await assertSuggestions(
+          `from a | change_point value on field as changePointType, pValue /`,
+          ['| ']
+        );
+      });
+    });
+  });
+});
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/change_point/index.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/commands/change_point/index.ts
@ -0,0 +1,147 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { ESQLCommand } from '@kbn/esql-ast';
+import { i18n } from '@kbn/i18n';
+import { ESQL_NUMBER_TYPES } from '../../../shared/esql_types';
+import { findFinalWord, isSingleItem } from '../../../shared/helpers';
+import { CommandSuggestParams } from '../../../definitions/types';
+import type { SuggestionRawDefinition } from '../../types';
+import { pipeCompleteItem } from '../../complete_items';
+import { buildVariablesDefinitions, TRIGGER_SUGGESTION_COMMAND } from '../../factories';
+
+export enum Position {
+  VALUE = 'value',
+  AFTER_VALUE = 'after_value',
+  ON_COLUMN = 'on_column',
+  AFTER_ON_CLAUSE = 'after_on_clause',
+  AS_TYPE_COLUMN = 'as_type_clause',
+  AS_P_VALUE_COLUMN = 'as_p_value_column',
+  AFTER_AS_CLAUSE = 'after_as_clause',
+}
+
+export const getPosition = (
+  innerText: string,
+  command: ESQLCommand<'change_point'>
+): Position | undefined => {
+  if (command.args.length < 2) {
+    if (innerText.match(/CHANGE_POINT\s+\S*$/i)) {
+      return Position.VALUE;
+    }
+
+    if (innerText.match(/CHANGE_POINT\s+\S+\s*\S*$/i)) {
+      return Position.AFTER_VALUE;
+    }
+  }
+
+  const lastArg = command.args[command.args.length - 1];
+
+  if (innerText.match(/on\s+\S*$/i)) {
+    return Position.ON_COLUMN;
+  }
+
+  if (isSingleItem(lastArg) && lastArg.name === 'on') {
+    if (innerText.match(/on\s+\S+\s+$/i)) {
+      return Position.AFTER_ON_CLAUSE;
+    }
+  }
+
+  if (innerText.match(/as\s+$/i)) {
+    return Position.AS_TYPE_COLUMN;
+  }
+
+  if (isSingleItem(lastArg) && lastArg.name === 'as') {
+    if (innerText.match(/as\s+\S+,\s*\S*$/i)) {
+      return Position.AS_P_VALUE_COLUMN;
+    }
+
+    if (innerText.match(/as\s+\S+,\s*\S+\s+$/i)) {
+      return Position.AFTER_AS_CLAUSE;
+    }
+  }
+};
+
+export const onSuggestion: SuggestionRawDefinition = {
+  label: 'ON',
+  text: 'ON ',
+  kind: 'Reference',
+  detail: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.onDoc', {
+    defaultMessage: 'On',
+  }),
+  sortText: '1',
+  command: TRIGGER_SUGGESTION_COMMAND,
+};
+
+export const asSuggestion: SuggestionRawDefinition = {
+  label: 'AS',
+  text: 'AS ',
+  kind: 'Reference',
+  detail: i18n.translate('kbn-esql-validation-autocomplete.esql.definitions.asDoc', {
+    defaultMessage: 'As',
+  }),
+  sortText: '2',
+  command: TRIGGER_SUGGESTION_COMMAND,
+};
+
+export async function suggest({
+  innerText,
+  command,
+  getColumnsByType,
+}: CommandSuggestParams<'change_point'>): Promise<SuggestionRawDefinition[]> {
+  const pos = getPosition(innerText, command);
+
+  switch (pos) {
+    case Position.VALUE:
+      const numericFields = await getColumnsByType(ESQL_NUMBER_TYPES, [], {
+        advanceCursor: true,
+        openSuggestions: true,
+      });
+      const lastWord = findFinalWord(innerText);
+      if (lastWord !== '') {
+        numericFields.forEach((fieldSuggestion) => {
+          fieldSuggestion.rangeToReplace = {
+            start: innerText.length - lastWord.length + 1,
+            end: innerText.length + 1,
+          };
+        });
+      }
+      return numericFields;
+    case Position.AFTER_VALUE: {
+      return [onSuggestion, asSuggestion, pipeCompleteItem];
+    }
+    case Position.ON_COLUMN: {
+      const onFields = await getColumnsByType('any', [], {
+        advanceCursor: true,
+        openSuggestions: true,
+      });
+      return onFields;
+    }
+    case Position.AFTER_ON_CLAUSE:
+      return [asSuggestion, pipeCompleteItem];
+    case Position.AS_TYPE_COLUMN: {
+      // add comma and space
+      return buildVariablesDefinitions(['changePointType']).map((v) => ({
+        ...v,
+        text: v.text + ', ',
+        command: TRIGGER_SUGGESTION_COMMAND,
+      }));
+    }
+    case Position.AS_P_VALUE_COLUMN: {
+      return buildVariablesDefinitions(['pValue']).map((v) => ({
+        ...v,
+        command: TRIGGER_SUGGESTION_COMMAND,
+      }));
+    }
+    case Position.AFTER_AS_CLAUSE: {
+      return [pipeCompleteItem];
+    }
+    default:
+      return [];
+  }
+}
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/complete_items.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/autocomplete/complete_items.ts
@ -9,8 +9,7 @@

 import { i18n } from '@kbn/i18n';
 import type { ItemKind, SuggestionRawDefinition } from './types';
-import { operatorsDefinitions } from '../definitions/all_operators';
-import { getOperatorSuggestion, TRIGGER_SUGGESTION_COMMAND } from './factories';
+import { TRIGGER_SUGGESTION_COMMAND } from './factories';
 import { CommandDefinition, CommandTypeDefinition } from '../definitions/types';
 import { getCommandDefinition } from '../shared/helpers';
 import { buildDocumentation } from './documentation_util';
@ -22,11 +21,6 @@ const techPreviewLabel = i18n.translate(
  }
 );

-export function getAssignmentDefinitionCompletitionItem() {
-  const assignFn = operatorsDefinitions.find(({ name }) => name === '=')!;
-  return getOperatorSuggestion(assignFn);
-}
-
 export const getCommandAutocompleteDefinitions = (
  commands: Array<CommandDefinition<string>>
 ): SuggestionRawDefinition[] => {
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/definitions/commands.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/definitions/commands.ts
@ -57,9 +57,11 @@ import { suggest as suggestForShow } from '../autocomplete/commands/show';
 import { suggest as suggestForSort } from '../autocomplete/commands/sort';
 import { suggest as suggestForStats } from '../autocomplete/commands/stats';
 import { suggest as suggestForWhere } from '../autocomplete/commands/where';
+import { suggest as suggestForChangePoint } from '../autocomplete/commands/change_point';

 import { METADATA_FIELDS } from '../shared/constants';
 import { getMessageFromId } from '../validation/errors';
+import { isNumericType } from '../shared/esql_types';

 const statsValidator = (command: ESQLCommand) => {
  const messages: ESQLMessage[] = [];
@ -586,4 +588,90 @@ export const commandDefinitions: Array<CommandDefinition<any>> = [
    ],
    suggest: suggestForJoin,
  },
+  {
+    hidden: true,
+    name: 'change_point',
+    preview: true,
+    description: i18n.translate(
+      'kbn-esql-validation-autocomplete.esql.definitions.changePointDoc',
+      {
+        defaultMessage: 'Detect change point in the query results',
+      }
+    ),
+    declaration: `CHANGE_POINT <value> ON <field_name> AS <type>, <pvalue>`,
+    examples: [
+      '… | CHANGE_POINT value',
+      '… | CHANGE_POINT value ON timestamp',
+      '… | CHANGE_POINT value ON timestamp AS type, pvalue',
+    ],
+    validate: (command: ESQLCommand, references) => {
+      const messages: ESQLMessage[] = [];
+
+      // validate change point value column
+      const valueArg = command.args[0];
+      if (isColumnItem(valueArg)) {
+        const columnName = valueArg.name;
+        // look up for columns in variables and existing fields
+        let valueColumnType: string | undefined;
+        const variableRef = references.variables.get(columnName);
+        if (variableRef) {
+          valueColumnType = variableRef.find((v) => v.name === columnName)?.type;
+        } else {
+          const fieldRef = references.fields.get(columnName);
+          valueColumnType = fieldRef?.type;
+        }
+
+        if (valueColumnType && !isNumericType(valueColumnType)) {
+          messages.push({
+            location: command.location,
+            text: i18n.translate(
+              'kbn-esql-validation-autocomplete.esql.validation.changePointUnsupportedFieldType',
+              {
+                defaultMessage:
+                  'CHANGE_POINT only supports numeric types values, found [{columnName}] of type [{valueColumnType}]',
+                values: { columnName, valueColumnType },
+              }
+            ),
+            type: 'error',
+            code: 'changePointUnsupportedFieldType',
+          });
+        }
+      }
+
+      // validate ON column
+      const defaultOnColumnName = '@timestamp';
+      const onColumn = command.args.find((arg) => isOptionItem(arg) && arg.name === 'on');
+      const hasDefaultOnColumn = references.fields.has(defaultOnColumnName);
+      if (!onColumn && !hasDefaultOnColumn) {
+        messages.push({
+          location: command.location,
+          text: i18n.translate(
+            'kbn-esql-validation-autocomplete.esql.validation.changePointOnFieldMissing',
+            {
+              defaultMessage: '[CHANGE_POINT] Default {defaultOnColumnName} column is missing',
+              values: { defaultOnColumnName },
+            }
+          ),
+          type: 'error',
+          code: 'changePointOnFieldMissing',
+        });
+      }
+
+      // validate AS
+      const asArg = command.args.find((arg) => isOptionItem(arg) && arg.name === 'as');
+      if (asArg && isOptionItem(asArg)) {
+        // populate variable references to prevent the common check from failing with unknown column
+        asArg.args.forEach((arg, index) => {
+          if (isColumnItem(arg)) {
+            references.variables.set(arg.name, [
+              { name: arg.name, location: arg.location, type: index === 0 ? 'keyword' : 'long' },
+            ]);
+          }
+        });
+      }
+
+      return messages;
+    },
+    suggest: suggestForChangePoint,
+  },
 ];
--- a/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/validation/tests/validation.command.change_point.test.ts
+++ b/src/platform/packages/shared/kbn-esql-validation-autocomplete/src/validation/tests/validation.command.change_point.test.ts
@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { setup } from './helpers';
+import { fields } from '../../__tests__/helpers';
+
+describe('validation', () => {
+  describe('command', () => {
+    describe('CHANGE_POINT <value> [ ON <condition> AS <type>, <pvalue>]', () => {
+      beforeEach(() => {
+        jest.clearAllMocks();
+      });
+
+      describe('... <value> ...', () => {
+        test('validates the most basic query', async () => {
+          const { expectErrors } = await setup();
+          await expectErrors('FROM index | CHANGE_POINT longField', []);
+        });
+
+        test('validates the full query', async () => {
+          const { expectErrors } = await setup();
+          await expectErrors(
+            'FROM index | STATS field = AVG(longField) BY @timestamp=BUCKET(@timestamp, 8 hours) | CHANGE_POINT field ON @timestamp',
+            []
+          );
+        });
+
+        test('raises error on unknown field', async () => {
+          const { expectErrors } = await setup();
+
+          await expectErrors('FROM index | CHANGE_POINT notExistingField', [
+            'Unknown column [notExistingField]',
+          ]);
+        });
+
+        test('raises error on unsupported field time for value', async () => {
+          const { expectErrors } = await setup();
+
+          await expectErrors('FROM index | CHANGE_POINT keywordField', [
+            'CHANGE_POINT only supports numeric types values, found [keywordField] of type [keyword]',
+          ]);
+        });
+
+        test('raises error when the default @timestamp field is missing', async () => {
+          const { expectErrors, callbacks } = await setup();
+
+          // make sure that @timestamp field is not present
+          (callbacks.getColumnsFor as jest.Mock).mockResolvedValue(
+            fields.filter((v) => v.name !== '@timestamp')
+          );
+
+          await expectErrors('FROM index | CHANGE_POINT longField', [
+            `[CHANGE_POINT] Default @timestamp column is missing`,
+          ]);
+        });
+
+        test('allows manual input for ON field', async () => {
+          const { expectErrors } = await setup();
+
+          await expectErrors('FROM index | CHANGE_POINT longField ON keywordField', []);
+        });
+
+        test('allows renaming for change point type and pValue columns', async () => {
+          const { expectErrors } = await setup();
+          await expectErrors(
+            'FROM index | STATS field = AVG(longField) BY @timestamp=BUCKET(@timestamp, 8 hours) | CHANGE_POINT field ON @timestamp AS changePointType, pValue',
+            []
+          );
+        });
+
+        test('allows renaming for change point type and pValue columns without specifying the ON field', async () => {
+          const { expectErrors } = await setup();
+          await expectErrors('FROM index | CHANGE_POINT longField AS changePointType, pValue', []);
+        });
+
+        test('does not allow renaming for change point type only', async () => {
+          const { expectErrors } = await setup();
+          await expectErrors(
+            'FROM index | STATS field = AVG(longField) BY @timestamp=BUCKET(@timestamp, 8 hours) | CHANGE_POINT field ON @timestamp AS changePointType',
+            [`SyntaxError: mismatched input '<EOF>' expecting ','`]
+          );
+        });
+      });
+    });
+  });
+});