From 86fdbe5379bbe43036d503752ff502fc28718f4b Mon Sep 17 00:00:00 2001 From: Vadim Kibana <82822460+vadimkibana@users.noreply.github.com> Date: Mon, 7 Apr 2025 16:11:10 +0200 Subject: [PATCH] [ES|QL] Source AST node parsing improvements and source selector parsing (#217299) ## Summary - This PR introduces source selector (aka "component") parsing `FROM index::` - It also improves source cluster and index parsing `FROM :` - Previous cluster and index would be parsed as `string` now they are parsed as `ESQLStringLiteral` instead. This is more correct as any of those can take three forms, and `ESQLStringLiteral` handles all three forms: 1. unquoted string: `cluster:index` 2. single-double quoted string: `"cluster":"index"` 3. triple-double quote string: `"""cluster""":"""index""` - The `ESQLStringLiteral` now also supports *"unquoted strings"* in addition to single `"str"` and triple `"""str"""` quoted strings. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../components/from_command/index.tsx | 2 +- .../shared/kbn-esql-ast/src/ast/helpers.ts | 4 + .../kbn-esql-ast/src/builder/builder.test.ts | 61 +++++- .../kbn-esql-ast/src/builder/builder.ts | 97 ++++++--- .../src/mutate/commands/from/sources.test.ts | 36 ++- .../src/mutate/commands/from/sources.ts | 6 +- .../generic/commands/args/index.test.ts | 14 +- .../__tests__/ast_parser.source.test.ts | 93 +++++--- .../src/parser/__tests__/from.test.ts | 205 ++++++++++++++++++ .../src/parser/esql_ast_builder_listener.ts | 22 +- .../kbn-esql-ast/src/parser/factories.ts | 115 +++++----- .../kbn-esql-ast/src/parser/factories/from.ts | 35 +++ .../kbn-esql-ast/src/parser/factories/join.ts | 4 +- .../shared/kbn-esql-ast/src/parser/walkers.ts | 10 - .../__tests__/basic_pretty_printer.test.ts | 22 +- .../__tests__/wrapping_pretty_printer.test.ts | 10 + .../src/pretty_print/leaf_printer.ts | 16 +- .../packages/shared/kbn-esql-ast/src/types.ts | 25 ++- .../languages/esql/lib/hover/helpers.test.ts | 12 +- 19 files changed, 611 insertions(+), 178 deletions(-) create mode 100644 src/platform/packages/shared/kbn-esql-ast/src/parser/factories/from.ts diff --git a/examples/esql_ast_inspector/public/components/esql_inspector/components/preview/components/preview_ui/components/from_command/index.tsx b/examples/esql_ast_inspector/public/components/esql_inspector/components/preview/components/preview_ui/components/from_command/index.tsx index 3c10240eadfe..e5a605aae464 100644 --- a/examples/esql_ast_inspector/public/components/esql_inspector/components/preview/components/preview_ui/components/from_command/index.tsx +++ b/examples/esql_ast_inspector/public/components/esql_inspector/components/preview/components/preview_ui/components/from_command/index.tsx @@ -51,7 +51,7 @@ export const FromCommand: React.FC = () => { color="text" onClick={() => { const length = from.args.length; - const source = Builder.expression.source({ + const source = Builder.expression.source.node({ index: `source${length + 1}`, sourceType: 'index', }); diff --git a/src/platform/packages/shared/kbn-esql-ast/src/ast/helpers.ts b/src/platform/packages/shared/kbn-esql-ast/src/ast/helpers.ts index 44aae9489f6a..3857770edae7 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/ast/helpers.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/ast/helpers.ts @@ -20,6 +20,7 @@ import type { ESQLParamLiteral, ESQLProperNode, ESQLSource, + ESQLStringLiteral, } from '../types'; import { BinaryExpressionGroup } from './constants'; @@ -62,6 +63,9 @@ export const isFieldExpression = ( export const isLiteral = (node: unknown): node is ESQLLiteral => isProperNode(node) && node.type === 'literal'; +export const isStringLiteral = (node: unknown): node is ESQLStringLiteral => + isLiteral(node) && node.literalType === 'keyword'; + export const isIntegerLiteral = (node: unknown): node is ESQLIntegerLiteral => isLiteral(node) && node.literalType === 'integer'; diff --git a/src/platform/packages/shared/kbn-esql-ast/src/builder/builder.test.ts b/src/platform/packages/shared/kbn-esql-ast/src/builder/builder.test.ts index d6ed06863337..d2ce118f478b 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/builder/builder.test.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/builder/builder.test.ts @@ -25,7 +25,7 @@ describe('command', () => { const node = Builder.command({ name: 'from', args: [ - Builder.expression.source({ index: 'my_index', sourceType: 'index' }), + Builder.expression.source.node({ index: 'my_index', sourceType: 'index' }), Builder.option({ name: 'by', args: [ @@ -94,21 +94,39 @@ describe('function', () => { describe('source', () => { test('basic index', () => { - const node = Builder.expression.source({ index: 'my_index', sourceType: 'index' }); + const node = Builder.expression.source.node({ index: 'my_index', sourceType: 'index' }); const text = BasicPrettyPrinter.expression(node); expect(text).toBe('my_index'); }); test('basic index using shortcut', () => { - const node = Builder.expression.source('my_index'); + const node = Builder.expression.source.node('my_index'); const text = BasicPrettyPrinter.expression(node); expect(text).toBe('my_index'); }); + test('basic quoted index using shortcut', () => { + const node = Builder.expression.source.node(Builder.expression.literal.string('my_index')); + const text = BasicPrettyPrinter.expression(node); + + expect(text).toBe('"my_index"'); + }); + test('index with cluster', () => { - const node = Builder.expression.source({ + const node = Builder.expression.source.node({ + index: 'my_index', + sourceType: 'index', + cluster: Builder.expression.literal.string('my_cluster', { unquoted: true }), + }); + const text = BasicPrettyPrinter.expression(node); + + expect(text).toBe('my_cluster:my_index'); + }); + + test('index with cluster - plain text cluster', () => { + const node = Builder.expression.source.node({ index: 'my_index', sourceType: 'index', cluster: 'my_cluster', @@ -118,19 +136,38 @@ describe('source', () => { expect(text).toBe('my_cluster:my_index'); }); - test('can use .indexSource() shorthand to specify cluster', () => { - const node = Builder.expression.indexSource('my_index', 'my_cluster'); - const text = BasicPrettyPrinter.expression(node); - - expect(text).toBe('my_cluster:my_index'); - }); - test('policy index', () => { - const node = Builder.expression.source({ index: 'my_policy', sourceType: 'policy' }); + const node = Builder.expression.source.node({ index: 'my_policy', sourceType: 'policy' }); const text = BasicPrettyPrinter.expression(node); expect(text).toBe('my_policy'); }); + + describe('.index', () => { + test('can use .source.index() shorthand to specify cluster', () => { + const node = Builder.expression.source.index('my_index', 'my_cluster'); + const text = BasicPrettyPrinter.expression(node); + + expect(text).toBe('my_cluster:my_index'); + }); + + test('can use .source.index() and specify quotes around cluster', () => { + const node = Builder.expression.source.index( + 'my_index', + Builder.expression.literal.string('hello 👋') + ); + const text = BasicPrettyPrinter.expression(node); + + expect(text).toBe('"hello 👋":my_index'); + }); + + test('can use .source.index() shorthand to specify selector', () => { + const node = Builder.expression.source.index('my_index', '', 'my_selector'); + const text = BasicPrettyPrinter.expression(node); + + expect(text).toBe('my_index::my_selector'); + }); + }); }); describe('column', () => { diff --git a/src/platform/packages/shared/kbn-esql-ast/src/builder/builder.ts b/src/platform/packages/shared/kbn-esql-ast/src/builder/builder.ts index ffd94ebc788d..09e5506a74c9 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/builder/builder.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/builder/builder.ts @@ -9,6 +9,7 @@ /* eslint-disable @typescript-eslint/no-namespace */ +import { isStringLiteral } from '../ast/helpers'; import { LeafPrinter } from '../pretty_print'; import { ESQLAstComment, @@ -110,41 +111,73 @@ export namespace Builder { }; }; - export type SourceTemplate = { index: string } & Omit, 'name'>; + export namespace source { + export type SourceTemplate = { + cluster?: string | ESQLSource['cluster']; + index?: string | ESQLSource['index']; + selector?: string | ESQLSource['selector']; + } & Omit, 'name' | 'cluster' | 'index' | 'selector'> & + Partial>; - export const source = ( - indexOrTemplate: string | SourceTemplate, - fromParser?: Partial - ): ESQLSource => { - const template: SourceTemplate = - typeof indexOrTemplate === 'string' - ? { sourceType: 'index', index: indexOrTemplate } - : indexOrTemplate; - const { index, cluster } = template; - return { - ...template, - ...Builder.parserFields(fromParser), - type: 'source', - name: (cluster ? cluster + ':' : '') + index, - }; - }; + export const node = ( + indexOrTemplate: string | ESQLStringLiteral | SourceTemplate, + fromParser?: Partial + ): ESQLSource => { + const template: SourceTemplate = + typeof indexOrTemplate === 'string' || isStringLiteral(indexOrTemplate) + ? { sourceType: 'index', index: indexOrTemplate } + : indexOrTemplate; + const cluster: ESQLSource['cluster'] = !template.cluster + ? undefined + : typeof template.cluster === 'string' + ? Builder.expression.literal.string(template.cluster, { unquoted: true }) + : template.cluster; + const index: ESQLSource['index'] = !template.index + ? undefined + : typeof template.index === 'string' + ? Builder.expression.literal.string(template.index, { unquoted: true }) + : template.index; + const selector: ESQLSource['selector'] = !template.selector + ? undefined + : typeof template.selector === 'string' + ? Builder.expression.literal.string(template.selector, { unquoted: true }) + : template.selector; + const sourceNode: ESQLSource = { + ...template, + ...Builder.parserFields(fromParser), + type: 'source', + cluster, + index, + selector, + name: template.name ?? '', + }; - export const indexSource = ( - index: string, - cluster?: string, - template?: Omit, 'name' | 'index' | 'cluster'>, - fromParser?: Partial - ): ESQLSource => { - return { - ...template, - ...Builder.parserFields(fromParser), - index, - cluster, - sourceType: 'index', - type: 'source', - name: (cluster ? cluster + ':' : '') + index, + if (!sourceNode.name) { + sourceNode.name = LeafPrinter.source(sourceNode); + } + + return sourceNode; }; - }; + + export const index = ( + indexName: string, + cluster?: string | ESQLSource['cluster'], + selector?: string | ESQLSource['selector'], + template?: Omit, 'name' | 'index' | 'cluster'>, + fromParser?: Partial + ): ESQLSource => { + return Builder.expression.source.node( + { + ...template, + index: indexName, + cluster, + selector, + sourceType: 'index', + }, + fromParser + ); + }; + } export type ColumnTemplate = Omit, 'name' | 'quoted' | 'parts'>; diff --git a/src/platform/packages/shared/kbn-esql-ast/src/mutate/commands/from/sources.test.ts b/src/platform/packages/shared/kbn-esql-ast/src/mutate/commands/from/sources.test.ts index 866a6dd8bdb2..0c9819e8ca57 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/mutate/commands/from/sources.test.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/mutate/commands/from/sources.test.ts @@ -40,16 +40,26 @@ describe('commands.from.sources', () => { expect(list).toMatchObject([ { type: 'source', - index: 'index', + index: { + valueUnquoted: 'index', + }, }, { type: 'source', - index: 'index2', + index: { + valueUnquoted: 'index2', + }, }, { type: 'source', - index: 'index3', - cluster: 'cl', + index: { + valueUnquoted: 'index3', + }, + cluster: { + type: 'literal', + literalType: 'keyword', + valueUnquoted: 'cl', + }, }, ]); }); @@ -72,7 +82,9 @@ describe('commands.from.sources', () => { expect(source).toMatchObject({ type: 'source', name: 'index', - index: 'index', + index: { + valueUnquoted: 'index', + }, }); }); @@ -85,13 +97,21 @@ describe('commands.from.sources', () => { expect(source1).toMatchObject({ type: 'source', name: 's2', - index: 's2', + index: { + valueUnquoted: 's2', + }, }); expect(source2).toMatchObject({ type: 'source', name: 'c:s1', - index: 's1', - cluster: 'c', + index: { + valueUnquoted: 's1', + }, + cluster: { + type: 'literal', + literalType: 'keyword', + valueUnquoted: 'c', + }, }); }); }); diff --git a/src/platform/packages/shared/kbn-esql-ast/src/mutate/commands/from/sources.ts b/src/platform/packages/shared/kbn-esql-ast/src/mutate/commands/from/sources.ts index 8c38cd829ea0..c464324ec339 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/mutate/commands/from/sources.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/mutate/commands/from/sources.ts @@ -52,10 +52,10 @@ export const find = ( cluster?: string ): ESQLSource | undefined => { return findByPredicate(ast, (source) => { - if (index !== source.index) { + if (index !== source.index?.valueUnquoted) { return false; } - if (typeof cluster === 'string' && cluster !== source.cluster) { + if (cluster && typeof cluster === 'string' && cluster !== source.cluster?.valueUnquoted) { return false; } @@ -91,7 +91,7 @@ export const insert = ( return; } - const source = Builder.expression.indexSource(indexName, clusterName); + const source = Builder.expression.source.index(indexName, clusterName); if (index === -1) { generic.commands.args.append(command, source); diff --git a/src/platform/packages/shared/kbn-esql-ast/src/mutate/generic/commands/args/index.test.ts b/src/platform/packages/shared/kbn-esql-ast/src/mutate/generic/commands/args/index.test.ts index 29c0898b694d..452951fa137a 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/mutate/generic/commands/args/index.test.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/mutate/generic/commands/args/index.test.ts @@ -21,7 +21,7 @@ describe('generic.commands.args', () => { generic.commands.args.insert( command!, - Builder.expression.source({ index: 'test', sourceType: 'index' }), + Builder.expression.source.node({ index: 'test', sourceType: 'index' }), 123 ); @@ -37,7 +37,7 @@ describe('generic.commands.args', () => { generic.commands.args.insert( command!, - Builder.expression.source({ index: 'test', sourceType: 'index' }), + Builder.expression.source.node({ index: 'test', sourceType: 'index' }), 0 ); @@ -53,7 +53,7 @@ describe('generic.commands.args', () => { generic.commands.args.insert( command!, - Builder.expression.source({ index: 'test', sourceType: 'index' }), + Builder.expression.source.node({ index: 'test', sourceType: 'index' }), 1 ); @@ -70,7 +70,7 @@ describe('generic.commands.args', () => { generic.commands.args.insert( command!, - Builder.expression.source({ index: 'test', sourceType: 'index' }), + Builder.expression.source.node({ index: 'test', sourceType: 'index' }), 123 ); @@ -86,7 +86,7 @@ describe('generic.commands.args', () => { generic.commands.args.insert( command!, - Builder.expression.source({ index: 'test', sourceType: 'index' }), + Builder.expression.source.node({ index: 'test', sourceType: 'index' }), 0 ); @@ -102,7 +102,7 @@ describe('generic.commands.args', () => { generic.commands.args.insert( command!, - Builder.expression.source({ index: 'test', sourceType: 'index' }), + Builder.expression.source.node({ index: 'test', sourceType: 'index' }), 1 ); @@ -121,7 +121,7 @@ describe('generic.commands.args', () => { generic.commands.args.append( command!, - Builder.expression.source({ index: 'test', sourceType: 'index' }) + Builder.expression.source.node({ index: 'test', sourceType: 'index' }) ); const src2 = BasicPrettyPrinter.print(root); diff --git a/src/platform/packages/shared/kbn-esql-ast/src/parser/__tests__/ast_parser.source.test.ts b/src/platform/packages/shared/kbn-esql-ast/src/parser/__tests__/ast_parser.source.test.ts index cf537b70ef44..2152d8d094b3 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/parser/__tests__/ast_parser.source.test.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/parser/__tests__/ast_parser.source.test.ts @@ -22,14 +22,21 @@ describe('source nodes', () => { { type: 'source', name: 'cluster:index', - cluster: 'cluster', - index: 'index', + index: { + valueUnquoted: 'index', + }, + cluster: { + type: 'literal', + literalType: 'keyword', + valueUnquoted: 'cluster', + }, }, { type: 'source', name: 'cluster:index', - cluster: '', - index: 'cluster:index', + index: { + valueUnquoted: 'cluster:index', + }, }, ], }, @@ -48,8 +55,11 @@ describe('source nodes', () => { { type: 'source', name: '', - cluster: '', - index: '', + cluster: undefined, + index: { + valueUnquoted: '', + }, + selector: undefined, }, ], }, @@ -69,8 +79,11 @@ describe('source nodes', () => { { type: 'source', name: 'a', - cluster: '', - index: 'a', + cluster: undefined, + index: { + valueUnquoted: 'a', + }, + selector: undefined, }, ], }, @@ -89,8 +102,11 @@ describe('source nodes', () => { { type: 'source', name: 'a/b', - cluster: '', - index: 'a/b', + cluster: undefined, + index: { + valueUnquoted: 'a/b', + }, + selector: undefined, }, ], }, @@ -109,8 +125,11 @@ describe('source nodes', () => { { type: 'source', name: 'a.b-*', - cluster: '', - index: 'a.b-*', + cluster: undefined, + index: { + valueUnquoted: 'a.b-*', + }, + selector: undefined, }, ], }, @@ -131,8 +150,11 @@ describe('source nodes', () => { { type: 'source', name: 'a', - cluster: '', - index: 'a', + cluster: undefined, + index: { + valueUnquoted: 'a', + }, + selector: undefined, }, ], }, @@ -151,8 +173,11 @@ describe('source nodes', () => { { type: 'source', name: expect.any(String), - cluster: '', - index: 'a " \r \n \t \\ b', + cluster: undefined, + index: { + valueUnquoted: 'a " \r \n \t \\ b', + }, + selector: undefined, }, ], }, @@ -173,8 +198,11 @@ describe('source nodes', () => { { type: 'source', name: 'a', - cluster: '', - index: 'a', + cluster: undefined, + index: { + valueUnquoted: 'a', + }, + selector: undefined, }, ], }, @@ -193,8 +221,11 @@ describe('source nodes', () => { { type: 'source', name: 'a"b', - cluster: '', - index: 'a"b', + cluster: undefined, + index: { + valueUnquoted: 'a"b', + }, + selector: undefined, }, ], }, @@ -213,8 +244,10 @@ describe('source nodes', () => { { type: 'source', name: 'a:\\/b', - cluster: '', - index: 'a:\\/b', + cluster: undefined, + index: { + valueUnquoted: 'a:\\/b', + }, }, ], }, @@ -233,8 +266,10 @@ describe('source nodes', () => { { type: 'source', name: 'a👍b', - cluster: '', - index: 'a👍b', + cluster: undefined, + index: { + valueUnquoted: 'a👍b', + }, }, ], }, @@ -255,8 +290,14 @@ describe('source nodes', () => { { type: 'source', name: 'cluster:a', - cluster: 'cluster', - index: 'a', + index: { + valueUnquoted: 'a', + }, + cluster: { + type: 'literal', + literalType: 'keyword', + valueUnquoted: 'cluster', + }, }, ], }, diff --git a/src/platform/packages/shared/kbn-esql-ast/src/parser/__tests__/from.test.ts b/src/platform/packages/shared/kbn-esql-ast/src/parser/__tests__/from.test.ts index f2f0fded57ca..aedc7b1dc269 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/parser/__tests__/from.test.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/parser/__tests__/from.test.ts @@ -157,6 +157,211 @@ describe('FROM', () => { }, ]); }); + + describe('source', () => { + describe('index', () => { + it('can parse single-double quoted index', () => { + const text = 'FROM "index"'; + const { root, errors } = parse(text); + + expect(errors.length).toBe(0); + expect(root.commands).toMatchObject([ + { + type: 'command', + name: 'from', + args: [ + { + type: 'source', + sourceType: 'index', + index: { + type: 'literal', + literalType: 'keyword', + valueUnquoted: 'index', + }, + }, + ], + }, + ]); + }); + + it('can parse triple-double quoted index', () => { + const text = 'FROM """index"""'; + const { root, errors } = parse(text); + + expect(errors.length).toBe(0); + expect(root.commands).toMatchObject([ + { + type: 'command', + name: 'from', + args: [ + { + type: 'source', + sourceType: 'index', + index: { + type: 'literal', + literalType: 'keyword', + valueUnquoted: 'index', + }, + }, + ], + }, + ]); + }); + }); + + describe('cluster', () => { + it('can parse unquoted cluster', () => { + const text = 'FROM cluster:index'; + const { root, errors } = parse(text); + + expect(errors.length).toBe(0); + expect(root.commands).toMatchObject([ + { + type: 'command', + name: 'from', + args: [ + { + type: 'source', + index: { + valueUnquoted: 'index', + }, + sourceType: 'index', + cluster: { + type: 'literal', + literalType: 'keyword', + valueUnquoted: 'cluster', + unquoted: true, + }, + }, + ], + }, + ]); + }); + + it('can parse single-double quoted cluster', () => { + const text = 'FROM "cluster":index'; + const { root, errors } = parse(text); + + expect(errors.length).toBe(0); + expect(root.commands).toMatchObject([ + { + type: 'command', + name: 'from', + args: [ + { + type: 'source', + index: { + valueUnquoted: 'index', + }, + sourceType: 'index', + cluster: { + type: 'literal', + literalType: 'keyword', + valueUnquoted: 'cluster', + }, + }, + ], + }, + ]); + }); + + it('can parse triple-double quoted cluster', () => { + const text = 'FROM """cluster""":index'; + const { root, errors } = parse(text); + + expect(errors.length).toBe(0); + expect(root.commands).toMatchObject([ + { + type: 'command', + name: 'from', + args: [ + { + type: 'source', + index: { + valueUnquoted: 'index', + }, + sourceType: 'index', + cluster: { + type: 'literal', + literalType: 'keyword', + valueUnquoted: 'cluster', + }, + }, + ], + }, + ]); + }); + }); + + describe('selector', () => { + it('can parse source selector', () => { + const text = 'FROM index::selector'; + const { root, errors } = parse(text); + + expect(errors.length).toBe(0); + expect(root.commands).toMatchObject([ + { + type: 'command', + name: 'from', + args: [ + { + type: 'source', + index: { + valueUnquoted: 'index', + }, + sourceType: 'index', + selector: { + type: 'literal', + literalType: 'keyword', + valueUnquoted: 'selector', + unquoted: true, + }, + }, + ], + }, + ]); + }); + + it('can parse single and triple quoted selectors', () => { + const text = 'FROM index1::"selector1", index2::"""selector2"""'; + const { root, errors } = parse(text); + + expect(errors.length).toBe(0); + expect(root.commands).toMatchObject([ + { + type: 'command', + name: 'from', + args: [ + { + type: 'source', + index: { + valueUnquoted: 'index1', + }, + sourceType: 'index', + selector: { + type: 'literal', + literalType: 'keyword', + valueUnquoted: 'selector1', + }, + }, + { + type: 'source', + index: { + valueUnquoted: 'index2', + }, + sourceType: 'index', + selector: { + type: 'literal', + literalType: 'keyword', + valueUnquoted: 'selector2', + }, + }, + ], + }, + ]); + }); + }); + }); }); describe('when incorrectly formatted, returns errors', () => { diff --git a/src/platform/packages/shared/kbn-esql-ast/src/parser/esql_ast_builder_listener.ts b/src/platform/packages/shared/kbn-esql-ast/src/parser/esql_ast_builder_listener.ts index 8e1e9e837f41..49e87897ec3b 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/parser/esql_ast_builder_listener.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/parser/esql_ast_builder_listener.ts @@ -37,15 +37,13 @@ import { default as ESQLParserListener } from '../antlr/esql_parser_listener'; import { createCommand, createFunction, - createOption, createLiteral, textExistsAndIsValid, - createSource, + visitSource, createAstBaseItem, } from './factories'; import { getPosition } from './helpers'; import { - collectAllSourceIdentifiers, collectAllFields, collectAllAggFields, visitByOption, @@ -64,6 +62,7 @@ import { createStatsCommand } from './factories/stats'; import { createChangePointCommand } from './factories/change_point'; import { createWhereCommand } from './factories/where'; import { createRowCommand } from './factories/row'; +import { createFromCommand } from './factories/from'; export class ESQLAstBuilderListener implements ESQLParserListener { private ast: ESQLAst = []; @@ -123,18 +122,9 @@ export class ESQLAstBuilderListener implements ESQLParserListener { * @param ctx the parse tree */ exitFromCommand(ctx: FromCommandContext) { - const commandAst = createCommand('from', ctx); - this.ast.push(commandAst); - commandAst.args.push(...collectAllSourceIdentifiers(ctx)); - const metadataContext = ctx.indexPatternAndMetadataFields().metadata(); - if (metadataContext && metadataContext.METADATA()) { - const option = createOption( - metadataContext.METADATA().getText().toLowerCase(), - metadataContext - ); - commandAst.args.push(option); - option.args.push(...collectAllColumnIdentifiers(metadataContext)); - } + const command = createFromCommand(ctx); + + this.ast.push(command); } /** @@ -149,7 +139,7 @@ export class ESQLAstBuilderListener implements ESQLParserListener { sources: ctx .indexPatternAndMetadataFields() .getTypedRuleContexts(IndexPatternContext) - .map((sourceCtx) => createSource(sourceCtx)), + .map((sourceCtx) => visitSource(sourceCtx)), }; this.ast.push(node); node.args.push(...node.sources); diff --git a/src/platform/packages/shared/kbn-esql-ast/src/parser/factories.ts b/src/platform/packages/shared/kbn-esql-ast/src/parser/factories.ts index d1f17958bdf2..b90bc516be11 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/parser/factories.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/parser/factories.ts @@ -35,6 +35,7 @@ import { StringContext, InputNamedOrPositionalDoubleParamsContext, InputDoubleParamsContext, + SelectorStringContext, } from '../antlr/esql_parser'; import { DOUBLE_TICKS_REGEX, SINGLE_BACKTICK, TICKS_REGEX } from './constants'; import type { @@ -62,9 +63,11 @@ import type { BinaryExpressionOperator, ESQLCommand, ESQLParamKinds, + ESQLStringLiteral, } from '../types'; import { parseIdentifier, getPosition } from './helpers'; import { Builder, type AstNodeParserFields } from '../builder'; +import { LeafPrinter } from '../pretty_print'; export function nonNullable(v: T): v is NonNullable { return v != null; @@ -88,6 +91,18 @@ const createParserFields = (ctx: ParserRuleContext): AstNodeParserFields => ({ incomplete: Boolean(ctx.exception), }); +const createParserFieldsFromTerminalNode = (node: TerminalNode): AstNodeParserFields => { + const text = node.getText(); + const symbol = node.symbol; + const fields: AstNodeParserFields = { + text, + location: getPosition(symbol, symbol), + incomplete: false, + }; + + return fields; +}; + export const createCommand = < Name extends string, Cmd extends ESQLCommand = ESQLCommand @@ -138,7 +153,9 @@ export function createFakeMultiplyLiteral( }; } -export function createLiteralString(ctx: StringContext): ESQLLiteral { +export function createLiteralString( + ctx: Pick & ParserRuleContext +): ESQLStringLiteral { const quotedString = ctx.QUOTED_STRING()?.getText() ?? '""'; const isTripleQuoted = quotedString.startsWith('"""') && quotedString.endsWith('"""'); let valueUnquoted = isTripleQuoted ? quotedString.slice(3, -3) : quotedString.slice(1, -1); @@ -438,34 +455,6 @@ function sanitizeSourceString(ctx: ParserRuleContext) { return contextText; } -const unquoteIndexString = (indexString: string): string => { - const isStringQuoted = indexString[0] === '"'; - - if (!isStringQuoted) { - return indexString; - } - - // If wrapped by triple double quotes, simply remove them. - if (indexString.startsWith(`"""`) && indexString.endsWith(`"""`)) { - return indexString.slice(3, -3); - } - - // If wrapped by double quote, remove them and unescape the string. - if (indexString[indexString.length - 1] === '"') { - indexString = indexString.slice(1, -1); - indexString = indexString - .replace(/\\"/g, '"') - .replace(/\\r/g, '\r') - .replace(/\\n/g, '\n') - .replace(/\\t/g, '\t') - .replace(/\\\\/g, '\\'); - return indexString; - } - - // This should never happen, but if it does, return the original string. - return indexString; -}; - export function sanitizeIdentifierString(ctx: ParserRuleContext) { const result = getUnquotedText(ctx)?.getText() || @@ -507,38 +496,66 @@ export function createPolicy(token: Token, policy: string): ESQLSource { }; } -export function createSource( +const visitUnquotedOrQuotedString = (ctx: SelectorStringContext): ESQLStringLiteral => { + const unquotedCtx = ctx.UNQUOTED_SOURCE(); + + if (unquotedCtx) { + const valueUnquoted = unquotedCtx.getText(); + const quotedString = LeafPrinter.string({ valueUnquoted }); + + return Builder.expression.literal.string( + valueUnquoted, + { + name: quotedString, + unquoted: true, + }, + createParserFieldsFromTerminalNode(unquotedCtx) + ); + } + + return createLiteralString(ctx); +}; + +export function visitSource( ctx: ParserRuleContext, type: 'index' | 'policy' = 'index' ): ESQLSource { const text = sanitizeSourceString(ctx); - let cluster: string = ''; - let index: string = ''; + let cluster: ESQLStringLiteral | undefined; + let index: ESQLStringLiteral | undefined; + let selector: ESQLStringLiteral | undefined; if (ctx instanceof IndexPatternContext) { - const clusterString = ctx.clusterString(); - const indexString = ctx.indexString(); + const clusterStringCtx = ctx.clusterString(); + const indexStringCtx = ctx.indexString(); + const selectorStringCtx = ctx.selectorString(); - if (clusterString) { - cluster = clusterString.getText(); + if (clusterStringCtx) { + cluster = visitUnquotedOrQuotedString(clusterStringCtx); } - if (indexString) { - index = indexString.getText(); - index = unquoteIndexString(index); + if (indexStringCtx) { + index = visitUnquotedOrQuotedString(indexStringCtx); + } + if (selectorStringCtx) { + selector = visitUnquotedOrQuotedString(selectorStringCtx); } } - return { - type: 'source', - cluster, - index, - name: text, - sourceType: type, - location: getPosition(ctx.start, ctx.stop), - incomplete: Boolean(ctx.exception || text === ''), - text: ctx?.getText(), - }; + return Builder.expression.source.node( + { + sourceType: type, + cluster, + index, + selector, + name: text, + }, + { + location: getPosition(ctx.start, ctx.stop), + incomplete: Boolean(ctx.exception || text === ''), + text: ctx?.getText(), + } + ); } export function createColumnStar(ctx: TerminalNode): ESQLColumn { diff --git a/src/platform/packages/shared/kbn-esql-ast/src/parser/factories/from.ts b/src/platform/packages/shared/kbn-esql-ast/src/parser/factories/from.ts new file mode 100644 index 000000000000..a9f1949850f1 --- /dev/null +++ b/src/platform/packages/shared/kbn-esql-ast/src/parser/factories/from.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { FromCommandContext, IndexPatternContext } from '../../antlr/esql_parser'; +import { ESQLCommand } from '../../types'; +import { createCommand, createOption, visitSource } from '../factories'; +import { collectAllColumnIdentifiers } from '../walkers'; + +export const createFromCommand = (ctx: FromCommandContext): ESQLCommand<'from'> => { + const command = createCommand('from', ctx); + const indexPatternCtx = ctx.indexPatternAndMetadataFields(); + const metadataCtx = indexPatternCtx.metadata(); + const sources = indexPatternCtx + .getTypedRuleContexts(IndexPatternContext) + .map((sourceCtx) => visitSource(sourceCtx)); + + command.args.push(...sources); + + if (metadataCtx && metadataCtx.METADATA()) { + const name = metadataCtx.METADATA().getText().toLowerCase(); + const option = createOption(name, metadataCtx); + const optionArgs = collectAllColumnIdentifiers(metadataCtx); + + option.args.push(...optionArgs); + command.args.push(option); + } + + return command; +}; diff --git a/src/platform/packages/shared/kbn-esql-ast/src/parser/factories/join.ts b/src/platform/packages/shared/kbn-esql-ast/src/parser/factories/join.ts index 1ab7259a68f4..bfb3eba1e088 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/parser/factories/join.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/parser/factories/join.ts @@ -9,11 +9,11 @@ import { JoinCommandContext, JoinTargetContext } from '../../antlr/esql_parser'; import { ESQLAstItem, ESQLAstJoinCommand, ESQLIdentifier, ESQLSource } from '../../types'; -import { createCommand, createOption, createSource } from '../factories'; +import { createCommand, createOption, visitSource } from '../factories'; import { visitValueExpression } from '../walkers'; const createNodeFromJoinTarget = (ctx: JoinTargetContext): ESQLSource | ESQLIdentifier => { - return createSource(ctx._index); + return visitSource(ctx._index); }; export const createJoinCommand = (ctx: JoinCommandContext): ESQLAstJoinCommand => { diff --git a/src/platform/packages/shared/kbn-esql-ast/src/parser/walkers.ts b/src/platform/packages/shared/kbn-esql-ast/src/parser/walkers.ts index e27ed21375eb..de6b27c6ee39 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/parser/walkers.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/parser/walkers.ts @@ -29,7 +29,6 @@ import { type FieldContext, type FieldsContext, type AggFieldsContext, - type FromCommandContext, FunctionContext, IntegerLiteralContext, IsNullContext, @@ -57,7 +56,6 @@ import { type ValueExpressionContext, ValueExpressionDefaultContext, InlineCastContext, - IndexPatternContext, InlinestatsCommandContext, MatchExpressionContext, MatchBooleanExpressionContext, @@ -65,7 +63,6 @@ import { EntryExpressionContext, } from '../antlr/esql_parser'; import { - createSource, createColumn, createOption, nonNullable, @@ -110,13 +107,6 @@ import { firstItem, lastItem } from '../visitor/utils'; import { Builder } from '../builder'; import { getPosition } from './helpers'; -export function collectAllSourceIdentifiers(ctx: FromCommandContext): ESQLAstItem[] { - const fromContexts = ctx - .indexPatternAndMetadataFields() - .getTypedRuleContexts(IndexPatternContext); - return fromContexts.map((sourceCtx) => createSource(sourceCtx)); -} - function terminalNodeToParserRuleContext(node: TerminalNode): ParserRuleContext { const context = new ParserRuleContext(); context.start = node.symbol; diff --git a/src/platform/packages/shared/kbn-esql-ast/src/pretty_print/__tests__/basic_pretty_printer.test.ts b/src/platform/packages/shared/kbn-esql-ast/src/pretty_print/__tests__/basic_pretty_printer.test.ts index c902a94c6848..8b0fd806e59c 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/pretty_print/__tests__/basic_pretty_printer.test.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/pretty_print/__tests__/basic_pretty_printer.test.ts @@ -210,13 +210,31 @@ describe('single line query', () => { test('quoted source', () => { const { text } = reprint('FROM "quoted"'); - expect(text).toBe('FROM quoted'); + expect(text).toBe('FROM "quoted"'); }); test('triple-quoted source', () => { const { text } = reprint('FROM """quoted"""'); - expect(text).toBe('FROM quoted'); + expect(text).toBe('FROM "quoted"'); + }); + + test('source selector', () => { + const { text } = reprint('FROM index::selector'); + + expect(text).toBe('FROM index::selector'); + }); + + test('single-double quoted source selector', () => { + const { text } = reprint('FROM index::"selector"'); + + expect(text).toBe('FROM index::"selector"'); + }); + + test('triple-double quoted source selector', () => { + const { text } = reprint('FROM index::"""say "jump"!"""'); + + expect(text).toBe('FROM index::"say \\"jump\\"!"'); }); }); diff --git a/src/platform/packages/shared/kbn-esql-ast/src/pretty_print/__tests__/wrapping_pretty_printer.test.ts b/src/platform/packages/shared/kbn-esql-ast/src/pretty_print/__tests__/wrapping_pretty_printer.test.ts index 38a247c9dd93..b77624841cdb 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/pretty_print/__tests__/wrapping_pretty_printer.test.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/pretty_print/__tests__/wrapping_pretty_printer.test.ts @@ -425,6 +425,16 @@ FROM 👉 METADATA _id, _source`); }); + test('supports quoted source, quoted cluster name, and quoted index selector component', () => { + const query = `FROM "this is a cluster name" : "this is a quoted index name", "this is another quoted index" :: "and this is a quoted index selector"`; + const text = reprint(query, { pipeTab: ' ' }).text; + + expect('\n' + text).toBe(` +FROM + "this is a cluster name":"this is a quoted index name", + "this is another quoted index"::"and this is a quoted index selector"`); + }); + test('can break multiple options', () => { const query = 'from a | enrich policy ON match_field_which_is_very_long WITH new_name1 = field1, new_name2 = field2'; diff --git a/src/platform/packages/shared/kbn-esql-ast/src/pretty_print/leaf_printer.ts b/src/platform/packages/shared/kbn-esql-ast/src/pretty_print/leaf_printer.ts index 2677904346f0..c9a1a845dac9 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/pretty_print/leaf_printer.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/pretty_print/leaf_printer.ts @@ -29,11 +29,14 @@ const regexUnquotedIdPattern = /^([a-z\*_\@]{1})[a-z0-9_\*]*$/i; */ export const LeafPrinter = { source: (node: ESQLSource): string => { - const { index, name, cluster } = node; - let text = index || name || ''; + const { index, name, cluster, selector } = node; + let text = (index ? LeafPrinter.string(index) : name) || ''; if (cluster) { - text = `${cluster}:${text}`; + text = `${LeafPrinter.string(cluster)}:${text}`; + } + if (selector) { + text = `${text}::${LeafPrinter.string(selector)}`; } return text; @@ -82,8 +85,13 @@ export const LeafPrinter = { return formatted; }, - string: (node: ESQLStringLiteral) => { + string: (node: Pick) => { const str = node.valueUnquoted; + + if (node.unquoted === true) { + return str; + } + const strFormatted = '"' + str diff --git a/src/platform/packages/shared/kbn-esql-ast/src/types.ts b/src/platform/packages/shared/kbn-esql-ast/src/types.ts index 1c2b3fe219b8..fcb6642c8dfa 100644 --- a/src/platform/packages/shared/kbn-esql-ast/src/types.ts +++ b/src/platform/packages/shared/kbn-esql-ast/src/types.ts @@ -289,7 +289,7 @@ export interface ESQLSource extends ESQLAstBaseItem { * FROM [:] * ``` */ - cluster?: string; + cluster?: ESQLStringLiteral | undefined; /** * Represents the index part of the source identifier. Unescaped and unquoted. @@ -298,7 +298,16 @@ export interface ESQLSource extends ESQLAstBaseItem { * FROM [:] * ``` */ - index?: string; + index?: ESQLStringLiteral | undefined; + + /** + * Represents the selector (component) part of the source identifier. + * + * ``` + * FROM [::] + * ``` + */ + selector?: ESQLStringLiteral | undefined; } export interface ESQLColumn extends ESQLAstBaseItem { @@ -402,6 +411,18 @@ export interface ESQLStringLiteral extends ESQLAstBaseItem { value: string; valueUnquoted: string; + + /** + * Whether the string was parsed as "unqouted" and/or can be pretty-printed + * unquoted, i.e. in the source text it did not have any quotes (not single ", + * not triple """) quotes. This happens in FROM command source parsing, the + * cluster and selector can be unquoted strings: + * + * ``` + * FROM :index: + * ``` + */ + unquoted?: boolean; } // @internal diff --git a/src/platform/packages/shared/kbn-monaco/src/languages/esql/lib/hover/helpers.test.ts b/src/platform/packages/shared/kbn-monaco/src/languages/esql/lib/hover/helpers.test.ts index 495628aa1d4c..00cd39b4aa9e 100644 --- a/src/platform/packages/shared/kbn-monaco/src/languages/esql/lib/hover/helpers.test.ts +++ b/src/platform/packages/shared/kbn-monaco/src/languages/esql/lib/hover/helpers.test.ts @@ -14,8 +14,10 @@ describe('getVariablesHoverContent', () => { test('should return empty array if no variables are used in the query', async () => { const node = { type: 'source', - cluster: '', - index: 'logst*', + cluster: undefined, + index: { + valueUnquoted: 'logst*', + }, name: 'logst*', sourceType: 'index', location: { @@ -40,8 +42,10 @@ describe('getVariablesHoverContent', () => { test('should return empty array if no variables are given', () => { const node = { type: 'source', - cluster: '', - index: 'logst*', + cluster: undefined, + index: { + valueUnquoted: 'logst*', + }, name: 'logst*', sourceType: 'index', location: {