github-mirrors/kibana
1
0
Fork 0
mirror of https://github.com/elastic/kibana.git synced 2025-04-18 23:21:39 -04:00
Code Issues Projects Releases Packages Wiki Activity

[ES|QL] Source AST node parsing improvements and source selector parsing (#217299)

Browse source 
## Summary

- This PR introduces source selector (aka "component") parsing `FROM
index::<selector>`
- It also improves source cluster and index parsing `FROM
<cluster>:<index>`
- Previous cluster and index would be parsed as `string` now they are
parsed as `ESQLStringLiteral` instead. This is more correct as any of
those can take three forms, and `ESQLStringLiteral` handles all three
forms:
    1. unquoted string: `cluster:index`
    2. single-double quoted string: `"cluster":"index"`
    3. triple-double quote string: `"""cluster""":"""index""`
- The `ESQLStringLiteral` now also supports *"unquoted strings"* in
addition to single `"str"` and triple `"""str"""` quoted strings.


### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
This commit is contained in:
Vadim Kibana 2025-04-07 16:11:10 +02:00 • committed by GitHub
parent ec72d4a880
commit 86fdbe5379
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 611 additions and 178 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
examples/esql_ast_inspector/public/components/esql_inspector/components/preview/components/preview_ui/components/from_command/index.tsx
View file

@ -51,7 +51,7 @@ export const FromCommand: React.FC = () => {
color="text"
onClick={() => {
const length = from.args.length;
const source = Builder.expression.source({
const source = Builder.expression.source.node({
index: `source${length + 1}`,
sourceType: 'index',
});

4
src/platform/packages/shared/kbn-esql-ast/src/ast/helpers.ts
View file

@ -20,6 +20,7 @@ import type {
ESQLParamLiteral,
ESQLProperNode,
ESQLSource,
ESQLStringLiteral,
} from '../types';
import { BinaryExpressionGroup } from './constants';
@ -62,6 +63,9 @@ export const isFieldExpression = (
export const isLiteral = (node: unknown): node is ESQLLiteral =>
isProperNode(node) && node.type === 'literal';
export const isStringLiteral = (node: unknown): node is ESQLStringLiteral =>
isLiteral(node) && node.literalType === 'keyword';
export const isIntegerLiteral = (node: unknown): node is ESQLIntegerLiteral =>
isLiteral(node) && node.literalType === 'integer';

61
src/platform/packages/shared/kbn-esql-ast/src/builder/builder.test.ts
View file

@ -25,7 +25,7 @@ describe('command', () => {
const node = Builder.command({
name: 'from',
args: [
Builder.expression.source({ index: 'my_index', sourceType: 'index' }),
Builder.expression.source.node({ index: 'my_index', sourceType: 'index' }),
Builder.option({
name: 'by',
args: [
@ -94,21 +94,39 @@ describe('function', () => {
describe('source', () => {
test('basic index', () => {
const node = Builder.expression.source({ index: 'my_index', sourceType: 'index' });
const node = Builder.expression.source.node({ index: 'my_index', sourceType: 'index' });
const text = BasicPrettyPrinter.expression(node);
expect(text).toBe('my_index');
});
test('basic index using shortcut', () => {
const node = Builder.expression.source('my_index');
const node = Builder.expression.source.node('my_index');
const text = BasicPrettyPrinter.expression(node);
expect(text).toBe('my_index');
});
test('basic quoted index using shortcut', () => {
const node = Builder.expression.source.node(Builder.expression.literal.string('my_index'));
const text = BasicPrettyPrinter.expression(node);
expect(text).toBe('"my_index"');
});
test('index with cluster', () => {
const node = Builder.expression.source({
const node = Builder.expression.source.node({
index: 'my_index',
sourceType: 'index',
cluster: Builder.expression.literal.string('my_cluster', { unquoted: true }),
});
const text = BasicPrettyPrinter.expression(node);
expect(text).toBe('my_cluster:my_index');
});
test('index with cluster - plain text cluster', () => {
const node = Builder.expression.source.node({
index: 'my_index',
sourceType: 'index',
cluster: 'my_cluster',
@ -118,19 +136,38 @@ describe('source', () => {
expect(text).toBe('my_cluster:my_index');
});
test('can use .indexSource() shorthand to specify cluster', () => {
const node = Builder.expression.indexSource('my_index', 'my_cluster');
const text = BasicPrettyPrinter.expression(node);
expect(text).toBe('my_cluster:my_index');
});
test('policy index', () => {
const node = Builder.expression.source({ index: 'my_policy', sourceType: 'policy' });
const node = Builder.expression.source.node({ index: 'my_policy', sourceType: 'policy' });
const text = BasicPrettyPrinter.expression(node);
expect(text).toBe('my_policy');
});
describe('.index', () => {
test('can use .source.index() shorthand to specify cluster', () => {
const node = Builder.expression.source.index('my_index', 'my_cluster');
const text = BasicPrettyPrinter.expression(node);
expect(text).toBe('my_cluster:my_index');
});
test('can use .source.index() and specify quotes around cluster', () => {
const node = Builder.expression.source.index(
'my_index',
Builder.expression.literal.string('hello 👋')
);
const text = BasicPrettyPrinter.expression(node);
expect(text).toBe('"hello 👋":my_index');
});
test('can use .source.index() shorthand to specify selector', () => {
const node = Builder.expression.source.index('my_index', '', 'my_selector');
const text = BasicPrettyPrinter.expression(node);
expect(text).toBe('my_index::my_selector');
});
});
});
describe('column', () => {

97
src/platform/packages/shared/kbn-esql-ast/src/builder/builder.ts
View file

@ -9,6 +9,7 @@
/* eslint-disable @typescript-eslint/no-namespace */
import { isStringLiteral } from '../ast/helpers';
import { LeafPrinter } from '../pretty_print';
import {
ESQLAstComment,
@ -110,41 +111,73 @@ export namespace Builder {
};
};
export type SourceTemplate = { index: string } & Omit<AstNodeTemplate<ESQLSource>, 'name'>;
export namespace source {
export type SourceTemplate = {
cluster?: string | ESQLSource['cluster'];
index?: string | ESQLSource['index'];
selector?: string | ESQLSource['selector'];
} & Omit<AstNodeTemplate<ESQLSource>, 'name' | 'cluster' | 'index' | 'selector'> &
Partial<Pick<ESQLSource, 'name'>>;
export const source = (
indexOrTemplate: string | SourceTemplate,
fromParser?: Partial<AstNodeParserFields>
): ESQLSource => {
const template: SourceTemplate =
typeof indexOrTemplate === 'string'
? { sourceType: 'index', index: indexOrTemplate }
: indexOrTemplate;
const { index, cluster } = template;
return {
...template,
...Builder.parserFields(fromParser),
type: 'source',
name: (cluster ? cluster + ':' : '') + index,
};
};
export const node = (
indexOrTemplate: string | ESQLStringLiteral | SourceTemplate,
fromParser?: Partial<AstNodeParserFields>
): ESQLSource => {
const template: SourceTemplate =
typeof indexOrTemplate === 'string' || isStringLiteral(indexOrTemplate)
? { sourceType: 'index', index: indexOrTemplate }
: indexOrTemplate;
const cluster: ESQLSource['cluster'] = !template.cluster
? undefined
: typeof template.cluster === 'string'
? Builder.expression.literal.string(template.cluster, { unquoted: true })
: template.cluster;
const index: ESQLSource['index'] = !template.index
? undefined
: typeof template.index === 'string'
? Builder.expression.literal.string(template.index, { unquoted: true })
: template.index;
const selector: ESQLSource['selector'] = !template.selector
? undefined
: typeof template.selector === 'string'
? Builder.expression.literal.string(template.selector, { unquoted: true })
: template.selector;
const sourceNode: ESQLSource = {
...template,
...Builder.parserFields(fromParser),
type: 'source',
cluster,
index,
selector,
name: template.name ?? '',
};
export const indexSource = (
index: string,
cluster?: string,
template?: Omit<AstNodeTemplate<ESQLSource>, 'name' | 'index' | 'cluster'>,
fromParser?: Partial<AstNodeParserFields>
): ESQLSource => {
return {
...template,
...Builder.parserFields(fromParser),
index,
cluster,
sourceType: 'index',
type: 'source',
name: (cluster ? cluster + ':' : '') + index,
if (!sourceNode.name) {
sourceNode.name = LeafPrinter.source(sourceNode);
}
return sourceNode;
};
};
export const index = (
indexName: string,
cluster?: string | ESQLSource['cluster'],
selector?: string | ESQLSource['selector'],
template?: Omit<AstNodeTemplate<ESQLSource>, 'name' | 'index' | 'cluster'>,
fromParser?: Partial<AstNodeParserFields>
): ESQLSource => {
return Builder.expression.source.node(
{
...template,
index: indexName,
cluster,
selector,
sourceType: 'index',
},
fromParser
);
};
}
export type ColumnTemplate = Omit<AstNodeTemplate<ESQLColumn>, 'name' | 'quoted' | 'parts'>;

36
src/platform/packages/shared/kbn-esql-ast/src/mutate/commands/from/sources.test.ts
View file

@ -40,16 +40,26 @@ describe('commands.from.sources', () => {
expect(list).toMatchObject([
{
type: 'source',
index: 'index',
index: {
valueUnquoted: 'index',
},
},
{
type: 'source',
index: 'index2',
index: {
valueUnquoted: 'index2',
},
},
{
type: 'source',
index: 'index3',
cluster: 'cl',
index: {
valueUnquoted: 'index3',
},
cluster: {
type: 'literal',
literalType: 'keyword',
valueUnquoted: 'cl',
},
},
]);
});
@ -72,7 +82,9 @@ describe('commands.from.sources', () => {
expect(source).toMatchObject({
type: 'source',
name: 'index',
index: 'index',
index: {
valueUnquoted: 'index',
},
});
});
@ -85,13 +97,21 @@ describe('commands.from.sources', () => {
expect(source1).toMatchObject({
type: 'source',
name: 's2',
index: 's2',
index: {
valueUnquoted: 's2',
},
});
expect(source2).toMatchObject({
type: 'source',
name: 'c:s1',
index: 's1',
cluster: 'c',
index: {
valueUnquoted: 's1',
},
cluster: {
type: 'literal',
literalType: 'keyword',
valueUnquoted: 'c',
},
});
});
});

6
src/platform/packages/shared/kbn-esql-ast/src/mutate/commands/from/sources.ts
View file

@ -52,10 +52,10 @@ export const find = (
cluster?: string
): ESQLSource | undefined => {
return findByPredicate(ast, (source) => {
if (index !== source.index) {
if (index !== source.index?.valueUnquoted) {
return false;
}
if (typeof cluster === 'string' && cluster !== source.cluster) {
if (cluster && typeof cluster === 'string' && cluster !== source.cluster?.valueUnquoted) {
return false;
}
@ -91,7 +91,7 @@ export const insert = (
return;
}
const source = Builder.expression.indexSource(indexName, clusterName);
const source = Builder.expression.source.index(indexName, clusterName);
if (index === -1) {
generic.commands.args.append(command, source);

14
src/platform/packages/shared/kbn-esql-ast/src/mutate/generic/commands/args/index.test.ts
View file

@ -21,7 +21,7 @@ describe('generic.commands.args', () => {
generic.commands.args.insert(
command!,
Builder.expression.source({ index: 'test', sourceType: 'index' }),
Builder.expression.source.node({ index: 'test', sourceType: 'index' }),
123
);
@ -37,7 +37,7 @@ describe('generic.commands.args', () => {
generic.commands.args.insert(
command!,
Builder.expression.source({ index: 'test', sourceType: 'index' }),
Builder.expression.source.node({ index: 'test', sourceType: 'index' }),
0
);
@ -53,7 +53,7 @@ describe('generic.commands.args', () => {
generic.commands.args.insert(
command!,
Builder.expression.source({ index: 'test', sourceType: 'index' }),
Builder.expression.source.node({ index: 'test', sourceType: 'index' }),
1
);
@ -70,7 +70,7 @@ describe('generic.commands.args', () => {
generic.commands.args.insert(
command!,
Builder.expression.source({ index: 'test', sourceType: 'index' }),
Builder.expression.source.node({ index: 'test', sourceType: 'index' }),
123
);
@ -86,7 +86,7 @@ describe('generic.commands.args', () => {
generic.commands.args.insert(
command!,
Builder.expression.source({ index: 'test', sourceType: 'index' }),
Builder.expression.source.node({ index: 'test', sourceType: 'index' }),
0
);
@ -102,7 +102,7 @@ describe('generic.commands.args', () => {
generic.commands.args.insert(
command!,
Builder.expression.source({ index: 'test', sourceType: 'index' }),
Builder.expression.source.node({ index: 'test', sourceType: 'index' }),
1
);
@ -121,7 +121,7 @@ describe('generic.commands.args', () => {
generic.commands.args.append(
command!,
Builder.expression.source({ index: 'test', sourceType: 'index' })
Builder.expression.source.node({ index: 'test', sourceType: 'index' })
);
const src2 = BasicPrettyPrinter.print(root);

93
src/platform/packages/shared/kbn-esql-ast/src/parser/__tests__/ast_parser.source.test.ts
View file

@ -22,14 +22,21 @@ describe('source nodes', () => {
{
type: 'source',
name: 'cluster:index',
cluster: 'cluster',
index: 'index',
index: {
valueUnquoted: 'index',
},
cluster: {
type: 'literal',
literalType: 'keyword',
valueUnquoted: 'cluster',
},
},
{
type: 'source',
name: 'cluster:index',
cluster: '',
index: 'cluster:index',
index: {
valueUnquoted: 'cluster:index',
},
},
],
},
@ -48,8 +55,11 @@ describe('source nodes', () => {
{
type: 'source',
name: '<logs-{now/d}>',
cluster: '',
index: '<logs-{now/d}>',
cluster: undefined,
index: {
valueUnquoted: '<logs-{now/d}>',
},
selector: undefined,
},
],
},
@ -69,8 +79,11 @@ describe('source nodes', () => {
{
type: 'source',
name: 'a',
cluster: '',
index: 'a',
cluster: undefined,
index: {
valueUnquoted: 'a',
},
selector: undefined,
},
],
},
@ -89,8 +102,11 @@ describe('source nodes', () => {
{
type: 'source',
name: 'a/b',
cluster: '',
index: 'a/b',
cluster: undefined,
index: {
valueUnquoted: 'a/b',
},
selector: undefined,
},
],
},
@ -109,8 +125,11 @@ describe('source nodes', () => {
{
type: 'source',
name: 'a.b-*',
cluster: '',
index: 'a.b-*',
cluster: undefined,
index: {
valueUnquoted: 'a.b-*',
},
selector: undefined,
},
],
},
@ -131,8 +150,11 @@ describe('source nodes', () => {
{
type: 'source',
name: 'a',
cluster: '',
index: 'a',
cluster: undefined,
index: {
valueUnquoted: 'a',
},
selector: undefined,
},
],
},
@ -151,8 +173,11 @@ describe('source nodes', () => {
{
type: 'source',
name: expect.any(String),
cluster: '',
index: 'a " \r \n \t \\ b',
cluster: undefined,
index: {
valueUnquoted: 'a " \r \n \t \\ b',
},
selector: undefined,
},
],
},
@ -173,8 +198,11 @@ describe('source nodes', () => {
{
type: 'source',
name: 'a',
cluster: '',
index: 'a',
cluster: undefined,
index: {
valueUnquoted: 'a',
},
selector: undefined,
},
],
},
@ -193,8 +221,11 @@ describe('source nodes', () => {
{
type: 'source',
name: 'a"b',
cluster: '',
index: 'a"b',
cluster: undefined,
index: {
valueUnquoted: 'a"b',
},
selector: undefined,
},
],
},
@ -213,8 +244,10 @@ describe('source nodes', () => {
{
type: 'source',
name: 'a:\\/b',
cluster: '',
index: 'a:\\/b',
cluster: undefined,
index: {
valueUnquoted: 'a:\\/b',
},
},
],
},
@ -233,8 +266,10 @@ describe('source nodes', () => {
{
type: 'source',
name: 'a👍b',
cluster: '',
index: 'a👍b',
cluster: undefined,
index: {
valueUnquoted: 'a👍b',
},
},
],
},
@ -255,8 +290,14 @@ describe('source nodes', () => {
{
type: 'source',
name: 'cluster:a',
cluster: 'cluster',
index: 'a',
index: {
valueUnquoted: 'a',
},
cluster: {
type: 'literal',
literalType: 'keyword',
valueUnquoted: 'cluster',
},
},
],
},

205
src/platform/packages/shared/kbn-esql-ast/src/parser/__tests__/from.test.ts
View file

@ -157,6 +157,211 @@ describe('FROM', () => {
},
]);
});
describe('source', () => {
describe('index', () => {
it('can parse single-double quoted index', () => {
const text = 'FROM "index"';
const { root, errors } = parse(text);
expect(errors.length).toBe(0);
expect(root.commands).toMatchObject([
{
type: 'command',
name: 'from',
args: [
{
type: 'source',
sourceType: 'index',
index: {
type: 'literal',
literalType: 'keyword',
valueUnquoted: 'index',
},
},
],
},
]);
});
it('can parse triple-double quoted index', () => {
const text = 'FROM """index"""';
const { root, errors } = parse(text);
expect(errors.length).toBe(0);
expect(root.commands).toMatchObject([
{
type: 'command',
name: 'from',
args: [
{
type: 'source',
sourceType: 'index',
index: {
type: 'literal',
literalType: 'keyword',
valueUnquoted: 'index',
},
},
],
},
]);
});
});
describe('cluster', () => {
it('can parse unquoted cluster', () => {
const text = 'FROM cluster:index';
const { root, errors } = parse(text);
expect(errors.length).toBe(0);
expect(root.commands).toMatchObject([
{
type: 'command',
name: 'from',
args: [
{
type: 'source',
index: {
valueUnquoted: 'index',
},
sourceType: 'index',
cluster: {
type: 'literal',
literalType: 'keyword',
valueUnquoted: 'cluster',
unquoted: true,
},
},
],
},
]);
});
it('can parse single-double quoted cluster', () => {
const text = 'FROM "cluster":index';
const { root, errors } = parse(text);
expect(errors.length).toBe(0);
expect(root.commands).toMatchObject([
{
type: 'command',
name: 'from',
args: [
{
type: 'source',
index: {
valueUnquoted: 'index',
},
sourceType: 'index',
cluster: {
type: 'literal',
literalType: 'keyword',
valueUnquoted: 'cluster',
},
},
],
},
]);
});
it('can parse triple-double quoted cluster', () => {
const text = 'FROM """cluster""":index';
const { root, errors } = parse(text);
expect(errors.length).toBe(0);
expect(root.commands).toMatchObject([
{
type: 'command',
name: 'from',
args: [
{
type: 'source',
index: {
valueUnquoted: 'index',
},
sourceType: 'index',
cluster: {
type: 'literal',
literalType: 'keyword',
valueUnquoted: 'cluster',
},
},
],
},
]);
});
});
describe('selector', () => {
it('can parse source selector', () => {
const text = 'FROM index::selector';
const { root, errors } = parse(text);
expect(errors.length).toBe(0);
expect(root.commands).toMatchObject([
{
type: 'command',
name: 'from',
args: [
{
type: 'source',
index: {
valueUnquoted: 'index',
},
sourceType: 'index',
selector: {
type: 'literal',
literalType: 'keyword',
valueUnquoted: 'selector',
unquoted: true,
},
},
],
},
]);
});
it('can parse single and triple quoted selectors', () => {
const text = 'FROM index1::"selector1", index2::"""selector2"""';
const { root, errors } = parse(text);
expect(errors.length).toBe(0);
expect(root.commands).toMatchObject([
{
type: 'command',
name: 'from',
args: [
{
type: 'source',
index: {
valueUnquoted: 'index1',
},
sourceType: 'index',
selector: {
type: 'literal',
literalType: 'keyword',
valueUnquoted: 'selector1',
},
},
{
type: 'source',
index: {
valueUnquoted: 'index2',
},
sourceType: 'index',
selector: {
type: 'literal',
literalType: 'keyword',
valueUnquoted: 'selector2',
},
},
],
},
]);
});
});
});
});
describe('when incorrectly formatted, returns errors', () => {

22
src/platform/packages/shared/kbn-esql-ast/src/parser/esql_ast_builder_listener.ts
View file

@ -37,15 +37,13 @@ import { default as ESQLParserListener } from '../antlr/esql_parser_listener';
import {
createCommand,
createFunction,
createOption,
createLiteral,
textExistsAndIsValid,
createSource,
visitSource,
createAstBaseItem,
} from './factories';
import { getPosition } from './helpers';
import {
collectAllSourceIdentifiers,
collectAllFields,
collectAllAggFields,
visitByOption,
@ -64,6 +62,7 @@ import { createStatsCommand } from './factories/stats';
import { createChangePointCommand } from './factories/change_point';
import { createWhereCommand } from './factories/where';
import { createRowCommand } from './factories/row';
import { createFromCommand } from './factories/from';
export class ESQLAstBuilderListener implements ESQLParserListener {
private ast: ESQLAst = [];
@ -123,18 +122,9 @@ export class ESQLAstBuilderListener implements ESQLParserListener {
* @param ctx the parse tree
*/
exitFromCommand(ctx: FromCommandContext) {
const commandAst = createCommand('from', ctx);
this.ast.push(commandAst);
commandAst.args.push(...collectAllSourceIdentifiers(ctx));
const metadataContext = ctx.indexPatternAndMetadataFields().metadata();
if (metadataContext && metadataContext.METADATA()) {
const option = createOption(
metadataContext.METADATA().getText().toLowerCase(),
metadataContext
);
commandAst.args.push(option);
option.args.push(...collectAllColumnIdentifiers(metadataContext));
}
const command = createFromCommand(ctx);
this.ast.push(command);
}
/**
@ -149,7 +139,7 @@ export class ESQLAstBuilderListener implements ESQLParserListener {
sources: ctx
.indexPatternAndMetadataFields()
.getTypedRuleContexts(IndexPatternContext)
.map((sourceCtx) => createSource(sourceCtx)),
.map((sourceCtx) => visitSource(sourceCtx)),
};
this.ast.push(node);
node.args.push(...node.sources);

115
src/platform/packages/shared/kbn-esql-ast/src/parser/factories.ts
View file

@ -35,6 +35,7 @@ import {
StringContext,
InputNamedOrPositionalDoubleParamsContext,
InputDoubleParamsContext,
SelectorStringContext,
} from '../antlr/esql_parser';
import { DOUBLE_TICKS_REGEX, SINGLE_BACKTICK, TICKS_REGEX } from './constants';
import type {
@ -62,9 +63,11 @@ import type {
BinaryExpressionOperator,
ESQLCommand,
ESQLParamKinds,
ESQLStringLiteral,
} from '../types';
import { parseIdentifier, getPosition } from './helpers';
import { Builder, type AstNodeParserFields } from '../builder';
import { LeafPrinter } from '../pretty_print';
export function nonNullable<T>(v: T): v is NonNullable<T> {
return v != null;
@ -88,6 +91,18 @@ const createParserFields = (ctx: ParserRuleContext): AstNodeParserFields => ({
incomplete: Boolean(ctx.exception),
});
const createParserFieldsFromTerminalNode = (node: TerminalNode): AstNodeParserFields => {
const text = node.getText();
const symbol = node.symbol;
const fields: AstNodeParserFields = {
text,
location: getPosition(symbol, symbol),
incomplete: false,
};
return fields;
};
export const createCommand = <
Name extends string,
Cmd extends ESQLCommand<Name> = ESQLCommand<Name>
@ -138,7 +153,9 @@ export function createFakeMultiplyLiteral(
};
}
export function createLiteralString(ctx: StringContext): ESQLLiteral {
export function createLiteralString(
ctx: Pick<StringContext, 'QUOTED_STRING'> & ParserRuleContext
): ESQLStringLiteral {
const quotedString = ctx.QUOTED_STRING()?.getText() ?? '""';
const isTripleQuoted = quotedString.startsWith('"""') && quotedString.endsWith('"""');
let valueUnquoted = isTripleQuoted ? quotedString.slice(3, -3) : quotedString.slice(1, -1);
@ -438,34 +455,6 @@ function sanitizeSourceString(ctx: ParserRuleContext) {
return contextText;
}
const unquoteIndexString = (indexString: string): string => {
const isStringQuoted = indexString[0] === '"';
if (!isStringQuoted) {
return indexString;
}
// If wrapped by triple double quotes, simply remove them.
if (indexString.startsWith(`"""`) && indexString.endsWith(`"""`)) {
return indexString.slice(3, -3);
}
// If wrapped by double quote, remove them and unescape the string.
if (indexString[indexString.length - 1] === '"') {
indexString = indexString.slice(1, -1);
indexString = indexString
.replace(/\\"/g, '"')
.replace(/\\r/g, '\r')
.replace(/\\n/g, '\n')
.replace(/\\t/g, '\t')
.replace(/\\\\/g, '\\');
return indexString;
}
// This should never happen, but if it does, return the original string.
return indexString;
};
export function sanitizeIdentifierString(ctx: ParserRuleContext) {
const result =
getUnquotedText(ctx)?.getText() ||
@ -507,38 +496,66 @@ export function createPolicy(token: Token, policy: string): ESQLSource {
};
}
export function createSource(
const visitUnquotedOrQuotedString = (ctx: SelectorStringContext): ESQLStringLiteral => {
const unquotedCtx = ctx.UNQUOTED_SOURCE();
if (unquotedCtx) {
const valueUnquoted = unquotedCtx.getText();
const quotedString = LeafPrinter.string({ valueUnquoted });
return Builder.expression.literal.string(
valueUnquoted,
{
name: quotedString,
unquoted: true,
},
createParserFieldsFromTerminalNode(unquotedCtx)
);
}
return createLiteralString(ctx);
};
export function visitSource(
ctx: ParserRuleContext,
type: 'index' | 'policy' = 'index'
): ESQLSource {
const text = sanitizeSourceString(ctx);
let cluster: string = '';
let index: string = '';
let cluster: ESQLStringLiteral | undefined;
let index: ESQLStringLiteral | undefined;
let selector: ESQLStringLiteral | undefined;
if (ctx instanceof IndexPatternContext) {
const clusterString = ctx.clusterString();
const indexString = ctx.indexString();
const clusterStringCtx = ctx.clusterString();
const indexStringCtx = ctx.indexString();
const selectorStringCtx = ctx.selectorString();
if (clusterString) {
cluster = clusterString.getText();
if (clusterStringCtx) {
cluster = visitUnquotedOrQuotedString(clusterStringCtx);
}
if (indexString) {
index = indexString.getText();
index = unquoteIndexString(index);
if (indexStringCtx) {
index = visitUnquotedOrQuotedString(indexStringCtx);
}
if (selectorStringCtx) {
selector = visitUnquotedOrQuotedString(selectorStringCtx);
}
}
return {
type: 'source',
cluster,
index,
name: text,
sourceType: type,
location: getPosition(ctx.start, ctx.stop),
incomplete: Boolean(ctx.exception || text === ''),
text: ctx?.getText(),
};
return Builder.expression.source.node(
{
sourceType: type,
cluster,
index,
selector,
name: text,
},
{
location: getPosition(ctx.start, ctx.stop),
incomplete: Boolean(ctx.exception || text === ''),
text: ctx?.getText(),
}
);
}
export function createColumnStar(ctx: TerminalNode): ESQLColumn {

35
src/platform/packages/shared/kbn-esql-ast/src/parser/factories/from.ts Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
import { FromCommandContext, IndexPatternContext } from '../../antlr/esql_parser';
import { ESQLCommand } from '../../types';
import { createCommand, createOption, visitSource } from '../factories';
import { collectAllColumnIdentifiers } from '../walkers';
export const createFromCommand = (ctx: FromCommandContext): ESQLCommand<'from'> => {
const command = createCommand('from', ctx);
const indexPatternCtx = ctx.indexPatternAndMetadataFields();
const metadataCtx = indexPatternCtx.metadata();
const sources = indexPatternCtx
.getTypedRuleContexts(IndexPatternContext)
.map((sourceCtx) => visitSource(sourceCtx));
command.args.push(...sources);
if (metadataCtx && metadataCtx.METADATA()) {
const name = metadataCtx.METADATA().getText().toLowerCase();
const option = createOption(name, metadataCtx);
const optionArgs = collectAllColumnIdentifiers(metadataCtx);
option.args.push(...optionArgs);
command.args.push(option);
}
return command;
};

4
src/platform/packages/shared/kbn-esql-ast/src/parser/factories/join.ts
View file

@ -9,11 +9,11 @@
import { JoinCommandContext, JoinTargetContext } from '../../antlr/esql_parser';
import { ESQLAstItem, ESQLAstJoinCommand, ESQLIdentifier, ESQLSource } from '../../types';
import { createCommand, createOption, createSource } from '../factories';
import { createCommand, createOption, visitSource } from '../factories';
import { visitValueExpression } from '../walkers';
const createNodeFromJoinTarget = (ctx: JoinTargetContext): ESQLSource | ESQLIdentifier => {
return createSource(ctx._index);
return visitSource(ctx._index);
};
export const createJoinCommand = (ctx: JoinCommandContext): ESQLAstJoinCommand => {

10
src/platform/packages/shared/kbn-esql-ast/src/parser/walkers.ts
View file

@ -29,7 +29,6 @@ import {
type FieldContext,
type FieldsContext,
type AggFieldsContext,
type FromCommandContext,
FunctionContext,
IntegerLiteralContext,
IsNullContext,
@ -57,7 +56,6 @@ import {
type ValueExpressionContext,
ValueExpressionDefaultContext,
InlineCastContext,
IndexPatternContext,
InlinestatsCommandContext,
MatchExpressionContext,
MatchBooleanExpressionContext,
@ -65,7 +63,6 @@ import {
EntryExpressionContext,
} from '../antlr/esql_parser';
import {
createSource,
createColumn,
createOption,
nonNullable,
@ -110,13 +107,6 @@ import { firstItem, lastItem } from '../visitor/utils';
import { Builder } from '../builder';
import { getPosition } from './helpers';
export function collectAllSourceIdentifiers(ctx: FromCommandContext): ESQLAstItem[] {
const fromContexts = ctx
.indexPatternAndMetadataFields()
.getTypedRuleContexts(IndexPatternContext);
return fromContexts.map((sourceCtx) => createSource(sourceCtx));
}
function terminalNodeToParserRuleContext(node: TerminalNode): ParserRuleContext {
const context = new ParserRuleContext();
context.start = node.symbol;

22
src/platform/packages/shared/kbn-esql-ast/src/pretty_print/__tests__/basic_pretty_printer.test.ts
View file

@ -210,13 +210,31 @@ describe('single line query', () => {
test('quoted source', () => {
const { text } = reprint('FROM "quoted"');
expect(text).toBe('FROM quoted');
expect(text).toBe('FROM "quoted"');
});
test('triple-quoted source', () => {
const { text } = reprint('FROM """quoted"""');
expect(text).toBe('FROM quoted');
expect(text).toBe('FROM "quoted"');
});
test('source selector', () => {
const { text } = reprint('FROM index::selector');
expect(text).toBe('FROM index::selector');
});
test('single-double quoted source selector', () => {
const { text } = reprint('FROM index::"selector"');
expect(text).toBe('FROM index::"selector"');
});
test('triple-double quoted source selector', () => {
const { text } = reprint('FROM index::"""say "jump"!"""');
expect(text).toBe('FROM index::"say \\"jump\\"!"');
});
});

10
src/platform/packages/shared/kbn-esql-ast/src/pretty_print/__tests__/wrapping_pretty_printer.test.ts
View file

@ -425,6 +425,16 @@ FROM
👉 METADATA _id, _source`);
});
test('supports quoted source, quoted cluster name, and quoted index selector component', () => {
const query = `FROM "this is a cluster name" : "this is a quoted index name", "this is another quoted index" :: "and this is a quoted index selector"`;
const text = reprint(query, { pipeTab: ' ' }).text;
expect('\n' + text).toBe(`
FROM
"this is a cluster name":"this is a quoted index name",
"this is another quoted index"::"and this is a quoted index selector"`);
});
test('can break multiple options', () => {
const query =
'from a | enrich policy ON match_field_which_is_very_long WITH new_name1 = field1, new_name2 = field2';

16
src/platform/packages/shared/kbn-esql-ast/src/pretty_print/leaf_printer.ts
View file

@ -29,11 +29,14 @@ const regexUnquotedIdPattern = /^([a-z\*_\@]{1})[a-z0-9_\*]*$/i;
*/
export const LeafPrinter = {
source: (node: ESQLSource): string => {
const { index, name, cluster } = node;
let text = index || name || '';
const { index, name, cluster, selector } = node;
let text = (index ? LeafPrinter.string(index) : name) || '';
if (cluster) {
text = `${cluster}:${text}`;
text = `${LeafPrinter.string(cluster)}:${text}`;
}
if (selector) {
text = `${text}::${LeafPrinter.string(selector)}`;
}
return text;
@ -82,8 +85,13 @@ export const LeafPrinter = {
return formatted;
},
string: (node: ESQLStringLiteral) => {
string: (node: Pick<ESQLStringLiteral, 'valueUnquoted' | 'unquoted'>) => {
const str = node.valueUnquoted;
if (node.unquoted === true) {
return str;
}
const strFormatted =
'"' +
str

25
src/platform/packages/shared/kbn-esql-ast/src/types.ts
View file

@ -289,7 +289,7 @@ export interface ESQLSource extends ESQLAstBaseItem {
* FROM [<cluster>:]<index>
* ```
*/
cluster?: string;
cluster?: ESQLStringLiteral | undefined;
/**
* Represents the index part of the source identifier. Unescaped and unquoted.
@ -298,7 +298,16 @@ export interface ESQLSource extends ESQLAstBaseItem {
* FROM [<cluster>:]<index>
* ```
*/
index?: string;
index?: ESQLStringLiteral | undefined;
/**
* Represents the selector (component) part of the source identifier.
*
* ```
* FROM <index>[::<selector>]
* ```
*/
selector?: ESQLStringLiteral | undefined;
}
export interface ESQLColumn extends ESQLAstBaseItem {
@ -402,6 +411,18 @@ export interface ESQLStringLiteral extends ESQLAstBaseItem {
value: string;
valueUnquoted: string;
/**
* Whether the string was parsed as "unqouted" and/or can be pretty-printed
* unquoted, i.e. in the source text it did not have any quotes (not single ",
* not triple """) quotes. This happens in FROM command source parsing, the
* cluster and selector can be unquoted strings:
*
* ```
* FROM <cluster>:index:<selector>
* ```
*/
unquoted?: boolean;
}
// @internal

12
src/platform/packages/shared/kbn-monaco/src/languages/esql/lib/hover/helpers.test.ts
View file

@ -14,8 +14,10 @@ describe('getVariablesHoverContent', () => {
test('should return empty array if no variables are used in the query', async () => {
const node = {
type: 'source',
cluster: '',
index: 'logst*',
cluster: undefined,
index: {
valueUnquoted: 'logst*',
},
name: 'logst*',
sourceType: 'index',
location: {
@ -40,8 +42,10 @@ describe('getVariablesHoverContent', () => {
test('should return empty array if no variables are given', () => {
const node = {
type: 'source',
cluster: '',
index: 'logst*',
cluster: undefined,
index: {
valueUnquoted: 'logst*',
},
name: 'logst*',
sourceType: 'index',
location: {