[ES|QL] Grammar update, JOIN command grammar change (#208356)

## Summary

The `JOIN` command grammar has been changed to accept `ESQLSource` as
the first command argument instead of `ESQLIdentifier`. This PR
addresses that change throughout the "ast" and "validation" packages.


### Checklist


- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

src/platform/packages/shared/kbn-esql-ast/index.ts

@@ -37,6 +37,7 @@ export {
   isBinaryExpression,
   isWhereExpression,
   isFieldExpression,
+  isSource,
   isIdentifier,
   isIntegerLiteral,
   isLiteral,

src/platform/packages/shared/kbn-esql-ast/src/antlr/esql_lexer.g4

@@ -563,6 +563,10 @@ JOIN_AS : AS -> type(AS);
 JOIN_ON : ON -> type(ON), popMode, pushMode(EXPRESSION_MODE);
 USING : 'USING' -> popMode, pushMode(EXPRESSION_MODE);
 
+JOIN_UNQUOTED_SOURCE: UNQUOTED_SOURCE -> type(UNQUOTED_SOURCE);
+JOIN_QUOTED_SOURCE : QUOTED_STRING -> type(QUOTED_STRING);
+JOIN_COLON : COLON -> type(COLON);
+
 JOIN_UNQUOTED_IDENTIFER: UNQUOTED_IDENTIFIER -> type(UNQUOTED_IDENTIFIER);
 JOIN_QUOTED_IDENTIFIER : QUOTED_IDENTIFIER -> type(QUOTED_IDENTIFIER);
 

src/platform/packages/shared/kbn-esql-ast/src/antlr/esql_parser.g4

@@ -330,7 +330,7 @@ joinCommand
     ;
 
 joinTarget
-    : index=identifier (AS alias=identifier)?
+    : index=indexPattern (AS alias=identifier)?
     ;
 
 joinCondition

src/platform/packages/shared/kbn-esql-ast/src/antlr/esql_parser.ts

@@ -3506,7 +3506,7 @@ export default class esql_parser extends parser_config {
 			this.enterOuterAlt(localctx, 1);
 			{
 			this.state = 634;
-			localctx._index = this.identifier();
+			localctx._index = this.indexPattern();
 			this.state = 637;
 			this._errHandler.sync(this);
 			_la = this._input.LA(1);
@@ -3906,7 +3906,7 @@ export default class esql_parser extends parser_config {
 	0,0,0,621,622,5,17,0,0,622,625,3,54,27,0,623,624,5,33,0,0,624,626,3,34,
 	17,0,625,623,1,0,0,0,625,626,1,0,0,0,626,123,1,0,0,0,627,629,7,8,0,0,628,
 	627,1,0,0,0,628,629,1,0,0,0,629,630,1,0,0,0,630,631,5,20,0,0,631,632,3,
-	126,63,0,632,633,3,128,64,0,633,125,1,0,0,0,634,637,3,64,32,0,635,636,5,
+	126,63,0,632,633,3,128,64,0,633,125,1,0,0,0,634,637,3,40,20,0,635,636,5,
 	91,0,0,636,638,3,64,32,0,637,635,1,0,0,0,637,638,1,0,0,0,638,127,1,0,0,
 	0,639,640,5,95,0,0,640,645,3,130,65,0,641,642,5,39,0,0,642,644,3,130,65,
 	0,643,641,1,0,0,0,644,647,1,0,0,0,645,643,1,0,0,0,645,646,1,0,0,0,646,129,
@@ -6613,21 +6613,21 @@ export class JoinCommandContext extends ParserRuleContext {
 
 
 export class JoinTargetContext extends ParserRuleContext {
-	public _index!: IdentifierContext;
+	public _index!: IndexPatternContext;
 	public _alias!: IdentifierContext;
 	constructor(parser?: esql_parser, parent?: ParserRuleContext, invokingState?: number) {
 		super(parent, invokingState);
     	this.parser = parser;
 	}
-	public identifier_list(): IdentifierContext[] {
-		return this.getTypedRuleContexts(IdentifierContext) as IdentifierContext[];
-	}
-	public identifier(i: number): IdentifierContext {
-		return this.getTypedRuleContext(IdentifierContext, i) as IdentifierContext;
+	public indexPattern(): IndexPatternContext {
+		return this.getTypedRuleContext(IndexPatternContext, 0) as IndexPatternContext;
 	}
 	public AS(): TerminalNode {
 		return this.getToken(esql_parser.AS, 0);
 	}
+	public identifier(): IdentifierContext {
+		return this.getTypedRuleContext(IdentifierContext, 0) as IdentifierContext;
+	}
     public get ruleIndex(): number {
     	return esql_parser.RULE_joinTarget;
 	}

src/platform/packages/shared/kbn-esql-ast/src/ast/helpers.ts

@@ -19,6 +19,7 @@ import type {
   ESQLLiteral,
   ESQLParamLiteral,
   ESQLProperNode,
+  ESQLSource,
 } from '../types';
 import { BinaryExpressionGroup } from './constants';
 
@@ -73,6 +74,9 @@ export const isParamLiteral = (node: unknown): node is ESQLParamLiteral =>
 export const isColumn = (node: unknown): node is ESQLColumn =>
   isProperNode(node) && node.type === 'column';
 
+export const isSource = (node: unknown): node is ESQLSource =>
+  isProperNode(node) && node.type === 'source';
+
 export const isIdentifier = (node: unknown): node is ESQLIdentifier =>
   isProperNode(node) && node.type === 'identifier';
 

src/platform/packages/shared/kbn-esql-ast/src/mutate/commands/join/index.test.ts

@@ -25,7 +25,7 @@ describe('commands.where', () => {
           name: 'join',
           args: [
             {
-              type: 'identifier',
+              type: 'source',
               name: 'join_index1',
             },
             {},
@@ -36,7 +36,7 @@ describe('commands.where', () => {
           name: 'join',
           args: [
             {
-              type: 'identifier',
+              type: 'source',
               name: 'join_index2',
             },
             {},
@@ -60,7 +60,7 @@ describe('commands.where', () => {
         name: 'join',
         args: [
           {
-            type: 'identifier',
+            type: 'source',
             name: 'join_index2',
           },
           {},
@@ -71,7 +71,7 @@ describe('commands.where', () => {
         name: 'join',
         args: [
           {
-            type: 'identifier',
+            type: 'source',
             name: 'join_index1',
           },
           {},
@@ -91,7 +91,7 @@ describe('commands.where', () => {
         {
           target: {
             index: {
-              type: 'identifier',
+              type: 'source',
               name: 'join_index1',
             },
           },
@@ -99,7 +99,7 @@ describe('commands.where', () => {
         {
           target: {
             index: {
-              type: 'identifier',
+              type: 'source',
               name: 'join_index2',
             },
           },

src/platform/packages/shared/kbn-esql-ast/src/mutate/commands/join/index.ts

@@ -16,6 +16,7 @@ import type {
   ESQLAstQueryExpression,
   ESQLCommand,
   ESQLIdentifier,
+  ESQLSource,
 } from '../../../types';
 import * as generic from '../../generic';
 
@@ -43,6 +44,11 @@ export const byIndex = (ast: ESQLAstQueryExpression, index: number): ESQLCommand
   return [...list(ast)][index];
 };
 
+const getSource = (node: WalkerAstNode): ESQLSource =>
+  Walker.match(node, {
+    type: 'source',
+  }) as ESQLSource;
+
 const getIdentifier = (node: WalkerAstNode): ESQLIdentifier =>
   Walker.match(node, {
     type: 'identifier',
@@ -60,15 +66,15 @@ export const summarize = (query: ESQLAstQueryExpression): JoinCommandSummary[] =
 
   for (const command of list(query)) {
     const firstArg = command.args[0];
-    let index: ESQLIdentifier | undefined;
+    let index: ESQLSource | undefined;
     let alias: ESQLIdentifier | undefined;
     const conditions: ESQLAstExpression[] = [];
 
     if (isAsExpression(firstArg)) {
-      index = getIdentifier(firstArg.args[0]);
+      index = getSource(firstArg.args[0]);
       alias = getIdentifier(firstArg.args[1]);
     } else {
-      index = getIdentifier(firstArg);
+      index = getSource(firstArg);
     }
 
     const on = generic.commands.options.find(command, ({ name }) => name === 'on');
@@ -96,6 +102,6 @@ export interface JoinCommandSummary {
 }
 
 export interface JoinCommandTarget {
-  index: ESQLIdentifier;
+  index: ESQLSource;
   alias?: ESQLIdentifier;
 }

src/platform/packages/shared/kbn-esql-ast/src/parser/__tests__/comments.test.ts

@@ -456,7 +456,7 @@ FROM index`;
         name: 'join',
         args: [
           {
-            type: 'identifier',
+            type: 'source',
             name: 'abc',
             formatting: {
               top: [
@@ -591,7 +591,7 @@ FROM index`;
         name: 'join',
         args: [
           {
-            type: 'identifier',
+            type: 'source',
             name: 'abc',
             formatting: {
               left: [
@@ -846,7 +846,7 @@ FROM index`;
         name: 'join',
         args: [
           {
-            type: 'identifier',
+            type: 'source',
             name: 'abc',
             formatting: {
               right: [

src/platform/packages/shared/kbn-esql-ast/src/parser/__tests__/join.test.ts

@@ -49,7 +49,7 @@ describe('<TYPE> JOIN command', () => {
         commandType: 'lookup',
         args: [
           {
-            type: 'identifier',
+            type: 'source',
             name: 'languages_lookup',
           },
           {},
@@ -70,7 +70,7 @@ describe('<TYPE> JOIN command', () => {
             name: 'as',
             args: [
               {
-                type: 'identifier',
+                type: 'source',
                 name: 'languages_lookup',
               },
               {
@@ -160,7 +160,7 @@ describe('<TYPE> JOIN command', () => {
         commandType: 'lookup',
         args: [
           {
-            type: 'identifier',
+            type: 'source',
             name: 'languages_lookup',
           },
           {
@@ -180,7 +180,7 @@ describe('<TYPE> JOIN command', () => {
     it('correctly extracts node positions', () => {
       const text = `FROM employees | LOOKUP JOIN index AS alias ON on_1, on_2 | LIMIT 1`;
       const query = EsqlQuery.fromSrc(text);
-      const node1 = Walker.match(query.ast, { type: 'identifier', name: 'index' });
+      const node1 = Walker.match(query.ast, { type: 'source', name: 'index' });
       const node2 = Walker.match(query.ast, { type: 'identifier', name: 'alias' });
       const node3 = Walker.match(query.ast, { type: 'column', name: 'on_1' });
       const node4 = Walker.match(query.ast, { type: 'column', name: 'on_2' });

src/platform/packages/shared/kbn-esql-ast/src/parser/factories/join.ts

@@ -8,19 +8,26 @@
  */
 
 import { JoinCommandContext, JoinTargetContext } from '../../antlr/esql_parser';
-import { ESQLAstItem, ESQLBinaryExpression, ESQLCommand, ESQLIdentifier } from '../../types';
+import {
+  ESQLAstItem,
+  ESQLBinaryExpression,
+  ESQLCommand,
+  ESQLIdentifier,
+  ESQLSource,
+} from '../../types';
 import {
   createBinaryExpression,
   createCommand,
   createIdentifier,
   createOption,
+  createSource,
 } from '../factories';
 import { visitValueExpression } from '../walkers';
 
 const createNodeFromJoinTarget = (
   ctx: JoinTargetContext
-): ESQLIdentifier | ESQLBinaryExpression => {
-  const index = createIdentifier(ctx._index);
+): ESQLSource | ESQLIdentifier | ESQLBinaryExpression => {
+  const index = createSource(ctx._index);
   const aliasCtx = ctx._alias;
 
   if (!aliasCtx) {

src/platform/packages/shared/kbn-esql-ast/src/walker/walker.test.ts

@@ -86,17 +86,20 @@ describe('structurally can walk all nodes', () => {
     test('can traverse JOIN command', () => {
       const { ast } = parse('FROM index | LEFT JOIN a AS b ON c, d');
       const commands: ESQLCommand[] = [];
+      const sources: ESQLSource[] = [];
       const identifiers: ESQLIdentifier[] = [];
       const columns: ESQLColumn[] = [];
 
       walk(ast, {
         visitCommand: (cmd) => commands.push(cmd),
+        visitSource: (id) => sources.push(id),
         visitIdentifier: (id) => identifiers.push(id),
         visitColumn: (col) => columns.push(col),
       });
 
       expect(commands.map(({ name }) => name).sort()).toStrictEqual(['from', 'join']);
-      expect(identifiers.map(({ name }) => name).sort()).toStrictEqual(['a', 'as', 'b', 'c', 'd']);
+      expect(sources.map(({ name }) => name).sort()).toStrictEqual(['a', 'index']);
+      expect(identifiers.map(({ name }) => name).sort()).toStrictEqual(['as', 'b', 'c', 'd']);
       expect(columns.map(({ name }) => name).sort()).toStrictEqual(['c', 'd']);
     });
 
@@ -1109,8 +1112,8 @@ describe('Walker.match()', () => {
       name: 'join',
       commandType: 'left',
     })!;
-    const identifier1 = Walker.match(join1, {
-      type: 'identifier',
+    const source1 = Walker.match(join1, {
+      type: 'source',
       name: 'a',
     })!;
     const join2 = Walker.match(root, {
@@ -1118,15 +1121,15 @@ describe('Walker.match()', () => {
       name: 'join',
       commandType: 'right',
     })!;
-    const identifier2 = Walker.match(join2, {
-      type: 'identifier',
+    const source2 = Walker.match(join2, {
+      type: 'source',
       name: 'b',
     })!;
 
-    expect(identifier1).toMatchObject({
+    expect(source1).toMatchObject({
       name: 'a',
     });
-    expect(identifier2).toMatchObject({
+    expect(source2).toMatchObject({
       name: 'b',
     });
   });

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/validation/errors.ts

@@ -14,6 +14,7 @@ import type {
   ESQLFunction,
   ESQLLocation,
   ESQLMessage,
+  ESQLSource,
 } from '@kbn/esql-ast';
 import { ESQLIdentifier } from '@kbn/esql-ast/src/types';
 import type { ErrorTypes, ErrorValues } from './types';
@@ -546,7 +547,7 @@ export const errors = {
       nestedAgg: fn.name,
     }),
 
-  invalidJoinIndex: (identifier: ESQLIdentifier): ESQLMessage =>
+  invalidJoinIndex: (identifier: ESQLSource): ESQLMessage =>
     errors.byId('invalidJoinIndex', identifier.location, {
       identifier: identifier.name,
     }),

src/platform/packages/shared/kbn-esql-validation-autocomplete/src/validation/validation.ts

@@ -23,6 +23,7 @@ import {
   walk,
   isBinaryExpression,
   isIdentifier,
+  isSource,
 } from '@kbn/esql-ast';
 import type {
   ESQLAstField,
@@ -1151,22 +1152,22 @@ const validateJoinCommand = (
   }
 
   const target = args[0] as ESQLProperNode;
-  let index: ESQLIdentifier;
+  let index: ESQLSource;
   let alias: ESQLIdentifier | undefined;
 
   if (isBinaryExpression(target)) {
     if (target.name === 'as') {
       alias = target.args[1] as ESQLIdentifier;
-      index = target.args[0] as ESQLIdentifier;
+      index = target.args[0] as ESQLSource;
 
-      if (!isIdentifier(index) || !isIdentifier(alias)) {
+      if (!isSource(index) || !isIdentifier(alias)) {
         return [errors.unexpected(target.location)];
       }
     } else {
       return [errors.unexpected(target.location)];
     }
-  } else if (isIdentifier(target)) {
-    index = target as ESQLIdentifier;
+  } else if (isSource(target)) {
+    index = target as ESQLSource;
   } else {
     return [errors.unexpected(target.location)];
   }