ES|QL: enable EXPLAIN (snapshot only) (#129526)

x-pack/plugin/esql/qa/server/src/main/resources/explain.csv-spec

@@ -1,19 +0,0 @@
-explainFrom-Ignore
-explain [ from foo ];
-
-plan:keyword | type:keyword
-"?foo" | PARSED
-"org.elasticsearch.xpack.esql.analysis.VerificationException: Found 1 problem
- line 1:11: Unknown index [foo]" | ANALYZED
-;
-
-
-explainCompositeQuery-Ignore
-explain [ row a = 1 | where b > 0 ];
-
-plan:keyword | type:keyword
-"Filter[?b > 0[INTEGER]]
-\_Row[[1[INTEGER] AS a]]" | PARSED
-"org.elasticsearch.xpack.esql.analysis.VerificationException: Found 1 problem
- line 1:29: Unknown column [b]" | ANALYZED
-;

x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens

@@ -3,7 +3,7 @@ MULTILINE_COMMENT=2
 WS=3
 CHANGE_POINT=4
 ENRICH=5
-EXPLAIN=6
+DEV_EXPLAIN=6
 COMPLETION=7
 DISSECT=8
 EVAL=9
@@ -139,7 +139,6 @@ SHOW_MULTILINE_COMMENT=138
 SHOW_WS=139
 'change_point'=4
 'enrich'=5
-'explain'=6
 'completion'=7
 'dissect'=8
 'eval'=9

x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4

@@ -33,12 +33,12 @@ query
     ;
 
 sourceCommand
-    : explainCommand
+    : fromCommand
-    | fromCommand
     | rowCommand
     | showCommand
     // in development
     | {this.isDevVersion()}? timeSeriesCommand
+    | {this.isDevVersion()}? explainCommand
     ;
 
 processingCommand
@@ -239,11 +239,11 @@ commandOption
     ;
 
 explainCommand
-    : EXPLAIN subqueryExpression
+    : DEV_EXPLAIN subqueryExpression
     ;
 
 subqueryExpression
-    : OPENING_BRACKET query CLOSING_BRACKET
+    : LP query RP
     ;
 
 showCommand

x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens

@@ -3,7 +3,7 @@ MULTILINE_COMMENT=2
 WS=3
 CHANGE_POINT=4
 ENRICH=5
-EXPLAIN=6
+DEV_EXPLAIN=6
 COMPLETION=7
 DISSECT=8
 EVAL=9
@@ -139,7 +139,6 @@ SHOW_MULTILINE_COMMENT=138
 SHOW_WS=139
 'change_point'=4
 'enrich'=5
-'explain'=6
 'completion'=7
 'dissect'=8
 'eval'=9

x-pack/plugin/esql/src/main/antlr/lexer/Explain.g4

@@ -9,12 +9,12 @@ lexer grammar Explain;
 //
 // Explain
 //
-EXPLAIN : 'explain'           -> pushMode(EXPLAIN_MODE);
+DEV_EXPLAIN : {this.isDevVersion()}? 'explain'           -> pushMode(EXPLAIN_MODE);
-
 
 mode EXPLAIN_MODE;
-EXPLAIN_OPENING_BRACKET : OPENING_BRACKET -> type(OPENING_BRACKET), pushMode(DEFAULT_MODE);
+EXPLAIN_LP : LP -> type(LP), pushMode(DEFAULT_MODE);
 EXPLAIN_PIPE : PIPE -> type(PIPE), popMode;
+
 EXPLAIN_WS : WS -> channel(HIDDEN);
 EXPLAIN_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN);
 EXPLAIN_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN);

x-pack/plugin/esql/src/main/antlr/lexer/From.g4

@@ -23,6 +23,9 @@ FROM_COMMA : COMMA -> type(COMMA);
 FROM_ASSIGN : ASSIGN -> type(ASSIGN);
 METADATA : 'metadata';
 
+// we need this for EXPLAIN
+FROM_RP : RP -> type(RP), popMode;
+
 // in 8.14 ` were not allowed
 // this has been relaxed in 8.15 since " is used for quoting
 fragment UNQUOTED_SOURCE_PART

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -1210,7 +1210,12 @@ public class EsqlCapabilities {
          *
          * https://github.com/elastic/elasticsearch/issues/129322
          */
-        NO_PLAIN_STRINGS_IN_LITERALS;
+        NO_PLAIN_STRINGS_IN_LITERALS,
+
+        /**
+         * (Re)Added EXPLAIN command
+         */
+        EXPLAIN(Build.current().isSnapshot());
 
         private final boolean enabled;
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Explain.java

@@ -27,6 +27,12 @@ public class Explain extends LeafPlan implements TelemetryAware {
 
     private final LogicalPlan query;
 
+    private final List<Attribute> output = List.of(
+        new ReferenceAttribute(Source.EMPTY, "role", DataType.KEYWORD),
+        new ReferenceAttribute(Source.EMPTY, "type", DataType.KEYWORD),
+        new ReferenceAttribute(Source.EMPTY, "plan", DataType.KEYWORD)
+    );
+
     public Explain(Source source, LogicalPlan query) {
         super(source);
         this.query = query;
@@ -42,32 +48,13 @@ public class Explain extends LeafPlan implements TelemetryAware {
         throw new UnsupportedOperationException("not serialized");
     }
 
-    // TODO: implement again
+    public LogicalPlan query() {
-    // @Override
+        return query;
-    // public void execute(EsqlSession session, ActionListener<Result> listener) {
+    }
-    // ActionListener<String> analyzedStringListener = listener.map(
-    // analyzed -> new Result(
-    // output(),
-    // List.of(List.of(query.toString(), Type.PARSED.toString()), List.of(analyzed, Type.ANALYZED.toString()))
-    // )
-    // );
-    //
-    // session.analyzedPlan(
-    // query,
-    // ActionListener.wrap(
-    // analyzed -> analyzedStringListener.onResponse(analyzed.toString()),
-    // e -> analyzedStringListener.onResponse(e.toString())
-    // )
-    // );
-    //
-    // }
 
     @Override
     public List<Attribute> output() {
-        return List.of(
+        return output;
-            new ReferenceAttribute(Source.EMPTY, "plan", DataType.KEYWORD),
-            new ReferenceAttribute(Source.EMPTY, "type", DataType.KEYWORD)
-        );
     }
 
     @Override

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java

@@ -16,6 +16,7 @@ import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BlockUtils;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.DriverCompletionInfo;
 import org.elasticsearch.core.Releasables;
@@ -46,6 +47,8 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
 import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.UnresolvedStar;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.Holder;
 import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver;
 import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
@@ -66,6 +69,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
 import org.elasticsearch.xpack.esql.plan.logical.Drop;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
+import org.elasticsearch.xpack.esql.plan.logical.Explain;
 import org.elasticsearch.xpack.esql.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.plan.logical.Fork;
 import org.elasticsearch.xpack.esql.plan.logical.InlineStats;
@@ -89,7 +93,9 @@ import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
 import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
 import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize;
 import org.elasticsearch.xpack.esql.plan.physical.FragmentExec;
+import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec;
 import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
+import org.elasticsearch.xpack.esql.planner.PlannerUtils;
 import org.elasticsearch.xpack.esql.planner.mapper.Mapper;
 import org.elasticsearch.xpack.esql.planner.premapper.PreMapper;
 import org.elasticsearch.xpack.esql.plugin.TransportActionServices;
@@ -106,6 +112,7 @@ import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
+import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
 import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD;
 
 public class EsqlSession {
@@ -138,6 +145,10 @@ public class EsqlSession {
     private Set<String> configuredClusters;
     private final InferenceRunner inferenceRunner;
 
+    private boolean explainMode;
+    private String parsedPlanString;
+    private String optimizedLogicalPlanString;
+
     public EsqlSession(
         String sessionId,
         Configuration configuration,
@@ -178,22 +189,39 @@ public class EsqlSession {
     public void execute(EsqlQueryRequest request, EsqlExecutionInfo executionInfo, PlanRunner planRunner, ActionListener<Result> listener) {
         assert executionInfo != null : "Null EsqlExecutionInfo";
         LOGGER.debug("ESQL query:\n{}", request.query());
-        analyzedPlan(
+        LogicalPlan parsed = parse(request.query(), request.params());
-            parse(request.query(), request.params()),
+        Explain explain = findExplain(parsed);
-            executionInfo,
+        if (explain != null) {
-            request.filter(),
+            explainMode = true;
-            new EsqlCCSUtils.CssPartialErrorsActionListener(executionInfo, listener) {
+            if (explain == parsed) {
-                @Override
+                parsed = explain.query();
-                public void onResponse(LogicalPlan analyzedPlan) {
+                parsedPlanString = parsed.toString();
-                    preMapper.preMapper(
+            } else {
-                        analyzedPlan,
+                throw new VerificationException("EXPLAIN does not support downstream commands");
-                        listener.delegateFailureAndWrap(
-                            (l, p) -> executeOptimizedPlan(request, executionInfo, planRunner, optimizedPlan(p), l)
-                        )
-                    );
-                }
             }
-        );
+        }
+        analyzedPlan(parsed, executionInfo, request.filter(), new EsqlCCSUtils.CssPartialErrorsActionListener(executionInfo, listener) {
+            @Override
+            public void onResponse(LogicalPlan analyzedPlan) {
+                preMapper.preMapper(
+                    analyzedPlan,
+                    listener.delegateFailureAndWrap((l, p) -> executeOptimizedPlan(request, executionInfo, planRunner, optimizedPlan(p), l))
+                );
+            }
+        });
+    }
+
+    private Explain findExplain(LogicalPlan parsed) {
+        if (parsed instanceof Explain e) {
+            return e;
+        }
+        for (LogicalPlan child : parsed.children()) {
+            Explain result = findExplain(child);
+            if (result != null) {
+                return result;
+            }
+        }
+        return null;
     }
 
     /**
@@ -208,6 +236,20 @@ public class EsqlSession {
         ActionListener<Result> listener
     ) {
         PhysicalPlan physicalPlan = logicalPlanToPhysicalPlan(optimizedPlan, request);
+        if (explainMode) {
+            String physicalPlanString = physicalPlan.toString();
+            List<Attribute> fields = List.of(
+                new ReferenceAttribute(EMPTY, "role", DataType.KEYWORD),
+                new ReferenceAttribute(EMPTY, "type", DataType.KEYWORD),
+                new ReferenceAttribute(EMPTY, "plan", DataType.KEYWORD)
+            );
+            List<List<Object>> values = new ArrayList<>();
+            values.add(List.of("coordinator", "parsedPlan", parsedPlanString));
+            values.add(List.of("coordinator", "optimizedLogicalPlan", optimizedLogicalPlanString));
+            values.add(List.of("coordinator", "optimizedPhysicalPlan", physicalPlanString));
+            var blocks = BlockUtils.fromList(PlannerUtils.NON_BREAKING_BLOCK_FACTORY, values);
+            physicalPlan = new LocalSourceExec(Source.EMPTY, fields, LocalSupplier.of(blocks));
+        }
         // TODO: this could be snuck into the underlying listener
         EsqlCCSUtils.updateExecutionInfoAtEndOfPlanning(executionInfo);
         // execute any potential subplans
@@ -792,6 +834,7 @@ public class EsqlSession {
         if (optimizedPlan.optimized() == false) {
             throw new IllegalStateException("Expected optimized plan");
         }
+        optimizedLogicalPlanString = optimizedPlan.toString();
         var plan = mapper.map(optimizedPlan);
         LOGGER.debug("Physical plan:\n{}", plan);
         return plan;

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java

@@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Drop;
 import org.elasticsearch.xpack.esql.plan.logical.Enrich;
 import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
 import org.elasticsearch.xpack.esql.plan.logical.Eval;
+import org.elasticsearch.xpack.esql.plan.logical.Explain;
 import org.elasticsearch.xpack.esql.plan.logical.Filter;
 import org.elasticsearch.xpack.esql.plan.logical.Fork;
 import org.elasticsearch.xpack.esql.plan.logical.Grok;
@@ -53,6 +54,7 @@ public enum FeatureMetric {
     STATS(Aggregate.class::isInstance),
     WHERE(Filter.class::isInstance),
     ENRICH(Enrich.class::isInstance),
+    EXPLAIN(Explain.class::isInstance),
     MV_EXPAND(MvExpand.class::isInstance),
     SHOW(ShowInfo.class::isInstance),
     ROW(Row.class::isInstance),

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java

@@ -961,41 +961,21 @@ public class StatementParserTests extends AbstractStatementParserTests {
     }
 
     public void testSubquery() {
-        assertEquals(new Explain(EMPTY, PROCESSING_CMD_INPUT), statement("explain [ row a = 1 ]"));
+        assertEquals(new Explain(EMPTY, PROCESSING_CMD_INPUT), statement("explain ( row a = 1 )"));
     }
 
     public void testSubqueryWithPipe() {
-        assertEquals(
+        assertEquals(new Explain(EMPTY, PROCESSING_CMD_INPUT), statement("explain ( row a = 1 )"));
-            new Limit(EMPTY, integer(10), new Explain(EMPTY, PROCESSING_CMD_INPUT)),
-            statement("explain [ row a = 1 ] | limit 10")
-        );
-    }
-
-    public void testNestedSubqueries() {
-        assertEquals(
-            new Limit(
-                EMPTY,
-                integer(10),
-                new Explain(EMPTY, new Limit(EMPTY, integer(5), new Explain(EMPTY, new Limit(EMPTY, integer(1), PROCESSING_CMD_INPUT))))
-            ),
-            statement("explain [ explain [ row a = 1 | limit 1 ] | limit 5 ] | limit 10")
-        );
-    }
-
-    public void testSubquerySpacing() {
-        assertEquals(statement("explain [ explain [ from a ] | where b == 1 ]"), statement("explain[explain[from a]|where b==1]"));
     }
 
     public void testBlockComments() {
-        String query = " explain [ from foo ] | limit 10 ";
+        String query = " explain ( from foo )";
         LogicalPlan expected = statement(query);
 
         int wsIndex = query.indexOf(' ');
 
         do {
-            String queryWithComment = query.substring(0, wsIndex)
+            String queryWithComment = query.substring(0, wsIndex) + "/*explain ( \nfrom bar ) */" + query.substring(wsIndex + 1);
-                + "/*explain [ \nfrom bar ] | where a > b*/"
-                + query.substring(wsIndex + 1);
 
             assertEquals(expected, statement(queryWithComment));
 
@@ -1004,15 +984,13 @@ public class StatementParserTests extends AbstractStatementParserTests {
     }
 
     public void testSingleLineComments() {
-        String query = " explain [ from foo ] | limit 10 ";
+        String query = " explain ( from foo ) ";
         LogicalPlan expected = statement(query);
 
         int wsIndex = query.indexOf(' ');
 
         do {
-            String queryWithComment = query.substring(0, wsIndex)
+            String queryWithComment = query.substring(0, wsIndex) + "//explain ( from bar ) \n" + query.substring(wsIndex + 1);
-                + "//explain [ from bar ] | where a > b \n"
-                + query.substring(wsIndex + 1);
 
             assertEquals(expected, statement(queryWithComment));
 
@@ -1039,13 +1017,12 @@ public class StatementParserTests extends AbstractStatementParserTests {
             Tuple.tuple("a+b = c", "a+b"),
             Tuple.tuple("a//hi", "a"),
             Tuple.tuple("a/*hi*/", "a"),
-            Tuple.tuple("explain [ frm a ]", "frm")
+            Tuple.tuple("explain ( frm a )", "frm")
         )) {
             expectThrows(
                 ParsingException.class,
                 allOf(
                     containsString("mismatched input '" + queryWithUnexpectedCmd.v2() + "'"),
-                    containsString("'explain'"),
                     containsString("'from'"),
                     containsString("'row'")
                 ),
@@ -1058,13 +1035,12 @@ public class StatementParserTests extends AbstractStatementParserTests {
     public void testSuggestAvailableProcessingCommandsOnParsingError() {
         for (Tuple<String, String> queryWithUnexpectedCmd : List.of(
             Tuple.tuple("from a | filter b > 1", "filter"),
-            Tuple.tuple("from a | explain [ row 1 ]", "explain"),
+            Tuple.tuple("from a | explain ( row 1 )", "explain"),
             Tuple.tuple("from a | not-a-thing", "not-a-thing"),
             Tuple.tuple("from a | high5 a", "high5"),
             Tuple.tuple("from a | a+b = c", "a+b"),
             Tuple.tuple("from a | a//hi", "a"),
-            Tuple.tuple("from a | a/*hi*/", "a"),
+            Tuple.tuple("from a | a/*hi*/", "a")
-            Tuple.tuple("explain [ from a | evl b = c ]", "evl")
         )) {
             expectThrows(
                 ParsingException.class,
@@ -1108,7 +1084,7 @@ public class StatementParserTests extends AbstractStatementParserTests {
     public void testMetadataFieldOnOtherSources() {
         expectError("row a = 1 metadata _index", "line 1:20: extraneous input '_index' expecting <EOF>");
         expectError("show info metadata _index", "line 1:11: token recognition error at: 'm'");
-        expectError("explain [from foo] metadata _index", "line 1:20: mismatched input 'metadata' expecting {'|', ',', ']', 'metadata'}");
+        expectError("explain ( from foo ) metadata _index", "line 1:22: mismatched input 'metadata' expecting {'|', ',', ')', 'metadata'}");
     }
 
     public void testMetadataFieldMultipleDeclarations() {
@@ -3476,9 +3452,9 @@ public class StatementParserTests extends AbstractStatementParserTests {
         expectError("row a = 1 | where a not in [1", "line 1:28: missing '(' at '['");
         expectError("row a = 1 | where a not in 123", "line 1:28: missing '(' at '123'");
         // test for [
-        expectError("explain", "line 1:8: mismatched input '<EOF>' expecting '['");
+        expectError("explain", "line 1:8: mismatched input '<EOF>' expecting '('");
         expectError("explain ]", "line 1:9: token recognition error at: ']'");
-        expectError("explain [row x = 1", "line 1:19: missing ']' at '<EOF>'");
+        expectError("explain ( row x = 1", "line 1:20: missing ')' at '<EOF>'");
     }
 
     public void testRerankDefaultInferenceIdAndScoreAttribute() {

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/220_explain.yml

@@ -0,0 +1,101 @@
+---
+setup:
+  - requires:
+      test_runner_features: [capabilities, contains, allowed_warnings_regex]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: []
+          capabilities: [explain]
+      reason: "new EXPLAIN command"
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            properties:
+              color:
+                type: text
+                fields:
+                  keyword:
+                    type: keyword
+              description:
+                type: text
+                fields:
+                  keyword:
+                    type: keyword
+
+  - do:
+      bulk:
+        index: "test"
+        refresh: true
+        body:
+          - { "index": { } }
+          - { "color": "red", "description": "The color Red" }
+          - { "index": { } }
+          - { "color": "blue", "description": "The color Blue" }
+          - { "index": { } }
+          - { "color": "green", "description": "The color Green" }
+
+---
+explainRow:
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'EXPLAIN (row a = 1)'
+
+  - length: { columns: 3 }
+  - match: {columns.0.name: "role"}
+  - match: {columns.0.type: "keyword"}
+  - match: {columns.1.name: "type"}
+  - match: {columns.1.type: "keyword"}
+  - match: {columns.2.name: "plan"}
+  - match: {columns.2.type: "keyword"}
+  - length: { values: 3 }
+  - match: { values.0.0: "coordinator" }
+  - match: { values.0.1: "parsedPlan" }
+  - match: { values.1.0: "coordinator" }
+  - match: { values.1.1: "optimizedLogicalPlan" }
+  - match: { values.2.0: "coordinator" }
+  - match: { values.2.1: "optimizedPhysicalPlan" }
+
+
+---
+explainQuery:
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'EXPLAIN (from test | where color == "red" | eval b = 20)'
+
+  - length: { columns: 3 }
+  - match: {columns.0.name: "role"}
+  - match: {columns.0.type: "keyword"}
+  - match: {columns.1.name: "type"}
+  - match: {columns.1.type: "keyword"}
+  - match: {columns.2.name: "plan"}
+  - match: {columns.2.type: "keyword"}
+  - length: { values: 3 }
+  - match: { values.0.0: "coordinator" }
+  - match: { values.0.1: "parsedPlan" }
+  - match: { values.1.0: "coordinator" }
+  - match: { values.1.1: "optimizedLogicalPlan" }
+  - match: { values.2.0: "coordinator" }
+  - match: { values.2.1: "optimizedPhysicalPlan" }
+
+
+---
+explainDownstream:
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'EXPLAIN (row a = 1) | eval b = 2'
+      catch: "bad_request"
+
+  - match: { error.type: "verification_exception" }
+  - contains: { error.reason: "EXPLAIN does not support downstream commands" }