github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-06-27 17:10:22 -04:00
Code Issues Projects Releases Packages Wiki Activity

ES|QL: enable EXPLAIN (snapshot only) (#129526)

Browse source
This commit is contained in:
Luigi Dell'Aquila 2025-06-23 09:55:45 +02:00 • committed by GitHub
parent f1b2c8dd8e
commit a79bbffb0b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 2293 additions and 2177 deletions
Download patch file Download diff file Expand all files Collapse all files

19
x-pack/plugin/esql/qa/server/src/main/resources/explain.csv-spec
View file

@ -1,19 +0,0 @@
explainFrom-Ignore
explain [ from foo ];
plan:keyword | type:keyword
"?foo" | PARSED
"org.elasticsearch.xpack.esql.analysis.VerificationException: Found 1 problem
line 1:11: Unknown index [foo]" | ANALYZED
;
explainCompositeQuery-Ignore
explain [ row a = 1 | where b > 0 ];
plan:keyword | type:keyword
"Filter[?b > 0[INTEGER]]
\_Row[[1[INTEGER] AS a]]" | PARSED
"org.elasticsearch.xpack.esql.analysis.VerificationException: Found 1 problem
line 1:29: Unknown column [b]" | ANALYZED
;

3
x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.tokens generated
View file

@ -3,7 +3,7 @@ MULTILINE_COMMENT=2
WS=3
CHANGE_POINT=4
ENRICH=5
EXPLAIN=6
DEV_EXPLAIN=6
COMPLETION=7
DISSECT=8
EVAL=9
@ -139,7 +139,6 @@ SHOW_MULTILINE_COMMENT=138
SHOW_WS=139
'change_point'=4
'enrich'=5
'explain'=6
'completion'=7
'dissect'=8
'eval'=9

8
x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4
View file

@ -33,12 +33,12 @@ query
;
sourceCommand
: explainCommand
| fromCommand
: fromCommand
| rowCommand
| showCommand
// in development
| {this.isDevVersion()}? timeSeriesCommand
| {this.isDevVersion()}? explainCommand
;
processingCommand
@ -239,11 +239,11 @@ commandOption
;
explainCommand
: EXPLAIN subqueryExpression
: DEV_EXPLAIN subqueryExpression
;
subqueryExpression
: OPENING_BRACKET query CLOSING_BRACKET
: LP query RP
;
showCommand

3
x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.tokens generated
View file

@ -3,7 +3,7 @@ MULTILINE_COMMENT=2
WS=3
CHANGE_POINT=4
ENRICH=5
EXPLAIN=6
DEV_EXPLAIN=6
COMPLETION=7
DISSECT=8
EVAL=9
@ -139,7 +139,6 @@ SHOW_MULTILINE_COMMENT=138
SHOW_WS=139
'change_point'=4
'enrich'=5
'explain'=6
'completion'=7
'dissect'=8
'eval'=9

6
x-pack/plugin/esql/src/main/antlr/lexer/Explain.g4
View file

@ -9,12 +9,12 @@ lexer grammar Explain;
//
// Explain
//
EXPLAIN : 'explain' -> pushMode(EXPLAIN_MODE);
DEV_EXPLAIN : {this.isDevVersion()}? 'explain' -> pushMode(EXPLAIN_MODE);
mode EXPLAIN_MODE;
EXPLAIN_OPENING_BRACKET : OPENING_BRACKET -> type(OPENING_BRACKET), pushMode(DEFAULT_MODE);
EXPLAIN_LP : LP -> type(LP), pushMode(DEFAULT_MODE);
EXPLAIN_PIPE : PIPE -> type(PIPE), popMode;
EXPLAIN_WS : WS -> channel(HIDDEN);
EXPLAIN_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN);
EXPLAIN_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN);

3
x-pack/plugin/esql/src/main/antlr/lexer/From.g4
View file

@ -23,6 +23,9 @@ FROM_COMMA : COMMA -> type(COMMA);
FROM_ASSIGN : ASSIGN -> type(ASSIGN);
METADATA : 'metadata';
// we need this for EXPLAIN
FROM_RP : RP -> type(RP), popMode;
// in 8.14 ` were not allowed
// this has been relaxed in 8.15 since " is used for quoting
fragment UNQUOTED_SOURCE_PART

7
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
View file

@ -1210,7 +1210,12 @@ public class EsqlCapabilities {
*
* https://github.com/elastic/elasticsearch/issues/129322
*/
NO_PLAIN_STRINGS_IN_LITERALS;
NO_PLAIN_STRINGS_IN_LITERALS,
/**
* (Re)Added EXPLAIN command
*/
EXPLAIN(Build.current().isSnapshot());
private final boolean enabled;

11
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.interp generated
View file

File diff suppressed because one or more lines are too long

2280
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer.java generated
View file

File diff suppressed because it is too large Load diff

6
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp generated
View file

File diff suppressed because one or more lines are too long

1867
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java generated
View file

File diff suppressed because it is too large Load diff

33
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Explain.java
View file

@ -27,6 +27,12 @@ public class Explain extends LeafPlan implements TelemetryAware {
private final LogicalPlan query;
private final List<Attribute> output = List.of(
new ReferenceAttribute(Source.EMPTY, "role", DataType.KEYWORD),
new ReferenceAttribute(Source.EMPTY, "type", DataType.KEYWORD),
new ReferenceAttribute(Source.EMPTY, "plan", DataType.KEYWORD)
);
public Explain(Source source, LogicalPlan query) {
super(source);
this.query = query;
@ -42,32 +48,13 @@ public class Explain extends LeafPlan implements TelemetryAware {
throw new UnsupportedOperationException("not serialized");
}
// TODO: implement again
// @Override
// public void execute(EsqlSession session, ActionListener<Result> listener) {
// ActionListener<String> analyzedStringListener = listener.map(
// analyzed -> new Result(
// output(),
// List.of(List.of(query.toString(), Type.PARSED.toString()), List.of(analyzed, Type.ANALYZED.toString()))
// )
// );
//
// session.analyzedPlan(
// query,
// ActionListener.wrap(
// analyzed -> analyzedStringListener.onResponse(analyzed.toString()),
// e -> analyzedStringListener.onResponse(e.toString())
// )
// );
//
// }
public LogicalPlan query() {
return query;
}
@Override
public List<Attribute> output() {
return List.of(
new ReferenceAttribute(Source.EMPTY, "plan", DataType.KEYWORD),
new ReferenceAttribute(Source.EMPTY, "type", DataType.KEYWORD)
);
return output;
}
@Override

73
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
View file

@ -16,6 +16,7 @@ import org.elasticsearch.common.collect.Iterators;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.compute.data.Block;
import org.elasticsearch.compute.data.BlockUtils;
import org.elasticsearch.compute.data.Page;
import org.elasticsearch.compute.operator.DriverCompletionInfo;
import org.elasticsearch.core.Releasables;
@ -46,6 +47,8 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
import org.elasticsearch.xpack.esql.core.expression.UnresolvedStar;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.core.util.Holder;
import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver;
import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy;
@ -66,6 +69,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
import org.elasticsearch.xpack.esql.plan.logical.Drop;
import org.elasticsearch.xpack.esql.plan.logical.Enrich;
import org.elasticsearch.xpack.esql.plan.logical.Eval;
import org.elasticsearch.xpack.esql.plan.logical.Explain;
import org.elasticsearch.xpack.esql.plan.logical.Filter;
import org.elasticsearch.xpack.esql.plan.logical.Fork;
import org.elasticsearch.xpack.esql.plan.logical.InlineStats;
@ -89,7 +93,9 @@ import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation;
import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier;
import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize;
import org.elasticsearch.xpack.esql.plan.physical.FragmentExec;
import org.elasticsearch.xpack.esql.plan.physical.LocalSourceExec;
import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan;
import org.elasticsearch.xpack.esql.planner.PlannerUtils;
import org.elasticsearch.xpack.esql.planner.mapper.Mapper;
import org.elasticsearch.xpack.esql.planner.premapper.PreMapper;
import org.elasticsearch.xpack.esql.plugin.TransportActionServices;
@ -106,6 +112,7 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD;
public class EsqlSession {
@ -138,6 +145,10 @@ public class EsqlSession {
private Set<String> configuredClusters;
private final InferenceRunner inferenceRunner;
private boolean explainMode;
private String parsedPlanString;
private String optimizedLogicalPlanString;
public EsqlSession(
String sessionId,
Configuration configuration,
@ -178,22 +189,39 @@ public class EsqlSession {
public void execute(EsqlQueryRequest request, EsqlExecutionInfo executionInfo, PlanRunner planRunner, ActionListener<Result> listener) {
assert executionInfo != null : "Null EsqlExecutionInfo";
LOGGER.debug("ESQL query:\n{}", request.query());
analyzedPlan(
parse(request.query(), request.params()),
executionInfo,
request.filter(),
new EsqlCCSUtils.CssPartialErrorsActionListener(executionInfo, listener) {
@Override
public void onResponse(LogicalPlan analyzedPlan) {
preMapper.preMapper(
analyzedPlan,
listener.delegateFailureAndWrap(
(l, p) -> executeOptimizedPlan(request, executionInfo, planRunner, optimizedPlan(p), l)
)
);
}
LogicalPlan parsed = parse(request.query(), request.params());
Explain explain = findExplain(parsed);
if (explain != null) {
explainMode = true;
if (explain == parsed) {
parsed = explain.query();
parsedPlanString = parsed.toString();
} else {
throw new VerificationException("EXPLAIN does not support downstream commands");
}
);
}
analyzedPlan(parsed, executionInfo, request.filter(), new EsqlCCSUtils.CssPartialErrorsActionListener(executionInfo, listener) {
@Override
public void onResponse(LogicalPlan analyzedPlan) {
preMapper.preMapper(
analyzedPlan,
listener.delegateFailureAndWrap((l, p) -> executeOptimizedPlan(request, executionInfo, planRunner, optimizedPlan(p), l))
);
}
});
}
private Explain findExplain(LogicalPlan parsed) {
if (parsed instanceof Explain e) {
return e;
}
for (LogicalPlan child : parsed.children()) {
Explain result = findExplain(child);
if (result != null) {
return result;
}
}
return null;
}
/**
@ -208,6 +236,20 @@ public class EsqlSession {
ActionListener<Result> listener
) {
PhysicalPlan physicalPlan = logicalPlanToPhysicalPlan(optimizedPlan, request);
if (explainMode) {
String physicalPlanString = physicalPlan.toString();
List<Attribute> fields = List.of(
new ReferenceAttribute(EMPTY, "role", DataType.KEYWORD),
new ReferenceAttribute(EMPTY, "type", DataType.KEYWORD),
new ReferenceAttribute(EMPTY, "plan", DataType.KEYWORD)
);
List<List<Object>> values = new ArrayList<>();
values.add(List.of("coordinator", "parsedPlan", parsedPlanString));
values.add(List.of("coordinator", "optimizedLogicalPlan", optimizedLogicalPlanString));
values.add(List.of("coordinator", "optimizedPhysicalPlan", physicalPlanString));
var blocks = BlockUtils.fromList(PlannerUtils.NON_BREAKING_BLOCK_FACTORY, values);
physicalPlan = new LocalSourceExec(Source.EMPTY, fields, LocalSupplier.of(blocks));
}
// TODO: this could be snuck into the underlying listener
EsqlCCSUtils.updateExecutionInfoAtEndOfPlanning(executionInfo);
// execute any potential subplans
@ -792,6 +834,7 @@ public class EsqlSession {
if (optimizedPlan.optimized() == false) {
throw new IllegalStateException("Expected optimized plan");
}
optimizedLogicalPlanString = optimizedPlan.toString();
var plan = mapper.map(optimizedPlan);
LOGGER.debug("Physical plan:\n{}", plan);
return plan;

2
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java
View file

@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Drop;
import org.elasticsearch.xpack.esql.plan.logical.Enrich;
import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
import org.elasticsearch.xpack.esql.plan.logical.Eval;
import org.elasticsearch.xpack.esql.plan.logical.Explain;
import org.elasticsearch.xpack.esql.plan.logical.Filter;
import org.elasticsearch.xpack.esql.plan.logical.Fork;
import org.elasticsearch.xpack.esql.plan.logical.Grok;
@ -53,6 +54,7 @@ public enum FeatureMetric {
STATS(Aggregate.class::isInstance),
WHERE(Filter.class::isInstance),
ENRICH(Enrich.class::isInstance),
EXPLAIN(Explain.class::isInstance),
MV_EXPAND(MvExpand.class::isInstance),
SHOW(ShowInfo.class::isInstance),
ROW(Row.class::isInstance),

48
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
View file

@ -961,41 +961,21 @@ public class StatementParserTests extends AbstractStatementParserTests {
}
public void testSubquery() {
assertEquals(new Explain(EMPTY, PROCESSING_CMD_INPUT), statement("explain [ row a = 1 ]"));
assertEquals(new Explain(EMPTY, PROCESSING_CMD_INPUT), statement("explain ( row a = 1 )"));
}
public void testSubqueryWithPipe() {
assertEquals(
new Limit(EMPTY, integer(10), new Explain(EMPTY, PROCESSING_CMD_INPUT)),
statement("explain [ row a = 1 ] | limit 10")
);
}
public void testNestedSubqueries() {
assertEquals(
new Limit(
EMPTY,
integer(10),
new Explain(EMPTY, new Limit(EMPTY, integer(5), new Explain(EMPTY, new Limit(EMPTY, integer(1), PROCESSING_CMD_INPUT))))
),
statement("explain [ explain [ row a = 1 | limit 1 ] | limit 5 ] | limit 10")
);
}
public void testSubquerySpacing() {
assertEquals(statement("explain [ explain [ from a ] | where b == 1 ]"), statement("explain[explain[from a]|where b==1]"));
assertEquals(new Explain(EMPTY, PROCESSING_CMD_INPUT), statement("explain ( row a = 1 )"));
}
public void testBlockComments() {
String query = " explain [ from foo ] | limit 10 ";
String query = " explain ( from foo )";
LogicalPlan expected = statement(query);
int wsIndex = query.indexOf(' ');
do {
String queryWithComment = query.substring(0, wsIndex)
+ "/*explain [ \nfrom bar ] | where a > b*/"
+ query.substring(wsIndex + 1);
String queryWithComment = query.substring(0, wsIndex) + "/*explain ( \nfrom bar ) */" + query.substring(wsIndex + 1);
assertEquals(expected, statement(queryWithComment));
@ -1004,15 +984,13 @@ public class StatementParserTests extends AbstractStatementParserTests {
}
public void testSingleLineComments() {
String query = " explain [ from foo ] | limit 10 ";
String query = " explain ( from foo ) ";
LogicalPlan expected = statement(query);
int wsIndex = query.indexOf(' ');
do {
String queryWithComment = query.substring(0, wsIndex)
+ "//explain [ from bar ] | where a > b \n"
+ query.substring(wsIndex + 1);
String queryWithComment = query.substring(0, wsIndex) + "//explain ( from bar ) \n" + query.substring(wsIndex + 1);
assertEquals(expected, statement(queryWithComment));
@ -1039,13 +1017,12 @@ public class StatementParserTests extends AbstractStatementParserTests {
Tuple.tuple("a+b = c", "a+b"),
Tuple.tuple("a//hi", "a"),
Tuple.tuple("a/*hi*/", "a"),
Tuple.tuple("explain [ frm a ]", "frm")
Tuple.tuple("explain ( frm a )", "frm")
)) {
expectThrows(
ParsingException.class,
allOf(
containsString("mismatched input '" + queryWithUnexpectedCmd.v2() + "'"),
containsString("'explain'"),
containsString("'from'"),
containsString("'row'")
),
@ -1058,13 +1035,12 @@ public class StatementParserTests extends AbstractStatementParserTests {
public void testSuggestAvailableProcessingCommandsOnParsingError() {
for (Tuple<String, String> queryWithUnexpectedCmd : List.of(
Tuple.tuple("from a | filter b > 1", "filter"),
Tuple.tuple("from a | explain [ row 1 ]", "explain"),
Tuple.tuple("from a | explain ( row 1 )", "explain"),
Tuple.tuple("from a | not-a-thing", "not-a-thing"),
Tuple.tuple("from a | high5 a", "high5"),
Tuple.tuple("from a | a+b = c", "a+b"),
Tuple.tuple("from a | a//hi", "a"),
Tuple.tuple("from a | a/*hi*/", "a"),
Tuple.tuple("explain [ from a | evl b = c ]", "evl")
Tuple.tuple("from a | a/*hi*/", "a")
)) {
expectThrows(
ParsingException.class,
@ -1108,7 +1084,7 @@ public class StatementParserTests extends AbstractStatementParserTests {
public void testMetadataFieldOnOtherSources() {
expectError("row a = 1 metadata _index", "line 1:20: extraneous input '_index' expecting <EOF>");
expectError("show info metadata _index", "line 1:11: token recognition error at: 'm'");
expectError("explain [from foo] metadata _index", "line 1:20: mismatched input 'metadata' expecting {'|', ',', ']', 'metadata'}");
expectError("explain ( from foo ) metadata _index", "line 1:22: mismatched input 'metadata' expecting {'|', ',', ')', 'metadata'}");
}
public void testMetadataFieldMultipleDeclarations() {
@ -3476,9 +3452,9 @@ public class StatementParserTests extends AbstractStatementParserTests {
expectError("row a = 1 | where a not in [1", "line 1:28: missing '(' at '['");
expectError("row a = 1 | where a not in 123", "line 1:28: missing '(' at '123'");
// test for [
expectError("explain", "line 1:8: mismatched input '<EOF>' expecting '['");
expectError("explain", "line 1:8: mismatched input '<EOF>' expecting '('");
expectError("explain ]", "line 1:9: token recognition error at: ']'");
expectError("explain [row x = 1", "line 1:19: missing ']' at '<EOF>'");
expectError("explain ( row x = 1", "line 1:20: missing ')' at '<EOF>'");
}
public void testRerankDefaultInferenceIdAndScoreAttribute() {

101
x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/220_explain.yml Normal file
View file

@ -0,0 +1,101 @@
---
setup:
- requires:
test_runner_features: [capabilities, contains, allowed_warnings_regex]
capabilities:
- method: POST
path: /_query
parameters: []
capabilities: [explain]
reason: "new EXPLAIN command"
- do:
indices.create:
index: test
body:
mappings:
properties:
color:
type: text
fields:
keyword:
type: keyword
description:
type: text
fields:
keyword:
type: keyword
- do:
bulk:
index: "test"
refresh: true
body:
- { "index": { } }
- { "color": "red", "description": "The color Red" }
- { "index": { } }
- { "color": "blue", "description": "The color Blue" }
- { "index": { } }
- { "color": "green", "description": "The color Green" }
---
explainRow:
- do:
allowed_warnings_regex:
- "No limit defined, adding default limit of \\[.*\\]"
esql.query:
body:
query: 'EXPLAIN (row a = 1)'
- length: { columns: 3 }
- match: {columns.0.name: "role"}
- match: {columns.0.type: "keyword"}
- match: {columns.1.name: "type"}
- match: {columns.1.type: "keyword"}
- match: {columns.2.name: "plan"}
- match: {columns.2.type: "keyword"}
- length: { values: 3 }
- match: { values.0.0: "coordinator" }
- match: { values.0.1: "parsedPlan" }
- match: { values.1.0: "coordinator" }
- match: { values.1.1: "optimizedLogicalPlan" }
- match: { values.2.0: "coordinator" }
- match: { values.2.1: "optimizedPhysicalPlan" }
---
explainQuery:
- do:
allowed_warnings_regex:
- "No limit defined, adding default limit of \\[.*\\]"
esql.query:
body:
query: 'EXPLAIN (from test | where color == "red" | eval b = 20)'
- length: { columns: 3 }
- match: {columns.0.name: "role"}
- match: {columns.0.type: "keyword"}
- match: {columns.1.name: "type"}
- match: {columns.1.type: "keyword"}
- match: {columns.2.name: "plan"}
- match: {columns.2.type: "keyword"}
- length: { values: 3 }
- match: { values.0.0: "coordinator" }
- match: { values.0.1: "parsedPlan" }
- match: { values.1.0: "coordinator" }
- match: { values.1.1: "optimizedLogicalPlan" }
- match: { values.2.0: "coordinator" }
- match: { values.2.1: "optimizedPhysicalPlan" }
---
explainDownstream:
- do:
allowed_warnings_regex:
- "No limit defined, adding default limit of \\[.*\\]"
esql.query:
body:
query: 'EXPLAIN (row a = 1) | eval b = 2'
catch: "bad_request"
- match: { error.type: "verification_exception" }
- contains: { error.reason: "EXPLAIN does not support downstream commands" }