ESQL - match operator included in non-snapshot builds (#116819)

This commit is contained in:
Carlos Delgado 2024-11-21 07:45:22 +01:00 committed by GitHub
parent adcc5bed1e
commit ea4b41fca8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 1963 additions and 1837 deletions

View file

@ -0,0 +1,5 @@
pr: 116819
summary: ESQL - Add match operator (:)
area: Search
type: feature
issues: []

View file

@ -0,0 +1,49 @@
{
"comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.",
"type" : "operator",
"name" : "match_operator",
"description" : "Performs a match query on the specified field. Returns true if the provided query matches the row.",
"signatures" : [
{
"params" : [
{
"name" : "field",
"type" : "keyword",
"optional" : false,
"description" : "Field that the query will target."
},
{
"name" : "query",
"type" : "keyword",
"optional" : false,
"description" : "Text you wish to find in the provided field."
}
],
"variadic" : false,
"returnType" : "boolean"
},
{
"params" : [
{
"name" : "field",
"type" : "text",
"optional" : false,
"description" : "Field that the query will target."
},
{
"name" : "query",
"type" : "text",
"optional" : false,
"description" : "Text you wish to find in the provided field."
}
],
"variadic" : false,
"returnType" : "boolean"
}
],
"examples" : [
"FROM books \n| WHERE MATCH(author, \"Faulkner\")\n| KEEP book_no, author \n| SORT book_no \n| LIMIT 5;"
],
"preview" : true,
"snapshot_only" : false
}

View file

@ -0,0 +1,14 @@
<!--
This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
-->
### MATCH_OPERATOR
Performs a match query on the specified field. Returns true if the provided query matches the row.
```
FROM books
| WHERE MATCH(author, "Faulkner")
| KEEP book_no, author
| SORT book_no
| LIMIT 5;
```

View file

@ -16,6 +16,7 @@ Boolean operators for comparing against one or multiple expressions.
* <<esql-in-operator>>
* <<esql-like-operator>>
* <<esql-rlike-operator>>
* experimental:[] <<esql-search-operators>>
// end::op_list[]
include::binary.asciidoc[]
@ -26,3 +27,4 @@ include::cast.asciidoc[]
include::in.asciidoc[]
include::like.asciidoc[]
include::rlike.asciidoc[]
include::search.asciidoc[]

View file

@ -0,0 +1,23 @@
[discrete]
[[esql-search-operators]]
=== Search operators
The only search operator is match (`:`).
preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."]
The match operator performs a <<query-dsl-match-query,match query>> on the specified field. Returns true if the provided query matches the row.
[.text-center]
image::esql/functions/signature/match_operator.svg[Embedded,opts=inline]
include::types/match.asciidoc[]
[source.merge.styled,esql]
----
include::{esql-specs}/match-operator.csv-spec[tag=match-with-field]
----
[%header.monospaced.styled,format=dsv,separator=|]
|===
include::{esql-specs}/match-operator.csv-spec[tag=match-with-field-result]
|===

View file

@ -0,0 +1 @@
<svg version="1.1" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg" width="222" height="46" viewbox="0 0 222 46"><defs><style type="text/css">#guide .c{fill:none;stroke:#222222;}#guide .j{fill:#000000;font-family:Roboto Mono,Sans-serif;font-size:20px;}#guide .l{fill:#e4f4ff;stroke:#222222;}#guide .syn{fill:#8D8D8D;font-family:Roboto Mono,Sans-serif;font-size:20px;}</style></defs><path class="c" d="M0 31h5m80 0h10m32 0h10m80 0h5"/><rect class="l" x="5" y="5" width="80" height="36" rx="7"/><text class="j" x="15" y="31">field</text><rect class="l" x="95" y="5" width="32" height="36" rx="7"/><text class="syn" x="105" y="31">:</text><rect class="l" x="137" y="5" width="80" height="36" rx="7"/><text class="j" x="147" y="31">query</text></svg>

After

Width:  |  Height:  |  Size: 775 B

View file

@ -0,0 +1,10 @@
// This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
*Supported types*
[%header.monospaced.styled,format=dsv,separator=|]
|===
field | query | result
keyword | keyword | boolean
text | text | boolean
|===

View file

@ -14,9 +14,6 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.junit.annotations.TestLogging;
import org.elasticsearch.xpack.esql.VerificationException;
import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
import org.elasticsearch.xpack.esql.action.EsqlQueryRequest;
import org.elasticsearch.xpack.esql.action.EsqlQueryResponse;
import org.junit.Before;
import java.util.List;
@ -32,12 +29,6 @@ public class MatchOperatorIT extends AbstractEsqlIntegTestCase {
createAndPopulateIndex();
}
@Override
protected EsqlQueryResponse run(EsqlQueryRequest request) {
assumeTrue("match operator capability not available", EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.isEnabled());
return super.run(request);
}
public void testSimpleWhereMatch() {
var query = """
FROM test

View file

@ -112,8 +112,6 @@ WS
: [ \r\n\t]+ -> channel(HIDDEN)
;
COLON : ':';
//
// Expression - used by most command
//
@ -184,6 +182,7 @@ AND : 'and';
ASC : 'asc';
ASSIGN : '=';
CAST_OP : '::';
COLON : ':';
COMMA : ',';
DESC : 'desc';
DOT : '.';
@ -216,7 +215,6 @@ MINUS : '-';
ASTERISK : '*';
SLASH : '/';
PERCENT : '%';
EXPRESSION_COLON : {this.isDevVersion()}? COLON -> type(COLON);
NESTED_WHERE : WHERE -> type(WHERE);

View file

@ -26,16 +26,16 @@ UNKNOWN_CMD=25
LINE_COMMENT=26
MULTILINE_COMMENT=27
WS=28
COLON=29
PIPE=30
QUOTED_STRING=31
INTEGER_LITERAL=32
DECIMAL_LITERAL=33
BY=34
AND=35
ASC=36
ASSIGN=37
CAST_OP=38
PIPE=29
QUOTED_STRING=30
INTEGER_LITERAL=31
DECIMAL_LITERAL=32
BY=33
AND=34
ASC=35
ASSIGN=36
CAST_OP=37
COLON=38
COMMA=39
DESC=40
DOT=41
@ -142,13 +142,13 @@ CLOSING_METRICS_WS=128
'sort'=14
'stats'=15
'where'=16
':'=29
'|'=30
'by'=34
'and'=35
'asc'=36
'='=37
'::'=38
'|'=29
'by'=33
'and'=34
'asc'=35
'='=36
'::'=37
':'=38
','=39
'desc'=40
'.'=41

View file

@ -69,7 +69,7 @@ booleanExpression
| left=booleanExpression operator=OR right=booleanExpression #logicalBinary
| valueExpression (NOT)? IN LP valueExpression (COMMA valueExpression)* RP #logicalIn
| valueExpression IS NOT? NULL #isNull
| {this.isDevVersion()}? matchBooleanExpression #matchExpression
| matchBooleanExpression #matchExpression
;
regexBooleanExpression

View file

@ -26,16 +26,16 @@ UNKNOWN_CMD=25
LINE_COMMENT=26
MULTILINE_COMMENT=27
WS=28
COLON=29
PIPE=30
QUOTED_STRING=31
INTEGER_LITERAL=32
DECIMAL_LITERAL=33
BY=34
AND=35
ASC=36
ASSIGN=37
CAST_OP=38
PIPE=29
QUOTED_STRING=30
INTEGER_LITERAL=31
DECIMAL_LITERAL=32
BY=33
AND=34
ASC=35
ASSIGN=36
CAST_OP=37
COLON=38
COMMA=39
DESC=40
DOT=41
@ -142,13 +142,13 @@ CLOSING_METRICS_WS=128
'sort'=14
'stats'=15
'where'=16
':'=29
'|'=30
'by'=34
'and'=35
'asc'=36
'='=37
'::'=38
'|'=29
'by'=33
'and'=34
'asc'=35
'='=36
'::'=37
':'=38
','=39
'desc'=40
'.'=41

View file

@ -307,7 +307,7 @@ public class EsqlCapabilities {
/**
* Support for match operator as a colon. Previous support for match operator as MATCH has been removed
*/
MATCH_OPERATOR_COLON(Build.current().isSnapshot()),
MATCH_OPERATOR_COLON,
/**
* Removing support for the {@code META} keyword.

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -2317,8 +2317,6 @@ public class AnalyzerTests extends ESTestCase {
}
public void testFromEnrichAndMatchColonUsage() {
assumeTrue("Match operator is available just for snapshots", EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.isEnabled());
LogicalPlan plan = analyze("""
from *:test
| EVAL x = to_string(languages)

View file

@ -1159,8 +1159,6 @@ public class VerifierTests extends ESTestCase {
}
public void testMatchFilter() throws Exception {
assumeTrue("Match operator is available just for snapshots", EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.isEnabled());
assertEquals(
"1:19: first argument of [salary:\"100\"] must be [string], found value [salary] type [integer]",
error("from test | where salary:\"100\"")
@ -1190,7 +1188,6 @@ public class VerifierTests extends ESTestCase {
}
public void testMatchFunctionAndOperatorHaveCorrectErrorMessages() throws Exception {
assumeTrue("skipping because MATCH operator is not enabled", EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.isEnabled());
assertEquals(
"1:24: [MATCH] function cannot be used after LIMIT",
error("from test | limit 10 | where match(first_name, \"Anna\")")
@ -1271,7 +1268,6 @@ public class VerifierTests extends ESTestCase {
}
public void testMatchOperatornOnlyAllowedInWhere() throws Exception {
assumeTrue("skipping because MATCH operator is not enabled", EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.isEnabled());
checkFullTextFunctionsOnlyAllowedInWhere(":", "first_name:\"Anna\"", "operator");
}
@ -1317,8 +1313,6 @@ public class VerifierTests extends ESTestCase {
}
public void testMatchOperatorWithDisjunctions() {
assumeTrue("skipping because MATCH operator is not enabled", EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.isEnabled());
checkWithDisjunctions(":", "first_name : \"Anna\"", "operator");
}
@ -1374,7 +1368,6 @@ public class VerifierTests extends ESTestCase {
}
public void testMatchOperatorWithNonBooleanFunctions() {
assumeTrue("skipping because MATCH operator is not enabled", EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.isEnabled());
checkFullTextFunctionsWithNonBooleanFunctions(":", "first_name:\"Anna\"", "operator");
}
@ -1452,8 +1445,6 @@ public class VerifierTests extends ESTestCase {
"1:68: Unknown column [first_name]",
error("from test | stats max_salary = max(salary) by emp_no | where match(first_name, \"Anna\")")
);
assumeTrue("skipping because MATCH operator is not enabled", EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.isEnabled());
assertEquals(
"1:62: Unknown column [first_name]",
error("from test | stats max_salary = max(salary) by emp_no | where first_name : \"Anna\"")
@ -1473,8 +1464,6 @@ public class VerifierTests extends ESTestCase {
public void testMatchTargetsExistingField() throws Exception {
assertEquals("1:39: Unknown column [first_name]", error("from test | keep emp_no | where match(first_name, \"Anna\")"));
assumeTrue("skipping because MATCH operator is not enabled", EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.isEnabled());
assertEquals("1:33: Unknown column [first_name]", error("from test | keep emp_no | where first_name : \"Anna\""));
}

View file

@ -44,6 +44,7 @@ import org.elasticsearch.xpack.esql.core.type.EsField;
import org.elasticsearch.xpack.esql.core.util.NumericUtils;
import org.elasticsearch.xpack.esql.core.util.StringUtils;
import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
import org.elasticsearch.xpack.esql.expression.function.fulltext.Match;
import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest;
import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
@ -130,7 +131,9 @@ public abstract class AbstractFunctionTestCase extends ESTestCase {
entry("mod", Mod.class),
entry("neg", Neg.class),
entry("is_null", IsNull.class),
entry("is_not_null", IsNotNull.class)
entry("is_not_null", IsNotNull.class),
// Match operator is both a function and an operator
entry("match_operator", Match.class)
);
private static EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry().snapshotRegistry();
@ -813,6 +816,10 @@ public abstract class AbstractFunctionTestCase extends ESTestCase {
if (unaryOperator != null) {
return RailRoadDiagram.unaryOperator(unaryOperator);
}
String searchOperator = searchOperator(name);
if (searchOperator != null) {
return RailRoadDiagram.searchOperator(searchOperator);
}
FunctionDefinition definition = definition(name);
if (definition != null) {
return RailRoadDiagram.functionSignature(definition);
@ -862,7 +869,7 @@ public abstract class AbstractFunctionTestCase extends ESTestCase {
return;
}
String name = functionName();
if (binaryOperator(name) != null || unaryOperator(name) != null || likeOrInOperator(name)) {
if (binaryOperator(name) != null || unaryOperator(name) != null || searchOperator(name) != null || likeOrInOperator(name)) {
renderDocsForOperators(name);
return;
}
@ -1258,6 +1265,16 @@ public abstract class AbstractFunctionTestCase extends ESTestCase {
};
}
/**
* If this test is a for a search operator return its symbol, otherwise return {@code null}.
*/
private static String searchOperator(String name) {
return switch (name) {
case "match_operator" -> ":";
default -> null;
};
}
/**
* If this tests is for a unary operator return its symbol, otherwise return {@code null}.
* This is functionally the reverse of {@link ExpressionBuilder#visitArithmeticUnary}.

View file

@ -89,6 +89,18 @@ public class RailRoadDiagram {
return toSvg(new Sequence(expressions.toArray(Expression[]::new)));
}
/**
* Generate a railroad diagram for a search operator. The output would look like
* {@code field : value}.
*/
static String searchOperator(String operator) throws IOException {
List<Expression> expressions = new ArrayList<>();
expressions.add(new Literal("field"));
expressions.add(new Syntax(operator));
expressions.add(new Literal("query"));
return toSvg(new Sequence(expressions.toArray(Expression[]::new)));
}
/**
* Generate a railroad diagram for unary operator. The output would look like
* {@code -v}.

View file

@ -0,0 +1,41 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.expression.function.fulltext;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.FunctionName;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import java.util.LinkedList;
import java.util.List;
import java.util.function.Supplier;
/**
* This class is only used to generates docs for the match operator - all testing is done in {@link MatchTests}
*/
@FunctionName("match_operator")
public class MatchOperatorTests extends MatchTests {
public MatchOperatorTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testCaseSupplier) {
super(testCaseSupplier);
}
@ParametersFactory
public static Iterable<Object[]> parameters() {
// Have a minimal test so that we can generate the appropriate types in the docs
List<TestCaseSupplier> suppliers = new LinkedList<>();
addPositiveTestCase(List.of(DataType.KEYWORD, DataType.KEYWORD), suppliers);
addPositiveTestCase(List.of(DataType.TEXT, DataType.TEXT), suppliers);
addPositiveTestCase(List.of(DataType.KEYWORD, DataType.TEXT), suppliers);
addPositiveTestCase(List.of(DataType.TEXT, DataType.KEYWORD), suppliers);
return parameterSuppliersFromTypedData(suppliers);
}
}

View file

@ -36,19 +36,11 @@ public class MatchTests extends AbstractFunctionTestCase {
@ParametersFactory
public static Iterable<Object[]> parameters() {
Set<DataType> supportedTextParams = Set.of(DataType.KEYWORD, DataType.TEXT);
Set<DataType> supportedNumericParams = Set.of(DataType.DOUBLE, DataType.INTEGER);
Set<DataType> supportedFuzzinessParams = Set.of(DataType.INTEGER, DataType.KEYWORD, DataType.TEXT);
List<Set<DataType>> supportedPerPosition = List.of(
supportedTextParams,
supportedTextParams,
supportedNumericParams,
supportedFuzzinessParams
);
List<Set<DataType>> supportedPerPosition = supportedParams();
List<TestCaseSupplier> suppliers = new LinkedList<>();
for (DataType fieldType : DataType.stringTypes()) {
for (DataType queryType : DataType.stringTypes()) {
addPositiveTestCase(List.of(fieldType, queryType), supportedPerPosition, suppliers);
addPositiveTestCase(List.of(fieldType, queryType), suppliers);
addNonFieldTestCase(List.of(fieldType, queryType), supportedPerPosition, suppliers);
}
}
@ -61,11 +53,20 @@ public class MatchTests extends AbstractFunctionTestCase {
);
}
private static void addPositiveTestCase(
List<DataType> paramDataTypes,
List<Set<DataType>> supportedPerPosition,
List<TestCaseSupplier> suppliers
) {
protected static List<Set<DataType>> supportedParams() {
Set<DataType> supportedTextParams = Set.of(DataType.KEYWORD, DataType.TEXT);
Set<DataType> supportedNumericParams = Set.of(DataType.DOUBLE, DataType.INTEGER);
Set<DataType> supportedFuzzinessParams = Set.of(DataType.INTEGER, DataType.KEYWORD, DataType.TEXT);
List<Set<DataType>> supportedPerPosition = List.of(
supportedTextParams,
supportedTextParams,
supportedNumericParams,
supportedFuzzinessParams
);
return supportedPerPosition;
}
protected static void addPositiveTestCase(List<DataType> paramDataTypes, List<TestCaseSupplier> suppliers) {
// Positive case - creates an ES field from the field parameter type
suppliers.add(

View file

@ -26,7 +26,6 @@ import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.xpack.core.enrich.EnrichPolicy;
import org.elasticsearch.xpack.esql.EsqlTestUtils;
import org.elasticsearch.xpack.esql.EsqlTestUtils.TestSearchStats;
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
import org.elasticsearch.xpack.esql.analysis.Analyzer;
import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
@ -1093,8 +1092,6 @@ public class LocalPhysicalPlanOptimizerTests extends MapperServiceTestCase {
* estimatedRowSize[324]
*/
public void testSingleMatchFilterPushdown() {
assumeTrue("skipping because MATCH operator is not enabled", EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.isEnabled());
var plan = plannerOptimizer.plan("""
from test
| where first_name:"Anna"
@ -1125,8 +1122,6 @@ public class LocalPhysicalPlanOptimizerTests extends MapperServiceTestCase {
* [_doc{f}#22], limit[1000], sort[[FieldSort[field=emp_no{f}#12, direction=ASC, nulls=LAST]]] estimatedRowSize[336]
*/
public void testMultipleMatchFilterPushdown() {
assumeTrue("skipping because MATCH operator is not enabled", EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.isEnabled());
String query = """
from test
| where first_name:"Anna" and first_name:"Anneke"

View file

@ -2300,7 +2300,6 @@ public class StatementParserTests extends AbstractStatementParserTests {
}
public void testMatchOperatorConstantQueryString() {
assumeTrue("skipping because MATCH operator is not enabled", EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.isEnabled());
var plan = statement("FROM test | WHERE field:\"value\"");
var filter = as(plan, Filter.class);
var match = (Match) filter.condition();
@ -2310,7 +2309,6 @@ public class StatementParserTests extends AbstractStatementParserTests {
}
public void testInvalidMatchOperator() {
assumeTrue("skipping because MATCH operator is not enabled", EsqlCapabilities.Cap.MATCH_OPERATOR_COLON.isEnabled());
expectError("from test | WHERE field:", "line 1:25: mismatched input '<EOF>' expecting {QUOTED_STRING, ");
expectError(
"from test | WHERE field:CONCAT(\"hello\", \"world\")",