github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-06-28 01:22:26 -04:00
Code Issues Projects Releases Packages Wiki Activity

ESQL: Pushdown constructs doing case-insensitive regexes (#128393)

Browse source 
This introduces an optimization to pushdown to Lucense those language constructs that aim at case-insensitive regular expression matching, used with `LIKE` and `RLIKE` operators, such as:
* `| WHERE TO_LOWER(field) LIKE "abc*"`
* `| WHERE TO_UPPER(field) RLIKE "ABC.*"` 
 
These are now pushed as case-insensitive `wildcard` and `regexp` respectively queries down to Lucene.

Closes #127479
This commit is contained in:
Bogdan Pintea 2025-05-30 10:55:00 +02:00 • committed by GitHub
parent cc461afa0a
commit 0a8091605b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
33 changed files with 756 additions and 236 deletions
Download patch file Download diff file Expand all files Collapse all files

2
benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
View file

@ -48,9 +48,9 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs;
import org.elasticsearch.xpack.esql.expression.function.scalar.math.RoundTo;
import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;

6
docs/changelog/128393.yaml Normal file
View file

@ -0,0 +1,6 @@
pr: 128393
summary: Pushdown constructs doing case-insensitive regexes
area: ES|QL
type: enhancement
issues:
- 127479

1
server/src/main/java/org/elasticsearch/TransportVersions.java
View file

@ -272,6 +272,7 @@ public class TransportVersions {
public static final TransportVersion ML_INFERENCE_VERTEXAI_CHATCOMPLETION_ADDED = def(9_083_0_00);
public static final TransportVersion INFERENCE_CUSTOM_SERVICE_ADDED = def(9_084_0_00);
public static final TransportVersion ESQL_LIMIT_ROW_SIZE = def(9_085_0_00);
public static final TransportVersion ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY = def(9_086_0_00);
/*
* STOP! READ THIS FIRST! No, really,

4
x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/AbstractStringPattern.java
View file

@ -16,11 +16,11 @@ public abstract class AbstractStringPattern implements StringPattern {
private Automaton automaton;
public abstract Automaton createAutomaton();
public abstract Automaton createAutomaton(boolean ignoreCase);
private Automaton automaton() {
if (automaton == null) {
automaton = createAutomaton();
automaton = createAutomaton(false);
}
return automaton;
}

35
x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLike.java
View file

@ -1,35 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.core.expression.predicate.regex;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.tree.Source;
import java.io.IOException;
public abstract class RLike extends RegexMatch<RLikePattern> {
public RLike(Source source, Expression value, RLikePattern pattern) {
super(source, value, pattern, false);
}
public RLike(Source source, Expression field, RLikePattern rLikePattern, boolean caseInsensitive) {
super(source, field, rLikePattern, caseInsensitive);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public String getWriteableName() {
throw new UnsupportedOperationException();
}
}

5
x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RLikePattern.java
View file

@ -21,9 +21,10 @@ public class RLikePattern extends AbstractStringPattern {
}
@Override
public Automaton createAutomaton() {
public Automaton createAutomaton(boolean ignoreCase) {
int matchFlags = ignoreCase ? RegExp.CASE_INSENSITIVE : 0;
return Operations.determinize(
new RegExp(regexpPattern, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT).toAutomaton(),
new RegExp(regexpPattern, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT, matchFlags).toAutomaton(),
Operations.DEFAULT_DETERMINIZE_WORK_LIMIT
);
}

35
x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardLike.java
View file

@ -1,35 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.core.expression.predicate.regex;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.tree.Source;
import java.io.IOException;
public abstract class WildcardLike extends RegexMatch<WildcardPattern> {
public WildcardLike(Source source, Expression left, WildcardPattern pattern) {
this(source, left, pattern, false);
}
public WildcardLike(Source source, Expression left, WildcardPattern pattern, boolean caseInsensitive) {
super(source, left, pattern, caseInsensitive);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public String getWriteableName() {
throw new UnsupportedOperationException();
}
}

13
x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/WildcardPattern.java
View file

@ -10,10 +10,13 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.RegExp;
import org.elasticsearch.xpack.esql.core.util.StringUtils;
import java.util.Objects;
import static org.elasticsearch.xpack.esql.core.util.StringUtils.luceneWildcardToRegExp;
/**
* Similar to basic regex, supporting '?' wildcard for single character (same as regex ".")
* and '*' wildcard for multiple characters (same as regex ".*")
@ -37,8 +40,14 @@ public class WildcardPattern extends AbstractStringPattern {
}
@Override
public Automaton createAutomaton() {
return WildcardQuery.toAutomaton(new Term(null, wildcard), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
public Automaton createAutomaton(boolean ignoreCase) {
return ignoreCase
? Operations.determinize(
new RegExp(luceneWildcardToRegExp(wildcard), RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT, RegExp.CASE_INSENSITIVE)
.toAutomaton(),
Operations.DEFAULT_DETERMINIZE_WORK_LIMIT
)
: WildcardQuery.toAutomaton(new Term(null, wildcard), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
}
@Override

39
x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/StringUtils.java
View file

@ -7,6 +7,7 @@
package org.elasticsearch.xpack.esql.core.util;
import org.apache.lucene.document.InetAddressPoint;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.spell.LevenshteinDistance;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CollectionUtil;
@ -178,6 +179,44 @@ public final class StringUtils {
return regex.toString();
}
/**
* Translates a Lucene wildcard pattern to a Lucene RegExp one.
* @param wildcard Lucene wildcard pattern
* @return Lucene RegExp pattern
*/
public static String luceneWildcardToRegExp(String wildcard) {
StringBuilder regex = new StringBuilder();
for (int i = 0, wcLen = wildcard.length(); i < wcLen; i++) {
char c = wildcard.charAt(i); // this will work chunking through Unicode as long as all values matched are ASCII
switch (c) {
case WildcardQuery.WILDCARD_STRING -> regex.append(".*");
case WildcardQuery.WILDCARD_CHAR -> regex.append(".");
case WildcardQuery.WILDCARD_ESCAPE -> {
if (i + 1 < wcLen) {
// consume the wildcard escaping, consider the next char
char next = wildcard.charAt(i + 1);
i++;
switch (next) {
case WildcardQuery.WILDCARD_STRING, WildcardQuery.WILDCARD_CHAR, WildcardQuery.WILDCARD_ESCAPE ->
// escape `*`, `.`, `\`, since these are special chars in RegExp as well
regex.append("\\");
// default: unnecessary escaping -- just ignore the escaping
}
regex.append(next);
} else {
// "else fallthru, lenient parsing with a trailing \" -- according to WildcardQuery#toAutomaton
regex.append("\\\\");
}
}
case '$', '(', ')', '+', '.', '[', ']', '^', '{', '|', '}' -> regex.append("\\").append(c);
default -> regex.append(c);
}
}
return regex.toString();
}
/**
* Translates a like pattern to a Lucene wildcard.
* This methods pays attention to the custom escape char which gets converted into \ (used by Lucene).

19
x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/StringUtilsTests.java
View file

@ -9,7 +9,9 @@ package org.elasticsearch.xpack.esql.core.util;
import org.elasticsearch.test.ESTestCase;
import static org.elasticsearch.xpack.esql.core.util.StringUtils.luceneWildcardToRegExp;
import static org.elasticsearch.xpack.esql.core.util.StringUtils.wildcardToJavaPattern;
import static org.hamcrest.Matchers.is;
public class StringUtilsTests extends ESTestCase {
@ -55,4 +57,21 @@ public class StringUtilsTests extends ESTestCase {
public void testEscapedEscape() {
assertEquals("^\\\\\\\\$", wildcardToJavaPattern("\\\\\\\\", '\\'));
}
public void testLuceneWildcardToRegExp() {
assertThat(luceneWildcardToRegExp(""), is(""));
assertThat(luceneWildcardToRegExp("*"), is(".*"));
assertThat(luceneWildcardToRegExp("?"), is("."));
assertThat(luceneWildcardToRegExp("\\\\"), is("\\\\"));
assertThat(luceneWildcardToRegExp("foo?bar"), is("foo.bar"));
assertThat(luceneWildcardToRegExp("foo*bar"), is("foo.*bar"));
assertThat(luceneWildcardToRegExp("foo\\\\bar"), is("foo\\\\bar"));
assertThat(luceneWildcardToRegExp("foo*bar?baz"), is("foo.*bar.baz"));
assertThat(luceneWildcardToRegExp("foo\\*bar"), is("foo\\*bar"));
assertThat(luceneWildcardToRegExp("foo\\?bar\\?"), is("foo\\?bar\\?"));
assertThat(luceneWildcardToRegExp("foo\\?bar\\"), is("foo\\?bar\\\\"));
assertThat(luceneWildcardToRegExp("[](){}^$.|+"), is("\\[\\]\\(\\)\\{\\}\\^\\$\\.\\|\\+"));
assertThat(luceneWildcardToRegExp("foo\\\uD83D\uDC14bar"), is("foo\uD83D\uDC14bar"));
assertThat(luceneWildcardToRegExp("foo\uD83D\uDC14bar"), is("foo\uD83D\uDC14bar"));
}
}

12
x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/util/TestUtils.java
View file

@ -13,6 +13,7 @@ import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.core.type.EsField;
import java.util.Locale;
import java.util.regex.Pattern;
import static java.util.Collections.emptyMap;
@ -61,4 +62,15 @@ public final class TestUtils {
public static String stripThrough(String input) {
return WS_PATTERN.matcher(input).replaceAll(StringUtils.EMPTY);
}
/** Returns the input string, but with parts of it having the letter casing changed. */
public static String randomCasing(String input) {
StringBuilder sb = new StringBuilder(input.length());
for (int i = 0, inputLen = input.length(), step = (int) Math.sqrt(inputLen), chunkEnd; i < inputLen; i += step) {
chunkEnd = Math.min(i + step, inputLen);
var chunk = input.substring(i, chunkEnd);
sb.append(randomBoolean() ? chunk.toLowerCase(Locale.ROOT) : chunk.toUpperCase(Locale.ROOT));
}
return sb.toString();
}
}

4
x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java
View file

@ -63,8 +63,8 @@ import org.elasticsearch.xpack.esql.core.type.EsField;
import org.elasticsearch.xpack.esql.core.util.DateUtils;
import org.elasticsearch.xpack.esql.core.util.StringUtils;
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.Range;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;

104
x-pack/plugin/esql/qa/testFixtures/src/main/resources/where-like.csv-spec
View file

@ -319,3 +319,107 @@ warningRegex:java.lang.IllegalArgumentException: single-value function encounter
emp_no:integer | job_positions:keyword
10025 | Accountant
;
likeWithUpperTurnedInsensitive
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_UPPER(first_name) LIKE "GEOR*"
;
emp_no:integer |first_name:keyword
10001 |Georgi
10055 |Georgy
;
likeWithLowerTurnedInsensitive
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_LOWER(TO_UPPER(first_name)) LIKE "geor*"
;
emp_no:integer |first_name:keyword
10001 |Georgi
10055 |Georgy
;
likeWithLowerConflictingFolded
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_UPPER(first_name) LIKE "geor*"
;
emp_no:integer |first_name:keyword
;
likeWithLowerTurnedInsensitiveNotPushedDown
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_LOWER(first_name) LIKE "geor*" OR emp_no + 1 IN (10002, 10056)
;
emp_no:integer |first_name:keyword
10001 |Georgi
10055 |Georgy
;
rlikeWithUpperTurnedInsensitive
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_UPPER(first_name) RLIKE "GEOR.*"
;
emp_no:integer |first_name:keyword
10001 |Georgi
10055 |Georgy
;
rlikeWithLowerTurnedInsensitive
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_LOWER(TO_UPPER(first_name)) RLIKE "geor.*"
;
emp_no:integer |first_name:keyword
10001 |Georgi
10055 |Georgy
;
rlikeWithLowerConflictingFolded
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_UPPER(first_name) RLIKE "geor.*"
;
emp_no:integer |first_name:keyword
;
negatedRLikeWithLowerTurnedInsensitive
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_LOWER(TO_UPPER(first_name)) NOT RLIKE "geor.*"
| STATS c = COUNT()
;
c:long
88
;
rlikeWithLowerTurnedInsensitiveNotPushedDown
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_LOWER(first_name) RLIKE "geor.*" OR emp_no + 1 IN (10002, 10056)
;
emp_no:integer |first_name:keyword
10001 |Georgi
10055 |Georgy
;

4
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java
View file

@ -68,11 +68,11 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StYMin;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.ByteLength;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.Space;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.Trim;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.util.Delay;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull;

57
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java → x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RLike.java
View file

@ -5,21 +5,17 @@
* 2.0.
*/
package org.elasticsearch.xpack.esql.expression.function.scalar.string;
package org.elasticsearch.xpack.esql.expression.function.scalar.string.regex;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.compute.operator.EvalOperator;
import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
import org.elasticsearch.xpack.esql.core.querydsl.query.RegexQuery;
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
import org.elasticsearch.xpack.esql.expression.function.Example;
import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
import org.elasticsearch.xpack.esql.expression.function.Param;
@ -29,14 +25,9 @@ import org.elasticsearch.xpack.esql.planner.TranslatorHandler;
import java.io.IOException;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
public class RLike extends org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike
implements
EvaluatorMapper,
TranslationAware.SingleValueTranslationAware {
public class RLike extends RegexMatch<RLikePattern> {
public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "RLike", RLike::new);
public static final String NAME = "RLIKE";
@FunctionInfo(returnType = "boolean", description = """
Use `RLIKE` to filter data based on string patterns using using
@ -52,13 +43,13 @@ public class RLike extends org.elasticsearch.xpack.esql.core.expression.predicat
To reduce the overhead of escaping, we suggest using triple quotes strings `\"\"\"`
<<load-esql-example, file=string tag=rlikeEscapingTripleQuotes>>
""", operator = "RLIKE", examples = @Example(file = "docs", tag = "rlike"))
""", operator = NAME, examples = @Example(file = "docs", tag = "rlike"))
public RLike(
Source source,
@Param(name = "str", type = { "keyword", "text" }, description = "A literal value.") Expression value,
@Param(name = "pattern", type = { "keyword", "text" }, description = "A regular expression.") RLikePattern pattern
) {
super(source, value, pattern);
this(source, value, pattern, false);
}
public RLike(Source source, Expression field, RLikePattern rLikePattern, boolean caseInsensitive) {
@ -66,7 +57,12 @@ public class RLike extends org.elasticsearch.xpack.esql.core.expression.predicat
}
private RLike(StreamInput in) throws IOException {
this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), new RLikePattern(in.readString()));
this(
Source.readFrom((PlanStreamInput) in),
in.readNamedWriteable(Expression.class),
new RLikePattern(in.readString()),
deserializeCaseInsensitivity(in)
);
}
@Override
@ -74,6 +70,12 @@ public class RLike extends org.elasticsearch.xpack.esql.core.expression.predicat
source().writeTo(out);
out.writeNamedWriteable(field());
out.writeString(pattern().asJavaRegex());
serializeCaseInsensitivity(out);
}
@Override
public String name() {
return NAME;
}
@Override
@ -91,35 +93,10 @@ public class RLike extends org.elasticsearch.xpack.esql.core.expression.predicat
return new RLike(source(), newChild, pattern(), caseInsensitive());
}
@Override
protected TypeResolution resolveType() {
return isString(field(), sourceText(), DEFAULT);
}
@Override
public Boolean fold(FoldContext ctx) {
return (Boolean) EvaluatorMapper.super.fold(source(), ctx);
}
@Override
public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
return AutomataMatch.toEvaluator(source(), toEvaluator.apply(field()), pattern().createAutomaton());
}
@Override
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableFieldAttribute(field()) ? Translatable.YES : Translatable.NO;
}
@Override
public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
var fa = LucenePushdownPredicates.checkIsFieldAttribute(field());
// TODO: see whether escaping is needed
return new RegexQuery(source(), handler.nameOf(fa.exactAttribute()), pattern().asJavaRegex(), caseInsensitive());
}
@Override
public Expression singleValueField() {
return field();
}
}

93
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/RegexMatch.java Normal file
View file

@ -0,0 +1,93 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.expression.function.scalar.string.regex;
import org.apache.lucene.util.automaton.Automata;
import org.elasticsearch.TransportVersions;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.compute.operator.EvalOperator;
import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.AbstractStringPattern;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.AutomataMatch;
import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates;
import java.io.IOException;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
abstract class RegexMatch<P extends AbstractStringPattern> extends org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch<
P> implements EvaluatorMapper, TranslationAware.SingleValueTranslationAware {
abstract String name();
RegexMatch(Source source, Expression field, P pattern, boolean caseInsensitive) {
super(source, field, pattern, caseInsensitive);
}
@Override
protected TypeResolution resolveType() {
return isString(field(), sourceText(), DEFAULT);
}
@Override
public Boolean fold(FoldContext ctx) {
return (Boolean) EvaluatorMapper.super.fold(source(), ctx);
}
@Override
public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
return AutomataMatch.toEvaluator(
source(),
toEvaluator.apply(field()),
// The empty pattern will accept the empty string
pattern().pattern().isEmpty() ? Automata.makeEmptyString() : pattern().createAutomaton(caseInsensitive())
);
}
@Override
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableFieldAttribute(field()) ? Translatable.YES : Translatable.NO;
}
@Override
public Expression singleValueField() {
return field();
}
@Override
public String nodeString() {
return name() + "(" + field().nodeString() + ", \"" + pattern().pattern() + "\", " + caseInsensitive() + ")";
}
void serializeCaseInsensitivity(StreamOutput out) throws IOException {
if (out.getTransportVersion().before(TransportVersions.ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY)) {
if (caseInsensitive()) {
// The plan has been optimized to run a case-insensitive match, which the remote peer cannot be notified of. Simply avoiding
// the serialization of the boolean would result in wrong results.
throw new EsqlIllegalArgumentException(
name() + " with case insensitivity is not supported in peer node's version [{}]. Upgrade to version [{}] or newer.",
out.getTransportVersion(),
TransportVersions.ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY
);
} // else: write nothing, the remote peer can execute the case-sensitive query
} else {
out.writeBoolean(caseInsensitive());
}
}
static boolean deserializeCaseInsensitivity(StreamInput in) throws IOException {
return in.getTransportVersion().onOrAfter(TransportVersions.ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY) && in.readBoolean();
}
}

68
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java → x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/regex/WildcardLike.java
View file

@ -5,23 +5,18 @@
* 2.0.
*/
package org.elasticsearch.xpack.esql.expression.function.scalar.string;
package org.elasticsearch.xpack.esql.expression.function.scalar.string.regex;
import org.apache.lucene.util.automaton.Automata;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.compute.operator.EvalOperator;
import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
import org.elasticsearch.xpack.esql.core.querydsl.query.WildcardQuery;
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
import org.elasticsearch.xpack.esql.expression.function.Example;
import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
import org.elasticsearch.xpack.esql.expression.function.Param;
@ -31,18 +26,13 @@ import org.elasticsearch.xpack.esql.planner.TranslatorHandler;
import java.io.IOException;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
public class WildcardLike extends org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike
implements
EvaluatorMapper,
TranslationAware.SingleValueTranslationAware {
public class WildcardLike extends RegexMatch<WildcardPattern> {
public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
Expression.class,
"WildcardLike",
WildcardLike::new
);
public static final String NAME = "LIKE";
@FunctionInfo(returnType = "boolean", description = """
Use `LIKE` to filter data based on string patterns using wildcards. `LIKE`
@ -63,17 +53,26 @@ public class WildcardLike extends org.elasticsearch.xpack.esql.core.expression.p
To reduce the overhead of escaping, we suggest using triple quotes strings `\"\"\"`
<<load-esql-example, file=string tag=likeEscapingTripleQuotes>>
""", operator = "LIKE", examples = @Example(file = "docs", tag = "like"))
""", operator = NAME, examples = @Example(file = "docs", tag = "like"))
public WildcardLike(
Source source,
@Param(name = "str", type = { "keyword", "text" }, description = "A literal expression.") Expression left,
@Param(name = "pattern", type = { "keyword", "text" }, description = "Pattern.") WildcardPattern pattern
) {
super(source, left, pattern, false);
this(source, left, pattern, false);
}
public WildcardLike(Source source, Expression left, WildcardPattern pattern, boolean caseInsensitive) {
super(source, left, pattern, caseInsensitive);
}
private WildcardLike(StreamInput in) throws IOException {
this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), new WildcardPattern(in.readString()));
this(
Source.readFrom((PlanStreamInput) in),
in.readNamedWriteable(Expression.class),
new WildcardPattern(in.readString()),
deserializeCaseInsensitivity(in)
);
}
@Override
@ -81,6 +80,12 @@ public class WildcardLike extends org.elasticsearch.xpack.esql.core.expression.p
source().writeTo(out);
out.writeNamedWriteable(field());
out.writeString(pattern().pattern());
serializeCaseInsensitivity(out);
}
@Override
public String name() {
return NAME;
}
@Override
@ -89,33 +94,13 @@ public class WildcardLike extends org.elasticsearch.xpack.esql.core.expression.p
}
@Override
protected NodeInfo<org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike> info() {
return NodeInfo.create(this, WildcardLike::new, field(), pattern());
protected NodeInfo<WildcardLike> info() {
return NodeInfo.create(this, WildcardLike::new, field(), pattern(), caseInsensitive());
}
@Override
protected WildcardLike replaceChild(Expression newLeft) {
return new WildcardLike(source(), newLeft, pattern());
}
@Override
protected TypeResolution resolveType() {
return isString(field(), sourceText(), DEFAULT);
}
@Override
public Boolean fold(FoldContext ctx) {
return (Boolean) EvaluatorMapper.super.fold(source(), ctx);
}
@Override
public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
return AutomataMatch.toEvaluator(
source(),
toEvaluator.apply(field()),
// The empty pattern will accept the empty string
pattern().pattern().length() == 0 ? Automata.makeEmptyString() : pattern().createAutomaton()
);
return new WildcardLike(source(), newLeft, pattern(), caseInsensitive());
}
@Override
@ -134,9 +119,4 @@ public class WildcardLike extends org.elasticsearch.xpack.esql.core.expression.p
private Query translateField(String targetFieldName) {
return new WildcardQuery(source(), targetFieldName, pattern().asLuceneWildcard(), caseInsensitive());
}
@Override
public Expression singleValueField() {
return field();
}
}

38
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java
View file

@ -9,6 +9,7 @@ package org.elasticsearch.xpack.esql.optimizer;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.PropagateEmptyRelation;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStatsFilteredAggWithEval;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStringCasingWithInsensitiveRegexMatch;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferIsNotNull;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferNonNullAggConstraint;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.LocalPropagateEmptyRelation;
@ -33,8 +34,7 @@ import static org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer.operat
*/
public class LocalLogicalPlanOptimizer extends ParameterizedRuleExecutor<LogicalPlan, LocalLogicalOptimizerContext> {
private static final List<Batch<LogicalPlan>> RULES = replaceRules(
arrayAsArrayList(
private static final List<Batch<LogicalPlan>> RULES = arrayAsArrayList(
new Batch<>(
"Local rewrite",
Limiter.ONCE,
@ -43,9 +43,8 @@ public class LocalLogicalPlanOptimizer extends ParameterizedRuleExecutor<Logical
new InferIsNotNull(),
new InferNonNullAggConstraint()
),
operators(),
localOperators(),
cleanup()
)
);
public LocalLogicalPlanOptimizer(LocalLogicalOptimizerContext localLogicalOptimizerContext) {
@ -58,27 +57,26 @@ public class LocalLogicalPlanOptimizer extends ParameterizedRuleExecutor<Logical
}
@SuppressWarnings("unchecked")
private static List<Batch<LogicalPlan>> replaceRules(List<Batch<LogicalPlan>> listOfRules) {
List<Batch<LogicalPlan>> newBatches = new ArrayList<>(listOfRules.size());
for (var batch : listOfRules) {
var rules = batch.rules();
private static Batch<LogicalPlan> localOperators() {
var operators = operators();
var rules = operators().rules();
List<Rule<?, LogicalPlan>> newRules = new ArrayList<>(rules.length);
boolean updated = false;
// apply updates to existing rules that have different applicability locally
for (var r : rules) {
if (r instanceof PropagateEmptyRelation) {
newRules.add(new LocalPropagateEmptyRelation());
updated = true;
} else if (r instanceof ReplaceStatsFilteredAggWithEval) {
switch (r) {
case PropagateEmptyRelation ignoredPropagate -> newRules.add(new LocalPropagateEmptyRelation());
// skip it: once a fragment contains an Agg, this can no longer be pruned, which the rule can do
updated = true;
} else {
newRules.add(r);
case ReplaceStatsFilteredAggWithEval ignoredReplace -> {
}
default -> newRules.add(r);
}
}
batch = updated ? batch.with(newRules.toArray(Rule[]::new)) : batch;
newBatches.add(batch);
}
return newBatches;
// add rule that should only apply locally
newRules.add(new ReplaceStringCasingWithInsensitiveRegexMatch());
return operators.with(newRules.toArray(Rule[]::new));
}
public LogicalPlan localOptimize(LogicalPlan plan) {

2
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java
View file

@ -66,7 +66,7 @@ public class ReplaceStringCasingWithInsensitiveEquals extends OptimizerRules.Opt
return e;
}
private static Expression unwrapCase(Expression e) {
static Expression unwrapCase(Expression e) {
for (; e instanceof ChangeCase cc; e = cc.field()) {
}
return e;

51
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveRegexMatch.java Normal file
View file

@ -0,0 +1,51 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.optimizer.rules.logical;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.Literal;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.StringPattern;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.ChangeCase;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
import static org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStringCasingWithInsensitiveEquals.unwrapCase;
public class ReplaceStringCasingWithInsensitiveRegexMatch extends OptimizerRules.OptimizerExpressionRule<
RegexMatch<? extends StringPattern>> {
public ReplaceStringCasingWithInsensitiveRegexMatch() {
super(OptimizerRules.TransformDirection.DOWN);
}
@Override
protected Expression rule(RegexMatch<? extends StringPattern> regexMatch, LogicalOptimizerContext unused) {
Expression e = regexMatch;
if (regexMatch.field() instanceof ChangeCase changeCase) {
var pattern = regexMatch.pattern().pattern();
e = changeCase.caseType().matchesCase(pattern) ? insensitiveRegexMatch(regexMatch) : Literal.of(regexMatch, Boolean.FALSE);
}
return e;
}
private static Expression insensitiveRegexMatch(RegexMatch<? extends StringPattern> regexMatch) {
return switch (regexMatch) {
case RLike rlike -> new RLike(rlike.source(), unwrapCase(rlike.field()), rlike.pattern(), true);
case WildcardLike wildcardLike -> new WildcardLike(
wildcardLike.source(),
unwrapCase(wildcardLike.field()),
wildcardLike.pattern(),
true
);
default -> regexMatch;
};
}
}

4
x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java
View file

@ -42,8 +42,8 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionResolutionStrate
import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction;
import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression;
import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchOperator;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Or;

4
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java
View file

@ -20,8 +20,8 @@ import org.elasticsearch.xcontent.json.JsonXContent;
import org.elasticsearch.xpack.esql.CsvTestsDataLoader;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchOperator;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Or;

1
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeSerializationTests.java
View file

@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import java.io.IOException;

45
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java
View file

@ -20,13 +20,17 @@ import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.junit.AfterClass;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import static org.elasticsearch.xpack.esql.core.util.TestUtils.randomCasing;
import static org.elasticsearch.xpack.esql.expression.function.DocsV3Support.renderNegatedOperator;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
@ -39,12 +43,13 @@ public class RLikeTests extends AbstractScalarFunctionTestCase {
@ParametersFactory
public static Iterable<Object[]> parameters() {
return parameters(str -> {
final Function<String, String> escapeString = str -> {
for (String syntax : new String[] { "\\", ".", "?", "+", "*", "|", "{", "}", "[", "]", "(", ")", "\"", "<", ">", "#", "&" }) {
str = str.replace(syntax, "\\" + syntax);
}
return str;
}, () -> randomAlphaOfLength(1) + "?");
};
return parameters(escapeString, () -> randomAlphaOfLength(1) + "?");
}
static Iterable<Object[]> parameters(Function<String, String> escapeString, Supplier<String> optionalPattern) {
@ -88,24 +93,52 @@ public class RLikeTests extends AbstractScalarFunctionTestCase {
String text = textSupplier.get();
return new TextAndPattern(text, escapeString.apply(text));
}, true);
cases(cases, title + " matches self case insensitive", () -> {
String text = textSupplier.get();
return new TextAndPattern(randomCasing(text), escapeString.apply(text));
}, true, true);
cases(cases, title + " doesn't match self with trailing", () -> {
String text = textSupplier.get();
return new TextAndPattern(text, escapeString.apply(text) + randomAlphaOfLength(1));
}, false);
cases(cases, title + " doesn't match self with trailing case insensitive", () -> {
String text = textSupplier.get();
return new TextAndPattern(randomCasing(text), escapeString.apply(text) + randomAlphaOfLength(1));
}, true, false);
cases(cases, title + " matches self with optional trailing", () -> {
String text = randomAlphaOfLength(1);
return new TextAndPattern(text, escapeString.apply(text) + optionalPattern.get());
}, true);
cases(cases, title + " matches self with optional trailing case insensitive", () -> {
String text = randomAlphaOfLength(1);
return new TextAndPattern(randomCasing(text), escapeString.apply(text) + optionalPattern.get());
}, true, true);
if (canGenerateDifferent) {
cases(cases, title + " doesn't match different", () -> {
String text = textSupplier.get();
String different = escapeString.apply(randomValueOtherThan(text, textSupplier));
return new TextAndPattern(text, different);
}, false);
cases(cases, title + " doesn't match different case insensitive", () -> {
String text = textSupplier.get();
Predicate<String> predicate = t -> t.toLowerCase(Locale.ROOT).equals(text.toLowerCase(Locale.ROOT));
String different = escapeString.apply(randomValueOtherThanMany(predicate, textSupplier));
return new TextAndPattern(text, different);
}, true, false);
}
}
private static void cases(List<TestCaseSupplier> cases, String title, Supplier<TextAndPattern> textAndPattern, boolean expected) {
cases(cases, title, textAndPattern, false, expected);
}
private static void cases(
List<TestCaseSupplier> cases,
String title,
Supplier<TextAndPattern> textAndPattern,
boolean caseInsensitive,
boolean expected
) {
for (DataType type : DataType.stringTypes()) {
cases.add(new TestCaseSupplier(title + " with " + type.esType(), List.of(type, DataType.KEYWORD, DataType.BOOLEAN), () -> {
TextAndPattern v = textAndPattern.get();
@ -113,13 +146,14 @@ public class RLikeTests extends AbstractScalarFunctionTestCase {
List.of(
new TestCaseSupplier.TypedData(new BytesRef(v.text), type, "e"),
new TestCaseSupplier.TypedData(new BytesRef(v.pattern), DataType.KEYWORD, "pattern").forceLiteral(),
new TestCaseSupplier.TypedData(false, DataType.BOOLEAN, "caseInsensitive").forceLiteral()
new TestCaseSupplier.TypedData(caseInsensitive, DataType.BOOLEAN, "caseInsensitive").forceLiteral()
),
startsWith("AutomataMatchEvaluator[input=Attribute[channel=0], pattern=digraph Automaton {\n"),
DataType.BOOLEAN,
equalTo(expected)
);
}));
if (caseInsensitive == false) {
cases.add(new TestCaseSupplier(title + " with " + type.esType(), List.of(type, DataType.KEYWORD), () -> {
TextAndPattern v = textAndPattern.get();
return new TestCaseSupplier.TestCase(
@ -134,6 +168,7 @@ public class RLikeTests extends AbstractScalarFunctionTestCase {
}));
}
}
}
@Override
protected Expression build(Source source, List<Expression> args) {
@ -150,7 +185,9 @@ public class RLikeTests extends AbstractScalarFunctionTestCase {
return caseInsensitiveBool
? new RLike(source, expression, new RLikePattern(patternString), true)
: new RLike(source, expression, new RLikePattern(patternString));
: (randomBoolean()
? new RLike(source, expression, new RLikePattern(patternString))
: new RLike(source, expression, new RLikePattern(patternString), false));
}
@AfterClass

1
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeSerializationTests.java
View file

@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import java.io.IOException;

23
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java
View file

@ -20,10 +20,12 @@ import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
import org.elasticsearch.xpack.esql.expression.function.FunctionName;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.junit.AfterClass;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
import java.util.function.Supplier;
import static org.elasticsearch.xpack.esql.expression.function.DocsV3Support.renderNegatedOperator;
@ -38,12 +40,13 @@ public class WildcardLikeTests extends AbstractScalarFunctionTestCase {
@ParametersFactory
public static Iterable<Object[]> parameters() {
List<Object[]> cases = (List<Object[]>) RLikeTests.parameters(str -> {
for (String syntax : new String[] { "\\", "*" }) {
final Function<String, String> escapeString = str -> {
for (String syntax : new String[] { "\\", "*", "?" }) {
str = str.replace(syntax, "\\" + syntax);
}
return str;
}, () -> "*");
};
List<Object[]> cases = (List<Object[]>) RLikeTests.parameters(escapeString, () -> "*");
List<TestCaseSupplier> suppliers = new ArrayList<>();
addCases(suppliers);
@ -83,11 +86,15 @@ public class WildcardLikeTests extends AbstractScalarFunctionTestCase {
static Expression buildWildcardLike(Source source, List<Expression> args) {
Expression expression = args.get(0);
Literal pattern = (Literal) args.get(1);
if (args.size() > 2) {
Literal caseInsensitive = (Literal) args.get(2);
assertThat(caseInsensitive.fold(FoldContext.small()), equalTo(false));
}
return new WildcardLike(source, expression, new WildcardPattern(((BytesRef) pattern.fold(FoldContext.small())).utf8ToString()));
Literal caseInsensitive = args.size() > 2 ? (Literal) args.get(2) : null;
boolean caseInsesitiveBool = caseInsensitive != null && (boolean) caseInsensitive.fold(FoldContext.small());
WildcardPattern wildcardPattern = new WildcardPattern(((BytesRef) pattern.fold(FoldContext.small())).utf8ToString());
return caseInsesitiveBool
? new WildcardLike(source, expression, wildcardPattern, true)
: (randomBoolean()
? new WildcardLike(source, expression, wildcardPattern)
: new WildcardLike(source, expression, wildcardPattern, false));
}
@AfterClass

85
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java
View file

@ -32,6 +32,8 @@ import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case;
import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull;
import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
@ -79,6 +81,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizer
import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
@ -635,6 +638,88 @@ public class LocalLogicalPlanOptimizerTests extends ESTestCase {
var source = as(filter.child(), EsRelation.class);
}
/*
* Limit[1000[INTEGER],false]
* \_Filter[RLIKE(first_name{f}#4, "VALÜ*", true)]
* \_EsRelation[test][_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen..]
*/
public void testReplaceUpperStringCasinqgWithInsensitiveRLike() {
var plan = localPlan("FROM test | WHERE TO_UPPER(TO_LOWER(TO_UPPER(first_name))) RLIKE \"VALÜ*\"");
var limit = as(plan, Limit.class);
var filter = as(limit.child(), Filter.class);
var rlike = as(filter.condition(), RLike.class);
var field = as(rlike.field(), FieldAttribute.class);
assertThat(field.fieldName(), is("first_name"));
assertThat(rlike.pattern().pattern(), is("VALÜ*"));
assertThat(rlike.caseInsensitive(), is(true));
var source = as(filter.child(), EsRelation.class);
}
// same plan as above, but lower case pattern
public void testReplaceLowerStringCasingWithInsensitiveRLike() {
var plan = localPlan("FROM test | WHERE TO_LOWER(TO_UPPER(first_name)) RLIKE \"valü*\"");
var limit = as(plan, Limit.class);
var filter = as(limit.child(), Filter.class);
var rlike = as(filter.condition(), RLike.class);
var field = as(rlike.field(), FieldAttribute.class);
assertThat(field.fieldName(), is("first_name"));
assertThat(rlike.pattern().pattern(), is("valü*"));
assertThat(rlike.caseInsensitive(), is(true));
var source = as(filter.child(), EsRelation.class);
}
/**
* LocalRelation[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12, langu
* ages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8],EMPTY]
*/
public void testReplaceStringCasingAndRLikeWithLocalRelation() {
var plan = localPlan("FROM test | WHERE TO_LOWER(TO_UPPER(first_name)) RLIKE \"VALÜ*\"");
var local = as(plan, LocalRelation.class);
assertThat(local.supplier(), equalTo(LocalSupplier.EMPTY));
}
// same plan as in testReplaceUpperStringCasingWithInsensitiveRLike, but with LIKE instead of RLIKE
public void testReplaceUpperStringCasingWithInsensitiveLike() {
var plan = localPlan("FROM test | WHERE TO_UPPER(TO_LOWER(TO_UPPER(first_name))) LIKE \"VALÜ*\"");
var limit = as(plan, Limit.class);
var filter = as(limit.child(), Filter.class);
var wlike = as(filter.condition(), WildcardLike.class);
var field = as(wlike.field(), FieldAttribute.class);
assertThat(field.fieldName(), is("first_name"));
assertThat(wlike.pattern().pattern(), is("VALÜ*"));
assertThat(wlike.caseInsensitive(), is(true));
var source = as(filter.child(), EsRelation.class);
}
// same plan as above, but lower case pattern
public void testReplaceLowerStringCasingWithInsensitiveLike() {
var plan = localPlan("FROM test | WHERE TO_LOWER(TO_UPPER(first_name)) LIKE \"valü*\"");
var limit = as(plan, Limit.class);
var filter = as(limit.child(), Filter.class);
var wlike = as(filter.condition(), WildcardLike.class);
var field = as(wlike.field(), FieldAttribute.class);
assertThat(field.fieldName(), is("first_name"));
assertThat(wlike.pattern().pattern(), is("valü*"));
assertThat(wlike.caseInsensitive(), is(true));
var source = as(filter.child(), EsRelation.class);
}
/**
* LocalRelation[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12, langu
* ages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8],EMPTY]
*/
public void testReplaceStringCasingAndLikeWithLocalRelation() {
var plan = localPlan("FROM test | WHERE TO_LOWER(TO_UPPER(first_name)) LIKE \"VALÜ*\"");
var local = as(plan, LocalRelation.class);
assertThat(local.supplier(), equalTo(LocalSupplier.EMPTY));
}
private IsNotNull isNotNull(Expression field) {
return new IsNotNull(EMPTY, field);
}

173
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java
View file

@ -83,6 +83,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialWi
import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Or;
@ -205,7 +206,7 @@ import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.startsWith;
// @TestLogging(value = "org.elasticsearch.xpack.esql:DEBUG", reason = "debug")
// @TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug")
public class PhysicalPlanOptimizerTests extends ESTestCase {
private static final String PARAM_FORMATTING = "%1$s";
@ -2201,6 +2202,163 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
assertThat(source.query(), nullValue());
}
/*
* LimitExec[1000[INTEGER]]
* \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12, langu
* ages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8],false]
* \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12, langu
* ages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8]]
* \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen..]<[],[]>
* \_EsQueryExec[test], indexMode[standard], query[{"esql_single_value":{"field":"first_name","next":{"regexp":{"first_name":
* {"value":"foo*","flags_value":65791,"case_insensitive":true,"max_determinized_states":10000,"boost":0.0}}},
* "source":"TO_LOWER(first_name) RLIKE \"foo*\"@2:9"}}][_doc{f}#25], limit[1000], sort[] estimatedRowSize[332]
*/
private void doTestPushDownCaseChangeRegexMatch(String query, String expected) {
var plan = physicalPlan(query);
var optimized = optimizedPlan(plan);
var topLimit = as(optimized, LimitExec.class);
var exchange = asRemoteExchange(topLimit.child());
var project = as(exchange.child(), ProjectExec.class);
var fieldExtract = as(project.child(), FieldExtractExec.class);
var source = as(fieldExtract.child(), EsQueryExec.class);
var singleValue = as(source.query(), SingleValueQuery.Builder.class);
assertThat(stripThrough(singleValue.toString()), is(stripThrough(expected)));
}
public void testPushDownLowerCaseChangeRLike() {
doTestPushDownCaseChangeRegexMatch("""
FROM test
| WHERE TO_LOWER(first_name) RLIKE "foo*"
""", """
{
"esql_single_value": {
"field": "first_name",
"next": {
"regexp": {
"first_name": {
"value": "foo*",
"flags_value": 65791,
"case_insensitive": true,
"max_determinized_states": 10000,
"boost": 0.0
}
}
},
"source": "TO_LOWER(first_name) RLIKE \\"foo*\\"@2:9"
}
}
""");
}
public void testPushDownUpperCaseChangeRLike() {
doTestPushDownCaseChangeRegexMatch("""
FROM test
| WHERE TO_UPPER(first_name) RLIKE "FOO*"
""", """
{
"esql_single_value": {
"field": "first_name",
"next": {
"regexp": {
"first_name": {
"value": "FOO*",
"flags_value": 65791,
"case_insensitive": true,
"max_determinized_states": 10000,
"boost": 0.0
}
}
},
"source": "TO_UPPER(first_name) RLIKE \\"FOO*\\"@2:9"
}
}
""");
}
public void testPushDownLowerCaseChangeLike() {
doTestPushDownCaseChangeRegexMatch("""
FROM test
| WHERE TO_LOWER(first_name) LIKE "foo*"
""", """
{
"esql_single_value": {
"field": "first_name",
"next": {
"wildcard": {
"first_name": {
"wildcard": "foo*",
"case_insensitive": true,
"boost": 0.0
}
}
},
"source": "TO_LOWER(first_name) LIKE \\"foo*\\"@2:9"
}
}
""");
}
public void testPushDownUpperCaseChangeLike() {
doTestPushDownCaseChangeRegexMatch("""
FROM test
| WHERE TO_UPPER(first_name) LIKE "FOO*"
""", """
{
"esql_single_value": {
"field": "first_name",
"next": {
"wildcard": {
"first_name": {
"wildcard": "FOO*",
"case_insensitive": true,
"boost": 0.0
}
}
},
"source": "TO_UPPER(first_name) LIKE \\"FOO*\\"@2:9"
}
}
""");
}
/*
* LimitExec[1000[INTEGER]]
* \_ExchangeExec[[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, gender{f}#6, hire_date{f}#11, job{f}#12, job.raw{f}#13, lang
* uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9],false]
* \_ProjectExec[[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, gender{f}#6, hire_date{f}#11, job{f}#12, job.raw{f}#13, lang
* uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9]]
* \_FieldExtractExec[_meta_field{f}#10, gender{f}#6, hire_date{f}#11, jo..]<[],[]>
* \_LimitExec[1000[INTEGER]]
* \_FilterExec[LIKE(first_name{f}#5, "FOO*", true) OR IN(1[INTEGER],2[INTEGER],3[INTEGER],emp_no{f}#4 + 1[INTEGER])]
* \_FieldExtractExec[first_name{f}#5, emp_no{f}#4]<[],[]>
* \_EsQueryExec[test], indexMode[standard], query[][_doc{f}#26], limit[], sort[] estimatedRowSize[332]
*/
public void testChangeCaseAsInsensitiveWildcardLikeNotPushedDown() {
var esql = """
FROM test
| WHERE TO_UPPER(first_name) LIKE "FOO*" OR emp_no + 1 IN (1, 2, 3)
""";
var plan = physicalPlan(esql);
var optimized = optimizedPlan(plan);
var topLimit = as(optimized, LimitExec.class);
var exchange = asRemoteExchange(topLimit.child());
var project = as(exchange.child(), ProjectExec.class);
var fieldExtract = as(project.child(), FieldExtractExec.class);
var limit = as(fieldExtract.child(), LimitExec.class);
var filter = as(limit.child(), FilterExec.class);
fieldExtract = as(filter.child(), FieldExtractExec.class);
var source = as(fieldExtract.child(), EsQueryExec.class);
var or = as(filter.condition(), Or.class);
var wildcard = as(or.left(), WildcardLike.class);
assertThat(Expressions.name(wildcard.field()), is("first_name"));
assertThat(wildcard.pattern().pattern(), is("FOO*"));
assertThat(wildcard.caseInsensitive(), is(true));
}
public void testPushDownNotRLike() {
var plan = physicalPlan("""
from test
@ -2432,6 +2590,17 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES + KEYWORD_EST));
}
/*
* LimitExec[1000[INTEGER]]
* \_ExchangeExec[[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, gender{f}#6, hire_date{f}#11, job{f}#12, job.raw{f}#13, lang
* uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9, _index{m}#2],false]
* \_ProjectExec[[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, gender{f}#6, hire_date{f}#11, job{f}#12, job.raw{f}#13, lang
* uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9, _index{m}#2]]
* \_FieldExtractExec[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, ge..]<[],[]>
* \_EsQueryExec[test], indexMode[standard], query[{"wildcard":{"_index":{"wildcard":"test*","boost":0.0}}}][_doc{f}#27],
* limit[1000], sort[] estimatedRowSize[382]
*
*/
public void testPushDownMetadataIndexInWildcard() {
var plan = physicalPlan("""
from test metadata _index
@ -8097,7 +8266,7 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
var physical = mapper.map(logical);
// System.out.println("Physical\n" + physical);
if (assertSerialization) {
assertSerialization(physical);
assertSerialization(physical, config);
}
return physical;
}

4
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java
View file

@ -18,8 +18,8 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.Range;
import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Not;

2
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java
View file

@ -47,8 +47,8 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMedi
import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Or;
import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull;

4
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFiltersTests.java
View file

@ -18,8 +18,8 @@ import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.aggregate.Count;
import org.elasticsearch.xpack.esql.expression.function.fulltext.Match;
import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pow;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.Predicates;
import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;

4
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java
View file

@ -15,8 +15,8 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
import org.elasticsearch.xpack.esql.core.util.StringUtils;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;

4
x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java
View file

@ -28,8 +28,8 @@ import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction;
import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression;
import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchOperator;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Or;
import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;