ESQL: Pushdown constructs doing case-insensitive regexes (#128393)

This introduces an optimization to pushdown to Lucense those language constructs that aim at case-insensitive regular expression matching, used with `LIKE` and `RLIKE` operators, such as:
* `| WHERE TO_LOWER(field) LIKE "abc*"`
* `| WHERE TO_UPPER(field) RLIKE "ABC.*"` 
 
These are now pushed as case-insensitive `wildcard` and `regexp` respectively queries down to Lucene.

Closes #127479
This commit is contained in:
Bogdan Pintea 2025-05-30 10:55:00 +02:00 committed by GitHub
parent cc461afa0a
commit 0a8091605b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
33 changed files with 756 additions and 236 deletions

View file

@ -48,9 +48,9 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs;
import org.elasticsearch.xpack.esql.expression.function.scalar.math.RoundTo; import org.elasticsearch.xpack.esql.expression.function.scalar.math.RoundTo;
import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower; import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper; import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;

View file

@ -0,0 +1,6 @@
pr: 128393
summary: Pushdown constructs doing case-insensitive regexes
area: ES|QL
type: enhancement
issues:
- 127479

View file

@ -272,6 +272,7 @@ public class TransportVersions {
public static final TransportVersion ML_INFERENCE_VERTEXAI_CHATCOMPLETION_ADDED = def(9_083_0_00); public static final TransportVersion ML_INFERENCE_VERTEXAI_CHATCOMPLETION_ADDED = def(9_083_0_00);
public static final TransportVersion INFERENCE_CUSTOM_SERVICE_ADDED = def(9_084_0_00); public static final TransportVersion INFERENCE_CUSTOM_SERVICE_ADDED = def(9_084_0_00);
public static final TransportVersion ESQL_LIMIT_ROW_SIZE = def(9_085_0_00); public static final TransportVersion ESQL_LIMIT_ROW_SIZE = def(9_085_0_00);
public static final TransportVersion ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY = def(9_086_0_00);
/* /*
* STOP! READ THIS FIRST! No, really, * STOP! READ THIS FIRST! No, really,

View file

@ -16,11 +16,11 @@ public abstract class AbstractStringPattern implements StringPattern {
private Automaton automaton; private Automaton automaton;
public abstract Automaton createAutomaton(); public abstract Automaton createAutomaton(boolean ignoreCase);
private Automaton automaton() { private Automaton automaton() {
if (automaton == null) { if (automaton == null) {
automaton = createAutomaton(); automaton = createAutomaton(false);
} }
return automaton; return automaton;
} }

View file

@ -1,35 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.core.expression.predicate.regex;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.tree.Source;
import java.io.IOException;
public abstract class RLike extends RegexMatch<RLikePattern> {
public RLike(Source source, Expression value, RLikePattern pattern) {
super(source, value, pattern, false);
}
public RLike(Source source, Expression field, RLikePattern rLikePattern, boolean caseInsensitive) {
super(source, field, rLikePattern, caseInsensitive);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public String getWriteableName() {
throw new UnsupportedOperationException();
}
}

View file

@ -21,9 +21,10 @@ public class RLikePattern extends AbstractStringPattern {
} }
@Override @Override
public Automaton createAutomaton() { public Automaton createAutomaton(boolean ignoreCase) {
int matchFlags = ignoreCase ? RegExp.CASE_INSENSITIVE : 0;
return Operations.determinize( return Operations.determinize(
new RegExp(regexpPattern, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT).toAutomaton(), new RegExp(regexpPattern, RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT, matchFlags).toAutomaton(),
Operations.DEFAULT_DETERMINIZE_WORK_LIMIT Operations.DEFAULT_DETERMINIZE_WORK_LIMIT
); );
} }

View file

@ -1,35 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.core.expression.predicate.regex;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.tree.Source;
import java.io.IOException;
public abstract class WildcardLike extends RegexMatch<WildcardPattern> {
public WildcardLike(Source source, Expression left, WildcardPattern pattern) {
this(source, left, pattern, false);
}
public WildcardLike(Source source, Expression left, WildcardPattern pattern, boolean caseInsensitive) {
super(source, left, pattern, caseInsensitive);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public String getWriteableName() {
throw new UnsupportedOperationException();
}
}

View file

@ -10,10 +10,13 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.RegExp;
import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.core.util.StringUtils;
import java.util.Objects; import java.util.Objects;
import static org.elasticsearch.xpack.esql.core.util.StringUtils.luceneWildcardToRegExp;
/** /**
* Similar to basic regex, supporting '?' wildcard for single character (same as regex ".") * Similar to basic regex, supporting '?' wildcard for single character (same as regex ".")
* and '*' wildcard for multiple characters (same as regex ".*") * and '*' wildcard for multiple characters (same as regex ".*")
@ -37,8 +40,14 @@ public class WildcardPattern extends AbstractStringPattern {
} }
@Override @Override
public Automaton createAutomaton() { public Automaton createAutomaton(boolean ignoreCase) {
return WildcardQuery.toAutomaton(new Term(null, wildcard), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); return ignoreCase
? Operations.determinize(
new RegExp(luceneWildcardToRegExp(wildcard), RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT, RegExp.CASE_INSENSITIVE)
.toAutomaton(),
Operations.DEFAULT_DETERMINIZE_WORK_LIMIT
)
: WildcardQuery.toAutomaton(new Term(null, wildcard), Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
} }
@Override @Override

View file

@ -7,6 +7,7 @@
package org.elasticsearch.xpack.esql.core.util; package org.elasticsearch.xpack.esql.core.util;
import org.apache.lucene.document.InetAddressPoint; import org.apache.lucene.document.InetAddressPoint;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.spell.LevenshteinDistance; import org.apache.lucene.search.spell.LevenshteinDistance;
import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CollectionUtil; import org.apache.lucene.util.CollectionUtil;
@ -178,6 +179,44 @@ public final class StringUtils {
return regex.toString(); return regex.toString();
} }
/**
* Translates a Lucene wildcard pattern to a Lucene RegExp one.
* @param wildcard Lucene wildcard pattern
* @return Lucene RegExp pattern
*/
public static String luceneWildcardToRegExp(String wildcard) {
StringBuilder regex = new StringBuilder();
for (int i = 0, wcLen = wildcard.length(); i < wcLen; i++) {
char c = wildcard.charAt(i); // this will work chunking through Unicode as long as all values matched are ASCII
switch (c) {
case WildcardQuery.WILDCARD_STRING -> regex.append(".*");
case WildcardQuery.WILDCARD_CHAR -> regex.append(".");
case WildcardQuery.WILDCARD_ESCAPE -> {
if (i + 1 < wcLen) {
// consume the wildcard escaping, consider the next char
char next = wildcard.charAt(i + 1);
i++;
switch (next) {
case WildcardQuery.WILDCARD_STRING, WildcardQuery.WILDCARD_CHAR, WildcardQuery.WILDCARD_ESCAPE ->
// escape `*`, `.`, `\`, since these are special chars in RegExp as well
regex.append("\\");
// default: unnecessary escaping -- just ignore the escaping
}
regex.append(next);
} else {
// "else fallthru, lenient parsing with a trailing \" -- according to WildcardQuery#toAutomaton
regex.append("\\\\");
}
}
case '$', '(', ')', '+', '.', '[', ']', '^', '{', '|', '}' -> regex.append("\\").append(c);
default -> regex.append(c);
}
}
return regex.toString();
}
/** /**
* Translates a like pattern to a Lucene wildcard. * Translates a like pattern to a Lucene wildcard.
* This methods pays attention to the custom escape char which gets converted into \ (used by Lucene). * This methods pays attention to the custom escape char which gets converted into \ (used by Lucene).

View file

@ -9,7 +9,9 @@ package org.elasticsearch.xpack.esql.core.util;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import static org.elasticsearch.xpack.esql.core.util.StringUtils.luceneWildcardToRegExp;
import static org.elasticsearch.xpack.esql.core.util.StringUtils.wildcardToJavaPattern; import static org.elasticsearch.xpack.esql.core.util.StringUtils.wildcardToJavaPattern;
import static org.hamcrest.Matchers.is;
public class StringUtilsTests extends ESTestCase { public class StringUtilsTests extends ESTestCase {
@ -55,4 +57,21 @@ public class StringUtilsTests extends ESTestCase {
public void testEscapedEscape() { public void testEscapedEscape() {
assertEquals("^\\\\\\\\$", wildcardToJavaPattern("\\\\\\\\", '\\')); assertEquals("^\\\\\\\\$", wildcardToJavaPattern("\\\\\\\\", '\\'));
} }
public void testLuceneWildcardToRegExp() {
assertThat(luceneWildcardToRegExp(""), is(""));
assertThat(luceneWildcardToRegExp("*"), is(".*"));
assertThat(luceneWildcardToRegExp("?"), is("."));
assertThat(luceneWildcardToRegExp("\\\\"), is("\\\\"));
assertThat(luceneWildcardToRegExp("foo?bar"), is("foo.bar"));
assertThat(luceneWildcardToRegExp("foo*bar"), is("foo.*bar"));
assertThat(luceneWildcardToRegExp("foo\\\\bar"), is("foo\\\\bar"));
assertThat(luceneWildcardToRegExp("foo*bar?baz"), is("foo.*bar.baz"));
assertThat(luceneWildcardToRegExp("foo\\*bar"), is("foo\\*bar"));
assertThat(luceneWildcardToRegExp("foo\\?bar\\?"), is("foo\\?bar\\?"));
assertThat(luceneWildcardToRegExp("foo\\?bar\\"), is("foo\\?bar\\\\"));
assertThat(luceneWildcardToRegExp("[](){}^$.|+"), is("\\[\\]\\(\\)\\{\\}\\^\\$\\.\\|\\+"));
assertThat(luceneWildcardToRegExp("foo\\\uD83D\uDC14bar"), is("foo\uD83D\uDC14bar"));
assertThat(luceneWildcardToRegExp("foo\uD83D\uDC14bar"), is("foo\uD83D\uDC14bar"));
}
} }

View file

@ -13,6 +13,7 @@ import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.EsField;
import java.util.Locale;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import static java.util.Collections.emptyMap; import static java.util.Collections.emptyMap;
@ -61,4 +62,15 @@ public final class TestUtils {
public static String stripThrough(String input) { public static String stripThrough(String input) {
return WS_PATTERN.matcher(input).replaceAll(StringUtils.EMPTY); return WS_PATTERN.matcher(input).replaceAll(StringUtils.EMPTY);
} }
/** Returns the input string, but with parts of it having the letter casing changed. */
public static String randomCasing(String input) {
StringBuilder sb = new StringBuilder(input.length());
for (int i = 0, inputLen = input.length(), step = (int) Math.sqrt(inputLen), chunkEnd; i < inputLen; i += step) {
chunkEnd = Math.min(i + step, inputLen);
var chunk = input.substring(i, chunkEnd);
sb.append(randomBoolean() ? chunk.toLowerCase(Locale.ROOT) : chunk.toUpperCase(Locale.ROOT));
}
return sb.toString();
}
} }

View file

@ -63,8 +63,8 @@ import org.elasticsearch.xpack.esql.core.type.EsField;
import org.elasticsearch.xpack.esql.core.util.DateUtils; import org.elasticsearch.xpack.esql.core.util.DateUtils;
import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.core.util.StringUtils;
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.Range; import org.elasticsearch.xpack.esql.expression.predicate.Range;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;

View file

@ -319,3 +319,107 @@ warningRegex:java.lang.IllegalArgumentException: single-value function encounter
emp_no:integer | job_positions:keyword emp_no:integer | job_positions:keyword
10025 | Accountant 10025 | Accountant
; ;
likeWithUpperTurnedInsensitive
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_UPPER(first_name) LIKE "GEOR*"
;
emp_no:integer |first_name:keyword
10001 |Georgi
10055 |Georgy
;
likeWithLowerTurnedInsensitive
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_LOWER(TO_UPPER(first_name)) LIKE "geor*"
;
emp_no:integer |first_name:keyword
10001 |Georgi
10055 |Georgy
;
likeWithLowerConflictingFolded
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_UPPER(first_name) LIKE "geor*"
;
emp_no:integer |first_name:keyword
;
likeWithLowerTurnedInsensitiveNotPushedDown
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_LOWER(first_name) LIKE "geor*" OR emp_no + 1 IN (10002, 10056)
;
emp_no:integer |first_name:keyword
10001 |Georgi
10055 |Georgy
;
rlikeWithUpperTurnedInsensitive
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_UPPER(first_name) RLIKE "GEOR.*"
;
emp_no:integer |first_name:keyword
10001 |Georgi
10055 |Georgy
;
rlikeWithLowerTurnedInsensitive
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_LOWER(TO_UPPER(first_name)) RLIKE "geor.*"
;
emp_no:integer |first_name:keyword
10001 |Georgi
10055 |Georgy
;
rlikeWithLowerConflictingFolded
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_UPPER(first_name) RLIKE "geor.*"
;
emp_no:integer |first_name:keyword
;
negatedRLikeWithLowerTurnedInsensitive
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_LOWER(TO_UPPER(first_name)) NOT RLIKE "geor.*"
| STATS c = COUNT()
;
c:long
88
;
rlikeWithLowerTurnedInsensitiveNotPushedDown
FROM employees
| KEEP emp_no, first_name
| SORT emp_no
| WHERE TO_LOWER(first_name) RLIKE "geor.*" OR emp_no + 1 IN (10002, 10056)
;
emp_no:integer |first_name:keyword
10001 |Georgi
10055 |Georgy
;

View file

@ -68,11 +68,11 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StYMin;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.ByteLength; import org.elasticsearch.xpack.esql.expression.function.scalar.string.ByteLength;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Length;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RTrim;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.Space; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Space;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.Trim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Trim;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.util.Delay; import org.elasticsearch.xpack.esql.expression.function.scalar.util.Delay;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull;

View file

@ -5,21 +5,17 @@
* 2.0. * 2.0.
*/ */
package org.elasticsearch.xpack.esql.expression.function.scalar.string; package org.elasticsearch.xpack.esql.expression.function.scalar.string.regex;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.compute.operator.EvalOperator;
import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
import org.elasticsearch.xpack.esql.core.querydsl.query.RegexQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.RegexQuery;
import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.Example;
import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.Param;
@ -29,14 +25,9 @@ import org.elasticsearch.xpack.esql.planner.TranslatorHandler;
import java.io.IOException; import java.io.IOException;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; public class RLike extends RegexMatch<RLikePattern> {
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
public class RLike extends org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLike
implements
EvaluatorMapper,
TranslationAware.SingleValueTranslationAware {
public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "RLike", RLike::new); public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "RLike", RLike::new);
public static final String NAME = "RLIKE";
@FunctionInfo(returnType = "boolean", description = """ @FunctionInfo(returnType = "boolean", description = """
Use `RLIKE` to filter data based on string patterns using using Use `RLIKE` to filter data based on string patterns using using
@ -52,13 +43,13 @@ public class RLike extends org.elasticsearch.xpack.esql.core.expression.predicat
To reduce the overhead of escaping, we suggest using triple quotes strings `\"\"\"` To reduce the overhead of escaping, we suggest using triple quotes strings `\"\"\"`
<<load-esql-example, file=string tag=rlikeEscapingTripleQuotes>> <<load-esql-example, file=string tag=rlikeEscapingTripleQuotes>>
""", operator = "RLIKE", examples = @Example(file = "docs", tag = "rlike")) """, operator = NAME, examples = @Example(file = "docs", tag = "rlike"))
public RLike( public RLike(
Source source, Source source,
@Param(name = "str", type = { "keyword", "text" }, description = "A literal value.") Expression value, @Param(name = "str", type = { "keyword", "text" }, description = "A literal value.") Expression value,
@Param(name = "pattern", type = { "keyword", "text" }, description = "A regular expression.") RLikePattern pattern @Param(name = "pattern", type = { "keyword", "text" }, description = "A regular expression.") RLikePattern pattern
) { ) {
super(source, value, pattern); this(source, value, pattern, false);
} }
public RLike(Source source, Expression field, RLikePattern rLikePattern, boolean caseInsensitive) { public RLike(Source source, Expression field, RLikePattern rLikePattern, boolean caseInsensitive) {
@ -66,7 +57,12 @@ public class RLike extends org.elasticsearch.xpack.esql.core.expression.predicat
} }
private RLike(StreamInput in) throws IOException { private RLike(StreamInput in) throws IOException {
this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), new RLikePattern(in.readString())); this(
Source.readFrom((PlanStreamInput) in),
in.readNamedWriteable(Expression.class),
new RLikePattern(in.readString()),
deserializeCaseInsensitivity(in)
);
} }
@Override @Override
@ -74,6 +70,12 @@ public class RLike extends org.elasticsearch.xpack.esql.core.expression.predicat
source().writeTo(out); source().writeTo(out);
out.writeNamedWriteable(field()); out.writeNamedWriteable(field());
out.writeString(pattern().asJavaRegex()); out.writeString(pattern().asJavaRegex());
serializeCaseInsensitivity(out);
}
@Override
public String name() {
return NAME;
} }
@Override @Override
@ -91,35 +93,10 @@ public class RLike extends org.elasticsearch.xpack.esql.core.expression.predicat
return new RLike(source(), newChild, pattern(), caseInsensitive()); return new RLike(source(), newChild, pattern(), caseInsensitive());
} }
@Override
protected TypeResolution resolveType() {
return isString(field(), sourceText(), DEFAULT);
}
@Override
public Boolean fold(FoldContext ctx) {
return (Boolean) EvaluatorMapper.super.fold(source(), ctx);
}
@Override
public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
return AutomataMatch.toEvaluator(source(), toEvaluator.apply(field()), pattern().createAutomaton());
}
@Override
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableFieldAttribute(field()) ? Translatable.YES : Translatable.NO;
}
@Override @Override
public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) { public Query asQuery(LucenePushdownPredicates pushdownPredicates, TranslatorHandler handler) {
var fa = LucenePushdownPredicates.checkIsFieldAttribute(field()); var fa = LucenePushdownPredicates.checkIsFieldAttribute(field());
// TODO: see whether escaping is needed // TODO: see whether escaping is needed
return new RegexQuery(source(), handler.nameOf(fa.exactAttribute()), pattern().asJavaRegex(), caseInsensitive()); return new RegexQuery(source(), handler.nameOf(fa.exactAttribute()), pattern().asJavaRegex(), caseInsensitive());
} }
@Override
public Expression singleValueField() {
return field();
}
} }

View file

@ -0,0 +1,93 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.expression.function.scalar.string.regex;
import org.apache.lucene.util.automaton.Automata;
import org.elasticsearch.TransportVersions;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.compute.operator.EvalOperator;
import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException;
import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.AbstractStringPattern;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.AutomataMatch;
import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates;
import java.io.IOException;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
abstract class RegexMatch<P extends AbstractStringPattern> extends org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch<
P> implements EvaluatorMapper, TranslationAware.SingleValueTranslationAware {
abstract String name();
RegexMatch(Source source, Expression field, P pattern, boolean caseInsensitive) {
super(source, field, pattern, caseInsensitive);
}
@Override
protected TypeResolution resolveType() {
return isString(field(), sourceText(), DEFAULT);
}
@Override
public Boolean fold(FoldContext ctx) {
return (Boolean) EvaluatorMapper.super.fold(source(), ctx);
}
@Override
public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
return AutomataMatch.toEvaluator(
source(),
toEvaluator.apply(field()),
// The empty pattern will accept the empty string
pattern().pattern().isEmpty() ? Automata.makeEmptyString() : pattern().createAutomaton(caseInsensitive())
);
}
@Override
public Translatable translatable(LucenePushdownPredicates pushdownPredicates) {
return pushdownPredicates.isPushableFieldAttribute(field()) ? Translatable.YES : Translatable.NO;
}
@Override
public Expression singleValueField() {
return field();
}
@Override
public String nodeString() {
return name() + "(" + field().nodeString() + ", \"" + pattern().pattern() + "\", " + caseInsensitive() + ")";
}
void serializeCaseInsensitivity(StreamOutput out) throws IOException {
if (out.getTransportVersion().before(TransportVersions.ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY)) {
if (caseInsensitive()) {
// The plan has been optimized to run a case-insensitive match, which the remote peer cannot be notified of. Simply avoiding
// the serialization of the boolean would result in wrong results.
throw new EsqlIllegalArgumentException(
name() + " with case insensitivity is not supported in peer node's version [{}]. Upgrade to version [{}] or newer.",
out.getTransportVersion(),
TransportVersions.ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY
);
} // else: write nothing, the remote peer can execute the case-sensitive query
} else {
out.writeBoolean(caseInsensitive());
}
}
static boolean deserializeCaseInsensitivity(StreamInput in) throws IOException {
return in.getTransportVersion().onOrAfter(TransportVersions.ESQL_REGEX_MATCH_WITH_CASE_INSENSITIVITY) && in.readBoolean();
}
}

View file

@ -5,23 +5,18 @@
* 2.0. * 2.0.
*/ */
package org.elasticsearch.xpack.esql.expression.function.scalar.string; package org.elasticsearch.xpack.esql.expression.function.scalar.string.regex;
import org.apache.lucene.util.automaton.Automata;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.compute.operator.EvalOperator;
import org.elasticsearch.xpack.esql.capabilities.TranslationAware;
import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
import org.elasticsearch.xpack.esql.core.querydsl.query.Query; import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
import org.elasticsearch.xpack.esql.core.querydsl.query.WildcardQuery; import org.elasticsearch.xpack.esql.core.querydsl.query.WildcardQuery;
import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.Example;
import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.Param;
@ -31,18 +26,13 @@ import org.elasticsearch.xpack.esql.planner.TranslatorHandler;
import java.io.IOException; import java.io.IOException;
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; public class WildcardLike extends RegexMatch<WildcardPattern> {
import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString;
public class WildcardLike extends org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike
implements
EvaluatorMapper,
TranslationAware.SingleValueTranslationAware {
public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
Expression.class, Expression.class,
"WildcardLike", "WildcardLike",
WildcardLike::new WildcardLike::new
); );
public static final String NAME = "LIKE";
@FunctionInfo(returnType = "boolean", description = """ @FunctionInfo(returnType = "boolean", description = """
Use `LIKE` to filter data based on string patterns using wildcards. `LIKE` Use `LIKE` to filter data based on string patterns using wildcards. `LIKE`
@ -63,17 +53,26 @@ public class WildcardLike extends org.elasticsearch.xpack.esql.core.expression.p
To reduce the overhead of escaping, we suggest using triple quotes strings `\"\"\"` To reduce the overhead of escaping, we suggest using triple quotes strings `\"\"\"`
<<load-esql-example, file=string tag=likeEscapingTripleQuotes>> <<load-esql-example, file=string tag=likeEscapingTripleQuotes>>
""", operator = "LIKE", examples = @Example(file = "docs", tag = "like")) """, operator = NAME, examples = @Example(file = "docs", tag = "like"))
public WildcardLike( public WildcardLike(
Source source, Source source,
@Param(name = "str", type = { "keyword", "text" }, description = "A literal expression.") Expression left, @Param(name = "str", type = { "keyword", "text" }, description = "A literal expression.") Expression left,
@Param(name = "pattern", type = { "keyword", "text" }, description = "Pattern.") WildcardPattern pattern @Param(name = "pattern", type = { "keyword", "text" }, description = "Pattern.") WildcardPattern pattern
) { ) {
super(source, left, pattern, false); this(source, left, pattern, false);
}
public WildcardLike(Source source, Expression left, WildcardPattern pattern, boolean caseInsensitive) {
super(source, left, pattern, caseInsensitive);
} }
private WildcardLike(StreamInput in) throws IOException { private WildcardLike(StreamInput in) throws IOException {
this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), new WildcardPattern(in.readString())); this(
Source.readFrom((PlanStreamInput) in),
in.readNamedWriteable(Expression.class),
new WildcardPattern(in.readString()),
deserializeCaseInsensitivity(in)
);
} }
@Override @Override
@ -81,6 +80,12 @@ public class WildcardLike extends org.elasticsearch.xpack.esql.core.expression.p
source().writeTo(out); source().writeTo(out);
out.writeNamedWriteable(field()); out.writeNamedWriteable(field());
out.writeString(pattern().pattern()); out.writeString(pattern().pattern());
serializeCaseInsensitivity(out);
}
@Override
public String name() {
return NAME;
} }
@Override @Override
@ -89,33 +94,13 @@ public class WildcardLike extends org.elasticsearch.xpack.esql.core.expression.p
} }
@Override @Override
protected NodeInfo<org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike> info() { protected NodeInfo<WildcardLike> info() {
return NodeInfo.create(this, WildcardLike::new, field(), pattern()); return NodeInfo.create(this, WildcardLike::new, field(), pattern(), caseInsensitive());
} }
@Override @Override
protected WildcardLike replaceChild(Expression newLeft) { protected WildcardLike replaceChild(Expression newLeft) {
return new WildcardLike(source(), newLeft, pattern()); return new WildcardLike(source(), newLeft, pattern(), caseInsensitive());
}
@Override
protected TypeResolution resolveType() {
return isString(field(), sourceText(), DEFAULT);
}
@Override
public Boolean fold(FoldContext ctx) {
return (Boolean) EvaluatorMapper.super.fold(source(), ctx);
}
@Override
public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
return AutomataMatch.toEvaluator(
source(),
toEvaluator.apply(field()),
// The empty pattern will accept the empty string
pattern().pattern().length() == 0 ? Automata.makeEmptyString() : pattern().createAutomaton()
);
} }
@Override @Override
@ -134,9 +119,4 @@ public class WildcardLike extends org.elasticsearch.xpack.esql.core.expression.p
private Query translateField(String targetFieldName) { private Query translateField(String targetFieldName) {
return new WildcardQuery(source(), targetFieldName, pattern().asLuceneWildcard(), caseInsensitive()); return new WildcardQuery(source(), targetFieldName, pattern().asLuceneWildcard(), caseInsensitive());
} }
@Override
public Expression singleValueField() {
return field();
}
} }

View file

@ -9,6 +9,7 @@ package org.elasticsearch.xpack.esql.optimizer;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.PropagateEmptyRelation; import org.elasticsearch.xpack.esql.optimizer.rules.logical.PropagateEmptyRelation;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStatsFilteredAggWithEval; import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStatsFilteredAggWithEval;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStringCasingWithInsensitiveRegexMatch;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferIsNotNull; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferIsNotNull;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferNonNullAggConstraint; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.InferNonNullAggConstraint;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.LocalPropagateEmptyRelation; import org.elasticsearch.xpack.esql.optimizer.rules.logical.local.LocalPropagateEmptyRelation;
@ -33,8 +34,7 @@ import static org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer.operat
*/ */
public class LocalLogicalPlanOptimizer extends ParameterizedRuleExecutor<LogicalPlan, LocalLogicalOptimizerContext> { public class LocalLogicalPlanOptimizer extends ParameterizedRuleExecutor<LogicalPlan, LocalLogicalOptimizerContext> {
private static final List<Batch<LogicalPlan>> RULES = replaceRules( private static final List<Batch<LogicalPlan>> RULES = arrayAsArrayList(
arrayAsArrayList(
new Batch<>( new Batch<>(
"Local rewrite", "Local rewrite",
Limiter.ONCE, Limiter.ONCE,
@ -43,9 +43,8 @@ public class LocalLogicalPlanOptimizer extends ParameterizedRuleExecutor<Logical
new InferIsNotNull(), new InferIsNotNull(),
new InferNonNullAggConstraint() new InferNonNullAggConstraint()
), ),
operators(), localOperators(),
cleanup() cleanup()
)
); );
public LocalLogicalPlanOptimizer(LocalLogicalOptimizerContext localLogicalOptimizerContext) { public LocalLogicalPlanOptimizer(LocalLogicalOptimizerContext localLogicalOptimizerContext) {
@ -58,27 +57,26 @@ public class LocalLogicalPlanOptimizer extends ParameterizedRuleExecutor<Logical
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
private static List<Batch<LogicalPlan>> replaceRules(List<Batch<LogicalPlan>> listOfRules) { private static Batch<LogicalPlan> localOperators() {
List<Batch<LogicalPlan>> newBatches = new ArrayList<>(listOfRules.size()); var operators = operators();
for (var batch : listOfRules) { var rules = operators().rules();
var rules = batch.rules();
List<Rule<?, LogicalPlan>> newRules = new ArrayList<>(rules.length); List<Rule<?, LogicalPlan>> newRules = new ArrayList<>(rules.length);
boolean updated = false;
// apply updates to existing rules that have different applicability locally
for (var r : rules) { for (var r : rules) {
if (r instanceof PropagateEmptyRelation) { switch (r) {
newRules.add(new LocalPropagateEmptyRelation()); case PropagateEmptyRelation ignoredPropagate -> newRules.add(new LocalPropagateEmptyRelation());
updated = true;
} else if (r instanceof ReplaceStatsFilteredAggWithEval) {
// skip it: once a fragment contains an Agg, this can no longer be pruned, which the rule can do // skip it: once a fragment contains an Agg, this can no longer be pruned, which the rule can do
updated = true; case ReplaceStatsFilteredAggWithEval ignoredReplace -> {
} else { }
newRules.add(r); default -> newRules.add(r);
} }
} }
batch = updated ? batch.with(newRules.toArray(Rule[]::new)) : batch;
newBatches.add(batch); // add rule that should only apply locally
} newRules.add(new ReplaceStringCasingWithInsensitiveRegexMatch());
return newBatches;
return operators.with(newRules.toArray(Rule[]::new));
} }
public LogicalPlan localOptimize(LogicalPlan plan) { public LogicalPlan localOptimize(LogicalPlan plan) {

View file

@ -66,7 +66,7 @@ public class ReplaceStringCasingWithInsensitiveEquals extends OptimizerRules.Opt
return e; return e;
} }
private static Expression unwrapCase(Expression e) { static Expression unwrapCase(Expression e) {
for (; e instanceof ChangeCase cc; e = cc.field()) { for (; e instanceof ChangeCase cc; e = cc.field()) {
} }
return e; return e;

View file

@ -0,0 +1,51 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
package org.elasticsearch.xpack.esql.optimizer.rules.logical;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.Literal;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.StringPattern;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.ChangeCase;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
import static org.elasticsearch.xpack.esql.optimizer.rules.logical.ReplaceStringCasingWithInsensitiveEquals.unwrapCase;
public class ReplaceStringCasingWithInsensitiveRegexMatch extends OptimizerRules.OptimizerExpressionRule<
RegexMatch<? extends StringPattern>> {
public ReplaceStringCasingWithInsensitiveRegexMatch() {
super(OptimizerRules.TransformDirection.DOWN);
}
@Override
protected Expression rule(RegexMatch<? extends StringPattern> regexMatch, LogicalOptimizerContext unused) {
Expression e = regexMatch;
if (regexMatch.field() instanceof ChangeCase changeCase) {
var pattern = regexMatch.pattern().pattern();
e = changeCase.caseType().matchesCase(pattern) ? insensitiveRegexMatch(regexMatch) : Literal.of(regexMatch, Boolean.FALSE);
}
return e;
}
private static Expression insensitiveRegexMatch(RegexMatch<? extends StringPattern> regexMatch) {
return switch (regexMatch) {
case RLike rlike -> new RLike(rlike.source(), unwrapCase(rlike.field()), rlike.pattern(), true);
case WildcardLike wildcardLike -> new WildcardLike(
wildcardLike.source(),
unwrapCase(wildcardLike.field()),
wildcardLike.pattern(),
true
);
default -> regexMatch;
};
}
}

View file

@ -42,8 +42,8 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionResolutionStrate
import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction;
import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression;
import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchOperator; import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchOperator;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or;

View file

@ -20,8 +20,8 @@ import org.elasticsearch.xcontent.json.JsonXContent;
import org.elasticsearch.xpack.esql.CsvTestsDataLoader; import org.elasticsearch.xpack.esql.CsvTestsDataLoader;
import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchOperator; import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchOperator;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or;

View file

@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import java.io.IOException; import java.io.IOException;

View file

@ -20,13 +20,17 @@ import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.junit.AfterClass; import org.junit.AfterClass;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Locale;
import java.util.function.Function; import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier; import java.util.function.Supplier;
import static org.elasticsearch.xpack.esql.core.util.TestUtils.randomCasing;
import static org.elasticsearch.xpack.esql.expression.function.DocsV3Support.renderNegatedOperator; import static org.elasticsearch.xpack.esql.expression.function.DocsV3Support.renderNegatedOperator;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.nullValue;
@ -39,12 +43,13 @@ public class RLikeTests extends AbstractScalarFunctionTestCase {
@ParametersFactory @ParametersFactory
public static Iterable<Object[]> parameters() { public static Iterable<Object[]> parameters() {
return parameters(str -> { final Function<String, String> escapeString = str -> {
for (String syntax : new String[] { "\\", ".", "?", "+", "*", "|", "{", "}", "[", "]", "(", ")", "\"", "<", ">", "#", "&" }) { for (String syntax : new String[] { "\\", ".", "?", "+", "*", "|", "{", "}", "[", "]", "(", ")", "\"", "<", ">", "#", "&" }) {
str = str.replace(syntax, "\\" + syntax); str = str.replace(syntax, "\\" + syntax);
} }
return str; return str;
}, () -> randomAlphaOfLength(1) + "?"); };
return parameters(escapeString, () -> randomAlphaOfLength(1) + "?");
} }
static Iterable<Object[]> parameters(Function<String, String> escapeString, Supplier<String> optionalPattern) { static Iterable<Object[]> parameters(Function<String, String> escapeString, Supplier<String> optionalPattern) {
@ -88,24 +93,52 @@ public class RLikeTests extends AbstractScalarFunctionTestCase {
String text = textSupplier.get(); String text = textSupplier.get();
return new TextAndPattern(text, escapeString.apply(text)); return new TextAndPattern(text, escapeString.apply(text));
}, true); }, true);
cases(cases, title + " matches self case insensitive", () -> {
String text = textSupplier.get();
return new TextAndPattern(randomCasing(text), escapeString.apply(text));
}, true, true);
cases(cases, title + " doesn't match self with trailing", () -> { cases(cases, title + " doesn't match self with trailing", () -> {
String text = textSupplier.get(); String text = textSupplier.get();
return new TextAndPattern(text, escapeString.apply(text) + randomAlphaOfLength(1)); return new TextAndPattern(text, escapeString.apply(text) + randomAlphaOfLength(1));
}, false); }, false);
cases(cases, title + " doesn't match self with trailing case insensitive", () -> {
String text = textSupplier.get();
return new TextAndPattern(randomCasing(text), escapeString.apply(text) + randomAlphaOfLength(1));
}, true, false);
cases(cases, title + " matches self with optional trailing", () -> { cases(cases, title + " matches self with optional trailing", () -> {
String text = randomAlphaOfLength(1); String text = randomAlphaOfLength(1);
return new TextAndPattern(text, escapeString.apply(text) + optionalPattern.get()); return new TextAndPattern(text, escapeString.apply(text) + optionalPattern.get());
}, true); }, true);
cases(cases, title + " matches self with optional trailing case insensitive", () -> {
String text = randomAlphaOfLength(1);
return new TextAndPattern(randomCasing(text), escapeString.apply(text) + optionalPattern.get());
}, true, true);
if (canGenerateDifferent) { if (canGenerateDifferent) {
cases(cases, title + " doesn't match different", () -> { cases(cases, title + " doesn't match different", () -> {
String text = textSupplier.get(); String text = textSupplier.get();
String different = escapeString.apply(randomValueOtherThan(text, textSupplier)); String different = escapeString.apply(randomValueOtherThan(text, textSupplier));
return new TextAndPattern(text, different); return new TextAndPattern(text, different);
}, false); }, false);
cases(cases, title + " doesn't match different case insensitive", () -> {
String text = textSupplier.get();
Predicate<String> predicate = t -> t.toLowerCase(Locale.ROOT).equals(text.toLowerCase(Locale.ROOT));
String different = escapeString.apply(randomValueOtherThanMany(predicate, textSupplier));
return new TextAndPattern(text, different);
}, true, false);
} }
} }
private static void cases(List<TestCaseSupplier> cases, String title, Supplier<TextAndPattern> textAndPattern, boolean expected) { private static void cases(List<TestCaseSupplier> cases, String title, Supplier<TextAndPattern> textAndPattern, boolean expected) {
cases(cases, title, textAndPattern, false, expected);
}
private static void cases(
List<TestCaseSupplier> cases,
String title,
Supplier<TextAndPattern> textAndPattern,
boolean caseInsensitive,
boolean expected
) {
for (DataType type : DataType.stringTypes()) { for (DataType type : DataType.stringTypes()) {
cases.add(new TestCaseSupplier(title + " with " + type.esType(), List.of(type, DataType.KEYWORD, DataType.BOOLEAN), () -> { cases.add(new TestCaseSupplier(title + " with " + type.esType(), List.of(type, DataType.KEYWORD, DataType.BOOLEAN), () -> {
TextAndPattern v = textAndPattern.get(); TextAndPattern v = textAndPattern.get();
@ -113,13 +146,14 @@ public class RLikeTests extends AbstractScalarFunctionTestCase {
List.of( List.of(
new TestCaseSupplier.TypedData(new BytesRef(v.text), type, "e"), new TestCaseSupplier.TypedData(new BytesRef(v.text), type, "e"),
new TestCaseSupplier.TypedData(new BytesRef(v.pattern), DataType.KEYWORD, "pattern").forceLiteral(), new TestCaseSupplier.TypedData(new BytesRef(v.pattern), DataType.KEYWORD, "pattern").forceLiteral(),
new TestCaseSupplier.TypedData(false, DataType.BOOLEAN, "caseInsensitive").forceLiteral() new TestCaseSupplier.TypedData(caseInsensitive, DataType.BOOLEAN, "caseInsensitive").forceLiteral()
), ),
startsWith("AutomataMatchEvaluator[input=Attribute[channel=0], pattern=digraph Automaton {\n"), startsWith("AutomataMatchEvaluator[input=Attribute[channel=0], pattern=digraph Automaton {\n"),
DataType.BOOLEAN, DataType.BOOLEAN,
equalTo(expected) equalTo(expected)
); );
})); }));
if (caseInsensitive == false) {
cases.add(new TestCaseSupplier(title + " with " + type.esType(), List.of(type, DataType.KEYWORD), () -> { cases.add(new TestCaseSupplier(title + " with " + type.esType(), List.of(type, DataType.KEYWORD), () -> {
TextAndPattern v = textAndPattern.get(); TextAndPattern v = textAndPattern.get();
return new TestCaseSupplier.TestCase( return new TestCaseSupplier.TestCase(
@ -134,6 +168,7 @@ public class RLikeTests extends AbstractScalarFunctionTestCase {
})); }));
} }
} }
}
@Override @Override
protected Expression build(Source source, List<Expression> args) { protected Expression build(Source source, List<Expression> args) {
@ -150,7 +185,9 @@ public class RLikeTests extends AbstractScalarFunctionTestCase {
return caseInsensitiveBool return caseInsensitiveBool
? new RLike(source, expression, new RLikePattern(patternString), true) ? new RLike(source, expression, new RLikePattern(patternString), true)
: new RLike(source, expression, new RLikePattern(patternString)); : (randomBoolean()
? new RLike(source, expression, new RLikePattern(patternString))
: new RLike(source, expression, new RLikePattern(patternString), false));
} }
@AfterClass @AfterClass

View file

@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import java.io.IOException; import java.io.IOException;

View file

@ -20,10 +20,12 @@ import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
import org.elasticsearch.xpack.esql.expression.function.FunctionName; import org.elasticsearch.xpack.esql.expression.function.FunctionName;
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.junit.AfterClass; import org.junit.AfterClass;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.function.Function;
import java.util.function.Supplier; import java.util.function.Supplier;
import static org.elasticsearch.xpack.esql.expression.function.DocsV3Support.renderNegatedOperator; import static org.elasticsearch.xpack.esql.expression.function.DocsV3Support.renderNegatedOperator;
@ -38,12 +40,13 @@ public class WildcardLikeTests extends AbstractScalarFunctionTestCase {
@ParametersFactory @ParametersFactory
public static Iterable<Object[]> parameters() { public static Iterable<Object[]> parameters() {
List<Object[]> cases = (List<Object[]>) RLikeTests.parameters(str -> { final Function<String, String> escapeString = str -> {
for (String syntax : new String[] { "\\", "*" }) { for (String syntax : new String[] { "\\", "*", "?" }) {
str = str.replace(syntax, "\\" + syntax); str = str.replace(syntax, "\\" + syntax);
} }
return str; return str;
}, () -> "*"); };
List<Object[]> cases = (List<Object[]>) RLikeTests.parameters(escapeString, () -> "*");
List<TestCaseSupplier> suppliers = new ArrayList<>(); List<TestCaseSupplier> suppliers = new ArrayList<>();
addCases(suppliers); addCases(suppliers);
@ -83,11 +86,15 @@ public class WildcardLikeTests extends AbstractScalarFunctionTestCase {
static Expression buildWildcardLike(Source source, List<Expression> args) { static Expression buildWildcardLike(Source source, List<Expression> args) {
Expression expression = args.get(0); Expression expression = args.get(0);
Literal pattern = (Literal) args.get(1); Literal pattern = (Literal) args.get(1);
if (args.size() > 2) { Literal caseInsensitive = args.size() > 2 ? (Literal) args.get(2) : null;
Literal caseInsensitive = (Literal) args.get(2); boolean caseInsesitiveBool = caseInsensitive != null && (boolean) caseInsensitive.fold(FoldContext.small());
assertThat(caseInsensitive.fold(FoldContext.small()), equalTo(false));
} WildcardPattern wildcardPattern = new WildcardPattern(((BytesRef) pattern.fold(FoldContext.small())).utf8ToString());
return new WildcardLike(source, expression, new WildcardPattern(((BytesRef) pattern.fold(FoldContext.small())).utf8ToString())); return caseInsesitiveBool
? new WildcardLike(source, expression, wildcardPattern, true)
: (randomBoolean()
? new WildcardLike(source, expression, wildcardPattern)
: new WildcardLike(source, expression, wildcardPattern, false));
} }
@AfterClass @AfterClass

View file

@ -32,6 +32,8 @@ import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case;
import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull;
import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
@ -79,6 +81,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizer
import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY;
import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.not;
@ -635,6 +638,88 @@ public class LocalLogicalPlanOptimizerTests extends ESTestCase {
var source = as(filter.child(), EsRelation.class); var source = as(filter.child(), EsRelation.class);
} }
/*
* Limit[1000[INTEGER],false]
* \_Filter[RLIKE(first_name{f}#4, "VALÜ*", true)]
* \_EsRelation[test][_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen..]
*/
public void testReplaceUpperStringCasinqgWithInsensitiveRLike() {
var plan = localPlan("FROM test | WHERE TO_UPPER(TO_LOWER(TO_UPPER(first_name))) RLIKE \"VALÜ*\"");
var limit = as(plan, Limit.class);
var filter = as(limit.child(), Filter.class);
var rlike = as(filter.condition(), RLike.class);
var field = as(rlike.field(), FieldAttribute.class);
assertThat(field.fieldName(), is("first_name"));
assertThat(rlike.pattern().pattern(), is("VALÜ*"));
assertThat(rlike.caseInsensitive(), is(true));
var source = as(filter.child(), EsRelation.class);
}
// same plan as above, but lower case pattern
public void testReplaceLowerStringCasingWithInsensitiveRLike() {
var plan = localPlan("FROM test | WHERE TO_LOWER(TO_UPPER(first_name)) RLIKE \"valü*\"");
var limit = as(plan, Limit.class);
var filter = as(limit.child(), Filter.class);
var rlike = as(filter.condition(), RLike.class);
var field = as(rlike.field(), FieldAttribute.class);
assertThat(field.fieldName(), is("first_name"));
assertThat(rlike.pattern().pattern(), is("valü*"));
assertThat(rlike.caseInsensitive(), is(true));
var source = as(filter.child(), EsRelation.class);
}
/**
* LocalRelation[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12, langu
* ages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8],EMPTY]
*/
public void testReplaceStringCasingAndRLikeWithLocalRelation() {
var plan = localPlan("FROM test | WHERE TO_LOWER(TO_UPPER(first_name)) RLIKE \"VALÜ*\"");
var local = as(plan, LocalRelation.class);
assertThat(local.supplier(), equalTo(LocalSupplier.EMPTY));
}
// same plan as in testReplaceUpperStringCasingWithInsensitiveRLike, but with LIKE instead of RLIKE
public void testReplaceUpperStringCasingWithInsensitiveLike() {
var plan = localPlan("FROM test | WHERE TO_UPPER(TO_LOWER(TO_UPPER(first_name))) LIKE \"VALÜ*\"");
var limit = as(plan, Limit.class);
var filter = as(limit.child(), Filter.class);
var wlike = as(filter.condition(), WildcardLike.class);
var field = as(wlike.field(), FieldAttribute.class);
assertThat(field.fieldName(), is("first_name"));
assertThat(wlike.pattern().pattern(), is("VALÜ*"));
assertThat(wlike.caseInsensitive(), is(true));
var source = as(filter.child(), EsRelation.class);
}
// same plan as above, but lower case pattern
public void testReplaceLowerStringCasingWithInsensitiveLike() {
var plan = localPlan("FROM test | WHERE TO_LOWER(TO_UPPER(first_name)) LIKE \"valü*\"");
var limit = as(plan, Limit.class);
var filter = as(limit.child(), Filter.class);
var wlike = as(filter.condition(), WildcardLike.class);
var field = as(wlike.field(), FieldAttribute.class);
assertThat(field.fieldName(), is("first_name"));
assertThat(wlike.pattern().pattern(), is("valü*"));
assertThat(wlike.caseInsensitive(), is(true));
var source = as(filter.child(), EsRelation.class);
}
/**
* LocalRelation[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12, langu
* ages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8],EMPTY]
*/
public void testReplaceStringCasingAndLikeWithLocalRelation() {
var plan = localPlan("FROM test | WHERE TO_LOWER(TO_UPPER(first_name)) LIKE \"VALÜ*\"");
var local = as(plan, LocalRelation.class);
assertThat(local.supplier(), equalTo(LocalSupplier.EMPTY));
}
private IsNotNull isNotNull(Expression field) { private IsNotNull isNotNull(Expression field) {
return new IsNotNull(EMPTY, field); return new IsNotNull(EMPTY, field);
} }

View file

@ -83,6 +83,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialWi
import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower; import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper; import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or;
@ -205,7 +206,7 @@ import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.Matchers.startsWith; import static org.hamcrest.Matchers.startsWith;
// @TestLogging(value = "org.elasticsearch.xpack.esql:DEBUG", reason = "debug") // @TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug")
public class PhysicalPlanOptimizerTests extends ESTestCase { public class PhysicalPlanOptimizerTests extends ESTestCase {
private static final String PARAM_FORMATTING = "%1$s"; private static final String PARAM_FORMATTING = "%1$s";
@ -2201,6 +2202,163 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
assertThat(source.query(), nullValue()); assertThat(source.query(), nullValue());
} }
/*
* LimitExec[1000[INTEGER]]
* \_ExchangeExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12, langu
* ages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8],false]
* \_ProjectExec[[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gender{f}#5, hire_date{f}#10, job{f}#11, job.raw{f}#12, langu
* ages{f}#6, last_name{f}#7, long_noidx{f}#13, salary{f}#8]]
* \_FieldExtractExec[_meta_field{f}#9, emp_no{f}#3, first_name{f}#4, gen..]<[],[]>
* \_EsQueryExec[test], indexMode[standard], query[{"esql_single_value":{"field":"first_name","next":{"regexp":{"first_name":
* {"value":"foo*","flags_value":65791,"case_insensitive":true,"max_determinized_states":10000,"boost":0.0}}},
* "source":"TO_LOWER(first_name) RLIKE \"foo*\"@2:9"}}][_doc{f}#25], limit[1000], sort[] estimatedRowSize[332]
*/
private void doTestPushDownCaseChangeRegexMatch(String query, String expected) {
var plan = physicalPlan(query);
var optimized = optimizedPlan(plan);
var topLimit = as(optimized, LimitExec.class);
var exchange = asRemoteExchange(topLimit.child());
var project = as(exchange.child(), ProjectExec.class);
var fieldExtract = as(project.child(), FieldExtractExec.class);
var source = as(fieldExtract.child(), EsQueryExec.class);
var singleValue = as(source.query(), SingleValueQuery.Builder.class);
assertThat(stripThrough(singleValue.toString()), is(stripThrough(expected)));
}
public void testPushDownLowerCaseChangeRLike() {
doTestPushDownCaseChangeRegexMatch("""
FROM test
| WHERE TO_LOWER(first_name) RLIKE "foo*"
""", """
{
"esql_single_value": {
"field": "first_name",
"next": {
"regexp": {
"first_name": {
"value": "foo*",
"flags_value": 65791,
"case_insensitive": true,
"max_determinized_states": 10000,
"boost": 0.0
}
}
},
"source": "TO_LOWER(first_name) RLIKE \\"foo*\\"@2:9"
}
}
""");
}
public void testPushDownUpperCaseChangeRLike() {
doTestPushDownCaseChangeRegexMatch("""
FROM test
| WHERE TO_UPPER(first_name) RLIKE "FOO*"
""", """
{
"esql_single_value": {
"field": "first_name",
"next": {
"regexp": {
"first_name": {
"value": "FOO*",
"flags_value": 65791,
"case_insensitive": true,
"max_determinized_states": 10000,
"boost": 0.0
}
}
},
"source": "TO_UPPER(first_name) RLIKE \\"FOO*\\"@2:9"
}
}
""");
}
public void testPushDownLowerCaseChangeLike() {
doTestPushDownCaseChangeRegexMatch("""
FROM test
| WHERE TO_LOWER(first_name) LIKE "foo*"
""", """
{
"esql_single_value": {
"field": "first_name",
"next": {
"wildcard": {
"first_name": {
"wildcard": "foo*",
"case_insensitive": true,
"boost": 0.0
}
}
},
"source": "TO_LOWER(first_name) LIKE \\"foo*\\"@2:9"
}
}
""");
}
public void testPushDownUpperCaseChangeLike() {
doTestPushDownCaseChangeRegexMatch("""
FROM test
| WHERE TO_UPPER(first_name) LIKE "FOO*"
""", """
{
"esql_single_value": {
"field": "first_name",
"next": {
"wildcard": {
"first_name": {
"wildcard": "FOO*",
"case_insensitive": true,
"boost": 0.0
}
}
},
"source": "TO_UPPER(first_name) LIKE \\"FOO*\\"@2:9"
}
}
""");
}
/*
* LimitExec[1000[INTEGER]]
* \_ExchangeExec[[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, gender{f}#6, hire_date{f}#11, job{f}#12, job.raw{f}#13, lang
* uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9],false]
* \_ProjectExec[[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, gender{f}#6, hire_date{f}#11, job{f}#12, job.raw{f}#13, lang
* uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9]]
* \_FieldExtractExec[_meta_field{f}#10, gender{f}#6, hire_date{f}#11, jo..]<[],[]>
* \_LimitExec[1000[INTEGER]]
* \_FilterExec[LIKE(first_name{f}#5, "FOO*", true) OR IN(1[INTEGER],2[INTEGER],3[INTEGER],emp_no{f}#4 + 1[INTEGER])]
* \_FieldExtractExec[first_name{f}#5, emp_no{f}#4]<[],[]>
* \_EsQueryExec[test], indexMode[standard], query[][_doc{f}#26], limit[], sort[] estimatedRowSize[332]
*/
public void testChangeCaseAsInsensitiveWildcardLikeNotPushedDown() {
var esql = """
FROM test
| WHERE TO_UPPER(first_name) LIKE "FOO*" OR emp_no + 1 IN (1, 2, 3)
""";
var plan = physicalPlan(esql);
var optimized = optimizedPlan(plan);
var topLimit = as(optimized, LimitExec.class);
var exchange = asRemoteExchange(topLimit.child());
var project = as(exchange.child(), ProjectExec.class);
var fieldExtract = as(project.child(), FieldExtractExec.class);
var limit = as(fieldExtract.child(), LimitExec.class);
var filter = as(limit.child(), FilterExec.class);
fieldExtract = as(filter.child(), FieldExtractExec.class);
var source = as(fieldExtract.child(), EsQueryExec.class);
var or = as(filter.condition(), Or.class);
var wildcard = as(or.left(), WildcardLike.class);
assertThat(Expressions.name(wildcard.field()), is("first_name"));
assertThat(wildcard.pattern().pattern(), is("FOO*"));
assertThat(wildcard.caseInsensitive(), is(true));
}
public void testPushDownNotRLike() { public void testPushDownNotRLike() {
var plan = physicalPlan(""" var plan = physicalPlan("""
from test from test
@ -2432,6 +2590,17 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES + KEYWORD_EST)); assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES + KEYWORD_EST));
} }
/*
* LimitExec[1000[INTEGER]]
* \_ExchangeExec[[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, gender{f}#6, hire_date{f}#11, job{f}#12, job.raw{f}#13, lang
* uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9, _index{m}#2],false]
* \_ProjectExec[[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, gender{f}#6, hire_date{f}#11, job{f}#12, job.raw{f}#13, lang
* uages{f}#7, last_name{f}#8, long_noidx{f}#14, salary{f}#9, _index{m}#2]]
* \_FieldExtractExec[_meta_field{f}#10, emp_no{f}#4, first_name{f}#5, ge..]<[],[]>
* \_EsQueryExec[test], indexMode[standard], query[{"wildcard":{"_index":{"wildcard":"test*","boost":0.0}}}][_doc{f}#27],
* limit[1000], sort[] estimatedRowSize[382]
*
*/
public void testPushDownMetadataIndexInWildcard() { public void testPushDownMetadataIndexInWildcard() {
var plan = physicalPlan(""" var plan = physicalPlan("""
from test metadata _index from test metadata _index
@ -8097,7 +8266,7 @@ public class PhysicalPlanOptimizerTests extends ESTestCase {
var physical = mapper.map(logical); var physical = mapper.map(logical);
// System.out.println("Physical\n" + physical); // System.out.println("Physical\n" + physical);
if (assertSerialization) { if (assertSerialization) {
assertSerialization(physical); assertSerialization(physical, config);
} }
return physical; return physical;
} }

View file

@ -18,8 +18,8 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.Range; import org.elasticsearch.xpack.esql.expression.predicate.Range;
import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.expression.predicate.logical.Not;

View file

@ -47,8 +47,8 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMedi
import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim; import org.elasticsearch.xpack.esql.expression.function.scalar.string.LTrim;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or;
import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull;

View file

@ -18,8 +18,8 @@ import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count;
import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match;
import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pow; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Pow;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.Predicates; import org.elasticsearch.xpack.esql.expression.predicate.Predicates;
import org.elasticsearch.xpack.esql.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.expression.predicate.logical.And;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;

View file

@ -15,8 +15,8 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch;
import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern;
import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.core.util.StringUtils;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.expression.predicate.nulls.IsNotNull;
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;

View file

@ -28,8 +28,8 @@ import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction;
import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression;
import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchOperator; import org.elasticsearch.xpack.esql.expression.function.fulltext.MatchOperator;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.RLike;
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.regex.WildcardLike;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.expression.predicate.logical.Not;
import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or;
import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;