ESQL: Clone ql for esql (#108773)

Part of https://github.com/elastic/elasticsearch/issues/106679

* Copy the `ql` project into a different project _just for esql_, call it `esql-core`.
* Make `esql` depend only on the latter.
* Fix `EsqlNodeSubclassTests`; I'm confused why this didn't bite us earlier.
* Update the warning regexes in some csv tests as the exceptions have other package names now.

**Note to reviewers:** Exclude the first commit when viewing the diff,
as that contains only the actual copying of `ql`. The remaining commits
are the actually meaningful ones. _The `build.gradle` files probably
require the most attention._

benchmarks/build.gradle

@@ -38,7 +38,7 @@ dependencies {
     exclude group: 'net.sf.jopt-simple', module: 'jopt-simple'
   }
   api(project(':modules:aggregations'))
-  api(project(':x-pack:plugin:ql'))
+  api(project(':x-pack:plugin:esql-core'))
   api(project(':x-pack:plugin:esql'))
   api(project(':x-pack:plugin:esql:compute'))
   implementation project(path: ':libs:elasticsearch-vec')

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -22,6 +22,12 @@ import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.compute.operator.Operator;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs;
@@ -31,12 +37,6 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.planner.Layout;
 import org.elasticsearch.xpack.esql.type.EsqlDataTypes;
-import org.elasticsearch.xpack.ql.expression.FieldAttribute;
-import org.elasticsearch.xpack.ql.expression.Literal;
-import org.elasticsearch.xpack.ql.expression.predicate.regex.RLikePattern;
-import org.elasticsearch.xpack.ql.tree.Source;
-import org.elasticsearch.xpack.ql.type.DataTypes;
-import org.elasticsearch.xpack.ql.type.EsField;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;

docs/reference/esql/functions/examples/to_integer.asciidoc

@@ -19,6 +19,6 @@ The header will provide information on the source of the failure:
 
 A following header will contain the failure reason and the offending value:
 
-`"org.elasticsearch.xpack.ql.InvalidArgumentException: [501379200000] out of [integer] range"`
+`"org.elasticsearch.xpack.esql.core.InvalidArgumentException: [501379200000] out of [integer] range"`
 
 

x-pack/plugin/esql-core/README.md

@@ -0,0 +1,14 @@
+# ES|QL core
+
+This project originated as a copy of the `ql` x-pack plugin.
+It contains some fundamental classes used in `esql`, like `Node`, its subclasses `Expression`, `QueryPlan`, and the plan optimizer code.
+Originally, `ql` shared classes between ES|QL, SQL and EQL, but ES|QL diverged far enough to justify a split.
+
+## Warning
+
+- **Consider the contents of this project untested.**
+  There may be some tests in `sql` and `eql` that may have indirectly covered the initial version of this (when it was copied from `ql`);
+  but neither do these tests apply to `esql`, nor do they even run against this.
+- **Consider this project technical debt.**
+  The contents of this project need to be consolidated with the `esql` plugin.
+  In particular, there is a significant amount of code (or code paths) that are not used/executed at all in `esql`.

x-pack/plugin/esql-core/build.gradle

@@ -0,0 +1,24 @@
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.internal-test-artifact'
+
+esplugin {
+  name 'x-pack-esql-core'
+  description 'Elasticsearch infrastructure plugin for ESQL'
+  classname 'org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin'
+  extendedPlugins = ['x-pack-core']
+}
+
+base {
+  archivesName = 'x-pack-esql-core'
+}
+
+dependencies {
+  api "org.antlr:antlr4-runtime:${versions.antlr4}"
+  api project(path: xpackModule('mapper-version'))
+  compileOnly project(path: xpackModule('core'))
+  testApi(project(xpackModule('esql-core:test-fixtures'))) {
+    exclude group: 'org.elasticsearch.plugin', module: 'esql-core'
+  }
+  testImplementation project(':test:framework')
+  testImplementation(testArtifact(project(xpackModule('core'))))
+}

x-pack/plugin/esql-core/licenses/antlr4-runtime-LICENSE.txt

@@ -0,0 +1,26 @@
+[The "BSD license"]
+Copyright (c) 2015 Terence Parr, Sam Harwell
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+ 1. Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in the
+    documentation and/or other materials provided with the distribution.
+ 3. The name of the author may not be used to endorse or promote products
+    derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/InvalidArgumentException.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core;
+
+/**
+ * Exception thrown when unable to continue processing client request,
+ * in cases such as invalid query parameter or failure to apply requested processing to given data.
+ * It's meant as a generic equivalent to QlIllegalArgumentException (that's a server exception).
+ * TODO: reason for [E|S|ES]QL specializations of QlIllegalArgumentException?
+ * TODO: the intended use of ql.ParsingException, vs its [E|S|ES]QL equivalents, subclassed from the respective XxxClientException?
+ *       Same for PlanningException.
+ */
+public class InvalidArgumentException extends QlClientException {
+
+    public InvalidArgumentException(String message, Object... args) {
+        super(message, args);
+    }
+
+    public InvalidArgumentException(Throwable cause, String message, Object... args) {
+        super(cause, message, args);
+    }
+
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/ParsingException.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core;
+
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
+
+public class ParsingException extends QlClientException {
+    private final int line;
+    private final int charPositionInLine;
+
+    public ParsingException(String message, Exception cause, int line, int charPositionInLine) {
+        super(message, cause);
+        this.line = line;
+        this.charPositionInLine = charPositionInLine;
+    }
+
+    public ParsingException(String message, Object... args) {
+        this(Source.EMPTY, message, args);
+    }
+
+    public ParsingException(Source source, String message, Object... args) {
+        super(message, args);
+        this.line = source.source().getLineNumber();
+        this.charPositionInLine = source.source().getColumnNumber();
+    }
+
+    public ParsingException(Exception cause, Source source, String message, Object... args) {
+        super(cause, message, args);
+        this.line = source.source().getLineNumber();
+        this.charPositionInLine = source.source().getColumnNumber();
+    }
+
+    public int getLineNumber() {
+        return line;
+    }
+
+    public int getColumnNumber() {
+        return charPositionInLine + 1;
+    }
+
+    public String getErrorMessage() {
+        return super.getMessage();
+    }
+
+    @Override
+    public String getMessage() {
+        return format("line {}:{}: {}", getLineNumber(), getColumnNumber(), getErrorMessage());
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/QlClientException.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core;
+
+import org.elasticsearch.rest.RestStatus;
+
+/**
+ * Exception thrown by performing client (or user) code.
+ * Typically it means the given action or query is incorrect and needs fixing.
+ */
+public class QlClientException extends QlException {
+
+    protected QlClientException(String message, Object... args) {
+        super(message, args);
+    }
+
+    protected QlClientException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+
+    protected QlClientException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    protected QlClientException(Throwable cause, String message, Object... args) {
+        super(cause, message, args);
+    }
+
+    protected QlClientException(Throwable cause) {
+        super(cause);
+    }
+
+    @Override
+    public RestStatus status() {
+        return RestStatus.BAD_REQUEST;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/QlException.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core;
+
+import org.elasticsearch.ElasticsearchException;
+
+/**
+ * Base class for all QL exceptions. Useful as a catch-all.
+ */
+public abstract class QlException extends ElasticsearchException {
+    public QlException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+
+    public QlException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public QlException(String message, Object... args) {
+        super(message, args);
+    }
+
+    public QlException(Throwable cause, String message, Object... args) {
+        super(message, cause, args);
+    }
+
+    public QlException(Throwable cause) {
+        super(cause);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/QlIllegalArgumentException.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core;
+
+public class QlIllegalArgumentException extends QlServerException {
+    public QlIllegalArgumentException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+
+    public QlIllegalArgumentException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public QlIllegalArgumentException(String message, Object... args) {
+        super(message, args);
+    }
+
+    public QlIllegalArgumentException(Throwable cause, String message, Object... args) {
+        super(cause, message, args);
+    }
+
+    public QlIllegalArgumentException(String message) {
+        super(message);
+    }
+
+    public QlIllegalArgumentException(Throwable cause) {
+        super(cause);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/QlServerException.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core;
+
+/**
+ * Exception triggered inside server-side code.
+ * Typically a validation error or worse, a bug.
+ */
+public abstract class QlServerException extends QlException {
+
+    protected QlServerException(String message, Object... args) {
+        super(message, args);
+    }
+
+    protected QlServerException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+
+    protected QlServerException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    protected QlServerException(Throwable cause, String message, Object... args) {
+        super(cause, message, args);
+    }
+
+    protected QlServerException(Throwable cause) {
+        super(cause);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/AnalyzerRules.java

@@ -0,0 +1,259 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.analyzer;
+
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute;
+import org.elasticsearch.xpack.esql.core.expression.function.Function;
+import org.elasticsearch.xpack.esql.core.expression.function.FunctionDefinition;
+import org.elasticsearch.xpack.esql.core.expression.function.FunctionRegistry;
+import org.elasticsearch.xpack.esql.core.expression.function.UnresolvedFunction;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic;
+import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.Equals;
+import org.elasticsearch.xpack.esql.core.plan.logical.Filter;
+import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.core.rule.ParameterizedRule;
+import org.elasticsearch.xpack.esql.core.rule.Rule;
+import org.elasticsearch.xpack.esql.core.session.Configuration;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.InvalidMappedField;
+import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
+
+public final class AnalyzerRules {
+
+    public static class AddMissingEqualsToBoolField extends AnalyzerRule<Filter> {
+
+        @Override
+        protected LogicalPlan rule(Filter filter) {
+            if (filter.resolved() == false) {
+                return filter;
+            }
+            // check the condition itself
+            Expression condition = replaceRawBoolFieldWithEquals(filter.condition());
+            // otherwise look for binary logic
+            if (condition == filter.condition()) {
+                condition = condition.transformUp(
+                    BinaryLogic.class,
+                    b -> b.replaceChildren(asList(replaceRawBoolFieldWithEquals(b.left()), replaceRawBoolFieldWithEquals(b.right())))
+                );
+            }
+
+            if (condition != filter.condition()) {
+                filter = filter.with(condition);
+            }
+            return filter;
+        }
+
+        private static Expression replaceRawBoolFieldWithEquals(Expression e) {
+            if (e instanceof FieldAttribute && e.dataType() == BOOLEAN) {
+                e = new Equals(e.source(), e, Literal.of(e, Boolean.TRUE));
+            }
+            return e;
+        }
+
+        @Override
+        protected boolean skipResolved() {
+            return false;
+        }
+    }
+
+    public abstract static class AnalyzerRule<SubPlan extends LogicalPlan> extends Rule<SubPlan, LogicalPlan> {
+
+        // transformUp (post-order) - that is first children and then the node
+        // but with a twist; only if the tree is not resolved or analyzed
+        @Override
+        public final LogicalPlan apply(LogicalPlan plan) {
+            return plan.transformUp(typeToken(), t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t));
+        }
+
+        protected abstract LogicalPlan rule(SubPlan plan);
+
+        protected boolean skipResolved() {
+            return true;
+        }
+    }
+
+    public abstract static class ParameterizedAnalyzerRule<SubPlan extends LogicalPlan, P> extends ParameterizedRule<
+        SubPlan,
+        LogicalPlan,
+        P> {
+
+        // transformUp (post-order) - that is first children and then the node
+        // but with a twist; only if the tree is not resolved or analyzed
+        public final LogicalPlan apply(LogicalPlan plan, P context) {
+            return plan.transformUp(typeToken(), t -> t.analyzed() || skipResolved() && t.resolved() ? t : rule(t, context));
+        }
+
+        protected abstract LogicalPlan rule(SubPlan plan, P context);
+
+        protected boolean skipResolved() {
+            return true;
+        }
+    }
+
+    public abstract static class BaseAnalyzerRule extends AnalyzerRule<LogicalPlan> {
+
+        @Override
+        protected LogicalPlan rule(LogicalPlan plan) {
+            if (plan.childrenResolved() == false) {
+                return plan;
+            }
+            return doRule(plan);
+        }
+
+        protected abstract LogicalPlan doRule(LogicalPlan plan);
+    }
+
+    public static Function resolveFunction(UnresolvedFunction uf, Configuration configuration, FunctionRegistry functionRegistry) {
+        Function f = null;
+        if (uf.analyzed()) {
+            f = uf;
+        } else if (uf.childrenResolved() == false) {
+            f = uf;
+        } else {
+            String functionName = functionRegistry.resolveAlias(uf.name());
+            if (functionRegistry.functionExists(functionName) == false) {
+                f = uf.missing(functionName, functionRegistry.listFunctions());
+            } else {
+                FunctionDefinition def = functionRegistry.resolveFunction(functionName);
+                f = uf.buildResolved(configuration, def);
+            }
+        }
+        return f;
+    }
+
+    public static List<Attribute> maybeResolveAgainstList(
+        UnresolvedAttribute u,
+        Collection<Attribute> attrList,
+        java.util.function.Function<Attribute, Attribute> fieldInspector
+    ) {
+        // first take into account the qualified version
+        final String qualifier = u.qualifier();
+        final String name = u.name();
+        final boolean qualified = u.qualifier() != null;
+
+        Predicate<Attribute> predicate = a -> {
+            return qualified ? Objects.equals(qualifier, a.qualifiedName()) :
+            // if the field is unqualified
+            // first check the names directly
+                (Objects.equals(name, a.name()))
+                    // but also if the qualifier might not be quoted and if there's any ambiguity with nested fields
+                    || Objects.equals(name, a.qualifiedName());
+
+        };
+        return maybeResolveAgainstList(predicate, () -> u, attrList, false, fieldInspector);
+    }
+
+    public static List<Attribute> maybeResolveAgainstList(
+        Predicate<Attribute> matcher,
+        Supplier<UnresolvedAttribute> unresolved,
+        Collection<Attribute> attrList,
+        boolean isPattern,
+        java.util.function.Function<Attribute, Attribute> fieldInspector
+    ) {
+        List<Attribute> matches = new ArrayList<>();
+
+        for (Attribute attribute : attrList) {
+            if (attribute.synthetic() == false) {
+                boolean match = matcher.test(attribute);
+                if (match) {
+                    matches.add(attribute);
+                }
+            }
+        }
+
+        if (matches.isEmpty()) {
+            return matches;
+        }
+
+        UnresolvedAttribute ua = unresolved.get();
+        // found exact match or multiple if pattern
+        if (matches.size() == 1 || isPattern) {
+            // NB: only add the location if the match is univocal; b/c otherwise adding the location will overwrite any preexisting one
+            matches.replaceAll(e -> fieldInspector.apply(e));
+            return matches;
+        }
+
+        // report ambiguity
+        List<String> refs = matches.stream().sorted((a, b) -> {
+            int lineDiff = a.sourceLocation().getLineNumber() - b.sourceLocation().getLineNumber();
+            int colDiff = a.sourceLocation().getColumnNumber() - b.sourceLocation().getColumnNumber();
+            return lineDiff != 0 ? lineDiff : (colDiff != 0 ? colDiff : a.qualifiedName().compareTo(b.qualifiedName()));
+        })
+            .map(
+                a -> "line "
+                    + a.sourceLocation().toString().substring(1)
+                    + " ["
+                    + (a.qualifier() != null ? "\"" + a.qualifier() + "\".\"" + a.name() + "\"" : a.name())
+                    + "]"
+            )
+            .toList();
+
+        return singletonList(
+            ua.withUnresolvedMessage(
+                "Reference ["
+                    + ua.qualifiedName()
+                    + "] is ambiguous (to disambiguate use quotes or qualifiers); "
+                    + "matches any of "
+                    + refs
+            )
+        );
+    }
+
+    public static Attribute handleSpecialFields(UnresolvedAttribute u, Attribute named, boolean allowCompound) {
+        // if it's a object/compound type, keep it unresolved with a nice error message
+        if (named instanceof FieldAttribute fa) {
+
+            // incompatible mappings
+            if (fa.field() instanceof InvalidMappedField imf) {
+                named = u.withUnresolvedMessage("Cannot use field [" + fa.name() + "] due to ambiguities being " + imf.errorMessage());
+            }
+            // unsupported types
+            else if (DataTypes.isUnsupported(fa.dataType())) {
+                UnsupportedEsField unsupportedField = (UnsupportedEsField) fa.field();
+                if (unsupportedField.hasInherited()) {
+                    named = u.withUnresolvedMessage(
+                        "Cannot use field ["
+                            + fa.name()
+                            + "] with unsupported type ["
+                            + unsupportedField.getOriginalType()
+                            + "] in hierarchy (field ["
+                            + unsupportedField.getInherited()
+                            + "])"
+                    );
+                } else {
+                    named = u.withUnresolvedMessage(
+                        "Cannot use field [" + fa.name() + "] with unsupported type [" + unsupportedField.getOriginalType() + "]"
+                    );
+                }
+            }
+            // compound fields
+            else if (allowCompound == false && DataTypes.isPrimitive(fa.dataType()) == false) {
+                named = u.withUnresolvedMessage(
+                    "Cannot use field [" + fa.name() + "] type [" + fa.dataType().typeName() + "] only its subfields"
+                );
+            }
+        }
+        // make sure to copy the resolved attribute with the proper location
+        return named.withLocation(u.source());
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/PreAnalyzer.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.analyzer;
+
+import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
+import org.elasticsearch.xpack.esql.core.plan.logical.UnresolvedRelation;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+
+// Since the pre-analyzer only inspect (and does NOT transform) the tree
+// it is not built as a rule executor.
+// Further more it applies 'the rules' only once and needs to return some
+// state back.
+public class PreAnalyzer {
+
+    public static class PreAnalysis {
+        public static final PreAnalysis EMPTY = new PreAnalysis(emptyList());
+
+        public final List<TableInfo> indices;
+
+        public PreAnalysis(List<TableInfo> indices) {
+            this.indices = indices;
+        }
+    }
+
+    public PreAnalysis preAnalyze(LogicalPlan plan) {
+        if (plan.analyzed()) {
+            return PreAnalysis.EMPTY;
+        }
+
+        return doPreAnalyze(plan);
+    }
+
+    private static PreAnalysis doPreAnalyze(LogicalPlan plan) {
+        List<TableInfo> indices = new ArrayList<>();
+
+        plan.forEachUp(UnresolvedRelation.class, p -> indices.add(new TableInfo(p.table(), p.frozen())));
+
+        // mark plan as preAnalyzed (if it were marked, there would be no analysis)
+        plan.forEachUp(LogicalPlan::setPreAnalyzed);
+
+        return new PreAnalysis(indices);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/TableInfo.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.analyzer;
+
+import org.elasticsearch.xpack.esql.core.plan.TableIdentifier;
+
+public class TableInfo {
+
+    private final TableIdentifier id;
+    private final boolean isFrozen;
+
+    public TableInfo(TableIdentifier id, boolean isFrozen) {
+        this.id = id;
+        this.isFrozen = isFrozen;
+    }
+
+    public TableIdentifier id() {
+        return id;
+    }
+
+    public boolean isFrozen() {
+        return isFrozen;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/analyzer/VerifierChecks.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.analyzer;
+
+import org.elasticsearch.xpack.esql.core.common.Failure;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.plan.logical.Filter;
+import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan;
+
+import java.util.Set;
+
+import static org.elasticsearch.xpack.esql.core.common.Failure.fail;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
+
+public final class VerifierChecks {
+
+    public static void checkFilterConditionType(LogicalPlan p, Set<Failure> localFailures) {
+        if (p instanceof Filter) {
+            Expression condition = ((Filter) p).condition();
+            if (condition.dataType() != BOOLEAN) {
+                localFailures.add(fail(condition, "Condition expression needs to be boolean, found [{}]", condition.dataType()));
+            }
+        }
+    }
+
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/async/AsyncTaskManagementService.java

@@ -0,0 +1,362 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.async;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.support.ListenerTimeouts;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.index.engine.DocumentMissingException;
+import org.elasticsearch.index.engine.VersionConflictEngineException;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskAwareRequest;
+import org.elasticsearch.tasks.TaskId;
+import org.elasticsearch.tasks.TaskManager;
+import org.elasticsearch.threadpool.Scheduler;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xpack.core.ClientHelper;
+import org.elasticsearch.xpack.core.async.AsyncExecutionId;
+import org.elasticsearch.xpack.core.async.AsyncTask;
+import org.elasticsearch.xpack.core.async.AsyncTaskIndexService;
+import org.elasticsearch.xpack.core.async.StoredAsyncResponse;
+import org.elasticsearch.xpack.core.async.StoredAsyncTask;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.elasticsearch.core.Strings.format;
+
+/**
+ * Service for managing EQL requests
+ */
+public class AsyncTaskManagementService<
+    Request extends TaskAwareRequest,
+    Response extends ActionResponse,
+    T extends StoredAsyncTask<Response>> {
+
+    private static final Logger logger = LogManager.getLogger(AsyncTaskManagementService.class);
+
+    private final TaskManager taskManager;
+    private final String action;
+    private final AsyncTaskIndexService<StoredAsyncResponse<Response>> asyncTaskIndexService;
+    private final AsyncOperation<Request, Response, T> operation;
+    private final ThreadPool threadPool;
+    private final ClusterService clusterService;
+    private final Class<T> taskClass;
+
+    public interface AsyncOperation<
+        Request extends TaskAwareRequest,
+        Response extends ActionResponse,
+        T extends CancellableTask & AsyncTask> {
+
+        T createTask(
+            Request request,
+            long id,
+            String type,
+            String action,
+            TaskId parentTaskId,
+            Map<String, String> headers,
+            Map<String, String> originHeaders,
+            AsyncExecutionId asyncExecutionId
+        );
+
+        void execute(Request request, T task, ActionListener<Response> listener);
+
+        Response initialResponse(T task);
+
+        Response readResponse(StreamInput inputStream) throws IOException;
+    }
+
+    /**
+     * Wrapper for EqlSearchRequest that creates an async version of EqlSearchTask
+     */
+    private class AsyncRequestWrapper implements TaskAwareRequest {
+        private final Request request;
+        private final String doc;
+        private final String node;
+
+        AsyncRequestWrapper(Request request, String node) {
+            this.request = request;
+            this.doc = UUIDs.randomBase64UUID();
+            this.node = node;
+        }
+
+        @Override
+        public void setParentTask(TaskId taskId) {
+            request.setParentTask(taskId);
+        }
+
+        @Override
+        public TaskId getParentTask() {
+            return request.getParentTask();
+        }
+
+        @Override
+        public void setRequestId(long requestId) {
+            request.setRequestId(requestId);
+        }
+
+        @Override
+        public long getRequestId() {
+            return request.getRequestId();
+        }
+
+        @Override
+        public Task createTask(long id, String type, String actionName, TaskId parentTaskId, Map<String, String> headers) {
+            Map<String, String> originHeaders = ClientHelper.getPersistableSafeSecurityHeaders(
+                threadPool.getThreadContext(),
+                clusterService.state()
+            );
+            return operation.createTask(
+                request,
+                id,
+                type,
+                actionName,
+                parentTaskId,
+                headers,
+                originHeaders,
+                new AsyncExecutionId(doc, new TaskId(node, id))
+            );
+        }
+
+        @Override
+        public String getDescription() {
+            return request.getDescription();
+        }
+    }
+
+    public AsyncTaskManagementService(
+        String index,
+        Client client,
+        String origin,
+        NamedWriteableRegistry registry,
+        TaskManager taskManager,
+        String action,
+        AsyncOperation<Request, Response, T> operation,
+        Class<T> taskClass,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        BigArrays bigArrays
+    ) {
+        this.taskManager = taskManager;
+        this.action = action;
+        this.operation = operation;
+        this.taskClass = taskClass;
+        this.asyncTaskIndexService = new AsyncTaskIndexService<>(
+            index,
+            clusterService,
+            threadPool.getThreadContext(),
+            client,
+            origin,
+            i -> new StoredAsyncResponse<>(operation::readResponse, i),
+            registry,
+            bigArrays
+        );
+        this.clusterService = clusterService;
+        this.threadPool = threadPool;
+    }
+
+    public void asyncExecute(
+        Request request,
+        TimeValue waitForCompletionTimeout,
+        TimeValue keepAlive,
+        boolean keepOnCompletion,
+        ActionListener<Response> listener
+    ) {
+        String nodeId = clusterService.localNode().getId();
+        try (var ignored = threadPool.getThreadContext().newTraceContext()) {
+            @SuppressWarnings("unchecked")
+            T searchTask = (T) taskManager.register("transport", action + "[a]", new AsyncRequestWrapper(request, nodeId));
+            boolean operationStarted = false;
+            try {
+                operation.execute(
+                    request,
+                    searchTask,
+                    wrapStoringListener(searchTask, waitForCompletionTimeout, keepAlive, keepOnCompletion, listener)
+                );
+                operationStarted = true;
+            } finally {
+                // If we didn't start operation for any reason, we need to clean up the task that we have created
+                if (operationStarted == false) {
+                    taskManager.unregister(searchTask);
+                }
+            }
+        }
+    }
+
+    private ActionListener<Response> wrapStoringListener(
+        T searchTask,
+        TimeValue waitForCompletionTimeout,
+        TimeValue keepAlive,
+        boolean keepOnCompletion,
+        ActionListener<Response> listener
+    ) {
+        AtomicReference<ActionListener<Response>> exclusiveListener = new AtomicReference<>(listener);
+        // This is will performed in case of timeout
+        Scheduler.ScheduledCancellable timeoutHandler = threadPool.schedule(() -> {
+            ActionListener<Response> acquiredListener = exclusiveListener.getAndSet(null);
+            if (acquiredListener != null) {
+                acquiredListener.onResponse(operation.initialResponse(searchTask));
+            }
+        }, waitForCompletionTimeout, threadPool.executor(ThreadPool.Names.SEARCH));
+
+        // This will be performed at the end of normal execution
+        return ActionListener.wrap(response -> {
+            ActionListener<Response> acquiredListener = exclusiveListener.getAndSet(null);
+            if (acquiredListener != null) {
+                // We finished before timeout
+                timeoutHandler.cancel();
+                if (keepOnCompletion) {
+                    storeResults(
+                        searchTask,
+                        new StoredAsyncResponse<>(response, threadPool.absoluteTimeInMillis() + keepAlive.getMillis()),
+                        ActionListener.running(() -> acquiredListener.onResponse(response))
+                    );
+                } else {
+                    taskManager.unregister(searchTask);
+                    searchTask.onResponse(response);
+                    acquiredListener.onResponse(response);
+                }
+            } else {
+                // We finished after timeout - saving results
+                storeResults(
+                    searchTask,
+                    new StoredAsyncResponse<>(response, threadPool.absoluteTimeInMillis() + keepAlive.getMillis()),
+                    ActionListener.running(response::decRef)
+                );
+            }
+        }, e -> {
+            ActionListener<Response> acquiredListener = exclusiveListener.getAndSet(null);
+            if (acquiredListener != null) {
+                // We finished before timeout
+                timeoutHandler.cancel();
+                if (keepOnCompletion) {
+                    storeResults(
+                        searchTask,
+                        new StoredAsyncResponse<>(e, threadPool.absoluteTimeInMillis() + keepAlive.getMillis()),
+                        ActionListener.running(() -> acquiredListener.onFailure(e))
+                    );
+                } else {
+                    taskManager.unregister(searchTask);
+                    searchTask.onFailure(e);
+                    acquiredListener.onFailure(e);
+                }
+            } else {
+                // We finished after timeout - saving exception
+                storeResults(searchTask, new StoredAsyncResponse<>(e, threadPool.absoluteTimeInMillis() + keepAlive.getMillis()));
+            }
+        });
+    }
+
+    private void storeResults(T searchTask, StoredAsyncResponse<Response> storedResponse) {
+        storeResults(searchTask, storedResponse, null);
+    }
+
+    private void storeResults(T searchTask, StoredAsyncResponse<Response> storedResponse, ActionListener<Void> finalListener) {
+        try {
+            asyncTaskIndexService.createResponseForEQL(
+                searchTask.getExecutionId().getDocId(),
+                searchTask.getOriginHeaders(),
+                threadPool.getThreadContext().getResponseHeaders(), // includes ESQL warnings
+                storedResponse,
+                ActionListener.wrap(
+                    // We should only unregister after the result is saved
+                    resp -> {
+                        // TODO: generalize the logging, not just eql
+                        logger.trace(() -> "stored eql search results for [" + searchTask.getExecutionId().getEncoded() + "]");
+                        taskManager.unregister(searchTask);
+                        if (storedResponse.getException() != null) {
+                            searchTask.onFailure(storedResponse.getException());
+                        } else {
+                            searchTask.onResponse(storedResponse.getResponse());
+                        }
+                        if (finalListener != null) {
+                            finalListener.onResponse(null);
+                        }
+                    },
+                    exc -> {
+                        taskManager.unregister(searchTask);
+                        searchTask.onFailure(exc);
+                        Throwable cause = ExceptionsHelper.unwrapCause(exc);
+                        if (cause instanceof DocumentMissingException == false
+                            && cause instanceof VersionConflictEngineException == false) {
+                            logger.error(
+                                // TODO: generalize the logging, not just eql
+                                () -> format("failed to store eql search results for [%s]", searchTask.getExecutionId().getEncoded()),
+                                exc
+                            );
+                        }
+                        if (finalListener != null) {
+                            finalListener.onFailure(exc);
+                        }
+                    }
+                )
+            );
+        } catch (Exception exc) {
+            taskManager.unregister(searchTask);
+            searchTask.onFailure(exc);
+            logger.error(() -> "failed to store eql search results for [" + searchTask.getExecutionId().getEncoded() + "]", exc);
+        }
+    }
+
+    /**
+     * Adds a self-unregistering listener to a task. It works as a normal listener except it retrieves a partial response and unregister
+     * itself from the task if timeout occurs. Returns false if the listener could not be added, if say for example the task completed.
+     * Otherwise, returns true.
+     */
+    public static <Response extends ActionResponse, Task extends StoredAsyncTask<Response>> boolean addCompletionListener(
+        ThreadPool threadPool,
+        Task task,
+        ActionListener<StoredAsyncResponse<Response>> listener,
+        TimeValue timeout
+    ) {
+        if (timeout.getMillis() <= 0) {
+            getCurrentResult(task, listener);
+            return true;
+        } else {
+            return task.addCompletionListener(
+                () -> ListenerTimeouts.wrapWithTimeout(
+                    threadPool,
+                    timeout,
+                    threadPool.executor(ThreadPool.Names.SEARCH),
+                    ActionListener.wrap(
+                        r -> listener.onResponse(new StoredAsyncResponse<>(r, task.getExpirationTimeMillis())),
+                        e -> listener.onResponse(new StoredAsyncResponse<>(e, task.getExpirationTimeMillis()))
+                    ),
+                    wrapper -> {
+                        // Timeout was triggered
+                        task.removeCompletionListener(wrapper);
+                        getCurrentResult(task, listener);
+                    }
+                )
+            );
+        }
+    }
+
+    private static <Response extends ActionResponse, Task extends StoredAsyncTask<Response>> void getCurrentResult(
+        Task task,
+        ActionListener<StoredAsyncResponse<Response>> listener
+    ) {
+        try {
+            listener.onResponse(new StoredAsyncResponse<>(task.getCurrentResult(), task.getExpirationTimeMillis()));
+        } catch (Exception ex) {
+            listener.onFailure(ex);
+        }
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/async/QlStatusResponse.java

@@ -0,0 +1,200 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.async;
+
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.xcontent.ToXContentObject;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.async.StoredAsyncResponse;
+import org.elasticsearch.xpack.core.search.action.SearchStatusResponse;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * A response for *QL search status request
+ */
+public class QlStatusResponse extends ActionResponse implements SearchStatusResponse, ToXContentObject {
+    private final String id;
+    private final boolean isRunning;
+    private final boolean isPartial;
+    private final Long startTimeMillis;
+    private final long expirationTimeMillis;
+    private final RestStatus completionStatus;
+
+    public interface AsyncStatus {
+        String id();
+
+        boolean isRunning();
+
+        boolean isPartial();
+    }
+
+    public QlStatusResponse(
+        String id,
+        boolean isRunning,
+        boolean isPartial,
+        Long startTimeMillis,
+        long expirationTimeMillis,
+        RestStatus completionStatus
+    ) {
+        this.id = id;
+        this.isRunning = isRunning;
+        this.isPartial = isPartial;
+        this.startTimeMillis = startTimeMillis;
+        this.expirationTimeMillis = expirationTimeMillis;
+        this.completionStatus = completionStatus;
+    }
+
+    /**
+     * Get status from the stored Ql search response
+     * @param storedResponse - a response from a stored search
+     * @param expirationTimeMillis – expiration time in milliseconds
+     * @param id – encoded async search id
+     * @return a status response
+     */
+    public static <S extends Writeable & AsyncStatus> QlStatusResponse getStatusFromStoredSearch(
+        StoredAsyncResponse<S> storedResponse,
+        long expirationTimeMillis,
+        String id
+    ) {
+        S searchResponse = storedResponse.getResponse();
+        if (searchResponse != null) {
+            assert searchResponse.isRunning() == false : "Stored Ql search response must have a completed status!";
+            return new QlStatusResponse(
+                searchResponse.id(),
+                false,
+                searchResponse.isPartial(),
+                null,  // we don't store in the index the start time for completed response
+                expirationTimeMillis,
+                RestStatus.OK
+            );
+        } else {
+            Exception exc = storedResponse.getException();
+            assert exc != null : "Stored Ql response must either have a search response or an exception!";
+            return new QlStatusResponse(
+                id,
+                false,
+                false,
+                null, // we don't store in the index the start time for completed response
+                expirationTimeMillis,
+                ExceptionsHelper.status(exc)
+            );
+        }
+    }
+
+    public QlStatusResponse(StreamInput in) throws IOException {
+        this.id = in.readString();
+        this.isRunning = in.readBoolean();
+        this.isPartial = in.readBoolean();
+        this.startTimeMillis = in.readOptionalLong();
+        this.expirationTimeMillis = in.readLong();
+        this.completionStatus = (this.isRunning == false) ? RestStatus.readFrom(in) : null;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(id);
+        out.writeBoolean(isRunning);
+        out.writeBoolean(isPartial);
+        out.writeOptionalLong(startTimeMillis);
+        out.writeLong(expirationTimeMillis);
+        if (isRunning == false) {
+            RestStatus.writeTo(out, completionStatus);
+        }
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        {
+            builder.field("id", id);
+            builder.field("is_running", isRunning);
+            builder.field("is_partial", isPartial);
+            if (startTimeMillis != null) { // start time is available only for a running eql search
+                builder.timeField("start_time_in_millis", "start_time", startTimeMillis);
+            }
+            builder.timeField("expiration_time_in_millis", "expiration_time", expirationTimeMillis);
+            if (isRunning == false) { // completion status is available only for a completed eql search
+                builder.field("completion_status", completionStatus.getStatus());
+            }
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        QlStatusResponse other = (QlStatusResponse) obj;
+        return id.equals(other.id)
+            && isRunning == other.isRunning
+            && isPartial == other.isPartial
+            && Objects.equals(startTimeMillis, other.startTimeMillis)
+            && expirationTimeMillis == other.expirationTimeMillis
+            && Objects.equals(completionStatus, other.completionStatus);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, isRunning, isPartial, startTimeMillis, expirationTimeMillis, completionStatus);
+    }
+
+    /**
+     * Returns the id of the eql search status request.
+     */
+    public String getId() {
+        return id;
+    }
+
+    /**
+     * Returns {@code true} if the eql search is still running in the cluster,
+     * or {@code false} if the search has been completed.
+     */
+    public boolean isRunning() {
+        return isRunning;
+    }
+
+    /**
+     * Returns {@code true} if the eql search results are partial.
+     * This could be either because eql search hasn't finished yet,
+     * or if it finished and some shards have failed or timed out.
+     */
+    public boolean isPartial() {
+        return isPartial;
+    }
+
+    /**
+     * Returns a timestamp when the eql search task started, in milliseconds since epoch.
+     * For a completed eql search returns {@code null}, as we don't store start time for completed searches.
+     */
+    public Long getStartTime() {
+        return startTimeMillis;
+    }
+
+    /**
+     * Returns a timestamp when the eql search will be expired, in milliseconds since epoch.
+     */
+    @Override
+    public long getExpirationTime() {
+        return expirationTimeMillis;
+    }
+
+    /**
+     * For a completed eql search returns the completion status.
+     * For a still running eql search returns {@code null}.
+     */
+    public RestStatus getCompletionStatus() {
+        return completionStatus;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/capabilities/Resolvable.java

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.capabilities;
+
+public interface Resolvable {
+
+    boolean resolved();
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/capabilities/Resolvables.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.capabilities;
+
+public abstract class Resolvables {
+
+    public static boolean resolved(Iterable<? extends Resolvable> resolvables) {
+        for (Resolvable resolvable : resolvables) {
+            if (resolvable.resolved() == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/capabilities/Unresolvable.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.capabilities;
+
+public interface Unresolvable extends Resolvable {
+
+    String UNRESOLVED_PREFIX = "?";
+
+    @Override
+    default boolean resolved() {
+        return false;
+    }
+
+    String unresolvedMessage();
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/capabilities/UnresolvedException.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.capabilities;
+
+import org.elasticsearch.xpack.esql.core.QlServerException;
+
+/**
+ * Thrown when we accidentally attempt to resolve something on on an unresolved entity. Throwing this
+ * is always a bug.
+ */
+public class UnresolvedException extends QlServerException {
+    public UnresolvedException(String action, Object target) {
+        super("Invalid call to {} on an unresolved object {}", action, target);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/common/Failure.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.common;
+
+import org.elasticsearch.xpack.esql.core.tree.Location;
+import org.elasticsearch.xpack.esql.core.tree.Node;
+import org.elasticsearch.xpack.esql.core.util.StringUtils;
+
+import java.util.Collection;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
+
+public class Failure {
+
+    private final Node<?> node;
+    private final String message;
+
+    public Failure(Node<?> node, String message) {
+        this.node = node;
+        this.message = message;
+    }
+
+    public Node<?> node() {
+        return node;
+    }
+
+    public String message() {
+        return message;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(node);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        Failure other = (Failure) obj;
+        return Objects.equals(node, other.node);
+    }
+
+    @Override
+    public String toString() {
+        return message;
+    }
+
+    public static Failure fail(Node<?> source, String message, Object... args) {
+        return new Failure(source, format(message, args));
+    }
+
+    public static String failMessage(Collection<Failure> failures) {
+        return failures.stream().map(f -> {
+            Location l = f.node().source().source();
+            return "line " + l.getLineNumber() + ":" + l.getColumnNumber() + ": " + f.message();
+        })
+            .collect(
+                Collectors.joining(
+                    StringUtils.NEW_LINE,
+                    format("Found {} problem{}\n", failures.size(), failures.size() > 1 ? "s" : StringUtils.EMPTY),
+                    StringUtils.EMPTY
+                )
+            );
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/common/Failures.java

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.common;
+
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.Objects;
+
+/**
+ * Glorified list for managing {@link Failure}s.
+ */
+public class Failures {
+
+    private final Collection<Failure> failures;
+
+    public Failures() {
+        this.failures = new LinkedHashSet<>();
+    }
+
+    public Failures add(Failure failure) {
+        if (failure != null) {
+            failures.add(failure);
+        }
+        return this;
+    }
+
+    public boolean hasFailures() {
+        return failures.size() > 0;
+    }
+
+    public Collection<Failure> failures() {
+        return failures;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(failures);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        Failures failures1 = (Failures) o;
+        return Objects.equals(failures, failures1.failures);
+    }
+
+    @Override
+    public String toString() {
+        return failures.isEmpty() ? "[]" : Failure.failMessage(failures);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/AggRef.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.execution.search;
+
+/**
+ * Reference to a ES aggregation (which can be either a GROUP BY or Metric agg).
+ */
+public abstract class AggRef implements FieldExtraction {
+
+    @Override
+    public void collectFields(QlSourceBuilder sourceBuilder) {
+        // Aggregations do not need any special fields
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        return true;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/FieldExtraction.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.execution.search;
+
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+
+/**
+ * An interface for something that needs to extract field(s) from a result.
+ */
+public interface FieldExtraction {
+
+    /**
+     * Add whatever is necessary to the {@link SearchSourceBuilder}
+     * in order to fetch the field. This can include tracking the score,
+     * {@code _source} fields, doc values fields, and script fields.
+     */
+    void collectFields(QlSourceBuilder sourceBuilder);
+
+    /**
+     * Is this aggregation supported in an "aggregation only" query
+     * ({@code true}) or should it force a scroll query ({@code false})?
+     */
+    boolean supportedByAggsOnlyQuery();
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/QlSourceBuilder.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.execution.search;
+
+import org.elasticsearch.script.Script;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
+
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A {@code SqlSourceBuilder} is a builder object passed to objects implementing
+ * {@link FieldExtraction} that can "build" whatever needs to be extracted from
+ * the resulting ES document as a field.
+ */
+public class QlSourceBuilder {
+    // The LinkedHashMaps preserve the order of the fields in the response
+    private final Set<FieldAndFormat> fetchFields = new LinkedHashSet<>();
+    private final Map<String, Script> scriptFields = new LinkedHashMap<>();
+
+    boolean trackScores = false;
+
+    public QlSourceBuilder() {}
+
+    /**
+     * Turns on returning the {@code _score} for documents.
+     */
+    public void trackScores() {
+        this.trackScores = true;
+    }
+
+    /**
+     * Retrieve the requested field using the "fields" API
+     */
+    public void addFetchField(String field, String format) {
+        fetchFields.add(new FieldAndFormat(field, format));
+    }
+
+    /**
+     * Return the given field as a script field with the supplied script
+     */
+    public void addScriptField(String name, Script script) {
+        scriptFields.put(name, script);
+    }
+
+    /**
+     * Collect the necessary fields, modifying the {@code SearchSourceBuilder}
+     * to retrieve them from the document.
+     */
+    public void build(SearchSourceBuilder sourceBuilder) {
+        sourceBuilder.trackScores(this.trackScores);
+        fetchFields.forEach(field -> sourceBuilder.fetchField(new FieldAndFormat(field.field, field.format, null)));
+        scriptFields.forEach(sourceBuilder::scriptField);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/extractor/AbstractFieldHitExtractor.java

@@ -0,0 +1,270 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.execution.search.extractor;
+
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.document.DocumentField;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
+import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+
+import java.io.IOException;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * Extractor for ES fields. Works for both 'normal' fields but also nested ones (which require hitName to be set).
+ * The latter is used as metadata in assembling the results in the tabular response.
+ */
+public abstract class AbstractFieldHitExtractor implements HitExtractor {
+
+    private final String fieldName, hitName;
+    private final DataType dataType;
+    private final ZoneId zoneId;
+
+    protected MultiValueSupport multiValueSupport;
+
+    public enum MultiValueSupport {
+        NONE,
+        LENIENT,
+        FULL
+    }
+
+    protected AbstractFieldHitExtractor(String name, DataType dataType, ZoneId zoneId) {
+        this(name, dataType, zoneId, null, MultiValueSupport.NONE);
+    }
+
+    protected AbstractFieldHitExtractor(String name, DataType dataType, ZoneId zoneId, MultiValueSupport multiValueSupport) {
+        this(name, dataType, zoneId, null, multiValueSupport);
+    }
+
+    protected AbstractFieldHitExtractor(
+        String name,
+        DataType dataType,
+        ZoneId zoneId,
+        String hitName,
+        MultiValueSupport multiValueSupport
+    ) {
+        this.fieldName = name;
+        this.dataType = dataType;
+        this.zoneId = zoneId;
+        this.multiValueSupport = multiValueSupport;
+        this.hitName = hitName;
+
+        if (hitName != null) {
+            if (name.contains(hitName) == false) {
+                throw new QlIllegalArgumentException("Hitname [{}] specified but not part of the name [{}]", hitName, name);
+            }
+        }
+    }
+
+    @SuppressWarnings("this-escape")
+    protected AbstractFieldHitExtractor(StreamInput in) throws IOException {
+        fieldName = in.readString();
+        String typeName = in.readOptionalString();
+        dataType = typeName != null ? loadTypeFromName(typeName) : null;
+        hitName = in.readOptionalString();
+        if (in.getTransportVersion().before(TransportVersions.V_8_6_0)) {
+            this.multiValueSupport = in.readBoolean() ? MultiValueSupport.LENIENT : MultiValueSupport.NONE;
+        } else {
+            this.multiValueSupport = in.readEnum(MultiValueSupport.class);
+        }
+        zoneId = readZoneId(in);
+    }
+
+    protected DataType loadTypeFromName(String typeName) {
+        return DataTypes.fromTypeName(typeName);
+    }
+
+    protected abstract ZoneId readZoneId(StreamInput in) throws IOException;
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(fieldName);
+        out.writeOptionalString(dataType == null ? null : dataType.typeName());
+        out.writeOptionalString(hitName);
+        if (out.getTransportVersion().before(TransportVersions.V_8_6_0)) {
+            out.writeBoolean(multiValueSupport != MultiValueSupport.NONE);
+        } else {
+            out.writeEnum(multiValueSupport);
+        }
+
+    }
+
+    @Override
+    public Object extract(SearchHit hit) {
+        Object value = null;
+        DocumentField field = null;
+        if (hitName != null) {
+            value = unwrapFieldsMultiValue(extractNestedField(hit));
+        } else {
+            field = hit.field(fieldName);
+            if (field != null) {
+                value = unwrapFieldsMultiValue(field.getValues());
+            }
+        }
+        return value;
+    }
+
+    /*
+     * For a path of fields like root.nested1.nested2.leaf where nested1 and nested2 are nested field types,
+     * fieldName is root.nested1.nested2.leaf, while hitName is root.nested1.nested2
+     * We first look for root.nested1.nested2 or root.nested1 or root in the SearchHit until we find something.
+     * If the DocumentField lives under "root.nested1" the remaining path to search for (in the DocumentField itself) is nested2.
+     * After this step is done, what remains to be done is just getting the leaf values.
+     */
+    @SuppressWarnings("unchecked")
+    private Object extractNestedField(SearchHit hit) {
+        Object value;
+        DocumentField field;
+        String tempHitname = hitName;
+        List<String> remainingPath = new ArrayList<>();
+        // first, search for the "root" DocumentField under which the remaining path of nested document values is
+        while ((field = hit.field(tempHitname)) == null) {
+            int indexOfDot = tempHitname.lastIndexOf('.');
+            if (indexOfDot < 0) {// there is no such field in the hit
+                return null;
+            }
+            remainingPath.add(0, tempHitname.substring(indexOfDot + 1));
+            tempHitname = tempHitname.substring(0, indexOfDot);
+        }
+        // then dig into DocumentField's structure until we reach the field we are interested into
+        if (remainingPath.size() > 0) {
+            List<Object> values = field.getValues();
+            Iterator<String> pathIterator = remainingPath.iterator();
+            while (pathIterator.hasNext()) {
+                String pathElement = pathIterator.next();
+                Map<String, List<Object>> elements = (Map<String, List<Object>>) values.get(0);
+                values = elements.get(pathElement);
+                /*
+                 * if this path is not found it means we hit another nested document (inner_root_1.inner_root_2.nested_field_2)
+                 * something like this
+                 * "root_field_1.root_field_2.nested_field_1" : [
+                 *     {
+                 *       "inner_root_1.inner_root_2.nested_field_2" : [
+                 *         {
+                 *           "leaf_field" : [
+                 *             "abc2"
+                 *           ]
+                 * So, start re-building the path until the right one is found, ie inner_root_1.inner_root_2......
+                 */
+                while (values == null) {
+                    pathElement += "." + pathIterator.next();
+                    values = elements.get(pathElement);
+                }
+            }
+            value = ((Map<String, Object>) values.get(0)).get(fieldName.substring(hitName.length() + 1));
+        } else {
+            value = field.getValues();
+        }
+        return value;
+    }
+
+    protected Object unwrapFieldsMultiValue(Object values) {
+        if (values == null) {
+            return null;
+        }
+        if (values instanceof Map && hitName != null) {
+            // extract the sub-field from a nested field (dep.dep_name -> dep_name)
+            return unwrapFieldsMultiValue(((Map<?, ?>) values).get(fieldName.substring(hitName.length() + 1)));
+        }
+        if (values instanceof List<?> list) {
+            if (list.isEmpty()) {
+                return null;
+            } else {
+                if (isPrimitive(list) == false) {
+                    if (list.size() == 1 || multiValueSupport == MultiValueSupport.LENIENT) {
+                        return unwrapFieldsMultiValue(list.get(0));
+                    } else if (multiValueSupport == MultiValueSupport.FULL) {
+                        List<Object> unwrappedValues = new ArrayList<>();
+                        for (Object value : list) {
+                            unwrappedValues.add(unwrapFieldsMultiValue(value));
+                        }
+                        values = unwrappedValues;
+                    } else {
+                        // missing `field_multi_value_leniency` setting
+                        throw new InvalidArgumentException("Arrays (returned by [{}]) are not supported", fieldName);
+                    }
+                }
+            }
+        }
+
+        Object unwrapped = unwrapCustomValue(values);
+        if (unwrapped != null && isListOfNulls(unwrapped) == false) {
+            return unwrapped;
+        }
+
+        return values;
+    }
+
+    private static boolean isListOfNulls(Object unwrapped) {
+        if (unwrapped instanceof List<?> list) {
+            if (list.size() == 0) {
+                return false;
+            }
+            for (Object o : list) {
+                if (o != null) {
+                    return false;
+                }
+            }
+            return true;
+        }
+        return false;
+    }
+
+    protected abstract Object unwrapCustomValue(Object values);
+
+    protected abstract boolean isPrimitive(List<?> list);
+
+    @Override
+    public String hitName() {
+        return hitName;
+    }
+
+    public String fieldName() {
+        return fieldName;
+    }
+
+    public ZoneId zoneId() {
+        return zoneId;
+    }
+
+    public DataType dataType() {
+        return dataType;
+    }
+
+    public MultiValueSupport multiValueSupport() {
+        return multiValueSupport;
+    }
+
+    @Override
+    public String toString() {
+        return fieldName + "@" + hitName + "@" + zoneId;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || obj.getClass() != getClass()) {
+            return false;
+        }
+        AbstractFieldHitExtractor other = (AbstractFieldHitExtractor) obj;
+        return fieldName.equals(other.fieldName) && hitName.equals(other.hitName) && multiValueSupport == other.multiValueSupport;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(fieldName, hitName, multiValueSupport);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/extractor/BucketExtractor.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.execution.search.extractor;
+
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
+
+/**
+ * Extracts an aggregation value from a {@link Bucket}.
+ */
+public interface BucketExtractor extends NamedWriteable {
+
+    Object extract(Bucket bucket);
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/extractor/BucketExtractors.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.execution.search.extractor;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public final class BucketExtractors {
+
+    private BucketExtractors() {}
+
+    /**
+     * All of the named writeables needed to deserialize the instances of
+     * {@linkplain BucketExtractor}s.
+     */
+    public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.add(new Entry(BucketExtractor.class, ComputingExtractor.NAME, ComputingExtractor::new));
+        entries.add(new Entry(BucketExtractor.class, ConstantExtractor.NAME, ConstantExtractor::new));
+        return entries;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/extractor/ComputingExtractor.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.execution.search.extractor;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
+import org.elasticsearch.xpack.esql.core.expression.gen.processor.HitExtractorProcessor;
+import org.elasticsearch.xpack.esql.core.expression.gen.processor.Processor;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Hit/BucketExtractor that delegates to a processor. The difference between this class
+ * and {@link HitExtractorProcessor} is that the latter is used inside a
+ * {@link Processor} tree as a leaf (and thus can effectively parse the
+ * {@link SearchHit} while this class is used when scrolling and passing down
+ * the results.
+ *
+ * In the future, the processor might be used across the board for all columns
+ * to reduce API complexity (and keep the {@link HitExtractor} only as an
+ * internal implementation detail).
+ */
+public class ComputingExtractor implements HitExtractor, BucketExtractor {
+    /**
+     * Stands for {@code comPuting}. We try to use short names for {@link HitExtractor}s
+     * to save a few bytes when when we send them back to the user.
+     */
+    static final String NAME = "p";
+    private final Processor processor;
+    private final String hitName;
+
+    public ComputingExtractor(Processor processor) {
+        this(processor, null);
+    }
+
+    public ComputingExtractor(Processor processor, String hitName) {
+        this.processor = processor;
+        this.hitName = hitName;
+    }
+
+    // Visibility required for tests
+    public ComputingExtractor(StreamInput in) throws IOException {
+        processor = in.readNamedWriteable(Processor.class);
+        hitName = in.readOptionalString();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(processor);
+        out.writeOptionalString(hitName);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    public Processor processor() {
+        return processor;
+    }
+
+    public Object extract(Object input) {
+        return processor.process(input);
+    }
+
+    @Override
+    public Object extract(Bucket bucket) {
+        return processor.process(bucket);
+    }
+
+    @Override
+    public Object extract(SearchHit hit) {
+        return processor.process(hit);
+    }
+
+    @Override
+    public String hitName() {
+        return hitName;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || obj.getClass() != getClass()) {
+            return false;
+        }
+        ComputingExtractor other = (ComputingExtractor) obj;
+        return Objects.equals(processor, other.processor) && Objects.equals(hitName, other.hitName);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(processor, hitName);
+    }
+
+    @Override
+    public String toString() {
+        return processor.toString();
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/extractor/ConstantExtractor.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.execution.search.extractor;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Returns the a constant for every search hit against which it is run.
+ */
+public class ConstantExtractor implements HitExtractor, BucketExtractor {
+    /**
+     * Stands for {@code constant}. We try to use short names for {@link HitExtractor}s
+     * to save a few bytes when when we send them back to the user.
+     */
+    static final String NAME = "c";
+    private final Object constant;
+
+    public ConstantExtractor(Object constant) {
+        this.constant = constant;
+    }
+
+    ConstantExtractor(StreamInput in) throws IOException {
+        constant = in.readGenericValue();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeGenericValue(constant);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    @Override
+    public Object extract(SearchHit hit) {
+        return constant;
+    }
+
+    @Override
+    public Object extract(Bucket bucket) {
+        return constant;
+    }
+
+    @Override
+    public String hitName() {
+        return null;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || obj.getClass() != getClass()) {
+            return false;
+        }
+        ConstantExtractor other = (ConstantExtractor) obj;
+        return Objects.equals(constant, other.constant);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(constant);
+    }
+
+    @Override
+    public String toString() {
+        return "^" + constant;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/extractor/HitExtractor.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.execution.search.extractor;
+
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.search.SearchHit;
+
+/**
+ * Extracts a column value from a {@link SearchHit}.
+ */
+public interface HitExtractor extends NamedWriteable {
+    /**
+     * Extract the value from a hit.
+     */
+    Object extract(SearchHit hit);
+
+    /**
+     * Name of the inner hit needed by this extractor if it needs one, {@code null} otherwise.
+     */
+    @Nullable
+    String hitName();
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/extractor/HitExtractors.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.execution.search.extractor;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public final class HitExtractors {
+
+    private HitExtractors() {}
+
+    /**
+     * All of the named writeables needed to deserialize the instances of
+     * {@linkplain HitExtractor}.
+     */
+    public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
+        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
+        entries.add(new Entry(HitExtractor.class, ConstantExtractor.NAME, ConstantExtractor::new));
+        entries.add(new Entry(HitExtractor.class, ComputingExtractor.NAME, ComputingExtractor::new));
+        return entries;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/execution/search/extractor/TotalHitsExtractor.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.execution.search.extractor;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
+import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
+
+import java.io.IOException;
+
+public class TotalHitsExtractor extends ConstantExtractor {
+
+    public TotalHitsExtractor(Long constant) {
+        super(constant);
+    }
+
+    TotalHitsExtractor(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public Object extract(MultiBucketsAggregation.Bucket bucket) {
+        return validate(super.extract(bucket));
+    }
+
+    @Override
+    public Object extract(SearchHit hit) {
+        return validate(super.extract(hit));
+    }
+
+    private static Object validate(Object value) {
+        if (Number.class.isInstance(value) == false) {
+            throw new QlIllegalArgumentException(
+                "Inconsistent total hits count handling, expected a numeric value but found a {}: {}",
+                value == null ? null : value.getClass().getSimpleName(),
+                value
+            );
+        }
+        if (((Number) value).longValue() < 0) {
+            throw new QlIllegalArgumentException(
+                "Inconsistent total hits count handling, expected a non-negative value but found {}",
+                value
+            );
+        }
+        return value;
+    }
+
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Alias.java

@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+
+/**
+ * An {@code Alias} is a {@code NamedExpression} that gets renamed to something else through the Alias.
+ *
+ * For example, in the statement {@code 5 + 2 AS x}, {@code x} is an alias which is points to {@code ADD(5, 2)}.
+ *
+ * And in {@code SELECT col AS x} "col" is a named expression that gets renamed to "x" through an alias.
+ *
+ */
+public class Alias extends NamedExpression {
+
+    private final Expression child;
+    private final String qualifier;
+
+    /**
+     * Postpone attribute creation until it is actually created.
+     * Being immutable, create only one instance.
+     */
+    private Attribute lazyAttribute;
+
+    public Alias(Source source, String name, Expression child) {
+        this(source, name, null, child, null);
+    }
+
+    public Alias(Source source, String name, String qualifier, Expression child) {
+        this(source, name, qualifier, child, null);
+    }
+
+    public Alias(Source source, String name, String qualifier, Expression child, NameId id) {
+        this(source, name, qualifier, child, id, false);
+    }
+
+    public Alias(Source source, String name, String qualifier, Expression child, NameId id, boolean synthetic) {
+        super(source, name, singletonList(child), id, synthetic);
+        this.child = child;
+        this.qualifier = qualifier;
+    }
+
+    @Override
+    protected NodeInfo<Alias> info() {
+        return NodeInfo.create(this, Alias::new, name(), qualifier, child, id(), synthetic());
+    }
+
+    public Alias replaceChild(Expression child) {
+        return new Alias(source(), name(), qualifier, child, id(), synthetic());
+    }
+
+    @Override
+    public Alias replaceChildren(List<Expression> newChildren) {
+        return new Alias(source(), name(), qualifier, newChildren.get(0), id(), synthetic());
+    }
+
+    public Expression child() {
+        return child;
+    }
+
+    public String qualifier() {
+        return qualifier;
+    }
+
+    public String qualifiedName() {
+        return qualifier == null ? name() : qualifier + "." + name();
+    }
+
+    @Override
+    public Nullability nullable() {
+        return child.nullable();
+    }
+
+    @Override
+    public DataType dataType() {
+        return child.dataType();
+    }
+
+    @Override
+    public Attribute toAttribute() {
+        if (lazyAttribute == null) {
+            lazyAttribute = resolved()
+                ? new ReferenceAttribute(source(), name(), dataType(), qualifier, nullable(), id(), synthetic())
+                : new UnresolvedAttribute(source(), name(), qualifier);
+        }
+        return lazyAttribute;
+    }
+
+    @Override
+    public String toString() {
+        return child + " AS " + name() + "#" + id();
+    }
+
+    @Override
+    public String nodeString() {
+        return child.nodeString() + " AS " + name();
+    }
+
+    /**
+     * If the given expression is an alias, return its child - otherwise return as is.
+     */
+    public static Expression unwrap(Expression e) {
+        return e instanceof Alias as ? as.child() : e;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Attribute.java

@@ -0,0 +1,170 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.util.List;
+import java.util.Objects;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.xpack.esql.core.util.StringUtils.splitQualifiedIndex;
+
+/**
+ * {@link Expression}s that can be materialized and describe properties of the derived table.
+ * In other words, an attribute represent a column in the results of a query.
+ *
+ * In the statement {@code SELECT ABS(foo), A, B+C FROM ...} the three named
+ * expressions {@code ABS(foo), A, B+C} get converted to attributes and the user can
+ * only see Attributes.
+ *
+ * In the statement {@code SELECT foo FROM TABLE WHERE foo > 10 + 1} only {@code foo} inside the SELECT
+ * is a named expression (an {@code Alias} will be created automatically for it).
+ * The rest are not as they are not part of the projection and thus are not part of the derived table.
+ */
+public abstract class Attribute extends NamedExpression {
+
+    // empty - such as a top level attribute in SELECT cause
+    // present - table name or a table name alias
+    private final String qualifier;
+    // cluster name in the qualifier (if any)
+    private final String cluster;
+
+    // can the attr be null - typically used in JOINs
+    private final Nullability nullability;
+
+    public Attribute(Source source, String name, String qualifier, NameId id) {
+        this(source, name, qualifier, Nullability.TRUE, id);
+    }
+
+    public Attribute(Source source, String name, String qualifier, Nullability nullability, NameId id) {
+        this(source, name, qualifier, nullability, id, false);
+    }
+
+    public Attribute(Source source, String name, String qualifier, Nullability nullability, NameId id, boolean synthetic) {
+        super(source, name, emptyList(), id, synthetic);
+        if (qualifier != null) {
+            Tuple<String, String> splitQualifier = splitQualifiedIndex(qualifier);
+            this.cluster = splitQualifier.v1();
+            this.qualifier = splitQualifier.v2();
+        } else {
+            this.cluster = null;
+            this.qualifier = null;
+        }
+        this.nullability = nullability;
+    }
+
+    @Override
+    public final Expression replaceChildren(List<Expression> newChildren) {
+        throw new UnsupportedOperationException("this type of node doesn't have any children to replace");
+    }
+
+    public String qualifier() {
+        return qualifier;
+    }
+
+    public String qualifiedName() {
+        return qualifier == null ? name() : qualifier + "." + name();
+    }
+
+    @Override
+    public Nullability nullable() {
+        return nullability;
+    }
+
+    @Override
+    public AttributeSet references() {
+        return new AttributeSet(this);
+    }
+
+    public Attribute withLocation(Source source) {
+        return Objects.equals(source(), source) ? this : clone(source, name(), dataType(), qualifier(), nullable(), id(), synthetic());
+    }
+
+    public Attribute withQualifier(String qualifier) {
+        return Objects.equals(qualifier(), qualifier)
+            ? this
+            : clone(source(), name(), dataType(), qualifier, nullable(), id(), synthetic());
+    }
+
+    public Attribute withName(String name) {
+        return Objects.equals(name(), name) ? this : clone(source(), name, dataType(), qualifier(), nullable(), id(), synthetic());
+    }
+
+    public Attribute withNullability(Nullability nullability) {
+        return Objects.equals(nullable(), nullability)
+            ? this
+            : clone(source(), name(), dataType(), qualifier(), nullability, id(), synthetic());
+    }
+
+    public Attribute withId(NameId id) {
+        return clone(source(), name(), dataType(), qualifier(), nullable(), id, synthetic());
+    }
+
+    public Attribute withDataType(DataType type) {
+        return Objects.equals(dataType(), type) ? this : clone(source(), name(), type, qualifier(), nullable(), id(), synthetic());
+    }
+
+    protected abstract Attribute clone(
+        Source source,
+        String name,
+        DataType type,
+        String qualifier,
+        Nullability nullability,
+        NameId id,
+        boolean synthetic
+    );
+
+    @Override
+    public Attribute toAttribute() {
+        return this;
+    }
+
+    @Override
+    public int semanticHash() {
+        return id().hashCode();
+    }
+
+    @Override
+    public boolean semanticEquals(Expression other) {
+        return other instanceof Attribute ? id().equals(((Attribute) other).id()) : false;
+    }
+
+    @Override
+    protected Expression canonicalize() {
+        return clone(Source.EMPTY, name(), dataType(), qualifier, nullability, id(), synthetic());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), qualifier, nullability);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (super.equals(obj)) {
+            Attribute other = (Attribute) obj;
+            return Objects.equals(qualifier, other.qualifier) && Objects.equals(nullability, other.nullability);
+        }
+
+        return false;
+    }
+
+    @Override
+    public String toString() {
+        return qualifiedName() + "{" + label() + "}" + "#" + id();
+    }
+
+    @Override
+    public String nodeString() {
+        return toString();
+    }
+
+    protected abstract String label();
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeMap.java

@@ -0,0 +1,406 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
+
+import java.util.AbstractSet;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.stream.Stream;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Dedicated map for checking {@link Attribute} equality.
+ * This is typically the case when comparing the initial declaration of an Attribute, such as {@link FieldAttribute} with
+ * references to it, namely {@link ReferenceAttribute}.
+ * Using plain object equality, the two references are difference due to their type however semantically, they are the same.
+ * Expressions support semantic equality through {@link Expression#semanticEquals(Expression)} - this map is dedicated solution
+ * for attributes as its common case picked up by the plan rules.
+ * <p>
+ * The map implementation is mutable thus consumers need to be careful NOT to modify the content unless they have ownership.
+ * Worth noting the {@link #combine(AttributeMap)}, {@link #intersect(AttributeMap)} and {@link #subtract(AttributeMap)} methods which
+ * return copies, decoupled from the input maps. In other words the returned maps can be modified without affecting the input or vice-versa.
+ */
+public final class AttributeMap<E> implements Map<Attribute, E> {
+
+    static class AttributeWrapper {
+
+        private final Attribute attr;
+
+        AttributeWrapper(Attribute attr) {
+            this.attr = attr;
+        }
+
+        @Override
+        public int hashCode() {
+            return attr.semanticHash();
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            return obj instanceof AttributeWrapper aw ? attr.semanticEquals(aw.attr) : false;
+        }
+
+        @Override
+        public String toString() {
+            return attr.toString();
+        }
+    }
+
+    /**
+     * Set that does unwrapping of keys inside the keySet and iterator.
+     */
+    private abstract static class UnwrappingSet<W, U> extends AbstractSet<U> {
+        private final Set<W> set;
+
+        UnwrappingSet(Set<W> originalSet) {
+            set = originalSet;
+        }
+
+        @Override
+        public Iterator<U> iterator() {
+            return new Iterator<>() {
+                final Iterator<W> i = set.iterator();
+
+                @Override
+                public boolean hasNext() {
+                    return i.hasNext();
+                }
+
+                @Override
+                public U next() {
+                    return unwrap(i.next());
+                }
+
+                @Override
+                public void remove() {
+                    i.remove();
+                }
+            };
+        }
+
+        protected abstract U unwrap(W next);
+
+        @Override
+        public Stream<U> stream() {
+            return set.stream().map(this::unwrap);
+        }
+
+        @Override
+        public Stream<U> parallelStream() {
+            return set.parallelStream().map(this::unwrap);
+        }
+
+        @Override
+        public int size() {
+            return set.size();
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            return set.equals(o);
+        }
+
+        @Override
+        public int hashCode() {
+            return set.hashCode();
+        }
+
+        @Override
+        public Object[] toArray() {
+            Object[] array = set.toArray();
+            for (int i = 0; i < array.length; i++) {
+                array[i] = ((AttributeWrapper) array[i]).attr;
+            }
+            return array;
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        public <A> A[] toArray(A[] a) {
+            // collection is immutable so use that to our advantage
+            if (a.length < size()) {
+                a = (A[]) java.lang.reflect.Array.newInstance(a.getClass().getComponentType(), size());
+            }
+            int i = 0;
+            Object[] result = a;
+            for (U u : this) {
+                result[i++] = u;
+            }
+            // array larger than size, mark the ending element as null
+            if (a.length > size()) {
+                a[size()] = null;
+            }
+            return a;
+        }
+
+        @Override
+        public String toString() {
+            return set.toString();
+        }
+    }
+
+    @SuppressWarnings("rawtypes")
+    private static final AttributeMap EMPTY = new AttributeMap<>(emptyMap());
+
+    @SuppressWarnings("unchecked")
+    public static <E> AttributeMap<E> emptyAttributeMap() {
+        return EMPTY;
+    }
+
+    private final Map<AttributeWrapper, E> delegate;
+
+    private AttributeMap(Map<AttributeWrapper, E> other) {
+        delegate = other;
+    }
+
+    public AttributeMap() {
+        delegate = new LinkedHashMap<>();
+    }
+
+    public AttributeMap(Attribute key, E value) {
+        delegate = new LinkedHashMap<>();
+        add(key, value);
+    }
+
+    public AttributeMap<E> combine(AttributeMap<E> other) {
+        AttributeMap<E> combine = new AttributeMap<>();
+        combine.addAll(this);
+        combine.addAll(other);
+
+        return combine;
+    }
+
+    public AttributeMap<E> subtract(AttributeMap<E> other) {
+        AttributeMap<E> diff = new AttributeMap<>();
+        for (Entry<AttributeWrapper, E> entry : this.delegate.entrySet()) {
+            if (other.delegate.containsKey(entry.getKey()) == false) {
+                diff.delegate.put(entry.getKey(), entry.getValue());
+            }
+        }
+
+        return diff;
+    }
+
+    public AttributeMap<E> intersect(AttributeMap<E> other) {
+        AttributeMap<E> smaller = (other.size() > size() ? this : other);
+        AttributeMap<E> larger = (smaller == this ? other : this);
+
+        AttributeMap<E> intersect = new AttributeMap<>();
+        for (Entry<AttributeWrapper, E> entry : smaller.delegate.entrySet()) {
+            if (larger.delegate.containsKey(entry.getKey())) {
+                intersect.delegate.put(entry.getKey(), entry.getValue());
+            }
+        }
+
+        return intersect;
+    }
+
+    public boolean subsetOf(AttributeMap<E> other) {
+        if (this.size() > other.size()) {
+            return false;
+        }
+        for (AttributeWrapper aw : delegate.keySet()) {
+            if (other.delegate.containsKey(aw) == false) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    public void add(Attribute key, E value) {
+        put(key, value);
+    }
+
+    public void addAll(AttributeMap<E> other) {
+        putAll(other);
+    }
+
+    public Set<String> attributeNames() {
+        Set<String> s = Sets.newLinkedHashSetWithExpectedSize(size());
+
+        for (AttributeWrapper aw : delegate.keySet()) {
+            s.add(aw.attr.name());
+        }
+        return s;
+    }
+
+    @Override
+    public int size() {
+        return delegate.size();
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return delegate.isEmpty();
+    }
+
+    @Override
+    public boolean containsKey(Object key) {
+        return key instanceof NamedExpression ne ? delegate.containsKey(new AttributeWrapper(ne.toAttribute())) : false;
+    }
+
+    @Override
+    public boolean containsValue(Object value) {
+        return delegate.containsValue(value);
+    }
+
+    @Override
+    public E get(Object key) {
+        return key instanceof NamedExpression ne ? delegate.get(new AttributeWrapper(ne.toAttribute())) : null;
+    }
+
+    @Override
+    public E getOrDefault(Object key, E defaultValue) {
+        return key instanceof NamedExpression ne
+            ? delegate.getOrDefault(new AttributeWrapper(ne.toAttribute()), defaultValue)
+            : defaultValue;
+    }
+
+    public E resolve(Object key) {
+        return resolve(key, null);
+    }
+
+    public E resolve(Object key, E defaultValue) {
+        E value = defaultValue;
+        E candidate = null;
+        int allowedLookups = 1000;
+        while ((candidate = get(key)) != null || containsKey(key)) {
+            // instead of circling around, return
+            if (candidate == key) {
+                return candidate;
+            }
+            if (--allowedLookups == 0) {
+                throw new QlIllegalArgumentException("Potential cycle detected");
+            }
+            key = candidate;
+            value = candidate;
+        }
+        return value;
+    }
+
+    @Override
+    public E put(Attribute key, E value) {
+        return delegate.put(new AttributeWrapper(key), value);
+    }
+
+    @Override
+    public void putAll(Map<? extends Attribute, ? extends E> m) {
+        for (Entry<? extends Attribute, ? extends E> entry : m.entrySet()) {
+            put(entry.getKey(), entry.getValue());
+        }
+    }
+
+    @Override
+    public E remove(Object key) {
+        return key instanceof NamedExpression ne ? delegate.remove(new AttributeWrapper(ne.toAttribute())) : null;
+    }
+
+    @Override
+    public void clear() {
+        delegate.clear();
+    }
+
+    @Override
+    public Set<Attribute> keySet() {
+        return new UnwrappingSet<>(delegate.keySet()) {
+            @Override
+            protected Attribute unwrap(AttributeWrapper next) {
+                return next.attr;
+            }
+        };
+    }
+
+    @Override
+    public Collection<E> values() {
+        return delegate.values();
+    }
+
+    @Override
+    public Set<Entry<Attribute, E>> entrySet() {
+        return new UnwrappingSet<>(delegate.entrySet()) {
+            @Override
+            protected Entry<Attribute, E> unwrap(final Entry<AttributeWrapper, E> next) {
+                return new Entry<>() {
+                    @Override
+                    public Attribute getKey() {
+                        return next.getKey().attr;
+                    }
+
+                    @Override
+                    public E getValue() {
+                        return next.getValue();
+                    }
+
+                    @Override
+                    public E setValue(E value) {
+                        return next.setValue(value);
+                    }
+                };
+            }
+        };
+    }
+
+    @Override
+    public void forEach(BiConsumer<? super Attribute, ? super E> action) {
+        delegate.forEach((k, v) -> action.accept(k.attr, v));
+    }
+
+    @Override
+    public int hashCode() {
+        return delegate.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj instanceof AttributeMap<?> am) {
+            obj = am.delegate;
+        }
+        return delegate.equals(obj);
+    }
+
+    @Override
+    public String toString() {
+        return delegate.toString();
+    }
+
+    public static <E> Builder<E> builder() {
+        return new Builder<>();
+    }
+
+    public static <E> Builder<E> builder(AttributeMap<E> map) {
+        return new Builder<E>().putAll(map);
+    }
+
+    public static class Builder<E> {
+        private AttributeMap<E> map = new AttributeMap<>();
+
+        private Builder() {}
+
+        public Builder<E> put(Attribute attr, E value) {
+            map.add(attr, value);
+            return this;
+        }
+
+        public Builder<E> putAll(AttributeMap<E> m) {
+            map.addAll(m);
+            return this;
+        }
+
+        public AttributeMap<E> build() {
+            return map;
+        }
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/AttributeSet.java

@@ -0,0 +1,190 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.Spliterator;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+
+/**
+ * Set variant of {@link AttributeMap} - please see that class Javadoc.
+ */
+public class AttributeSet implements Set<Attribute> {
+
+    public static final AttributeSet EMPTY = new AttributeSet(AttributeMap.emptyAttributeMap());
+
+    // use the same name as in HashSet
+    private static final Object PRESENT = new Object();
+
+    private final AttributeMap<Object> delegate;
+
+    public AttributeSet() {
+        delegate = new AttributeMap<>();
+    }
+
+    public AttributeSet(Attribute attr) {
+        delegate = new AttributeMap<>(attr, PRESENT);
+    }
+
+    public AttributeSet(Collection<? extends Attribute> attr) {
+        delegate = new AttributeMap<>();
+
+        for (Attribute a : attr) {
+            delegate.add(a, PRESENT);
+        }
+    }
+
+    private AttributeSet(AttributeMap<Object> delegate) {
+        this.delegate = delegate;
+    }
+
+    public AttributeSet combine(AttributeSet other) {
+        return new AttributeSet(delegate.combine(other.delegate));
+    }
+
+    public AttributeSet subtract(AttributeSet other) {
+        return new AttributeSet(delegate.subtract(other.delegate));
+    }
+
+    public AttributeSet intersect(AttributeSet other) {
+        return new AttributeSet(delegate.intersect(other.delegate));
+    }
+
+    public boolean subsetOf(AttributeSet other) {
+        return delegate.subsetOf(other.delegate);
+    }
+
+    public Set<String> names() {
+        return delegate.attributeNames();
+    }
+
+    @Override
+    public void forEach(Consumer<? super Attribute> action) {
+        delegate.forEach((k, v) -> action.accept(k));
+    }
+
+    @Override
+    public int size() {
+        return delegate.size();
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return delegate.isEmpty();
+    }
+
+    @Override
+    public boolean contains(Object o) {
+        return delegate.containsKey(o);
+    }
+
+    @Override
+    public boolean containsAll(Collection<?> c) {
+        for (Object o : c) {
+            if (delegate.containsKey(o) == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public Iterator<Attribute> iterator() {
+        return delegate.keySet().iterator();
+    }
+
+    @Override
+    public Object[] toArray() {
+        return delegate.keySet().toArray();
+    }
+
+    @Override
+    public <T> T[] toArray(T[] a) {
+        return delegate.keySet().toArray(a);
+    }
+
+    @Override
+    public boolean add(Attribute e) {
+        return delegate.put(e, PRESENT) != null;
+    }
+
+    @Override
+    public boolean remove(Object o) {
+        return delegate.remove(o) != null;
+    }
+
+    public void addAll(AttributeSet other) {
+        delegate.addAll(other.delegate);
+    }
+
+    @Override
+    public boolean addAll(Collection<? extends Attribute> c) {
+        int size = delegate.size();
+        for (var e : c) {
+            delegate.put(e, PRESENT);
+        }
+        return delegate.size() != size;
+    }
+
+    @Override
+    public boolean retainAll(Collection<?> c) {
+        return delegate.keySet().removeIf(e -> c.contains(e) == false);
+    }
+
+    @Override
+    public boolean removeAll(Collection<?> c) {
+        int size = delegate.size();
+        for (var e : c) {
+            delegate.remove(e);
+        }
+        return delegate.size() != size;
+    }
+
+    @Override
+    public void clear() {
+        delegate.clear();
+    }
+
+    @Override
+    public Spliterator<Attribute> spliterator() {
+        return delegate.keySet().spliterator();
+    }
+
+    @Override
+    public boolean removeIf(Predicate<? super Attribute> filter) {
+        return delegate.keySet().removeIf(filter);
+    }
+
+    @Override
+    public Stream<Attribute> stream() {
+        return delegate.keySet().stream();
+    }
+
+    @Override
+    public Stream<Attribute> parallelStream() {
+        return delegate.keySet().parallelStream();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return delegate.equals(o);
+    }
+
+    @Override
+    public int hashCode() {
+        return delegate.hashCode();
+    }
+
+    @Override
+    public String toString() {
+        return delegate.keySet().toString();
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/EmptyAttribute.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.util.StringUtils;
+
+/**
+ * Marker for optional attributes. Acting as a dummy placeholder to avoid using null
+ * in the tree (which is not allowed).
+ */
+public class EmptyAttribute extends Attribute {
+
+    public EmptyAttribute(Source source) {
+        super(source, StringUtils.EMPTY, null, null);
+    }
+
+    @Override
+    protected Attribute clone(
+        Source source,
+        String name,
+        DataType type,
+        String qualifier,
+        Nullability nullability,
+        NameId id,
+        boolean synthetic
+    ) {
+        return this;
+    }
+
+    @Override
+    protected String label() {
+        return "e";
+    }
+
+    @Override
+    public boolean resolved() {
+        return true;
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.NULL;
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this);
+    }
+
+    @Override
+    public int hashCode() {
+        return EmptyAttribute.class.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        return true;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java

@@ -0,0 +1,199 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.capabilities.Resolvable;
+import org.elasticsearch.xpack.esql.core.capabilities.Resolvables;
+import org.elasticsearch.xpack.esql.core.tree.Node;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.StringUtils;
+
+import java.util.List;
+import java.util.function.Supplier;
+
+/**
+ * In a SQL statement, an Expression is whatever a user specifies inside an
+ * action, so for instance:
+ *
+ * {@code SELECT a, b, ABS(c) FROM i}
+ *
+ * a, b, ABS(c), and i are all Expressions, with ABS(c) being a Function
+ * (which is a type of expression) with a single child, c.
+ */
+public abstract class Expression extends Node<Expression> implements Resolvable {
+
+    public static class TypeResolution {
+        private final boolean failed;
+        private final String message;
+
+        public static final TypeResolution TYPE_RESOLVED = new TypeResolution(false, StringUtils.EMPTY);
+
+        public TypeResolution(String message) {
+            this(true, message);
+        }
+
+        private TypeResolution(boolean unresolved, String message) {
+            this.failed = unresolved;
+            this.message = message;
+        }
+
+        public boolean unresolved() {
+            return failed;
+        }
+
+        public boolean resolved() {
+            return failed == false;
+        }
+
+        public TypeResolution and(TypeResolution other) {
+            return failed ? this : other;
+        }
+
+        public TypeResolution and(Supplier<TypeResolution> other) {
+            return failed ? this : other.get();
+        }
+
+        public String message() {
+            return message;
+        }
+
+        @Override
+        public String toString() {
+            return resolved() ? "" : message;
+        }
+    }
+
+    private TypeResolution lazyTypeResolution = null;
+    private Boolean lazyChildrenResolved = null;
+    private Expression lazyCanonical = null;
+    private AttributeSet lazyReferences = null;
+
+    public Expression(Source source, List<Expression> children) {
+        super(source, children);
+    }
+
+    // whether the expression can be evaluated statically (folded) or not
+    public boolean foldable() {
+        return false;
+    }
+
+    public Object fold() {
+        throw new QlIllegalArgumentException("Should not fold expression");
+    }
+
+    public abstract Nullability nullable();
+
+    // the references/inputs/leaves of the expression tree
+    public AttributeSet references() {
+        if (lazyReferences == null) {
+            lazyReferences = Expressions.references(children());
+        }
+        return lazyReferences;
+    }
+
+    public boolean childrenResolved() {
+        if (lazyChildrenResolved == null) {
+            lazyChildrenResolved = Boolean.valueOf(Resolvables.resolved(children()));
+        }
+        return lazyChildrenResolved;
+    }
+
+    /**
+     * Does the tree rooted at this expression have valid types at all nodes?
+     * <p>
+     *     For example, {@code SIN(1.2)} has a valid type and should return
+     *     {@link TypeResolution#TYPE_RESOLVED} to signal "this type is fine".
+     *     Another example, {@code SIN("cat")} has an invalid type in the
+     *     tree. The value passed to the {@code SIN} function is a string which
+     *     doesn't make any sense. So this method should return a "failure"
+     *     resolution which it can build by calling {@link TypeResolution#TypeResolution(String)}.
+     * </p>
+     * <p>
+     *     Take {@code SIN(1.2) + COS(ATAN("cat"))}, this tree should also
+     *     fail, specifically because {@code ATAN("cat")} is invalid. This should
+     *     fail even though {@code +} is perfectly valid when run on the results
+     *     of {@code SIN} and {@code COS}. And {@code COS} can operate on the results
+     *     of any valid call to {@code ATAN}. For this method to return a "valid"
+     *     result the <strong>whole</strong> tree rooted at this expression must
+     *     be valid.
+     * </p>
+     */
+    public final TypeResolution typeResolved() {
+        if (lazyTypeResolution == null) {
+            lazyTypeResolution = resolveType();
+        }
+        return lazyTypeResolution;
+    }
+
+    /**
+     * The implementation of {@link #typeResolved}, which is just a caching wrapper
+     * around this method. See it's javadoc for what this method should return.
+     * <p>
+     *     Implementations will rarely interact with the {@link TypeResolution}
+     *     class directly, instead usually calling the utility methods on {@link TypeResolutions}.
+     * </p>
+     * <p>
+     *     Implementations should fail if {@link #childrenResolved()} returns {@code false}.
+     * </p>
+     */
+    protected TypeResolution resolveType() {
+        return TypeResolution.TYPE_RESOLVED;
+    }
+
+    public final Expression canonical() {
+        if (lazyCanonical == null) {
+            lazyCanonical = canonicalize();
+        }
+        return lazyCanonical;
+    }
+
+    protected Expression canonicalize() {
+        if (children().isEmpty()) {
+            return this;
+        }
+        List<Expression> canonicalChildren = Expressions.canonicalize(children());
+        // check if replacement is really needed
+        if (children().equals(canonicalChildren)) {
+            return this;
+        }
+        return replaceChildrenSameSize(canonicalChildren);
+    }
+
+    public boolean semanticEquals(Expression other) {
+        return canonical().equals(other.canonical());
+    }
+
+    public int semanticHash() {
+        return canonical().hashCode();
+    }
+
+    @Override
+    public boolean resolved() {
+        return childrenResolved() && typeResolved().resolved();
+    }
+
+    /**
+     * The {@link DataType} returned by executing the tree rooted at this
+     * expression. If {@link #typeResolved()} returns an error then the behavior
+     * of this method is undefined. It <strong>may</strong> return a valid
+     * type. Or it may throw an exception. Or it may return a totally nonsensical
+     * type.
+     */
+    public abstract DataType dataType();
+
+    @Override
+    public String toString() {
+        return sourceText();
+    }
+
+    @Override
+    public String propertiesToString(boolean skipIfChild) {
+        return super.propertiesToString(false);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ExpressionSet.java

@@ -0,0 +1,154 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static java.util.Collections.emptyList;
+
+/**
+ * @param <E> expression type
+ */
+public final class ExpressionSet<E extends Expression> implements Set<E> {
+
+    @SuppressWarnings("rawtypes")
+    public static final ExpressionSet EMPTY = new ExpressionSet<>(emptyList());
+
+    @SuppressWarnings("unchecked")
+    public static <T extends Expression> ExpressionSet<T> emptySet() {
+        return (ExpressionSet<T>) EMPTY;
+    }
+
+    // canonical to actual/original association
+    private final Map<Expression, E> map = new LinkedHashMap<>();
+
+    public ExpressionSet() {
+        super();
+    }
+
+    public ExpressionSet(Collection<? extends E> c) {
+        addAll(c);
+    }
+
+    // Returns the equivalent expression (if already exists in the set) or null if none is found
+    public E get(Expression e) {
+        return map.get(e.canonical());
+    }
+
+    @Override
+    public int size() {
+        return map.size();
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return map.isEmpty();
+    }
+
+    @Override
+    public boolean contains(Object o) {
+        if (o instanceof Expression) {
+            return map.containsKey(((Expression) o).canonical());
+        }
+        return false;
+    }
+
+    @Override
+    public boolean containsAll(Collection<?> c) {
+        for (Object o : c) {
+            if (contains(o) == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public Iterator<E> iterator() {
+        return map.values().iterator();
+    }
+
+    @Override
+    public boolean add(E e) {
+        return map.putIfAbsent(e.canonical(), e) == null;
+    }
+
+    @Override
+    public boolean addAll(Collection<? extends E> c) {
+        boolean result = true;
+        for (E o : c) {
+            result &= add(o);
+        }
+        return result;
+    }
+
+    @Override
+    public boolean retainAll(Collection<?> c) {
+        boolean modified = false;
+
+        Iterator<Expression> keys = map.keySet().iterator();
+
+        while (keys.hasNext()) {
+            Expression key = keys.next();
+            boolean found = false;
+            for (Object o : c) {
+                if (o instanceof Expression) {
+                    o = ((Expression) o).canonical();
+                }
+                if (key.equals(o)) {
+                    found = true;
+                    break;
+                }
+            }
+            if (found == false) {
+                keys.remove();
+            }
+        }
+        return modified;
+    }
+
+    @Override
+    public boolean remove(Object o) {
+        if (o instanceof Expression) {
+            return map.remove(((Expression) o).canonical()) != null;
+        }
+        return false;
+    }
+
+    @Override
+    public boolean removeAll(Collection<?> c) {
+        boolean modified = false;
+        for (Object o : c) {
+            modified |= remove(o);
+        }
+        return modified;
+    }
+
+    @Override
+    public void clear() {
+        map.clear();
+    }
+
+    @Override
+    public Object[] toArray() {
+        return map.values().toArray();
+    }
+
+    @Override
+    public <T> T[] toArray(T[] a) {
+        return map.values().toArray(a);
+    }
+
+    @Override
+    public String toString() {
+        return map.toString();
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java

@@ -0,0 +1,241 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.function.Function;
+import org.elasticsearch.xpack.esql.core.expression.gen.pipeline.AttributeInput;
+import org.elasticsearch.xpack.esql.core.expression.gen.pipeline.ConstantInput;
+import org.elasticsearch.xpack.esql.core.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Predicate;
+
+import static java.util.Collections.emptyList;
+
+public final class Expressions {
+
+    private Expressions() {}
+
+    public static NamedExpression wrapAsNamed(Expression exp) {
+        return exp instanceof NamedExpression ne ? ne : new Alias(exp.source(), exp.sourceText(), exp);
+    }
+
+    public static List<Attribute> asAttributes(List<? extends NamedExpression> named) {
+        if (named.isEmpty()) {
+            return emptyList();
+        }
+        List<Attribute> list = new ArrayList<>(named.size());
+        for (NamedExpression exp : named) {
+            list.add(exp.toAttribute());
+        }
+        return list;
+    }
+
+    public static AttributeMap<Expression> asAttributeMap(List<? extends NamedExpression> named) {
+        if (named.isEmpty()) {
+            return AttributeMap.emptyAttributeMap();
+        }
+
+        AttributeMap<Expression> map = new AttributeMap<>();
+        for (NamedExpression exp : named) {
+            map.add(exp.toAttribute(), exp);
+        }
+        return map;
+    }
+
+    public static boolean anyMatch(List<? extends Expression> exps, Predicate<? super Expression> predicate) {
+        for (Expression exp : exps) {
+            if (exp.anyMatch(predicate)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public static boolean match(List<? extends Expression> exps, Predicate<? super Expression> predicate) {
+        for (Expression exp : exps) {
+            if (predicate.test(exp)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Return the logical AND of a list of {@code Nullability}
+     * <pre>
+     *  UNKNOWN AND TRUE/FALSE/UNKNOWN = UNKNOWN
+     *  FALSE AND FALSE = FALSE
+     *  TRUE AND FALSE/TRUE = TRUE
+     * </pre>
+     */
+    public static Nullability nullable(List<? extends Expression> exps) {
+        Nullability value = Nullability.FALSE;
+        for (Expression exp : exps) {
+            switch (exp.nullable()) {
+                case UNKNOWN:
+                    return Nullability.UNKNOWN;
+                case TRUE:
+                    value = Nullability.TRUE;
+                    break;
+                default:
+                    // not nullable
+                    break;
+            }
+        }
+        return value;
+    }
+
+    public static List<Expression> canonicalize(List<? extends Expression> exps) {
+        List<Expression> canonical = new ArrayList<>(exps.size());
+        for (Expression exp : exps) {
+            canonical.add(exp.canonical());
+        }
+        return canonical;
+    }
+
+    public static boolean foldable(List<? extends Expression> exps) {
+        for (Expression exp : exps) {
+            if (exp.foldable() == false) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public static List<Object> fold(List<? extends Expression> exps) {
+        List<Object> folded = new ArrayList<>(exps.size());
+        for (Expression exp : exps) {
+            folded.add(exp.fold());
+        }
+
+        return folded;
+    }
+
+    public static AttributeSet references(List<? extends Expression> exps) {
+        if (exps.isEmpty()) {
+            return AttributeSet.EMPTY;
+        }
+
+        AttributeSet set = new AttributeSet();
+        for (Expression exp : exps) {
+            set.addAll(exp.references());
+        }
+        return set;
+    }
+
+    public static String name(Expression e) {
+        return e instanceof NamedExpression ne ? ne.name() : e.sourceText();
+    }
+
+    public static boolean isNull(Expression e) {
+        return e.dataType() == DataTypes.NULL || (e.foldable() && e.fold() == null);
+    }
+
+    public static List<String> names(Collection<? extends Expression> e) {
+        List<String> names = new ArrayList<>(e.size());
+        for (Expression ex : e) {
+            names.add(name(ex));
+        }
+
+        return names;
+    }
+
+    public static Attribute attribute(Expression e) {
+        if (e instanceof NamedExpression ne) {
+            return ne.toAttribute();
+        }
+        return null;
+    }
+
+    public static boolean isPresent(NamedExpression e) {
+        return e instanceof EmptyAttribute == false;
+    }
+
+    public static boolean equalsAsAttribute(Expression left, Expression right) {
+        if (left.semanticEquals(right) == false) {
+            Attribute l = attribute(left);
+            return (l != null && l.semanticEquals(attribute(right)));
+        }
+        return true;
+    }
+
+    public static List<Tuple<Attribute, Expression>> aliases(List<? extends NamedExpression> named) {
+        // an alias of same name and data type can be reused (by mistake): need to use a list to collect all refs (and later report them)
+        List<Tuple<Attribute, Expression>> aliases = new ArrayList<>();
+        for (NamedExpression ne : named) {
+            if (ne instanceof Alias as) {
+                aliases.add(new Tuple<>(ne.toAttribute(), as.child()));
+            }
+        }
+        return aliases;
+    }
+
+    public static boolean hasReferenceAttribute(Collection<Attribute> output) {
+        for (Attribute attribute : output) {
+            if (attribute instanceof ReferenceAttribute) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public static List<Attribute> onlyPrimitiveFieldAttributes(Collection<Attribute> attributes) {
+        List<Attribute> filtered = new ArrayList<>();
+        // add only primitives
+        // but filter out multi fields (allow only the top-level value)
+        Set<Attribute> seenMultiFields = new LinkedHashSet<>();
+
+        for (Attribute a : attributes) {
+            if (DataTypes.isUnsupported(a.dataType()) == false && DataTypes.isPrimitive(a.dataType())) {
+                if (a instanceof FieldAttribute fa) {
+                    // skip nested fields and seen multi-fields
+                    if (fa.isNested() == false && seenMultiFields.contains(fa.parent()) == false) {
+                        filtered.add(a);
+                        seenMultiFields.add(a);
+                    }
+                } else {
+                    filtered.add(a);
+                }
+            }
+        }
+
+        return filtered;
+    }
+
+    public static Pipe pipe(Expression e) {
+        if (e.foldable()) {
+            return new ConstantInput(e.source(), e, e.fold());
+        }
+        if (e instanceof NamedExpression ne) {
+            return new AttributeInput(e.source(), e, ne.toAttribute());
+        }
+        if (e instanceof Function f) {
+            return f.asPipe();
+        }
+        throw new QlIllegalArgumentException("Cannot create pipe for {}", e);
+    }
+
+    public static List<Pipe> pipe(List<Expression> expressions) {
+        List<Pipe> pipes = new ArrayList<>(expressions.size());
+        for (Expression e : expressions) {
+            pipes.add(pipe(e));
+        }
+        return pipes;
+    }
+
+    public static String id(Expression e) {
+        return Integer.toHexString(e.hashCode());
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java

@@ -0,0 +1,156 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.util.StringUtils;
+
+import java.util.Objects;
+
+/**
+ * Attribute for an ES field.
+ * To differentiate between the different type of fields this class offers:
+ * - name - the fully qualified name (foo.bar.tar)
+ * - path - the path pointing to the field name (foo.bar)
+ * - parent - the immediate parent of the field; useful for figuring out the type of field (nested vs object)
+ * - nestedParent - if nested, what's the parent (which might not be the immediate one)
+ */
+public class FieldAttribute extends TypedAttribute {
+
+    private final FieldAttribute parent;
+    private final FieldAttribute nestedParent;
+    private final String path;
+    private final EsField field;
+
+    public FieldAttribute(Source source, String name, EsField field) {
+        this(source, null, name, field);
+    }
+
+    public FieldAttribute(Source source, FieldAttribute parent, String name, EsField field) {
+        this(source, parent, name, field, null, Nullability.TRUE, null, false);
+    }
+
+    public FieldAttribute(
+        Source source,
+        FieldAttribute parent,
+        String name,
+        EsField field,
+        String qualifier,
+        Nullability nullability,
+        NameId id,
+        boolean synthetic
+    ) {
+        this(source, parent, name, field.getDataType(), field, qualifier, nullability, id, synthetic);
+    }
+
+    public FieldAttribute(
+        Source source,
+        FieldAttribute parent,
+        String name,
+        DataType type,
+        EsField field,
+        String qualifier,
+        Nullability nullability,
+        NameId id,
+        boolean synthetic
+    ) {
+        super(source, name, type, qualifier, nullability, id, synthetic);
+        this.path = parent != null ? parent.name() : StringUtils.EMPTY;
+        this.parent = parent;
+        this.field = field;
+
+        // figure out the last nested parent
+        FieldAttribute nestedPar = null;
+        if (parent != null) {
+            nestedPar = parent.nestedParent;
+            if (parent.dataType() == DataTypes.NESTED) {
+                nestedPar = parent;
+            }
+        }
+        this.nestedParent = nestedPar;
+    }
+
+    @Override
+    protected NodeInfo<FieldAttribute> info() {
+        return NodeInfo.create(this, FieldAttribute::new, parent, name(), dataType(), field, qualifier(), nullable(), id(), synthetic());
+    }
+
+    public FieldAttribute parent() {
+        return parent;
+    }
+
+    public String path() {
+        return path;
+    }
+
+    public String qualifiedPath() {
+        // return only the qualifier is there's no path
+        return qualifier() != null ? qualifier() + (Strings.hasText(path) ? "." + path : StringUtils.EMPTY) : path;
+    }
+
+    public boolean isNested() {
+        return nestedParent != null;
+    }
+
+    public FieldAttribute nestedParent() {
+        return nestedParent;
+    }
+
+    public EsField.Exact getExactInfo() {
+        return field.getExactInfo();
+    }
+
+    public FieldAttribute exactAttribute() {
+        EsField exactField = field.getExactField();
+        if (exactField.equals(field) == false) {
+            return innerField(exactField);
+        }
+        return this;
+    }
+
+    private FieldAttribute innerField(EsField type) {
+        return new FieldAttribute(source(), this, name() + "." + type.getName(), type, qualifier(), nullable(), id(), synthetic());
+    }
+
+    @Override
+    protected Attribute clone(
+        Source source,
+        String name,
+        DataType type,
+        String qualifier,
+        Nullability nullability,
+        NameId id,
+        boolean synthetic
+    ) {
+        FieldAttribute qualifiedParent = parent != null ? (FieldAttribute) parent.withQualifier(qualifier) : null;
+        return new FieldAttribute(source, qualifiedParent, name, field, qualifier, nullability, id, synthetic);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), path);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return super.equals(obj) && Objects.equals(path, ((FieldAttribute) obj).path);
+    }
+
+    @Override
+    protected String label() {
+        return "f";
+    }
+
+    public EsField field() {
+        return field;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Foldables.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
+
+public abstract class Foldables {
+
+    public static Object valueOf(Expression e) {
+        if (e.foldable()) {
+            return e.fold();
+        }
+        throw new QlIllegalArgumentException("Cannot determine value for {}", e);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/LeafExpression.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+
+public abstract class LeafExpression extends Expression {
+
+    protected LeafExpression(Source source) {
+        super(source, emptyList());
+    }
+
+    @Override
+    public final Expression replaceChildren(List<Expression> newChildren) {
+        throw new UnsupportedOperationException("this type of node doesn't have any children to replace");
+    }
+
+    public AttributeSet references() {
+        return AttributeSet.EMPTY;
+    }
+
+    @Override
+    protected Expression canonicalize() {
+        return this;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java

@@ -0,0 +1,116 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+
+import java.util.Objects;
+
+/**
+ * SQL Literal or constant.
+ */
+public class Literal extends LeafExpression {
+
+    public static final Literal TRUE = new Literal(Source.EMPTY, Boolean.TRUE, DataTypes.BOOLEAN);
+    public static final Literal FALSE = new Literal(Source.EMPTY, Boolean.FALSE, DataTypes.BOOLEAN);
+    public static final Literal NULL = new Literal(Source.EMPTY, null, DataTypes.NULL);
+
+    private final Object value;
+    private final DataType dataType;
+
+    public Literal(Source source, Object value, DataType dataType) {
+        super(source);
+        this.dataType = dataType;
+        this.value = value;
+    }
+
+    @Override
+    protected NodeInfo<? extends Literal> info() {
+        return NodeInfo.create(this, Literal::new, value, dataType);
+    }
+
+    public Object value() {
+        return value;
+    }
+
+    @Override
+    public boolean foldable() {
+        return true;
+    }
+
+    @Override
+    public Nullability nullable() {
+        return value == null ? Nullability.TRUE : Nullability.FALSE;
+    }
+
+    @Override
+    public DataType dataType() {
+        return dataType;
+    }
+
+    @Override
+    public boolean resolved() {
+        return true;
+    }
+
+    @Override
+    public Object fold() {
+        return value;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(dataType, value);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        Literal other = (Literal) obj;
+        return Objects.equals(value, other.value) && Objects.equals(dataType, other.dataType);
+    }
+
+    @Override
+    public String toString() {
+        return String.valueOf(value);
+    }
+
+    @Override
+    public String nodeString() {
+        return toString() + "[" + dataType + "]";
+    }
+
+    /**
+     * Utility method for creating a literal out of a foldable expression.
+     * Throws an exception if the expression is not foldable.
+     */
+    public static Literal of(Expression foldable) {
+        if (foldable.foldable() == false) {
+            throw new QlIllegalArgumentException("Foldable expression required for Literal creation; received unfoldable " + foldable);
+        }
+
+        if (foldable instanceof Literal) {
+            return (Literal) foldable;
+        }
+
+        return new Literal(foldable.source(), foldable.fold(), foldable.dataType());
+    }
+
+    public static Literal of(Expression source, Object value) {
+        return new Literal(source.source(), value, source.dataType());
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/MetadataAttribute.java

@@ -0,0 +1,99 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.mapper.IdFieldMapper;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+
+import java.util.Map;
+
+import static org.elasticsearch.core.Tuple.tuple;
+
+public class MetadataAttribute extends TypedAttribute {
+
+    private static final Map<String, Tuple<DataType, Boolean>> ATTRIBUTES_MAP = Map.of(
+        "_version",
+        tuple(DataTypes.LONG, false), // _version field is not searchable
+        "_index",
+        tuple(DataTypes.KEYWORD, true),
+        IdFieldMapper.NAME,
+        tuple(DataTypes.KEYWORD, false), // actually searchable, but fielddata access on the _id field is disallowed by default
+        SourceFieldMapper.NAME,
+        tuple(DataTypes.SOURCE, false)
+    );
+
+    private final boolean searchable;
+
+    public MetadataAttribute(
+        Source source,
+        String name,
+        DataType dataType,
+        String qualifier,
+        Nullability nullability,
+        NameId id,
+        boolean synthetic,
+        boolean searchable
+    ) {
+        super(source, name, dataType, qualifier, nullability, id, synthetic);
+        this.searchable = searchable;
+    }
+
+    public MetadataAttribute(Source source, String name, DataType dataType, boolean searchable) {
+        this(source, name, dataType, null, Nullability.TRUE, null, false, searchable);
+    }
+
+    @Override
+    protected MetadataAttribute clone(
+        Source source,
+        String name,
+        DataType type,
+        String qualifier,
+        Nullability nullability,
+        NameId id,
+        boolean synthetic
+    ) {
+        return new MetadataAttribute(source, name, type, qualifier, nullability, id, synthetic, searchable);
+    }
+
+    @Override
+    protected String label() {
+        return "m";
+    }
+
+    @Override
+    protected NodeInfo<? extends Expression> info() {
+        return NodeInfo.create(this, MetadataAttribute::new, name(), dataType(), qualifier(), nullable(), id(), synthetic(), searchable);
+    }
+
+    public boolean searchable() {
+        return searchable;
+    }
+
+    private MetadataAttribute withSource(Source source) {
+        return new MetadataAttribute(source, name(), dataType(), qualifier(), nullable(), id(), synthetic(), searchable());
+    }
+
+    public static MetadataAttribute create(Source source, String name) {
+        var t = ATTRIBUTES_MAP.get(name);
+        return t != null ? new MetadataAttribute(source, name, t.v1(), t.v2()) : null;
+    }
+
+    public static DataType dataType(String name) {
+        var t = ATTRIBUTES_MAP.get(name);
+        return t != null ? t.v1() : null;
+    }
+
+    public static boolean isSupported(String name) {
+        return ATTRIBUTES_MAP.containsKey(name);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NameId.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Unique identifier for a named expression.
+ * <p>
+ * We use an {@link AtomicLong} to guarantee that they are unique
+ * and that create reproducible values when run in subsequent
+ * tests. They don't produce reproducible values in production, but
+ * you rarely debug with them in production and commonly do so in
+ * tests.
+ */
+public class NameId {
+    private static final AtomicLong COUNTER = new AtomicLong();
+    private final long id;
+
+    public NameId() {
+        this.id = COUNTER.incrementAndGet();
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == this) {
+            return true;
+        }
+        if (obj == null || obj.getClass() != getClass()) {
+            return false;
+        }
+        NameId other = (NameId) obj;
+        return id == other.id;
+    }
+
+    @Override
+    public String toString() {
+        return Long.toString(id);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/NamedExpression.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * An expression that has a name. Named expressions can be used as a result
+ * (by converting to an attribute).
+ */
+public abstract class NamedExpression extends Expression {
+
+    private final String name;
+    private final NameId id;
+    private final boolean synthetic;
+
+    public NamedExpression(Source source, String name, List<Expression> children, NameId id) {
+        this(source, name, children, id, false);
+    }
+
+    public NamedExpression(Source source, String name, List<Expression> children, NameId id, boolean synthetic) {
+        super(source, children);
+        this.name = name;
+        this.id = id == null ? new NameId() : id;
+        this.synthetic = synthetic;
+    }
+
+    public String name() {
+        return name;
+    }
+
+    public NameId id() {
+        return id;
+    }
+
+    public boolean synthetic() {
+        return synthetic;
+    }
+
+    public abstract Attribute toAttribute();
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), name, synthetic);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        NamedExpression other = (NamedExpression) obj;
+        return Objects.equals(synthetic, other.synthetic)
+            /*
+             * It is important that the line below be `name`
+             * and not `name()` because subclasses might override
+             * `name()` in ways that are not compatible with
+             * equality. Specifically the `Unresolved` subclasses.
+             */
+            && Objects.equals(name, other.name)
+            && Objects.equals(children(), other.children());
+    }
+
+    @Override
+    public String toString() {
+        return super.toString() + "#" + id();
+    }
+
+    @Override
+    public String nodeString() {
+        return name();
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Nullability.java

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+public enum Nullability {
+    TRUE,    // Whether the expression can become null
+    FALSE,   // The expression can never become null
+    UNKNOWN // Cannot determine if the expression supports possible null folding
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Order.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.util.List;
+import java.util.Objects;
+
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isExact;
+
+public class Order extends Expression {
+
+    public enum OrderDirection {
+        ASC,
+        DESC
+    }
+
+    public enum NullsPosition {
+        FIRST,
+        LAST,
+        /**
+         * Nulls position has not been specified by the user and an appropriate default will be used.
+         *
+         * The default values are chosen such that it stays compatible with previous behavior. Unfortunately, this results in
+         * inconsistencies across different types of queries (see https://github.com/elastic/elasticsearch/issues/77068).
+         */
+        ANY;
+    }
+
+    private final Expression child;
+    private final OrderDirection direction;
+    private final NullsPosition nulls;
+
+    public Order(Source source, Expression child, OrderDirection direction, NullsPosition nulls) {
+        super(source, singletonList(child));
+        this.child = child;
+        this.direction = direction;
+        this.nulls = nulls == null ? NullsPosition.ANY : nulls;
+    }
+
+    @Override
+    protected NodeInfo<Order> info() {
+        return NodeInfo.create(this, Order::new, child, direction, nulls);
+    }
+
+    @Override
+    public Nullability nullable() {
+        return Nullability.FALSE;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        return isExact(child, "ORDER BY cannot be applied to field of data type [{}]: {}");
+    }
+
+    @Override
+    public DataType dataType() {
+        return child.dataType();
+    }
+
+    @Override
+    public Order replaceChildren(List<Expression> newChildren) {
+        return new Order(source(), newChildren.get(0), direction, nulls);
+    }
+
+    public Expression child() {
+        return child;
+    }
+
+    public OrderDirection direction() {
+        return direction;
+    }
+
+    public NullsPosition nullsPosition() {
+        return nulls;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(child, direction, nulls);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        Order other = (Order) obj;
+        return Objects.equals(direction, other.direction) && Objects.equals(nulls, other.nulls) && Objects.equals(child, other.child);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/ReferenceAttribute.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+/**
+ * Attribute based on a reference to an expression.
+ */
+public class ReferenceAttribute extends TypedAttribute {
+
+    public ReferenceAttribute(Source source, String name, DataType dataType) {
+        this(source, name, dataType, null, Nullability.FALSE, null, false);
+    }
+
+    public ReferenceAttribute(
+        Source source,
+        String name,
+        DataType dataType,
+        String qualifier,
+        Nullability nullability,
+        NameId id,
+        boolean synthetic
+    ) {
+        super(source, name, dataType, qualifier, nullability, id, synthetic);
+    }
+
+    @Override
+    protected Attribute clone(
+        Source source,
+        String name,
+        DataType dataType,
+        String qualifier,
+        Nullability nullability,
+        NameId id,
+        boolean synthetic
+    ) {
+        return new ReferenceAttribute(source, name, dataType, qualifier, nullability, id, synthetic);
+    }
+
+    @Override
+    protected NodeInfo<ReferenceAttribute> info() {
+        return NodeInfo.create(this, ReferenceAttribute::new, name(), dataType(), qualifier(), nullable(), id(), synthetic());
+    }
+
+    @Override
+    protected String label() {
+        return "r";
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java

@@ -0,0 +1,184 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression.TypeResolution;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+import org.elasticsearch.xpack.esql.core.type.EsField;
+
+import java.util.Locale;
+import java.util.StringJoiner;
+import java.util.function.Predicate;
+
+import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
+import static org.elasticsearch.xpack.esql.core.expression.Expressions.name;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.BOOLEAN;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.NULL;
+
+public final class TypeResolutions {
+
+    public enum ParamOrdinal {
+        DEFAULT,
+        FIRST,
+        SECOND,
+        THIRD,
+        FOURTH,
+        FIFTH;
+
+        public static ParamOrdinal fromIndex(int index) {
+            return switch (index) {
+                case 0 -> FIRST;
+                case 1 -> SECOND;
+                case 2 -> THIRD;
+                case 3 -> FOURTH;
+                case 4 -> FIFTH;
+                default -> DEFAULT;
+            };
+        }
+    }
+
+    private TypeResolutions() {}
+
+    public static TypeResolution isBoolean(Expression e, String operationName, ParamOrdinal paramOrd) {
+        return isType(e, dt -> dt == BOOLEAN, operationName, paramOrd, "boolean");
+    }
+
+    public static TypeResolution isInteger(Expression e, String operationName, ParamOrdinal paramOrd) {
+        return isType(e, DataType::isInteger, operationName, paramOrd, "integer");
+    }
+
+    public static TypeResolution isNumeric(Expression e, String operationName, ParamOrdinal paramOrd) {
+        return isType(e, DataType::isNumeric, operationName, paramOrd, "numeric");
+    }
+
+    public static TypeResolution isString(Expression e, String operationName, ParamOrdinal paramOrd) {
+        return isType(e, DataTypes::isString, operationName, paramOrd, "string");
+    }
+
+    public static TypeResolution isIP(Expression e, String operationName, ParamOrdinal paramOrd) {
+        return isType(e, dt -> dt == IP, operationName, paramOrd, "ip");
+    }
+
+    public static TypeResolution isDate(Expression e, String operationName, ParamOrdinal paramOrd) {
+        return isType(e, dt -> dt == DATETIME, operationName, paramOrd, "datetime");
+    }
+
+    public static TypeResolution isExact(Expression e, String message) {
+        if (e instanceof FieldAttribute fa) {
+            EsField.Exact exact = fa.getExactInfo();
+            if (exact.hasExact() == false) {
+                return new TypeResolution(format(null, message, e.dataType().typeName(), exact.errorMsg()));
+            }
+        }
+        return TypeResolution.TYPE_RESOLVED;
+    }
+
+    public static TypeResolution isExact(Expression e, String operationName, ParamOrdinal paramOrd) {
+        if (e instanceof FieldAttribute fa) {
+            EsField.Exact exact = fa.getExactInfo();
+            if (exact.hasExact() == false) {
+                return new TypeResolution(
+                    format(
+                        null,
+                        "[{}] cannot operate on {}field of data type [{}]: {}",
+                        operationName,
+                        paramOrd == null || paramOrd == DEFAULT ? "" : paramOrd.name().toLowerCase(Locale.ROOT) + " argument ",
+                        e.dataType().typeName(),
+                        exact.errorMsg()
+                    )
+                );
+            }
+        }
+        return TypeResolution.TYPE_RESOLVED;
+    }
+
+    public static TypeResolution isStringAndExact(Expression e, String operationName, ParamOrdinal paramOrd) {
+        TypeResolution resolution = isString(e, operationName, paramOrd);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+
+        return isExact(e, operationName, paramOrd);
+    }
+
+    public static TypeResolution isIPAndExact(Expression e, String operationName, ParamOrdinal paramOrd) {
+        TypeResolution resolution = isIP(e, operationName, paramOrd);
+        if (resolution.unresolved()) {
+            return resolution;
+        }
+
+        return isExact(e, operationName, paramOrd);
+    }
+
+    public static TypeResolution isFoldable(Expression e, String operationName, ParamOrdinal paramOrd) {
+        if (e.foldable() == false) {
+            return new TypeResolution(
+                format(
+                    null,
+                    "{}argument of [{}] must be a constant, received [{}]",
+                    paramOrd == null || paramOrd == DEFAULT ? "" : paramOrd.name().toLowerCase(Locale.ROOT) + " ",
+                    operationName,
+                    Expressions.name(e)
+                )
+            );
+        }
+        return TypeResolution.TYPE_RESOLVED;
+    }
+
+    public static TypeResolution isNotFoldable(Expression e, String operationName, ParamOrdinal paramOrd) {
+        if (e.foldable()) {
+            return new TypeResolution(
+                format(
+                    null,
+                    "{}argument of [{}] must be a table column, found constant [{}]",
+                    paramOrd == null || paramOrd == DEFAULT ? "" : paramOrd.name().toLowerCase(Locale.ROOT) + " ",
+                    operationName,
+                    Expressions.name(e)
+                )
+            );
+        }
+        return TypeResolution.TYPE_RESOLVED;
+    }
+
+    public static TypeResolution isType(
+        Expression e,
+        Predicate<DataType> predicate,
+        String operationName,
+        ParamOrdinal paramOrd,
+        String... acceptedTypes
+    ) {
+        return predicate.test(e.dataType()) || e.dataType() == NULL
+            ? TypeResolution.TYPE_RESOLVED
+            : new TypeResolution(
+                format(
+                    null,
+                    "{}argument of [{}] must be [{}], found value [{}] type [{}]",
+                    paramOrd == null || paramOrd == DEFAULT ? "" : paramOrd.name().toLowerCase(Locale.ROOT) + " ",
+                    operationName,
+                    acceptedTypesForErrorMsg(acceptedTypes),
+                    name(e),
+                    e.dataType().typeName()
+                )
+            );
+    }
+
+    private static String acceptedTypesForErrorMsg(String... acceptedTypes) {
+        StringJoiner sj = new StringJoiner(", ");
+        for (int i = 0; i < acceptedTypes.length - 1; i++) {
+            sj.add(acceptedTypes[i]);
+        }
+        if (acceptedTypes.length > 1) {
+            return sj.toString() + " or " + acceptedTypes[acceptedTypes.length - 1];
+        } else {
+            return acceptedTypes[0];
+        }
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypedAttribute.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.util.Objects;
+
+public abstract class TypedAttribute extends Attribute {
+
+    private final DataType dataType;
+
+    protected TypedAttribute(
+        Source source,
+        String name,
+        DataType dataType,
+        String qualifier,
+        Nullability nullability,
+        NameId id,
+        boolean synthetic
+    ) {
+        super(source, name, qualifier, nullability, id, synthetic);
+        this.dataType = dataType;
+    }
+
+    @Override
+    public DataType dataType() {
+        return dataType;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), dataType);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        return super.equals(obj) && Objects.equals(dataType, ((TypedAttribute) obj).dataType);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnaryExpression.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.util.List;
+import java.util.Objects;
+
+import static java.util.Collections.singletonList;
+
+public abstract class UnaryExpression extends Expression {
+
+    private final Expression child;
+
+    protected UnaryExpression(Source source, Expression child) {
+        super(source, singletonList(child));
+        this.child = child;
+    }
+
+    @Override
+    public final UnaryExpression replaceChildren(List<Expression> newChildren) {
+        return replaceChild(newChildren.get(0));
+    }
+
+    protected abstract UnaryExpression replaceChild(Expression newChild);
+
+    public Expression child() {
+        return child;
+    }
+
+    @Override
+    public boolean foldable() {
+        return child.foldable();
+    }
+
+    @Override
+    public Nullability nullable() {
+        return child.nullable();
+    }
+
+    @Override
+    public boolean resolved() {
+        return child.resolved();
+    }
+
+    @Override
+    public DataType dataType() {
+        return child.dataType();
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(child);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        UnaryExpression other = (UnaryExpression) obj;
+        return Objects.equals(child, other.child);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAlias.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.List;
+import java.util.Objects;
+
+import static java.util.Collections.singletonList;
+
+public class UnresolvedAlias extends UnresolvedNamedExpression {
+
+    private final Expression child;
+
+    public UnresolvedAlias(Source source, Expression child) {
+        super(source, singletonList(child));
+        this.child = child;
+    }
+
+    @Override
+    protected NodeInfo<UnresolvedAlias> info() {
+        return NodeInfo.create(this, UnresolvedAlias::new, child);
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new UnresolvedAlias(source(), newChildren.get(0));
+    }
+
+    public Expression child() {
+        return child;
+    }
+
+    @Override
+    public String unresolvedMessage() {
+        return "Unknown alias [" + name() + "]";
+    }
+
+    @Override
+    public Nullability nullable() {
+        throw new UnresolvedException("nullable", this);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(child);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        /*
+         * Intentionally not calling the superclass
+         * equals because it uses id which we always
+         * mutate when we make a clone.
+         */
+        if (obj == null || obj.getClass() != getClass()) {
+            return false;
+        }
+        return Objects.equals(child, ((UnresolvedAlias) obj).child);
+    }
+
+    @Override
+    public String toString() {
+        return child + " AS ?";
+    }
+
+    @Override
+    public String nodeString() {
+        return toString();
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedAttribute.java

@@ -0,0 +1,136 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable;
+import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
+
+import java.util.List;
+import java.util.Objects;
+
+// unfortunately we can't use UnresolvedNamedExpression
+public class UnresolvedAttribute extends Attribute implements Unresolvable {
+
+    private final String unresolvedMsg;
+    private final boolean customMessage;
+    private final Object resolutionMetadata;
+
+    public UnresolvedAttribute(Source source, String name) {
+        this(source, name, null);
+    }
+
+    public UnresolvedAttribute(Source source, String name, String qualifier) {
+        this(source, name, qualifier, null);
+    }
+
+    public UnresolvedAttribute(Source source, String name, String qualifier, String unresolvedMessage) {
+        this(source, name, qualifier, null, unresolvedMessage, null);
+    }
+
+    @SuppressWarnings("this-escape")
+    public UnresolvedAttribute(
+        Source source,
+        String name,
+        String qualifier,
+        NameId id,
+        String unresolvedMessage,
+        Object resolutionMetadata
+    ) {
+        super(source, name, qualifier, id);
+        this.customMessage = unresolvedMessage != null;
+        this.unresolvedMsg = unresolvedMessage == null ? errorMessage(qualifiedName(), null) : unresolvedMessage;
+        this.resolutionMetadata = resolutionMetadata;
+    }
+
+    @Override
+    protected NodeInfo<UnresolvedAttribute> info() {
+        return NodeInfo.create(this, UnresolvedAttribute::new, name(), qualifier(), id(), unresolvedMsg, resolutionMetadata);
+    }
+
+    public Object resolutionMetadata() {
+        return resolutionMetadata;
+    }
+
+    public boolean customMessage() {
+        return customMessage;
+    }
+
+    @Override
+    public boolean resolved() {
+        return false;
+    }
+
+    @Override
+    protected Attribute clone(
+        Source source,
+        String name,
+        DataType dataType,
+        String qualifier,
+        Nullability nullability,
+        NameId id,
+        boolean synthetic
+    ) {
+        return this;
+    }
+
+    public UnresolvedAttribute withUnresolvedMessage(String unresolvedMessage) {
+        return new UnresolvedAttribute(source(), name(), qualifier(), id(), unresolvedMessage, resolutionMetadata());
+    }
+
+    @Override
+    public DataType dataType() {
+        throw new UnresolvedException("dataType", this);
+    }
+
+    @Override
+    public String toString() {
+        return UNRESOLVED_PREFIX + qualifiedName();
+    }
+
+    @Override
+    protected String label() {
+        return UNRESOLVED_PREFIX;
+    }
+
+    @Override
+    public String nodeString() {
+        return toString();
+    }
+
+    @Override
+    public String unresolvedMessage() {
+        return unresolvedMsg;
+    }
+
+    public static String errorMessage(String name, List<String> potentialMatches) {
+        String msg = "Unknown column [" + name + "]";
+        if (CollectionUtils.isEmpty(potentialMatches) == false) {
+            msg += ", did you mean "
+                + (potentialMatches.size() == 1 ? "[" + potentialMatches.get(0) + "]" : "any of " + potentialMatches.toString())
+                + "?";
+        }
+        return msg;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), resolutionMetadata, unresolvedMsg);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (super.equals(obj)) {
+            UnresolvedAttribute ua = (UnresolvedAttribute) obj;
+            return Objects.equals(resolutionMetadata, ua.resolutionMetadata) && Objects.equals(unresolvedMsg, ua.unresolvedMsg);
+        }
+        return false;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedNamedExpression.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable;
+import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.util.List;
+
+public abstract class UnresolvedNamedExpression extends NamedExpression implements Unresolvable {
+
+    public UnresolvedNamedExpression(Source source, List<Expression> children) {
+        super(source, "<unresolved>", children, new NameId());
+    }
+
+    @Override
+    public boolean resolved() {
+        return false;
+    }
+
+    @Override
+    public String name() {
+        throw new UnresolvedException("name", this);
+    }
+
+    @Override
+    public NameId id() {
+        throw new UnresolvedException("id", this);
+    }
+
+    @Override
+    public DataType dataType() {
+        throw new UnresolvedException("data type", this);
+    }
+
+    @Override
+    public Attribute toAttribute() {
+        throw new UnresolvedException("attribute", this);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/UnresolvedStar.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression;
+
+import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.List;
+import java.util.Objects;
+
+import static java.util.Collections.emptyList;
+
+public class UnresolvedStar extends UnresolvedNamedExpression {
+
+    // typically used for nested fields or inner/dotted fields
+    private final UnresolvedAttribute qualifier;
+
+    public UnresolvedStar(Source source, UnresolvedAttribute qualifier) {
+        super(source, emptyList());
+        this.qualifier = qualifier;
+    }
+
+    @Override
+    protected NodeInfo<UnresolvedStar> info() {
+        return NodeInfo.create(this, UnresolvedStar::new, qualifier);
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        throw new UnsupportedOperationException("this type of node doesn't have any children to replace");
+    }
+
+    @Override
+    public Nullability nullable() {
+        throw new UnresolvedException("nullable", this);
+    }
+
+    public UnresolvedAttribute qualifier() {
+        return qualifier;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(qualifier);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        /*
+         * Intentionally not calling the superclass
+         * equals because it uses id which we always
+         * mutate when we make a clone. So we need
+         * to ignore it in equals for the transform
+         * tests to pass.
+         */
+        if (obj == null || obj.getClass() != getClass()) {
+            return false;
+        }
+
+        UnresolvedStar other = (UnresolvedStar) obj;
+        return Objects.equals(qualifier, other.qualifier);
+    }
+
+    private String message() {
+        return (qualifier() != null ? qualifier().qualifiedName() + "." : "") + "*";
+    }
+
+    @Override
+    public String unresolvedMessage() {
+        return "Cannot determine columns for [" + message() + "]";
+    }
+
+    @Override
+    public String nodeString() {
+        return toString();
+    }
+
+    @Override
+    public String toString() {
+        return UNRESOLVED_PREFIX + message();
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/DefaultFunctionTypeRegistry.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function;
+
+import org.elasticsearch.xpack.esql.core.expression.function.aggregate.AggregateFunction;
+import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction;
+
+public class DefaultFunctionTypeRegistry implements FunctionTypeRegistry {
+
+    public static final DefaultFunctionTypeRegistry INSTANCE = new DefaultFunctionTypeRegistry();
+
+    private enum Types {
+        AGGREGATE(AggregateFunction.class),
+        SCALAR(ScalarFunction.class);
+
+        private Class<? extends Function> baseClass;
+
+        Types(Class<? extends Function> base) {
+            this.baseClass = base;
+        }
+    }
+
+    @Override
+    public String type(Class<? extends Function> clazz) {
+        for (Types type : Types.values()) {
+            if (type.baseClass.isAssignableFrom(clazz)) {
+                return type.name();
+            }
+        }
+        return "UNKNOWN";
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/Function.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.Nullability;
+import org.elasticsearch.xpack.esql.core.expression.gen.pipeline.ConstantInput;
+import org.elasticsearch.xpack.esql.core.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.StringJoiner;
+
+/**
+ * Any SQL expression with parentheses, like {@code MAX()}, or {@code ABS()}. A
+ * function is always a {@code NamedExpression}.
+ */
+public abstract class Function extends Expression {
+
+    private final String functionName = getClass().getSimpleName().toUpperCase(Locale.ROOT);
+
+    private Pipe lazyPipe = null;
+
+    // TODO: Functions supporting distinct should add a dedicated constructor Location, List<Expression>, boolean
+    protected Function(Source source, List<Expression> children) {
+        super(source, children);
+    }
+
+    public final List<Expression> arguments() {
+        return children();
+    }
+
+    public String functionName() {
+        return functionName;
+    }
+
+    @Override
+    public Nullability nullable() {
+        return Expressions.nullable(children());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getClass(), children());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        Function other = (Function) obj;
+        return Objects.equals(children(), other.children());
+    }
+
+    public Pipe asPipe() {
+        if (lazyPipe == null) {
+            lazyPipe = foldable() ? new ConstantInput(source(), this, fold()) : makePipe();
+        }
+        return lazyPipe;
+    }
+
+    protected Pipe makePipe() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public String nodeString() {
+        StringJoiner sj = new StringJoiner(",", functionName() + "(", ")");
+        for (Expression ex : arguments()) {
+            sj.add(ex.nodeString());
+        }
+        return sj.toString();
+    }
+
+    public abstract ScriptTemplate asScript();
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/FunctionDefinition.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function;
+
+import org.elasticsearch.xpack.esql.core.session.Configuration;
+
+import java.util.List;
+
+import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
+
+public class FunctionDefinition {
+    /**
+     * Converts an {@link UnresolvedFunction} into the a proper {@link Function}.
+     * <p>
+     * Provides the basic signature (unresolved function + runtime configuration object) while
+     * allowing extensions through the vararg extras which subclasses should expand for their
+     * own purposes.
+     */
+    @FunctionalInterface
+    public interface Builder {
+        Function build(UnresolvedFunction uf, Configuration configuration, Object... extras);
+    }
+
+    private final String name;
+    private final List<String> aliases;
+    private final Class<? extends Function> clazz;
+    private final Builder builder;
+
+    public FunctionDefinition(String name, List<String> aliases, Class<? extends Function> clazz, Builder builder) {
+        this.name = name;
+        this.aliases = aliases;
+        this.clazz = clazz;
+        this.builder = builder;
+    }
+
+    public String name() {
+        return name;
+    }
+
+    public List<String> aliases() {
+        return aliases;
+    }
+
+    public Class<? extends Function> clazz() {
+        return clazz;
+    }
+
+    protected Builder builder() {
+        return builder;
+    }
+
+    @Override
+    public String toString() {
+        return format(null, "{}({})", name, aliases.isEmpty() ? "" : aliases.size() == 1 ? aliases.get(0) : aliases);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/FunctionRegistry.java

@@ -0,0 +1,463 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function;
+
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.xpack.esql.core.ParsingException;
+import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.session.Configuration;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.util.Check;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.function.BiFunction;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.unmodifiableList;
+import static java.util.stream.Collectors.toList;
+
+public class FunctionRegistry {
+
+    // Translation table for error messaging in the following function
+    private static final String[] NUM_NAMES = { "zero", "one", "two", "three", "four", "five", };
+
+    // list of functions grouped by type of functions (aggregate, statistics, math etc) and ordered alphabetically inside each group
+    // a single function will have one entry for itself with its name associated to its instance and, also, one entry for each alias
+    // it has with the alias name associated to the FunctionDefinition instance
+    private final Map<String, FunctionDefinition> defs = new LinkedHashMap<>();
+    private final Map<String, String> aliases = new HashMap<>();
+
+    public FunctionRegistry() {}
+
+    /**
+     * Register the given function definitions with this registry.
+     */
+    @SuppressWarnings("this-escape")
+    public FunctionRegistry(FunctionDefinition... functions) {
+        register(functions);
+    }
+
+    @SuppressWarnings("this-escape")
+    public FunctionRegistry(FunctionDefinition[]... groupFunctions) {
+        register(groupFunctions);
+    }
+
+    protected void register(FunctionDefinition[]... groupFunctions) {
+        for (FunctionDefinition[] group : groupFunctions) {
+            register(group);
+        }
+    }
+
+    protected void register(FunctionDefinition... functions) {
+        // temporary map to hold [function_name/alias_name : function instance]
+        Map<String, FunctionDefinition> batchMap = new HashMap<>();
+        for (FunctionDefinition f : functions) {
+            batchMap.put(f.name(), f);
+            for (String alias : f.aliases()) {
+                Object old = batchMap.put(alias, f);
+                if (old != null || defs.containsKey(alias)) {
+                    throw new QlIllegalArgumentException(
+                        "alias ["
+                            + alias
+                            + "] is used by "
+                            + "["
+                            + (old != null ? old : defs.get(alias).name())
+                            + "] and ["
+                            + f.name()
+                            + "]"
+                    );
+                }
+                aliases.put(alias, f.name());
+            }
+        }
+        // sort the temporary map by key name and add it to the global map of functions
+        defs.putAll(
+            batchMap.entrySet()
+                .stream()
+                .sorted(Map.Entry.comparingByKey())
+                .collect(
+                    Collectors.<
+                        Entry<String, FunctionDefinition>,
+                        String,
+                        FunctionDefinition,
+                        LinkedHashMap<String, FunctionDefinition>>toMap(
+                            Map.Entry::getKey,
+                            Map.Entry::getValue,
+                            (oldValue, newValue) -> oldValue,
+                            LinkedHashMap::new
+                        )
+                )
+        );
+    }
+
+    public FunctionDefinition resolveFunction(String functionName) {
+        FunctionDefinition def = defs.get(functionName);
+        if (def == null) {
+            throw new QlIllegalArgumentException("Cannot find function {}; this should have been caught during analysis", functionName);
+        }
+        return def;
+    }
+
+    protected String normalize(String name) {
+        return name.toUpperCase(Locale.ROOT);
+    }
+
+    public String resolveAlias(String alias) {
+        String normalized = normalize(alias);
+        return aliases.getOrDefault(normalized, normalized);
+    }
+
+    public boolean functionExists(String functionName) {
+        return defs.containsKey(functionName);
+    }
+
+    public Collection<FunctionDefinition> listFunctions() {
+        // It is worth double checking if we need this copy. These are immutable anyway.
+        return defs.values();
+    }
+
+    public Collection<FunctionDefinition> listFunctions(String pattern) {
+        // It is worth double checking if we need this copy. These are immutable anyway.
+        Pattern p = Strings.hasText(pattern) ? Pattern.compile(normalize(pattern)) : null;
+        return defs.entrySet()
+            .stream()
+            .filter(e -> p == null || p.matcher(e.getKey()).matches())
+            .map(e -> cloneDefinition(e.getKey(), e.getValue()))
+            .collect(toList());
+    }
+
+    protected FunctionDefinition cloneDefinition(String name, FunctionDefinition definition) {
+        return new FunctionDefinition(name, emptyList(), definition.clazz(), definition.builder());
+    }
+
+    protected interface FunctionBuilder {
+        Function build(Source source, List<Expression> children, Configuration cfg);
+    }
+
+    /**
+     * Main method to register a function.
+     *
+     * @param names Must always have at least one entry which is the method's primary name
+     */
+    @SuppressWarnings("overloads")
+    protected static FunctionDefinition def(Class<? extends Function> function, FunctionBuilder builder, String... names) {
+        Check.isTrue(names.length > 0, "At least one name must be provided for the function");
+        String primaryName = names[0];
+        List<String> aliases = Arrays.asList(names).subList(1, names.length);
+        FunctionDefinition.Builder realBuilder = (uf, cfg, extras) -> {
+            if (CollectionUtils.isEmpty(extras) == false) {
+                throw new ParsingException(
+                    uf.source(),
+                    "Unused parameters {} detected when building [{}]",
+                    Arrays.toString(extras),
+                    primaryName
+                );
+            }
+            try {
+                return builder.build(uf.source(), uf.children(), cfg);
+            } catch (QlIllegalArgumentException e) {
+                throw new ParsingException(e, uf.source(), "error building [{}]: {}", primaryName, e.getMessage());
+            }
+        };
+        return new FunctionDefinition(primaryName, unmodifiableList(aliases), function, realBuilder);
+    }
+
+    /**
+     * Build a {@linkplain FunctionDefinition} for a no-argument function.
+     */
+    protected static <T extends Function> FunctionDefinition def(
+        Class<T> function,
+        java.util.function.Function<Source, T> ctorRef,
+        String... names
+    ) {
+        FunctionBuilder builder = (source, children, cfg) -> {
+            if (false == children.isEmpty()) {
+                throw new QlIllegalArgumentException("expects no arguments");
+            }
+            return ctorRef.apply(source);
+        };
+        return def(function, builder, names);
+    }
+
+    /**
+     * Build a {@linkplain FunctionDefinition} for a unary function.
+     */
+    @SuppressWarnings("overloads")  // These are ambiguous if you aren't using ctor references but we always do
+    public static <T extends Function> FunctionDefinition def(
+        Class<T> function,
+        BiFunction<Source, Expression, T> ctorRef,
+        String... names
+    ) {
+        FunctionBuilder builder = (source, children, cfg) -> {
+            if (children.size() != 1) {
+                throw new QlIllegalArgumentException("expects exactly one argument");
+            }
+            return ctorRef.apply(source, children.get(0));
+        };
+        return def(function, builder, names);
+    }
+
+    /**
+     * Build a {@linkplain FunctionDefinition} for multi-arg/n-ary function.
+     */
+    @SuppressWarnings("overloads") // These are ambiguous if you aren't using ctor references but we always do
+    protected <T extends Function> FunctionDefinition def(Class<T> function, NaryBuilder<T> ctorRef, String... names) {
+        FunctionBuilder builder = (source, children, cfg) -> { return ctorRef.build(source, children); };
+        return def(function, builder, names);
+    }
+
+    protected interface NaryBuilder<T> {
+        T build(Source source, List<Expression> children);
+    }
+
+    /**
+     * Build a {@linkplain FunctionDefinition} for a binary function.
+     */
+    @SuppressWarnings("overloads")  // These are ambiguous if you aren't using ctor references but we always do
+    protected static <T extends Function> FunctionDefinition def(Class<T> function, BinaryBuilder<T> ctorRef, String... names) {
+        FunctionBuilder builder = (source, children, cfg) -> {
+            boolean isBinaryOptionalParamFunction = OptionalArgument.class.isAssignableFrom(function);
+            if (isBinaryOptionalParamFunction && (children.size() > 2 || children.size() < 1)) {
+                throw new QlIllegalArgumentException("expects one or two arguments");
+            } else if (isBinaryOptionalParamFunction == false && children.size() != 2) {
+                throw new QlIllegalArgumentException("expects exactly two arguments");
+            }
+
+            return ctorRef.build(source, children.get(0), children.size() == 2 ? children.get(1) : null);
+        };
+        return def(function, builder, names);
+    }
+
+    protected interface BinaryBuilder<T> {
+        T build(Source source, Expression left, Expression right);
+    }
+
+    /**
+     * Build a {@linkplain FunctionDefinition} for a ternary function.
+     */
+    @SuppressWarnings("overloads")  // These are ambiguous if you aren't using ctor references but we always do
+    protected static <T extends Function> FunctionDefinition def(Class<T> function, TernaryBuilder<T> ctorRef, String... names) {
+        FunctionBuilder builder = (source, children, cfg) -> {
+            boolean hasMinimumTwo = OptionalArgument.class.isAssignableFrom(function);
+            if (hasMinimumTwo && (children.size() > 3 || children.size() < 2)) {
+                throw new QlIllegalArgumentException("expects two or three arguments");
+            } else if (hasMinimumTwo == false && children.size() != 3) {
+                throw new QlIllegalArgumentException("expects exactly three arguments");
+            }
+            return ctorRef.build(source, children.get(0), children.get(1), children.size() == 3 ? children.get(2) : null);
+        };
+        return def(function, builder, names);
+    }
+
+    protected interface TernaryBuilder<T> {
+        T build(Source source, Expression one, Expression two, Expression three);
+    }
+
+    /**
+     * Build a {@linkplain FunctionDefinition} for a quaternary function.
+     */
+    @SuppressWarnings("overloads")  // These are ambiguous if you aren't using ctor references but we always do
+    protected static <T extends Function> FunctionDefinition def(Class<T> function, QuaternaryBuilder<T> ctorRef, String... names) {
+        FunctionBuilder builder = (source, children, cfg) -> {
+            if (OptionalArgument.class.isAssignableFrom(function)) {
+                if (children.size() > 4 || children.size() < 3) {
+                    throw new QlIllegalArgumentException("expects three or four arguments");
+                }
+            } else if (TwoOptionalArguments.class.isAssignableFrom(function)) {
+                if (children.size() > 4 || children.size() < 2) {
+                    throw new QlIllegalArgumentException("expects minimum two, maximum four arguments");
+                }
+            } else if (children.size() != 4) {
+                throw new QlIllegalArgumentException("expects exactly four arguments");
+            }
+            return ctorRef.build(
+                source,
+                children.get(0),
+                children.get(1),
+                children.size() > 2 ? children.get(2) : null,
+                children.size() > 3 ? children.get(3) : null
+            );
+        };
+        return def(function, builder, names);
+    }
+
+    protected interface QuaternaryBuilder<T> {
+        T build(Source source, Expression one, Expression two, Expression three, Expression four);
+    }
+
+    /**
+     * Build a {@linkplain FunctionDefinition} for a quinary function.
+     */
+    @SuppressWarnings("overloads")  // These are ambiguous if you aren't using ctor references but we always do
+    protected static <T extends Function> FunctionDefinition def(
+        Class<T> function,
+        QuinaryBuilder<T> ctorRef,
+        int numOptionalParams,
+        String... names
+    ) {
+        FunctionBuilder builder = (source, children, cfg) -> {
+            final int NUM_TOTAL_PARAMS = 5;
+            boolean hasOptionalParams = OptionalArgument.class.isAssignableFrom(function);
+            if (hasOptionalParams && (children.size() > NUM_TOTAL_PARAMS || children.size() < NUM_TOTAL_PARAMS - numOptionalParams)) {
+                throw new QlIllegalArgumentException(
+                    "expects between "
+                        + NUM_NAMES[NUM_TOTAL_PARAMS - numOptionalParams]
+                        + " and "
+                        + NUM_NAMES[NUM_TOTAL_PARAMS]
+                        + " arguments"
+                );
+            } else if (hasOptionalParams == false && children.size() != NUM_TOTAL_PARAMS) {
+                throw new QlIllegalArgumentException("expects exactly " + NUM_NAMES[NUM_TOTAL_PARAMS] + " arguments");
+            }
+            return ctorRef.build(
+                source,
+                children.size() > 0 ? children.get(0) : null,
+                children.size() > 1 ? children.get(1) : null,
+                children.size() > 2 ? children.get(2) : null,
+                children.size() > 3 ? children.get(3) : null,
+                children.size() > 4 ? children.get(4) : null
+            );
+        };
+        return def(function, builder, names);
+    }
+
+    protected interface QuinaryBuilder<T> {
+        T build(Source source, Expression one, Expression two, Expression three, Expression four, Expression five);
+    }
+
+    /**
+     * Build a {@linkplain FunctionDefinition} for functions with a mandatory argument followed by a varidic list.
+     */
+    @SuppressWarnings("overloads")  // These are ambiguous if you aren't using ctor references but we always do
+    protected static <T extends Function> FunctionDefinition def(Class<T> function, UnaryVariadicBuilder<T> ctorRef, String... names) {
+        FunctionBuilder builder = (source, children, cfg) -> {
+            boolean hasMinimumOne = OptionalArgument.class.isAssignableFrom(function);
+            if (hasMinimumOne && children.size() < 1) {
+                throw new QlIllegalArgumentException("expects at least one argument");
+            } else if (hasMinimumOne == false && children.size() < 2) {
+                throw new QlIllegalArgumentException("expects at least two arguments");
+            }
+            return ctorRef.build(source, children.get(0), children.subList(1, children.size()));
+        };
+        return def(function, builder, names);
+    }
+
+    protected interface UnaryVariadicBuilder<T> {
+        T build(Source source, Expression exp, List<Expression> variadic);
+    }
+
+    /**
+     * Build a {@linkplain FunctionDefinition} for a no-argument function that is configuration aware.
+     */
+    @SuppressWarnings("overloads")
+    protected static <T extends Function> FunctionDefinition def(Class<T> function, ConfigurationAwareBuilder<T> ctorRef, String... names) {
+        FunctionBuilder builder = (source, children, cfg) -> {
+            if (false == children.isEmpty()) {
+                throw new QlIllegalArgumentException("expects no arguments");
+            }
+            return ctorRef.build(source, cfg);
+        };
+        return def(function, builder, names);
+    }
+
+    protected interface ConfigurationAwareBuilder<T> {
+        T build(Source source, Configuration configuration);
+    }
+
+    /**
+     * Build a {@linkplain FunctionDefinition} for a one-argument function that is configuration aware.
+     */
+    @SuppressWarnings("overloads")
+    protected static <T extends Function> FunctionDefinition def(
+        Class<T> function,
+        UnaryConfigurationAwareBuilder<T> ctorRef,
+        String... names
+    ) {
+        FunctionBuilder builder = (source, children, cfg) -> {
+            if (children.size() > 1) {
+                throw new QlIllegalArgumentException("expects exactly one argument");
+            }
+            Expression ex = children.size() == 1 ? children.get(0) : null;
+            return ctorRef.build(source, ex, cfg);
+        };
+        return def(function, builder, names);
+    }
+
+    protected interface UnaryConfigurationAwareBuilder<T> {
+        T build(Source source, Expression exp, Configuration configuration);
+    }
+
+    /**
+     * Build a {@linkplain FunctionDefinition} for a binary function that is configuration aware.
+     */
+    @SuppressWarnings("overloads")  // These are ambiguous if you aren't using ctor references but we always do
+    protected static <T extends Function> FunctionDefinition def(
+        Class<T> function,
+        BinaryConfigurationAwareBuilder<T> ctorRef,
+        String... names
+    ) {
+        FunctionBuilder builder = (source, children, cfg) -> {
+            boolean isBinaryOptionalParamFunction = OptionalArgument.class.isAssignableFrom(function);
+            if (isBinaryOptionalParamFunction && (children.size() > 2 || children.size() < 1)) {
+                throw new QlIllegalArgumentException("expects one or two arguments");
+            } else if (isBinaryOptionalParamFunction == false && children.size() != 2) {
+                throw new QlIllegalArgumentException("expects exactly two arguments");
+            }
+            return ctorRef.build(source, children.get(0), children.size() == 2 ? children.get(1) : null, cfg);
+        };
+        return def(function, builder, names);
+    }
+
+    protected interface BinaryConfigurationAwareBuilder<T> {
+        T build(Source source, Expression left, Expression right, Configuration configuration);
+    }
+
+    /**
+     * Build a {@linkplain FunctionDefinition} for a ternary function that is configuration aware.
+     */
+    @SuppressWarnings("overloads")  // These are ambiguous if you aren't using ctor references but we always do
+    protected <T extends Function> FunctionDefinition def(Class<T> function, TernaryConfigurationAwareBuilder<T> ctorRef, String... names) {
+        FunctionBuilder builder = (source, children, cfg) -> {
+            boolean hasMinimumTwo = OptionalArgument.class.isAssignableFrom(function);
+            if (hasMinimumTwo && (children.size() > 3 || children.size() < 2)) {
+                throw new QlIllegalArgumentException("expects two or three arguments");
+            } else if (hasMinimumTwo == false && children.size() != 3) {
+                throw new QlIllegalArgumentException("expects exactly three arguments");
+            }
+            return ctorRef.build(source, children.get(0), children.get(1), children.size() == 3 ? children.get(2) : null, cfg);
+        };
+        return def(function, builder, names);
+    }
+
+    protected interface TernaryConfigurationAwareBuilder<T> {
+        T build(Source source, Expression one, Expression two, Expression three, Configuration configuration);
+    }
+
+    //
+    // Utility method for extra argument extraction.
+    //
+    protected static Boolean asBool(Object[] extras) {
+        if (CollectionUtils.isEmpty(extras)) {
+            return null;
+        }
+        if (extras.length != 1 || (extras[0] instanceof Boolean) == false) {
+            throw new QlIllegalArgumentException("Invalid number and types of arguments given to function definition");
+        }
+        return (Boolean) extras[0];
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/FunctionResolutionStrategy.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression.function;
+
+import org.elasticsearch.xpack.esql.core.session.Configuration;
+
+/**
+ * Strategy indicating the type of resolution to apply for resolving the actual function definition in a pluggable way.
+ */
+public interface FunctionResolutionStrategy {
+
+    /**
+     * Default behavior of standard function calls like {@code ABS(col)}.
+     */
+    FunctionResolutionStrategy DEFAULT = new FunctionResolutionStrategy() {
+    };
+
+    /**
+     * Build the real function from this one and resolution metadata.
+     */
+    default Function buildResolved(UnresolvedFunction uf, Configuration cfg, FunctionDefinition def) {
+        return def.builder().build(uf, cfg);
+    }
+
+    /**
+     * The kind of strategy being applied. Used when
+     * building the error message sent back to the user when
+     * they specify a function that doesn't exist.
+     */
+    default String kind() {
+        return "function";
+    }
+
+    /**
+     * Is {@code def} a valid alternative for function invocations
+     * of this kind. Used to filter the list of "did you mean"
+     * options sent back to the user when they specify a missing
+     * function.
+     */
+    default boolean isValidAlternative(FunctionDefinition def) {
+        return true;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/FunctionTypeRegistry.java

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression.function;
+
+public interface FunctionTypeRegistry {
+
+    String type(Class<? extends Function> clazz);
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/Functions.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.function.aggregate.AggregateFunction;
+import org.elasticsearch.xpack.esql.core.expression.function.grouping.GroupingFunction;
+
+public abstract class Functions {
+
+    public static boolean isAggregate(Expression e) {
+        return e instanceof AggregateFunction;
+    }
+
+    public static boolean isGrouping(Expression e) {
+        return e instanceof GroupingFunction;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/OptionalArgument.java

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression.function;
+
+/**
+ * Marker interface indicating that a function accepts one optional argument (typically the last one).
+ * This is used by the {@link FunctionRegistry} to perform validation of function declaration.
+ */
+public interface OptionalArgument {
+
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/TwoOptionalArguments.java

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression.function;
+
+/**
+ * Marker interface indicating that a function accepts two optional arguments (the last two).
+ * This is used by the {@link FunctionRegistry} to perform validation of function declaration.
+ */
+public interface TwoOptionalArguments {
+
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/UnresolvedFunction.java

@@ -0,0 +1,170 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function;
+
+import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable;
+import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Nullability;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.esql.core.session.Configuration;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.StringUtils;
+
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+public class UnresolvedFunction extends Function implements Unresolvable {
+
+    private final String name;
+    private final String unresolvedMsg;
+    private final FunctionResolutionStrategy resolution;
+
+    /**
+     * Flag to indicate analysis has been applied and there's no point in
+     * doing it again this is an optimization to prevent searching for a
+     * better unresolved message over and over again.
+     */
+    private final boolean analyzed;
+
+    public UnresolvedFunction(Source source, String name, FunctionResolutionStrategy resolutionStrategy, List<Expression> children) {
+        this(source, name, resolutionStrategy, children, false, null);
+    }
+
+    /**
+     * Constructor used for specifying a more descriptive message (typically
+     * 'did you mean') instead of the default one.
+     *
+     * @see #withMessage(String)
+     */
+    UnresolvedFunction(
+        Source source,
+        String name,
+        FunctionResolutionStrategy resolutionStrategy,
+        List<Expression> children,
+        boolean analyzed,
+        String unresolvedMessage
+    ) {
+        super(source, children);
+        this.name = name;
+        this.resolution = resolutionStrategy;
+        this.analyzed = analyzed;
+        this.unresolvedMsg = unresolvedMessage == null ? "Unknown " + resolutionStrategy.kind() + " [" + name + "]" : unresolvedMessage;
+    }
+
+    @Override
+    protected NodeInfo<UnresolvedFunction> info() {
+        return NodeInfo.create(this, UnresolvedFunction::new, name, resolution, children(), analyzed, unresolvedMsg);
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        return new UnresolvedFunction(source(), name, resolution, newChildren, analyzed, unresolvedMsg);
+    }
+
+    public UnresolvedFunction withMessage(String message) {
+        return new UnresolvedFunction(source(), name(), resolution, children(), true, message);
+    }
+
+    /**
+     * Build a function to replace this one after resolving the function.
+     */
+    public Function buildResolved(Configuration configuration, FunctionDefinition def) {
+        return resolution.buildResolved(this, configuration, def);
+    }
+
+    /**
+     * Build a marker {@link UnresolvedFunction} with an error message
+     * about the function being missing.
+     */
+    public UnresolvedFunction missing(String normalizedName, Iterable<FunctionDefinition> alternatives) {
+        // try to find alternatives
+        Set<String> names = new LinkedHashSet<>();
+        for (FunctionDefinition def : alternatives) {
+            if (resolution.isValidAlternative(def)) {
+                names.add(def.name());
+                names.addAll(def.aliases());
+            }
+        }
+
+        List<String> matches = StringUtils.findSimilar(normalizedName, names);
+        if (matches.isEmpty()) {
+            return this;
+        }
+        String matchesMessage = matches.size() == 1 ? "[" + matches.get(0) + "]" : "any of " + matches;
+        return withMessage("Unknown " + resolution.kind() + " [" + name + "], did you mean " + matchesMessage + "?");
+    }
+
+    @Override
+    public boolean resolved() {
+        return false;
+    }
+
+    public String name() {
+        return name;
+    }
+
+    public FunctionResolutionStrategy resolutionStrategy() {
+        return resolution;
+    }
+
+    public boolean analyzed() {
+        return analyzed;
+    }
+
+    @Override
+    public DataType dataType() {
+        throw new UnresolvedException("dataType", this);
+    }
+
+    @Override
+    public Nullability nullable() {
+        throw new UnresolvedException("nullable", this);
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        throw new UnresolvedException("script", this);
+    }
+
+    @Override
+    public String unresolvedMessage() {
+        return unresolvedMsg;
+    }
+
+    @Override
+    public String toString() {
+        return UNRESOLVED_PREFIX + name + children();
+    }
+
+    @Override
+    public String nodeString() {
+        return toString();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null || obj.getClass() != getClass()) {
+            return false;
+        }
+        UnresolvedFunction other = (UnresolvedFunction) obj;
+        return name.equals(other.name)
+            && resolution.equals(other.resolution)
+            && children().equals(other.children())
+            && analyzed == other.analyzed
+            && Objects.equals(unresolvedMsg, other.unresolvedMsg);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, resolution, children(), analyzed, unresolvedMsg);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/aggregate/AggregateFunction.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function.aggregate;
+
+import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.TypeResolutions;
+import org.elasticsearch.xpack.esql.core.expression.function.Function;
+import org.elasticsearch.xpack.esql.core.expression.gen.pipeline.AggNameInput;
+import org.elasticsearch.xpack.esql.core.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
+
+import java.util.List;
+import java.util.Objects;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT;
+
+/**
+ * A type of {@code Function} that takes multiple values and extracts a single value out of them. For example, {@code AVG()}.
+ */
+public abstract class AggregateFunction extends Function {
+
+    private final Expression field;
+    private final List<? extends Expression> parameters;
+
+    protected AggregateFunction(Source source, Expression field) {
+        this(source, field, emptyList());
+    }
+
+    protected AggregateFunction(Source source, Expression field, List<? extends Expression> parameters) {
+        super(source, CollectionUtils.combine(singletonList(field), parameters));
+        this.field = field;
+        this.parameters = parameters;
+    }
+
+    public Expression field() {
+        return field;
+    }
+
+    public List<? extends Expression> parameters() {
+        return parameters;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        return TypeResolutions.isExact(field, sourceText(), DEFAULT);
+    }
+
+    @Override
+    protected Pipe makePipe() {
+        // unresolved AggNameInput (should always get replaced by the folder)
+        return new AggNameInput(source(), this, sourceText());
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        throw new QlIllegalArgumentException("Aggregate functions cannot be scripted");
+    }
+
+    @Override
+    public int hashCode() {
+        // NB: the hashcode is currently used for key generation so
+        // to avoid clashes between aggs with the same arguments, add the class name as variation
+        return Objects.hash(getClass(), children());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (super.equals(obj)) {
+            AggregateFunction other = (AggregateFunction) obj;
+            return Objects.equals(other.field(), field()) && Objects.equals(other.parameters(), parameters());
+        }
+        return false;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/aggregate/CompoundAggregate.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression.function.aggregate;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+
+import java.util.List;
+
+/**
+ * Marker type for compound aggregates, that is an aggregate that provides multiple values (like Stats or Matrix)
+ */
+public interface CompoundAggregate {
+
+    Expression field();
+
+    List<? extends Expression> parameters();
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/aggregate/Count.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function.aggregate;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Count the number of documents matched ({@code COUNT})
+ * <strong>OR</strong> count the number of distinct values
+ * for a field that matched ({@code COUNT(DISTINCT}.
+ */
+public class Count extends AggregateFunction {
+
+    private final boolean distinct;
+
+    public Count(Source source, Expression field, boolean distinct) {
+        super(source, field);
+        this.distinct = distinct;
+    }
+
+    @Override
+    protected NodeInfo<Count> info() {
+        return NodeInfo.create(this, Count::new, field(), distinct);
+    }
+
+    @Override
+    public Count replaceChildren(List<Expression> newChildren) {
+        return new Count(source(), newChildren.get(0), distinct);
+    }
+
+    public boolean distinct() {
+        return distinct;
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.LONG;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), distinct());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (super.equals(obj)) {
+            Count other = (Count) obj;
+            return Objects.equals(other.distinct(), distinct());
+        }
+        return false;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/aggregate/EnclosedAgg.java

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function.aggregate;
+
+// Agg 'enclosed' by another agg. Used for agg that return multiple embedded aggs (like MatrixStats)
+public interface EnclosedAgg {
+
+    String innerName();
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/aggregate/InnerAggregate.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function.aggregate;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.Check;
+
+import java.util.List;
+import java.util.Objects;
+
+public class InnerAggregate extends AggregateFunction {
+
+    private final AggregateFunction inner;
+    private final CompoundAggregate outer;
+    private final String innerName;
+    // used when the result needs to be extracted from a map (like in MatrixAggs or Percentiles)
+    private final Expression innerKey;
+
+    public InnerAggregate(AggregateFunction inner, CompoundAggregate outer) {
+        this(inner.source(), inner, outer, null);
+    }
+
+    public InnerAggregate(Source source, AggregateFunction inner, CompoundAggregate outer, Expression innerKey) {
+        super(source, outer.field(), outer.parameters());
+        this.inner = inner;
+        this.outer = outer;
+        Check.isTrue(inner instanceof EnclosedAgg, "Inner function is not marked as Enclosed");
+        Check.isTrue(outer instanceof Expression, "CompoundAggregate is not an Expression");
+        this.innerName = ((EnclosedAgg) inner).innerName();
+        this.innerKey = innerKey;
+    }
+
+    @Override
+    protected NodeInfo<InnerAggregate> info() {
+        return NodeInfo.create(this, InnerAggregate::new, inner, outer, innerKey);
+    }
+
+    @Override
+    public Expression replaceChildren(List<Expression> newChildren) {
+        /* I can't figure out how rewriting this one's children ever worked because its
+         * are all twisted up in `outer`. Refusing to rewrite it doesn't break anything
+         * that I can see right now so lets just go with it and hope for the best.
+         * Maybe someone will make this make sense one day! */
+        throw new UnsupportedOperationException("can't be rewritten");
+    }
+
+    public AggregateFunction inner() {
+        return inner;
+    }
+
+    public CompoundAggregate outer() {
+        return outer;
+    }
+
+    public String innerName() {
+        return innerName;
+    }
+
+    public Expression innerKey() {
+        return innerKey;
+    }
+
+    @Override
+    public DataType dataType() {
+        return inner.dataType();
+    }
+
+    @Override
+    public String functionName() {
+        return inner.functionName();
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(inner, outer, innerKey);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (super.equals(obj)) {
+            InnerAggregate other = (InnerAggregate) obj;
+            return Objects.equals(inner, other.inner) && Objects.equals(outer, other.outer) && Objects.equals(innerKey, other.innerKey);
+        }
+        return false;
+    }
+
+    @Override
+    public String toString() {
+        return nodeName() + "[" + outer + ">" + inner.nodeName() + "]";
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/aggregate/SpatialAggregateFunction.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression.function.aggregate;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.Objects;
+
+/**
+ * All spatial aggregate functions extend this class to enable the planning of reading from doc values for higher performance.
+ * The AggregateMapper class will generate multiple aggregation functions for each combination, allowing the planner to
+ * select the best one.
+ */
+public abstract class SpatialAggregateFunction extends AggregateFunction {
+    protected final boolean useDocValues;
+
+    protected SpatialAggregateFunction(Source source, Expression field, boolean useDocValues) {
+        super(source, field);
+        this.useDocValues = useDocValues;
+    }
+
+    public abstract SpatialAggregateFunction withDocValues();
+
+    @Override
+    public int hashCode() {
+        // NB: the hashcode is currently used for key generation so
+        // to avoid clashes between aggs with the same arguments, add the class name as variation
+        return Objects.hash(getClass(), children(), useDocValues);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (super.equals(obj)) {
+            SpatialAggregateFunction other = (SpatialAggregateFunction) obj;
+            return Objects.equals(other.field(), field())
+                && Objects.equals(other.parameters(), parameters())
+                && Objects.equals(other.useDocValues, useDocValues);
+        }
+        return false;
+    }
+
+    public boolean useDocValues() {
+        return useDocValues;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/grouping/GroupingFunction.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function.grouping;
+
+import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.function.Function;
+import org.elasticsearch.xpack.esql.core.expression.gen.pipeline.AggNameInput;
+import org.elasticsearch.xpack.esql.core.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
+
+import java.util.List;
+import java.util.Objects;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
+
+/**
+ * A type of {@code Function} that creates groups or buckets.
+ */
+public abstract class GroupingFunction extends Function {
+
+    private final Expression field;
+    private final List<Expression> parameters;
+
+    protected GroupingFunction(Source source, Expression field) {
+        this(source, field, emptyList());
+    }
+
+    protected GroupingFunction(Source source, Expression field, List<Expression> parameters) {
+        super(source, CollectionUtils.combine(singletonList(field), parameters));
+        this.field = field;
+        this.parameters = parameters;
+    }
+
+    public Expression field() {
+        return field;
+    }
+
+    public List<Expression> parameters() {
+        return parameters;
+    }
+
+    @Override
+    protected Pipe makePipe() {
+        // unresolved AggNameInput (should always get replaced by the folder)
+        return new AggNameInput(source(), this, sourceText());
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        throw new QlIllegalArgumentException("Grouping functions cannot be scripted");
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (false == super.equals(obj)) {
+            return false;
+        }
+        GroupingFunction other = (GroupingFunction) obj;
+        return Objects.equals(other.field(), field()) && Objects.equals(other.parameters(), parameters());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(field(), parameters());
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/BaseSurrogateFunction.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression.function.scalar;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.List;
+
+public abstract class BaseSurrogateFunction extends ScalarFunction implements SurrogateFunction {
+
+    private ScalarFunction lazySubstitute;
+
+    public BaseSurrogateFunction(Source source) {
+        super(source);
+    }
+
+    public BaseSurrogateFunction(Source source, List<Expression> fields) {
+        super(source, fields);
+    }
+
+    @Override
+    public ScalarFunction substitute() {
+        if (lazySubstitute == null) {
+            lazySubstitute = makeSubstitute();
+        }
+        return lazySubstitute;
+    }
+
+    protected abstract ScalarFunction makeSubstitute();
+
+    @Override
+    public boolean foldable() {
+        return substitute().foldable();
+    }
+
+    @Override
+    public Object fold() {
+        return substitute().fold();
+    }
+
+    @Override
+    protected Pipe makePipe() {
+        return substitute().asPipe();
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        return substitute().asScript();
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/BinaryScalarFunction.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function.scalar;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.Scripts;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+public abstract class BinaryScalarFunction extends ScalarFunction {
+
+    private final Expression left, right;
+
+    protected BinaryScalarFunction(Source source, Expression left, Expression right) {
+        super(source, Arrays.asList(left, right));
+        this.left = left;
+        this.right = right;
+    }
+
+    @Override
+    public final BinaryScalarFunction replaceChildren(List<Expression> newChildren) {
+        Expression newLeft = newChildren.get(0);
+        Expression newRight = newChildren.get(1);
+
+        return left.equals(newLeft) && right.equals(newRight) ? this : replaceChildren(newLeft, newRight);
+    }
+
+    protected abstract BinaryScalarFunction replaceChildren(Expression newLeft, Expression newRight);
+
+    public Expression left() {
+        return left;
+    }
+
+    public Expression right() {
+        return right;
+    }
+
+    @Override
+    public boolean foldable() {
+        return left.foldable() && right.foldable();
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate leftScript = asScript(left());
+        ScriptTemplate rightScript = asScript(right());
+
+        return asScriptFrom(leftScript, rightScript);
+    }
+
+    protected ScriptTemplate asScriptFrom(ScriptTemplate leftScript, ScriptTemplate rightScript) {
+        return Scripts.binaryMethod(Scripts.classPackageAsPrefix(getClass()), scriptMethodName(), leftScript, rightScript, dataType());
+    }
+
+    protected String scriptMethodName() {
+        return getClass().getSimpleName().toLowerCase(Locale.ROOT);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/ConfigurationFunction.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression.function.scalar;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.session.Configuration;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.List;
+
+public abstract class ConfigurationFunction extends ScalarFunction {
+
+    private final Configuration configuration;
+
+    protected ConfigurationFunction(Source source, List<Expression> fields, Configuration configuration) {
+        super(source, fields);
+        this.configuration = configuration;
+    }
+
+    public Configuration configuration() {
+        return configuration;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/IntervalScripting.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression.function.scalar;
+
+// FIXME: accessor interface until making script generation pluggable
+public interface IntervalScripting {
+
+    String script();
+
+    String value();
+
+    String typeName();
+
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/ScalarFunction.java

@@ -0,0 +1,174 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function.scalar;
+
+import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.function.Function;
+import org.elasticsearch.xpack.esql.core.expression.function.aggregate.AggregateFunction;
+import org.elasticsearch.xpack.esql.core.expression.function.grouping.GroupingFunction;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.Params;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.ParamsBuilder;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.Scripts;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.DateUtils;
+
+import java.time.OffsetTime;
+import java.time.ZonedDateTime;
+import java.util.List;
+
+import static java.util.Collections.emptyList;
+import static org.elasticsearch.common.logging.LoggerMessageFormat.format;
+import static org.elasticsearch.xpack.esql.core.expression.gen.script.ParamsBuilder.paramsBuilder;
+import static org.elasticsearch.xpack.esql.core.expression.gen.script.Scripts.PARAM;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.DATETIME;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.LONG;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.UNSIGNED_LONG;
+
+/**
+ * A {@code ScalarFunction} is a {@code Function} that takes values from some
+ * operation and converts each to another value. An example would be
+ * {@code ABS()}, which takes one value at a time, applies a function to the
+ * value (abs) and returns a new value.
+ */
+public abstract class ScalarFunction extends Function {
+
+    protected ScalarFunction(Source source) {
+        super(source, emptyList());
+    }
+
+    protected ScalarFunction(Source source, List<Expression> fields) {
+        super(source, fields);
+    }
+
+    //
+    // Script generation
+    //
+    public ScriptTemplate asScript(Expression exp) {
+        if (exp.foldable()) {
+            return scriptWithFoldable(exp);
+        }
+
+        if (exp instanceof FieldAttribute) {
+            return scriptWithField((FieldAttribute) exp);
+        }
+
+        if (exp instanceof ScalarFunction) {
+            return scriptWithScalar((ScalarFunction) exp);
+        }
+
+        if (exp instanceof AggregateFunction) {
+            return scriptWithAggregate((AggregateFunction) exp);
+        }
+
+        if (exp instanceof GroupingFunction) {
+            return scriptWithGrouping((GroupingFunction) exp);
+        }
+        throw new QlIllegalArgumentException("Cannot evaluate script for expression {}", exp);
+    }
+
+    protected ScriptTemplate scriptWithFoldable(Expression foldable) {
+        Object fold = foldable.fold();
+
+        // FIXME: this needs to be refactored
+        //
+        // Custom type handling
+        //
+
+        // wrap intervals with dedicated methods for serialization
+        if (fold instanceof ZonedDateTime zdt) {
+            return new ScriptTemplate(
+                processScript("{sql}.asDateTime({})"),
+                paramsBuilder().variable(DateUtils.toString(zdt)).build(),
+                dataType()
+            );
+        }
+
+        if (fold instanceof IntervalScripting is) {
+            return new ScriptTemplate(
+                processScript(is.script()),
+                paramsBuilder().variable(is.value()).variable(is.typeName()).build(),
+                dataType()
+            );
+        }
+
+        if (fold instanceof OffsetTime ot) {
+            return new ScriptTemplate(processScript("{sql}.asTime({})"), paramsBuilder().variable(ot.toString()).build(), dataType());
+        }
+
+        if (fold != null && fold.getClass().getSimpleName().equals("GeoShape")) {
+            return new ScriptTemplate(processScript("{sql}.stWktToSql({})"), paramsBuilder().variable(fold.toString()).build(), dataType());
+        }
+
+        return new ScriptTemplate(processScript("{}"), paramsBuilder().variable(fold).build(), dataType());
+    }
+
+    protected ScriptTemplate scriptWithScalar(ScalarFunction scalar) {
+        ScriptTemplate nested = scalar.asScript();
+        return new ScriptTemplate(processScript(nested.template()), paramsBuilder().script(nested.params()).build(), dataType());
+    }
+
+    protected ScriptTemplate scriptWithAggregate(AggregateFunction aggregate) {
+        String template = PARAM;
+        ParamsBuilder paramsBuilder = paramsBuilder().agg(aggregate);
+
+        DataType nullSafeCastDataType = null;
+        DataType dataType = aggregate.dataType();
+        if (dataType.name().equals("DATE") || dataType == DATETIME ||
+        // Aggregations on date_nanos are returned as string
+            aggregate.field().dataType() == DATETIME) {
+
+            template = "{sql}.asDateTime({})";
+        } else if (dataType.isInteger()) {
+            // MAX, MIN need to retain field's data type, so that possible operations on integral types (like division) work
+            // correctly -> perform a cast in the aggs filtering script, the bucket selector for HAVING.
+            // SQL function classes not available in QL: filter by name
+            String fn = aggregate.functionName();
+            if ("MAX".equals(fn) || "MIN".equals(fn)) {
+                nullSafeCastDataType = dataType;
+            } else if ("SUM".equals(fn)) {
+                // SUM(integral_type) requires returning a LONG value
+                nullSafeCastDataType = LONG;
+            }
+        }
+        if (nullSafeCastDataType != null) {
+            template = "{ql}.nullSafeCastNumeric({},{})";
+            paramsBuilder.variable(nullSafeCastDataType.name());
+        }
+        return new ScriptTemplate(processScript(template), paramsBuilder.build(), dataType());
+    }
+
+    // This method isn't actually used at the moment, since there is no grouping function (ie HISTOGRAM)
+    // that currently results in a script being generated
+    protected ScriptTemplate scriptWithGrouping(GroupingFunction grouping) {
+        String template = PARAM;
+        return new ScriptTemplate(processScript(template), paramsBuilder().grouping(grouping).build(), dataType());
+    }
+
+    protected ScriptTemplate scriptWithField(FieldAttribute field) {
+        Params params = paramsBuilder().variable(field.exactAttribute().name()).build();
+        // unsigned_long fields get returned in scripts as plain longs, so a conversion is required
+        return field.dataType() != UNSIGNED_LONG
+            ? new ScriptTemplate(processScript(Scripts.DOC_VALUE), params, dataType())
+            : new ScriptTemplate(
+                processScript(format("{ql}.", "nullSafeCastToUnsignedLong({})", Scripts.DOC_VALUE)),
+                params,
+                UNSIGNED_LONG
+            );
+    }
+
+    protected String processScript(String script) {
+        return formatTemplate(script);
+    }
+
+    protected String formatTemplate(String template) {
+        return Scripts.formatTemplate(template);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/SurrogateFunction.java

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression.function.scalar;
+
+public interface SurrogateFunction {
+
+    ScalarFunction substitute();
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function.scalar;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.esql.core.expression.gen.pipeline.UnaryPipe;
+import org.elasticsearch.xpack.esql.core.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.List;
+
+import static java.util.Collections.singletonList;
+
+public abstract class UnaryScalarFunction extends ScalarFunction {
+
+    private final Expression field;
+
+    protected UnaryScalarFunction(Source source) {
+        super(source);
+        this.field = null;
+    }
+
+    protected UnaryScalarFunction(Source source, Expression field) {
+        super(source, singletonList(field));
+        this.field = field;
+    }
+
+    @Override
+    public final UnaryScalarFunction replaceChildren(List<Expression> newChildren) {
+        return replaceChild(newChildren.get(0));
+    }
+
+    protected abstract UnaryScalarFunction replaceChild(Expression newChild);
+
+    public Expression field() {
+        return field;
+    }
+
+    @Override
+    public final Pipe makePipe() {
+        return new UnaryPipe(source(), this, Expressions.pipe(field()), makeProcessor());
+    }
+
+    protected abstract Processor makeProcessor();
+
+    @Override
+    public boolean foldable() {
+        return field.foldable();
+    }
+
+    @Override
+    public Object fold() {
+        return makeProcessor().process(field().fold());
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        return asScript(field);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/string/BinaryComparisonCaseInsensitiveFunction.java

@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.Scripts;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+
+import java.util.Locale;
+import java.util.Objects;
+
+import static java.lang.String.format;
+import static java.util.Arrays.asList;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isStringAndExact;
+import static org.elasticsearch.xpack.esql.core.expression.gen.script.ParamsBuilder.paramsBuilder;
+
+public abstract class BinaryComparisonCaseInsensitiveFunction extends CaseInsensitiveScalarFunction {
+
+    private final Expression left, right;
+
+    protected BinaryComparisonCaseInsensitiveFunction(Source source, Expression left, Expression right, boolean caseInsensitive) {
+        super(source, asList(left, right), caseInsensitive);
+        this.left = left;
+        this.right = right;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (childrenResolved() == false) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution sourceResolution = isStringAndExact(left, sourceText(), FIRST);
+        if (sourceResolution.unresolved()) {
+            return sourceResolution;
+        }
+
+        return isStringAndExact(right, sourceText(), SECOND);
+    }
+
+    public Expression left() {
+        return left;
+    }
+
+    public Expression right() {
+        return right;
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.BOOLEAN;
+    }
+
+    @Override
+    public boolean foldable() {
+        return left.foldable() && right.foldable();
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate leftScript = asScript(left);
+        ScriptTemplate rightScript = asScript(right);
+
+        return asScriptFrom(leftScript, rightScript);
+    }
+
+    protected ScriptTemplate asScriptFrom(ScriptTemplate leftScript, ScriptTemplate rightScript) {
+        return new ScriptTemplate(
+            format(
+                Locale.ROOT,
+                formatTemplate("%s.%s(%s,%s,%s)"),
+                Scripts.classPackageAsPrefix(getClass()),
+                scriptMethodName(),
+                leftScript.template(),
+                rightScript.template(),
+                "{}"
+            ),
+            paramsBuilder().script(leftScript.params()).script(rightScript.params()).variable(isCaseInsensitive()).build(),
+            dataType()
+        );
+    }
+
+    protected String scriptMethodName() {
+        String simpleName = getClass().getSimpleName();
+        return Character.toLowerCase(simpleName.charAt(0)) + simpleName.substring(1);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(left, right, isCaseInsensitive());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        BinaryComparisonCaseInsensitiveFunction other = (BinaryComparisonCaseInsensitiveFunction) obj;
+        return Objects.equals(left, other.left)
+            && Objects.equals(right, other.right)
+            && Objects.equals(isCaseInsensitive(), other.isCaseInsensitive());
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/string/CaseInsensitiveScalarFunction.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.Scripts;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.List;
+import java.util.Objects;
+
+import static org.elasticsearch.xpack.esql.core.expression.gen.script.ParamsBuilder.paramsBuilder;
+
+public abstract class CaseInsensitiveScalarFunction extends ScalarFunction {
+
+    private final boolean caseInsensitive;
+
+    protected CaseInsensitiveScalarFunction(Source source, List<Expression> fields, boolean caseInsensitive) {
+        super(source, fields);
+        this.caseInsensitive = caseInsensitive;
+    }
+
+    public boolean isCaseInsensitive() {
+        return caseInsensitive;
+    }
+
+    @Override
+    public ScriptTemplate scriptWithField(FieldAttribute field) {
+        return new ScriptTemplate(
+            processScript(Scripts.DOC_VALUE),
+            paramsBuilder().variable(field.exactAttribute().name()).build(),
+            dataType()
+        );
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), isCaseInsensitive());
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        return super.equals(other) && Objects.equals(((CaseInsensitiveScalarFunction) other).caseInsensitive, caseInsensitive);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/string/StartsWith.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Expressions;
+import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.ParamsBuilder;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.ScriptTemplate;
+import org.elasticsearch.xpack.esql.core.expression.gen.script.Scripts;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.type.DataTypes;
+
+import java.util.Arrays;
+
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isStringAndExact;
+import static org.elasticsearch.xpack.esql.core.expression.function.scalar.string.StartsWithFunctionProcessor.doProcess;
+import static org.elasticsearch.xpack.esql.core.expression.gen.script.ParamsBuilder.paramsBuilder;
+
+/**
+ * Function that checks if first parameter starts with the second parameter. Both parameters should be strings
+ * and the function returns a boolean value.
+ */
+public abstract class StartsWith extends CaseInsensitiveScalarFunction {
+
+    private final Expression input;
+    private final Expression pattern;
+
+    public StartsWith(Source source, Expression input, Expression pattern, boolean caseInsensitive) {
+        super(source, Arrays.asList(input, pattern), caseInsensitive);
+        this.input = input;
+        this.pattern = pattern;
+    }
+
+    @Override
+    protected TypeResolution resolveType() {
+        if (childrenResolved() == false) {
+            return new TypeResolution("Unresolved children");
+        }
+
+        TypeResolution fieldResolution = isStringAndExact(input, sourceText(), FIRST);
+        if (fieldResolution.unresolved()) {
+            return fieldResolution;
+        }
+
+        return isStringAndExact(pattern, sourceText(), SECOND);
+    }
+
+    public Expression input() {
+        return input;
+    }
+
+    public Expression pattern() {
+        return pattern;
+    }
+
+    @Override
+    public Pipe makePipe() {
+        return new StartsWithFunctionPipe(source(), this, Expressions.pipe(input), Expressions.pipe(pattern), isCaseInsensitive());
+    }
+
+    @Override
+    public boolean foldable() {
+        return input.foldable() && pattern.foldable();
+    }
+
+    @Override
+    public Object fold() {
+        return doProcess(input.fold(), pattern.fold(), isCaseInsensitive());
+    }
+
+    @Override
+    public ScriptTemplate asScript() {
+        ScriptTemplate fieldScript = asScript(input);
+        ScriptTemplate patternScript = asScript(pattern);
+
+        return asScriptFrom(fieldScript, patternScript);
+    }
+
+    protected ScriptTemplate asScriptFrom(ScriptTemplate fieldScript, ScriptTemplate patternScript) {
+        ParamsBuilder params = paramsBuilder();
+
+        String template = formatTemplate("{ql}.startsWith(" + fieldScript.template() + ", " + patternScript.template() + ", {})");
+        params.script(fieldScript.params()).script(patternScript.params()).variable(isCaseInsensitive());
+
+        return new ScriptTemplate(template, params.build(), dataType());
+    }
+
+    @Override
+    public ScriptTemplate scriptWithField(FieldAttribute field) {
+        return new ScriptTemplate(
+            processScript(Scripts.DOC_VALUE),
+            paramsBuilder().variable(field.exactAttribute().name()).build(),
+            dataType()
+        );
+    }
+
+    @Override
+    public DataType dataType() {
+        return DataTypes.BOOLEAN;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/string/StartsWithFunctionPipe.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function.scalar.string;
+
+import org.elasticsearch.xpack.esql.core.execution.search.QlSourceBuilder;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.gen.pipeline.Pipe;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class StartsWithFunctionPipe extends Pipe {
+
+    private final Pipe input;
+    private final Pipe pattern;
+    private final boolean isCaseSensitive;
+
+    public StartsWithFunctionPipe(Source source, Expression expression, Pipe input, Pipe pattern, boolean isCaseSensitive) {
+        super(source, expression, Arrays.asList(input, pattern));
+        this.input = input;
+        this.pattern = pattern;
+        this.isCaseSensitive = isCaseSensitive;
+    }
+
+    @Override
+    public final Pipe replaceChildren(List<Pipe> newChildren) {
+        return replaceChildren(newChildren.get(0), newChildren.get(1));
+    }
+
+    @Override
+    public final Pipe resolveAttributes(AttributeResolver resolver) {
+        Pipe newField = input.resolveAttributes(resolver);
+        Pipe newPattern = pattern.resolveAttributes(resolver);
+        if (newField == input && newPattern == pattern) {
+            return this;
+        }
+        return replaceChildren(newField, newPattern);
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        return input.supportedByAggsOnlyQuery() && pattern.supportedByAggsOnlyQuery();
+    }
+
+    @Override
+    public boolean resolved() {
+        return input.resolved() && pattern.resolved();
+    }
+
+    protected Pipe replaceChildren(Pipe newField, Pipe newPattern) {
+        return new StartsWithFunctionPipe(source(), expression(), newField, newPattern, isCaseSensitive);
+    }
+
+    @Override
+    public final void collectFields(QlSourceBuilder sourceBuilder) {
+        input.collectFields(sourceBuilder);
+        pattern.collectFields(sourceBuilder);
+    }
+
+    @Override
+    protected NodeInfo<StartsWithFunctionPipe> info() {
+        return NodeInfo.create(this, StartsWithFunctionPipe::new, expression(), input, pattern, isCaseSensitive);
+    }
+
+    @Override
+    public StartsWithFunctionProcessor asProcessor() {
+        return new StartsWithFunctionProcessor(input.asProcessor(), pattern.asProcessor(), isCaseSensitive);
+    }
+
+    public Pipe input() {
+        return input;
+    }
+
+    public Pipe pattern() {
+        return pattern;
+    }
+
+    public boolean isCaseSensitive() {
+        return isCaseSensitive;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(input, pattern, isCaseSensitive);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        StartsWithFunctionPipe other = (StartsWithFunctionPipe) obj;
+        return Objects.equals(input, other.input)
+            && Objects.equals(pattern, other.pattern)
+            && Objects.equals(isCaseSensitive, other.isCaseSensitive);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/string/StartsWithFunctionProcessor.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.function.scalar.string;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.expression.gen.processor.Processor;
+
+import java.io.IOException;
+import java.util.Locale;
+import java.util.Objects;
+
+public class StartsWithFunctionProcessor implements Processor {
+
+    public static final String NAME = "sstw";
+
+    private final Processor source;
+    private final Processor pattern;
+    private final boolean caseInsensitive;
+
+    public StartsWithFunctionProcessor(Processor source, Processor pattern, boolean caseInsensitive) {
+        this.source = source;
+        this.pattern = pattern;
+        this.caseInsensitive = caseInsensitive;
+    }
+
+    public StartsWithFunctionProcessor(StreamInput in) throws IOException {
+        source = in.readNamedWriteable(Processor.class);
+        pattern = in.readNamedWriteable(Processor.class);
+        caseInsensitive = in.readBoolean();
+    }
+
+    @Override
+    public final void writeTo(StreamOutput out) throws IOException {
+        out.writeNamedWriteable(source);
+        out.writeNamedWriteable(pattern);
+        out.writeBoolean(caseInsensitive);
+    }
+
+    @Override
+    public Object process(Object input) {
+        return doProcess(source.process(input), pattern.process(input), isCaseInsensitive());
+    }
+
+    public static Object doProcess(Object source, Object pattern, boolean caseInsensitive) {
+        if (source == null) {
+            return null;
+        }
+        if (source instanceof String == false && source instanceof Character == false) {
+            throw new QlIllegalArgumentException("A string/char is required; received [{}]", source);
+        }
+        if (pattern == null) {
+            return null;
+        }
+        if (pattern instanceof String == false && pattern instanceof Character == false) {
+            throw new QlIllegalArgumentException("A string/char is required; received [{}]", pattern);
+        }
+
+        if (caseInsensitive == false) {
+            return source.toString().startsWith(pattern.toString());
+        } else {
+            return source.toString().toLowerCase(Locale.ROOT).startsWith(pattern.toString().toLowerCase(Locale.ROOT));
+        }
+    }
+
+    protected Processor source() {
+        return source;
+    }
+
+    protected Processor pattern() {
+        return pattern;
+    }
+
+    protected boolean isCaseInsensitive() {
+        return caseInsensitive;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        StartsWithFunctionProcessor other = (StartsWithFunctionProcessor) obj;
+        return Objects.equals(source(), other.source())
+            && Objects.equals(pattern(), other.pattern())
+            && Objects.equals(isCaseInsensitive(), other.isCaseInsensitive());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(source(), pattern(), isCaseInsensitive());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/whitelist/InternalQlScriptUtils.java

@@ -0,0 +1,170 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.core.expression.function.scalar.whitelist;
+
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.xpack.esql.core.expression.function.scalar.string.StartsWithFunctionProcessor;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogicProcessor.BinaryLogicOperation;
+import org.elasticsearch.xpack.esql.core.expression.predicate.logical.NotProcessor;
+import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.CheckNullProcessor.CheckNullOperation;
+import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.DefaultBinaryArithmeticOperation;
+import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.UnaryArithmeticProcessor.UnaryArithmeticOperation;
+import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparisonProcessor.BinaryComparisonOperation;
+import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.InProcessor;
+import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexProcessor.RegexOperation;
+import org.elasticsearch.xpack.esql.core.util.StringUtils;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.convert;
+import static org.elasticsearch.xpack.esql.core.type.DataTypeConverter.toUnsignedLong;
+import static org.elasticsearch.xpack.esql.core.type.DataTypes.fromTypeName;
+
+public class InternalQlScriptUtils {
+
+    //
+    // Utilities
+    //
+
+    // safe missing mapping/value extractor
+    public static <T> Object docValue(Map<String, ScriptDocValues<T>> doc, String fieldName) {
+        if (doc.containsKey(fieldName)) {
+            ScriptDocValues<T> docValues = doc.get(fieldName);
+            if (docValues.isEmpty() == false) {
+                return docValues.get(0);
+            }
+        }
+        return null;
+    }
+
+    public static boolean nullSafeFilter(Boolean filter) {
+        return filter == null ? false : filter.booleanValue();
+    }
+
+    public static double nullSafeSortNumeric(Number sort) {
+        return sort == null ? 0.0d : sort.doubleValue();
+    }
+
+    public static String nullSafeSortString(Object sort) {
+        return sort == null ? StringUtils.EMPTY : sort.toString();
+    }
+
+    public static Number nullSafeCastNumeric(Number number, String typeName) {
+        return number == null || Double.isNaN(number.doubleValue()) ? null : (Number) convert(number, fromTypeName(typeName));
+    }
+
+    public static Number nullSafeCastToUnsignedLong(Number number) {
+        return number == null || Double.isNaN(number.doubleValue()) ? null : toUnsignedLong(number);
+    }
+
+    //
+    // Operators
+    //
+
+    //
+    // Logical
+    //
+    public static Boolean eq(Object left, Object right) {
+        return BinaryComparisonOperation.EQ.apply(left, right);
+    }
+
+    public static Boolean nulleq(Object left, Object right) {
+        return BinaryComparisonOperation.NULLEQ.apply(left, right);
+    }
+
+    public static Boolean neq(Object left, Object right) {
+        return BinaryComparisonOperation.NEQ.apply(left, right);
+    }
+
+    public static Boolean lt(Object left, Object right) {
+        return BinaryComparisonOperation.LT.apply(left, right);
+    }
+
+    public static Boolean lte(Object left, Object right) {
+        return BinaryComparisonOperation.LTE.apply(left, right);
+    }
+
+    public static Boolean gt(Object left, Object right) {
+        return BinaryComparisonOperation.GT.apply(left, right);
+    }
+
+    public static Boolean gte(Object left, Object right) {
+        return BinaryComparisonOperation.GTE.apply(left, right);
+    }
+
+    public static Boolean in(Object value, List<Object> values) {
+        return InProcessor.apply(value, values);
+    }
+
+    public static Boolean and(Boolean left, Boolean right) {
+        return BinaryLogicOperation.AND.apply(left, right);
+    }
+
+    public static Boolean or(Boolean left, Boolean right) {
+        return BinaryLogicOperation.OR.apply(left, right);
+    }
+
+    public static Boolean not(Boolean expression) {
+        return NotProcessor.apply(expression);
+    }
+
+    public static Boolean isNull(Object expression) {
+        return CheckNullOperation.IS_NULL.test(expression);
+    }
+
+    public static Boolean isNotNull(Object expression) {
+        return CheckNullOperation.IS_NOT_NULL.test(expression);
+    }
+
+    //
+    // Regex
+    //
+    public static Boolean regex(String value, String pattern) {
+        return regex(value, pattern, Boolean.FALSE);
+    }
+
+    public static Boolean regex(String value, String pattern, Boolean caseInsensitive) {
+        // TODO: this needs to be improved to avoid creating the pattern on every call
+        return RegexOperation.match(value, pattern, caseInsensitive);
+    }
+
+    //
+    // Math
+    //
+    public static Number add(Number left, Number right) {
+        return (Number) DefaultBinaryArithmeticOperation.ADD.apply(left, right);
+    }
+
+    public static Number div(Number left, Number right) {
+        return (Number) DefaultBinaryArithmeticOperation.DIV.apply(left, right);
+    }
+
+    public static Number mod(Number left, Number right) {
+        return (Number) DefaultBinaryArithmeticOperation.MOD.apply(left, right);
+    }
+
+    public static Number mul(Number left, Number right) {
+        return (Number) DefaultBinaryArithmeticOperation.MUL.apply(left, right);
+    }
+
+    public static Number neg(Number value) {
+        return UnaryArithmeticOperation.NEGATE.apply(value);
+    }
+
+    public static Number sub(Number left, Number right) {
+        return (Number) DefaultBinaryArithmeticOperation.SUB.apply(left, right);
+    }
+
+    //
+    // String
+    //
+    public static Boolean startsWith(String s, String pattern, Boolean caseInsensitive) {
+        return (Boolean) StartsWithFunctionProcessor.doProcess(s, pattern, caseInsensitive);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/gen/pipeline/AggExtractorInput.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.gen.pipeline;
+
+import org.elasticsearch.xpack.esql.core.execution.search.QlSourceBuilder;
+import org.elasticsearch.xpack.esql.core.execution.search.extractor.BucketExtractor;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.gen.processor.BucketExtractorProcessor;
+import org.elasticsearch.xpack.esql.core.expression.gen.processor.ChainingProcessor;
+import org.elasticsearch.xpack.esql.core.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+public class AggExtractorInput extends LeafInput<BucketExtractor> {
+
+    private final Processor chained;
+
+    public AggExtractorInput(Source source, Expression expression, Processor processor, BucketExtractor context) {
+        super(source, expression, context);
+        this.chained = processor;
+    }
+
+    @Override
+    protected NodeInfo<AggExtractorInput> info() {
+        return NodeInfo.create(this, AggExtractorInput::new, expression(), chained, context());
+    }
+
+    @Override
+    public Processor asProcessor() {
+        Processor proc = new BucketExtractorProcessor(context());
+        return chained != null ? new ChainingProcessor(proc, chained) : proc;
+    }
+
+    @Override
+    public final boolean supportedByAggsOnlyQuery() {
+        return true;
+    }
+
+    @Override
+    public Pipe resolveAttributes(AttributeResolver resolver) {
+        return this;
+    }
+
+    @Override
+    public final void collectFields(QlSourceBuilder sourceBuilder) {
+        // Nothing to collect
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/gen/pipeline/AggNameInput.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.gen.pipeline;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+public class AggNameInput extends CommonNonExecutableInput<String> {
+    public AggNameInput(Source source, Expression expression, String context) {
+        super(source, expression, context);
+    }
+
+    @Override
+    protected NodeInfo<AggNameInput> info() {
+        return NodeInfo.create(this, AggNameInput::new, expression(), context());
+    }
+
+    @Override
+    public final boolean supportedByAggsOnlyQuery() {
+        return true;
+    }
+
+    @Override
+    public final boolean resolved() {
+        return false;
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/gen/pipeline/AggPathInput.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.gen.pipeline;
+
+import org.elasticsearch.xpack.esql.core.execution.search.AggRef;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.Objects;
+
+public class AggPathInput extends CommonNonExecutableInput<AggRef> {
+
+    // used in case the agg itself is not returned in a suitable format (like date aggs)
+    private final Processor action;
+
+    public AggPathInput(Expression expression, AggRef context) {
+        this(Source.EMPTY, expression, context, null);
+    }
+
+    /**
+     *
+     * Constructs a new <code>AggPathInput</code> instance.
+     * The action is used for handling corner-case results such as date histogram which returns
+     * a full date object for year which requires additional extraction.
+     */
+    public AggPathInput(Source source, Expression expression, AggRef context, Processor action) {
+        super(source, expression, context);
+        this.action = action;
+    }
+
+    @Override
+    protected NodeInfo<AggPathInput> info() {
+        return NodeInfo.create(this, AggPathInput::new, expression(), context(), action);
+    }
+
+    public Processor action() {
+        return action;
+    }
+
+    @Override
+    public boolean resolved() {
+        return true;
+    }
+
+    @Override
+    public final boolean supportedByAggsOnlyQuery() {
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(context(), action);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        AggPathInput other = (AggPathInput) obj;
+        return Objects.equals(context(), other.context()) && Objects.equals(action, other.action);
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/gen/pipeline/AttributeInput.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.gen.pipeline;
+
+import org.elasticsearch.xpack.esql.core.execution.search.QlSourceBuilder;
+import org.elasticsearch.xpack.esql.core.expression.Attribute;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+/**
+ * An input that must first be rewritten against the rest of the query
+ * before it can be further processed.
+ */
+public class AttributeInput extends NonExecutableInput<Attribute> {
+    public AttributeInput(Source source, Expression expression, Attribute context) {
+        super(source, expression, context);
+    }
+
+    @Override
+    protected NodeInfo<AttributeInput> info() {
+        return NodeInfo.create(this, AttributeInput::new, expression(), context());
+    }
+
+    @Override
+    public final boolean supportedByAggsOnlyQuery() {
+        return true;
+    }
+
+    @Override
+    public Pipe resolveAttributes(AttributeResolver resolver) {
+        return new ReferenceInput(source(), expression(), resolver.resolve(context()));
+    }
+
+    @Override
+    public final void collectFields(QlSourceBuilder sourceBuilder) {
+        // Nothing to extract
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/gen/pipeline/BinaryPipe.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.gen.pipeline;
+
+import org.elasticsearch.xpack.esql.core.execution.search.QlSourceBuilder;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public abstract class BinaryPipe extends Pipe {
+
+    private final Pipe left, right;
+
+    public BinaryPipe(Source source, Expression expression, Pipe left, Pipe right) {
+        super(source, expression, Arrays.asList(left, right));
+        this.left = left;
+        this.right = right;
+    }
+
+    @Override
+    public final Pipe replaceChildren(List<Pipe> newChildren) {
+        return replaceChildren(newChildren.get(0), newChildren.get(1));
+    }
+
+    public Pipe left() {
+        return left;
+    }
+
+    public Pipe right() {
+        return right;
+    }
+
+    @Override
+    public boolean supportedByAggsOnlyQuery() {
+        return left.supportedByAggsOnlyQuery() || right.supportedByAggsOnlyQuery();
+    }
+
+    @Override
+    public final Pipe resolveAttributes(AttributeResolver resolver) {
+        Pipe newLeft = left.resolveAttributes(resolver);
+        Pipe newRight = right.resolveAttributes(resolver);
+        if (newLeft == left && newRight == right) {
+            return this;
+        }
+        return replaceChildren(newLeft, newRight);
+    }
+
+    /**
+     * Build a copy of this object with new left and right children. Used by
+     * {@link #resolveAttributes(AttributeResolver)}.
+     */
+    protected abstract BinaryPipe replaceChildren(Pipe left, Pipe right);
+
+    @Override
+    public boolean resolved() {
+        return left().resolved() && right().resolved();
+    }
+
+    @Override
+    public final void collectFields(QlSourceBuilder sourceBuilder) {
+        left.collectFields(sourceBuilder);
+        right.collectFields(sourceBuilder);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(left(), right());
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        BinaryPipe other = (BinaryPipe) obj;
+        return Objects.equals(left(), other.left()) && Objects.equals(right(), other.right());
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/gen/pipeline/CommonNonExecutableInput.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.gen.pipeline;
+
+import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
+import org.elasticsearch.xpack.esql.core.execution.search.QlSourceBuilder;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+/**
+ * Implementation common to most subclasses of
+ * {@link NonExecutableInput} but not shared by all.
+ */
+abstract class CommonNonExecutableInput<T> extends NonExecutableInput<T> {
+    CommonNonExecutableInput(Source source, Expression expression, T context) {
+        super(source, expression, context);
+    }
+
+    @Override
+    public final Processor asProcessor() {
+        throw new QlIllegalArgumentException("Unresolved input - needs resolving first");
+    }
+
+    @Override
+    public final Pipe resolveAttributes(AttributeResolver resolver) {
+        return this;
+    }
+
+    @Override
+    public final void collectFields(QlSourceBuilder sourceBuilder) {
+        // Nothing to extract
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/gen/pipeline/ConstantInput.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.gen.pipeline;
+
+import org.elasticsearch.xpack.esql.core.execution.search.QlSourceBuilder;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.gen.processor.ConstantProcessor;
+import org.elasticsearch.xpack.esql.core.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+public class ConstantInput extends LeafInput<Object> {
+
+    public ConstantInput(Source source, Expression expression, Object context) {
+        super(source, expression, context);
+    }
+
+    @Override
+    protected NodeInfo<ConstantInput> info() {
+        return NodeInfo.create(this, ConstantInput::new, expression(), context());
+    }
+
+    @Override
+    public Processor asProcessor() {
+        return new ConstantProcessor(context());
+    }
+
+    @Override
+    public final boolean supportedByAggsOnlyQuery() {
+        return false;
+    }
+
+    @Override
+    public Pipe resolveAttributes(AttributeResolver resolver) {
+        return this;
+    }
+
+    @Override
+    public final void collectFields(QlSourceBuilder sourceBuilder) {
+        // Nothing to collect
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/gen/pipeline/HitExtractorInput.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.gen.pipeline;
+
+import org.elasticsearch.xpack.esql.core.execution.search.QlSourceBuilder;
+import org.elasticsearch.xpack.esql.core.execution.search.extractor.HitExtractor;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.gen.processor.HitExtractorProcessor;
+import org.elasticsearch.xpack.esql.core.expression.gen.processor.Processor;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+public class HitExtractorInput extends LeafInput<HitExtractor> {
+
+    public HitExtractorInput(Source source, Expression expression, HitExtractor context) {
+        super(source, expression, context);
+    }
+
+    @Override
+    protected NodeInfo<HitExtractorInput> info() {
+        return NodeInfo.create(this, HitExtractorInput::new, expression(), context());
+    }
+
+    @Override
+    public Processor asProcessor() {
+        return new HitExtractorProcessor(context());
+    }
+
+    @Override
+    public final boolean supportedByAggsOnlyQuery() {
+        return true;
+    }
+
+    @Override
+    public Pipe resolveAttributes(AttributeResolver resolver) {
+        return this;
+    }
+
+    @Override
+    public final void collectFields(QlSourceBuilder sourceBuilder) {
+        // No fields to collect
+    }
+}

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/gen/pipeline/LeafInput.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.esql.core.expression.gen.pipeline;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import java.util.List;
+import java.util.Objects;
+
+import static java.util.Collections.emptyList;
+
+public abstract class LeafInput<T> extends Pipe {
+
+    private T context;
+
+    public LeafInput(Source source, Expression expression, T context) {
+        super(source, expression, emptyList());
+        this.context = context;
+    }
+
+    @Override
+    public final Pipe replaceChildren(List<Pipe> newChildren) {
+        throw new UnsupportedOperationException("this type of node doesn't have any children to replace");
+    }
+
+    public T context() {
+        return context;
+    }
+
+    @Override
+    public boolean resolved() {
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(expression(), context);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+
+        LeafInput<?> other = (LeafInput<?>) obj;
+        return Objects.equals(context(), other.context()) && Objects.equals(expression(), other.expression());
+    }
+}