Merge main into multi-project

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.template.yml

@@ -18,6 +18,7 @@ steps:
               - rhel-8
               - rhel-9
               - almalinux-8
+              - almalinux-9
         agents:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}

.buildkite/pipelines/periodic-packaging.yml

@@ -19,6 +19,7 @@ steps:
               - rhel-8
               - rhel-9
               - almalinux-8
+              - almalinux-9
         agents:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}
@@ -268,8 +269,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.6 / packaging-tests-upgrade"
+      - label: "{{matrix.image}} / 8.15.5 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.6
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.5
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -282,7 +283,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.15.6
+          BWC_VERSION: 8.15.5
 
       - label: "{{matrix.image}} / 8.16.2 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.2

.buildkite/pipelines/periodic-platform-support.yml

@@ -18,6 +18,7 @@ steps:
               - rhel-8
               - rhel-9
               - almalinux-8
+              - almalinux-9
         agents:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}

.buildkite/pipelines/periodic.yml

@@ -287,8 +287,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.6 / bwc
+      - label: 8.15.5 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.6#bwcTest
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.5#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -297,7 +297,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.15.6
+          BWC_VERSION: 8.15.5
         retry:
           automatic:
             - exit_status: "-1"
@@ -448,7 +448,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -490,7 +490,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.15.6", "8.16.2", "8.17.0", "8.18.0", "9.0.0"]
+            BWC_VERSION: ["8.16.2", "8.17.0", "8.18.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -21,6 +21,7 @@ steps:
               - rhel-8
               - rhel-9
               - almalinux-8
+              - almalinux-9
             PACKAGING_TASK:
               - docker
               - docker-cloud-ess

.ci/bwcVersions

@@ -14,7 +14,7 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.6"
+  - "8.15.5"
   - "8.16.2"
   - "8.17.0"
   - "8.18.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,4 @@
 BWC_VERSION:
-  - "8.15.6"
   - "8.16.2"
   - "8.17.0"
   - "8.18.0"

branches.json

@@ -13,9 +13,6 @@
     {
       "branch": "8.x"
     },
-    {
-      "branch": "8.15"
-    },
     {
       "branch": "7.17"
     }

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java

@@ -17,8 +17,6 @@ import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 
 import java.io.File;
-import java.util.Arrays;
-import java.util.Map;
 
 /**
  * This plugin configures formatting for Java source using Spotless
@@ -66,8 +64,7 @@ public class FormattingPrecommitPlugin implements Plugin<Project> {
                 java.importOrderFile(new File(elasticsearchWorkspace, importOrderPath));
 
                 // Most formatting is done through the Eclipse formatter
-                java.eclipse().withP2Mirrors(Map.of("https://download.eclipse.org/", "https://mirror.umd.edu/eclipse/"))
+                java.eclipse().configFile(new File(elasticsearchWorkspace, formatterConfigPath));
-                    .configFile(new File(elasticsearchWorkspace, formatterConfigPath));
 
                 // Ensure blank lines are actually empty. Since formatters are applied in
                 // order, apply this one last, otherwise non-empty blank lines can creep

build.gradle

@@ -301,7 +301,10 @@ allprojects {
     if (project.path.contains(":distribution:docker")) {
       enabled = false
     }
-
+    if (project.path.contains(":libs:cli")) {
+      // ensure we resolve p2 dependencies for the spotless eclipse formatter
+      dependsOn "spotlessJavaCheck"
+    }
   }
 
   plugins.withId('lifecycle-base') {

docs/changelog/116423.yaml

@@ -0,0 +1,5 @@
+pr: 116423
+summary: Support mTLS for the Elastic Inference Service integration inside the inference API
+area: Machine Learning
+type: feature
+issues: []

docs/changelog/117839.yaml

@@ -0,0 +1,5 @@
+pr: 117839
+summary: Add match support for `semantic_text` fields
+area: "Search"
+type: enhancement
+issues: []

docs/changelog/117840.yaml

@@ -0,0 +1,5 @@
+pr: 117840
+summary: Fix timeout ingesting an empty string into a `semantic_text` field
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/118035.yaml

@@ -0,0 +1,6 @@
+pr: 118035
+summary: Include hidden indices in `DeprecationInfoAction`
+area: Indices APIs
+type: bug
+issues:
+ - 118020

docs/changelog/118102.yaml

@@ -0,0 +1,5 @@
+pr: 118102
+summary: "ESQL: Enterprise license enforcement for CCS"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/118173.yaml

@@ -0,0 +1,5 @@
+pr: 118173
+summary: ES|QL categorize with multiple groupings
+area: Machine Learning
+type: feature
+issues: []

docs/changelog/118194.yaml

@@ -0,0 +1,5 @@
+pr: 118194
+summary: Retry on `ClusterBlockException` on transform destination index
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/118291.yaml

@@ -0,0 +1,5 @@
+pr: 118291
+summary: Adding a migration reindex cancel API
+area: Data streams
+type: enhancement
+issues: []

docs/changelog/118435.yaml

@@ -0,0 +1,6 @@
+pr: 118435
+summary: '`_score` should not be a reserved attribute in ES|QL'
+area: ES|QL
+type: enhancement
+issues:
+ - 118460

docs/reference/connector/docs/connectors-release-notes.asciidoc

@@ -9,8 +9,76 @@
 Prior to version *8.16.0*, the connector release notes were published as part of the {enterprise-search-ref}/changelog.html[Enterprise Search documentation].
 ====
 
-*Release notes*:
+[discrete]
+[[es-connectors-release-notes-8-17-0]]
+=== 8.17.0
 
-* <<es-connectors-release-notes-8-16-0>>
+No notable changes in this release.
 
-include::release-notes/connectors-release-notes-8.16.0.asciidoc[]
+[discrete]
+[[es-connectors-release-notes-8-16-1]]
+=== 8.16.1
+
+[discrete]
+[[es-connectors-release-notes-8-16-1-bug-fixes]]
+==== Bug fixes
+
+* Fixed a bug in the Outlook Connector where having deactivated users could cause the sync to fail.
+See https://github.com/elastic/connectors/pull/2967[*PR 2967*].
+* Fixed a bug where the Confluence connector was not downloading some blog post documents due to unexpected response format.
+See https://github.com/elastic/connectors/pull/2984[*PR 2984*].
+
+[discrete]
+[[es-connectors-release-notes-8-16-0]]
+=== 8.16.0
+
+[discrete]
+[[es-connectors-release-notes-deprecation-notice]]
+==== Deprecation notices
+
+* *Direct index access for connectors and sync jobs*
++
+IMPORTANT: Directly accessing connector and sync job state through `.elastic-connectors*` indices is deprecated, and will be disallowed entirely in a future release.
+
+* Instead, the Elasticsearch Connector APIs should be used. Connectors framework code now uses the <<connector-apis,Connector APIs>> by default.
+See https://github.com/elastic/connectors/pull/2884[*PR 2902*].
+
+* *Docker `enterprise-search` namespace deprecation*
++
+IMPORTANT: The `enterprise-search` Docker namespace is deprecated and will be discontinued in a future release. 
++
+Starting in `8.16.0`, Docker images are being transitioned to the new `integrations` namespace, which will become the sole location for future releases. This affects the https://github.com/elastic/connectors[Elastic Connectors] and https://github.com/elastic/data-extraction-service[Elastic Data Extraction Service].
++
+During this transition period, images are published to both namespaces:
++
+** *Example*:
++
+Deprecated namespace::
+`docker.elastic.co/enterprise-search/elastic-connectors:v8.16.0`
++
+New namespace::
+`docker.elastic.co/integrations/elastic-connectors:v8.16.0`
++
+Users should migrate to the new `integrations` namespace as soon as possible to ensure continued access to future releases.
+
+[discrete]
+[[es-connectors-release-notes-8-16-0-enhancements]]
+==== Enhancements
+
+* Docker images now use Chainguard's Wolfi base image (`docker.elastic.co/wolfi/jdk:openjdk-11-dev`), replacing the previous `ubuntu:focal` base.
+
+* The Sharepoint Online connector now works with the `Sites.Selected` permission instead of the broader permission `Sites.Read.All`.
+See https://github.com/elastic/connectors/pull/2762[*PR 2762*].
+
+* Starting in 8.16.0, connectors will start using proper SEMVER, with `MAJOR.MINOR.PATCH`, which aligns with Elasticsearch/Kibana versions. This drops the previous `.BUILD` suffix, which we used to release connectors between Elastic stack releases. Going forward, these inter-stack-release releases will be suffixed instead with `+<timestamp>`, aligning with Elastic Agent and conforming to SEMVER. 
+See https://github.com/elastic/connectors/pull/2749[*PR 2749*].
+
+* Connector logs now use UTC timestamps, instead of machine-local timestamps. This only impacts logging output.
+See https://github.com/elastic/connectors/pull/2695[*PR 2695*].
+
+[discrete]
+[[es-connectors-release-notes-8-16-0-bug-fixes]]
+==== Bug fixes
+
+* The Dropbox connector now fetches the files from team shared folders.
+See https://github.com/elastic/connectors/pull/2718[*PR 2718*].

docs/reference/connector/docs/release-notes/connectors-release-notes-8.16.0.asciidoc

@@ -1,53 +0,0 @@
-[[es-connectors-release-notes-8-16-0]]
-=== 8.16.0 connectors release notes
-
-[discrete]
-[[es-connectors-release-notes-deprecation-notice]]
-==== Deprecation notices
-
-* *Direct index access for connectors and sync jobs*
-+
-IMPORTANT: Directly accessing connector and sync job state through `.elastic-connectors*` indices is deprecated, and will be disallowed entirely in a future release.
-
-* Instead, the Elasticsearch Connector APIs should be used. Connectors framework code now uses the <<connector-apis,Connector APIs>> by default.
-See https://github.com/elastic/connectors/pull/2884[*PR 2902*].
-
-* *Docker `enterprise-search` namespace deprecation*
-+
-IMPORTANT: The `enterprise-search` Docker namespace is deprecated and will be discontinued in a future release. 
-+
-Starting in `8.16.0`, Docker images are being transitioned to the new `integrations` namespace, which will become the sole location for future releases. This affects the https://github.com/elastic/connectors[Elastic Connectors] and https://github.com/elastic/data-extraction-service[Elastic Data Extraction Service].
-+
-During this transition period, images are published to both namespaces:
-+
-** *Example*:
-+
-Deprecated namespace::
-`docker.elastic.co/enterprise-search/elastic-connectors:v8.16.0`
-+
-New namespace::
-`docker.elastic.co/integrations/elastic-connectors:v8.16.0`
-+
-Users should migrate to the new `integrations` namespace as soon as possible to ensure continued access to future releases.
-
-[discrete]
-[[es-connectors-release-notes-8-16-0-enhancements]]
-==== Enhancements
-
-* Docker images now use Chainguard's Wolfi base image (`docker.elastic.co/wolfi/jdk:openjdk-11-dev`), replacing the previous `ubuntu:focal` base.
-
-* The Sharepoint Online connector now works with the `Sites.Selected` permission instead of the broader permission `Sites.Read.All`.
-See https://github.com/elastic/connectors/pull/2762[*PR 2762*].
-
-* Starting in 8.16.0, connectors will start using proper SEMVER, with `MAJOR.MINOR.PATCH`, which aligns with Elasticsearch/Kibana versions. This drops the previous `.BUILD` suffix, which we used to release connectors between Elastic stack releases. Going forward, these inter-stack-release releases will be suffixed instead with `+<timestamp>`, aligning with Elastic Agent and conforming to SEMVER. 
-See https://github.com/elastic/connectors/pull/2749[*PR 2749*].
-
-* Connector logs now use UTC timestamps, instead of machine-local timestamps. This only impacts logging output.
-See https://github.com/elastic/connectors/pull/2695[*PR 2695*].
-
-[discrete]
-[[es-connectors-release-notes-8-16-0-bug-fixes]]
-==== Bug fixes
-
-* The Dropbox connector now fetches the files from team shared folders.
-See https://github.com/elastic/connectors/pull/2718[*PR 2718*].

docs/reference/docs/update.asciidoc

@@ -71,7 +71,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=refresh]
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing]
 
 `_source`::
-(Optional, list) Set to `false` to disable source retrieval (default: `true`).
+(Optional, list) Set to `true` to enable source retrieval (default: `false`).
 You can also specify a comma-separated list of the fields you want to retrieve.
 
 `_source_excludes`::

docs/reference/index-modules.asciidoc

@@ -113,7 +113,7 @@ Index mode supports the following values:
 
 `standard`::: Standard indexing with default settings.
 
-`tsds`::: _(data streams only)_ Index mode optimized for storage of metrics. For more information, see <<tsds-index-settings>>.
+`time_series`::: _(data streams only)_ Index mode optimized for storage of metrics. For more information, see <<tsds-index-settings>>.
 
 `logsdb`::: _(data streams only)_ Index mode optimized for <<logs-data-stream,logs>>.
 

docs/reference/inference/inference-apis.asciidoc

@@ -48,21 +48,21 @@ When adaptive allocations are enabled:
 
 For more information about adaptive allocations and resources, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] documentation.
 
-//[discrete]
+[discrete]
-//[[default-enpoints]]
+[[default-enpoints]]
-//=== Default {infer} endpoints
+=== Default {infer} endpoints
 
-//Your {es} deployment contains some preconfigured {infer} endpoints that makes it easier for you to use them when defining `semantic_text` fields or {infer} processors.
+Your {es} deployment contains preconfigured {infer} endpoints which makes them easier to use when defining `semantic_text` fields or using {infer} processors.
-//The following list contains the default {infer} endpoints listed by `inference_id`:
+The following list contains the default {infer} endpoints listed by `inference_id`:
 
-//* `.elser-2-elasticsearch`: uses the {ml-docs}/ml-nlp-elser.html[ELSER] built-in trained model for `sparse_embedding` tasks (recommended for English language texts)
+* `.elser-2-elasticsearch`: uses the {ml-docs}/ml-nlp-elser.html[ELSER] built-in trained model for `sparse_embedding` tasks (recommended for English language texts)
-//* `.multilingual-e5-small-elasticsearch`: uses the {ml-docs}/ml-nlp-e5.html[E5] built-in trained model for `text_embedding` tasks (recommended for non-English language texts)
+* `.multilingual-e5-small-elasticsearch`: uses the {ml-docs}/ml-nlp-e5.html[E5] built-in trained model for `text_embedding` tasks (recommended for non-English language texts)
 
-//Use the `inference_id` of the endpoint in a <<semantic-text,`semantic_text`>> field definition or when creating an <<inference-processor,{infer} processor>>.
+Use the `inference_id` of the endpoint in a <<semantic-text,`semantic_text`>> field definition or when creating an <<inference-processor,{infer} processor>>.
-//The API call will automatically download and deploy the model which might take a couple of minutes.
+The API call will automatically download and deploy the model which might take a couple of minutes.
-//Default {infer} enpoints have {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[adaptive allocations] enabled.
+Default {infer} enpoints have {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[adaptive allocations] enabled.
-//For these models, the minimum number of allocations is `0`. 
+For these models, the minimum number of allocations is `0`. 
-//If there is no {infer} activity that uses the endpoint, the number of allocations will scale down to `0` automatically after 15 minutes.
+If there is no {infer} activity that uses the endpoint, the number of allocations will scale down to `0` automatically after 15 minutes.
 
 
 [discrete]

docs/reference/mapping/params/index-prefixes.asciidoc

@@ -54,3 +54,30 @@ PUT my-index-000001
   }
 }
 --------------------------------
+
+`index_prefixes` parameter instructs {ES} to create a subfield "._index_prefix". This
+field will be used to do fast prefix queries. When doing highlighting, add "._index_prefix"
+subfield to the `matched_fields` parameter to highlight the main field based on the
+found matches of the prefix field, like in the request below:
+
+[source,console]
+--------------------------------
+GET my-index-000001/_search
+{
+  "query": {
+    "prefix": {
+      "full_name": {
+        "value": "ki"
+      }
+    }
+  },
+  "highlight": {
+    "fields": {
+      "full_name": {
+        "matched_fields": ["full_name._index_prefix"]
+      }
+    }
+  }
+}
+--------------------------------
+// TEST[continued]

docs/reference/mapping/types/search-as-you-type.asciidoc

@@ -97,11 +97,21 @@ GET my-index-000001/_search
         "my_field._3gram"
       ]
     }
+  },
+  "highlight": {
+    "fields": {
+      "my_field": {
+        "matched_fields": ["my_field._index_prefix"] <1>
+      }
+    }
   }
 }
 --------------------------------------------------
 // TEST[continued]
 
+<1> Adding "my_field._index_prefix" to the `matched_fields` allows to highlight
+    "my_field" also based on matches from "my_field._index_prefix" field.
+
 [source,console-result]
 --------------------------------------------------
 {
@@ -126,6 +136,11 @@ GET my-index-000001/_search
         "_score" : 0.8630463,
         "_source" : {
           "my_field" : "quick brown fox jump lazy dog"
+        },
+        "highlight": {
+          "my_field": [
+            "quick <em>brown fox jump lazy</em> dog"
+          ]
         }
       }
     ]

docs/reference/search/search-your-data/ccs-version-compat-matrix.asciidoc

@@ -1,25 +1,26 @@
 |====
-| 20+^h| Remote cluster version
+| 21+^h| Remote cluster version
 h| Local cluster version
-            |  6.8        | 7.1–7.16   | 7.17       | 8.0        | 8.1        | 8.2        | 8.3       | 8.4       | 8.5       | 8.6        | 8.7        | 8.8        | 8.9        | 8.10       | 8.11       | 8.12      | 8.13      | 8.14      | 8.15      | 8.16
+            |  6.8        | 7.1–7.16   | 7.17       | 8.0        | 8.1        | 8.2        | 8.3       | 8.4       | 8.5       | 8.6        | 8.7        | 8.8        | 8.9        | 8.10       | 8.11       | 8.12      | 8.13      | 8.14      | 8.15      | 8.16      | 8.17
-| 6.8       |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}
+| 6.8       |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}
-| 7.1–7.16  |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}
+| 7.1–7.16  |  {yes-icon} | {yes-icon} | {yes-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}
-| 7.17      |  {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 7.17      |  {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.0       |  {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.0       |  {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.1       |  {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.1       |  {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.2       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.2       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.3       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.3       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.4       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.4       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon}| {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.5       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.5       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {yes-icon}| {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.6       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.6       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {yes-icon}| {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.7       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.7       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.8       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.8       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.9       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.9       |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.10      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.10      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.11      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.11      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.12      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.12      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.13      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.13      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.14      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.14      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.15      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}
+| 8.15      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {yes-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
-| 8.16      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {yes-icon} | {yes-icon}
+| 8.16      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {yes-icon} | {yes-icon}  | {yes-icon}
+| 8.17      |  {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {no-icon} | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon}  | {no-icon} | {no-icon} | {yes-icon}  | {yes-icon}
 |====

docs/reference/search/search-your-data/semantic-search-inference.asciidoc

@@ -45,7 +45,7 @@ include::{es-ref-dir}/tab-widgets/inference-api/infer-api-task-widget.asciidoc[]
 ==== Create the index mapping
 
 The mapping of the destination index - the index that contains the embeddings that the model will create based on your input text - must be created.
-The destination index must have a field with the <<dense-vector, `dense_vector`>> field type for most models and the <<sparse-vector, `sparse_vector`>> field type for the sparse vector models like in the case of the `elser` service to index the output of the used model.
+The destination index must have a field with the <<dense-vector, `dense_vector`>> field type for most models and the <<sparse-vector, `sparse_vector`>> field type for the sparse vector models like in the case of the `elasticsearch` service to index the output of the used model.
 
 include::{es-ref-dir}/tab-widgets/inference-api/infer-api-mapping-widget.asciidoc[]
 

docs/reference/tab-widgets/inference-api/infer-api-requirements.asciidoc

@@ -8,7 +8,7 @@ the Cohere service.
 // tag::elser[]
 
 ELSER is a model trained by Elastic. If you have an {es} deployment, there is no
-further requirement for using the {infer} API with the `elser` service.
+further requirement for using the {infer} API with the `elasticsearch` service.
 
 // end::elser[]
 

libs/entitlement/qa/build.gradle

@@ -13,8 +13,8 @@ apply plugin: 'elasticsearch.internal-test-artifact'
 
 dependencies {
   javaRestTestImplementation project(':libs:entitlement:qa:common')
-  clusterPlugins project(':libs:entitlement:qa:entitlement-allowed')
+  clusterModules project(':libs:entitlement:qa:entitlement-allowed')
-  clusterPlugins project(':libs:entitlement:qa:entitlement-allowed-nonmodular')
+  clusterModules project(':libs:entitlement:qa:entitlement-allowed-nonmodular')
   clusterPlugins project(':libs:entitlement:qa:entitlement-denied')
   clusterPlugins project(':libs:entitlement:qa:entitlement-denied-nonmodular')
 }

libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsAllowedIT.java

@@ -28,8 +28,8 @@ public class EntitlementsAllowedIT extends ESRestTestCase {
 
     @ClassRule
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
-        .plugin("entitlement-allowed")
+        .module("entitlement-allowed")
-        .plugin("entitlement-allowed-nonmodular")
+        .module("entitlement-allowed-nonmodular")
         .systemProperty("es.entitlements.enabled", "true")
         .setting("xpack.security.enabled", "false")
         .build();

libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java

@@ -15,7 +15,6 @@ import com.sun.tools.attach.AttachNotSupportedException;
 import com.sun.tools.attach.VirtualMachine;
 
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.entitlement.initialization.EntitlementInitialization;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
@@ -29,7 +28,9 @@ import java.util.function.Function;
 
 public class EntitlementBootstrap {
 
-    public record BootstrapArgs(Collection<Tuple<Path, Boolean>> pluginData, Function<Class<?>, String> pluginResolver) {}
+    public record PluginData(Path pluginPath, boolean isModular, boolean isExternalPlugin) {}
+
+    public record BootstrapArgs(Collection<PluginData> pluginData, Function<Class<?>, String> pluginResolver) {}
 
     private static BootstrapArgs bootstrapArgs;
 
@@ -40,11 +41,11 @@ public class EntitlementBootstrap {
     /**
      * Activates entitlement checking. Once this method returns, calls to methods protected by Entitlements from classes without a valid
      * policy will throw {@link org.elasticsearch.entitlement.runtime.api.NotEntitledException}.
-     * @param pluginData a collection of (plugin path, boolean), that holds the paths of all the installed Elasticsearch modules and
+     * @param pluginData a collection of (plugin path, boolean, boolean), that holds the paths of all the installed Elasticsearch modules
-     *                   plugins, and whether they are Java modular or not.
+     *                   and plugins, whether they are Java modular or not, and whether they are Elasticsearch modules or external plugins.
      * @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
      */
-    public static void bootstrap(Collection<Tuple<Path, Boolean>> pluginData, Function<Class<?>, String> pluginResolver) {
+    public static void bootstrap(Collection<PluginData> pluginData, Function<Class<?>, String> pluginResolver) {
         logger.debug("Loading entitlement agent");
         if (EntitlementBootstrap.bootstrapArgs != null) {
             throw new IllegalStateException("plugin data is already set");

libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java

@@ -9,7 +9,6 @@
 
 package org.elasticsearch.entitlement.initialization;
 
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.core.internal.provider.ProviderLocator;
 import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap;
 import org.elasticsearch.entitlement.bridge.EntitlementChecker;
@@ -96,25 +95,25 @@ public class EntitlementInitialization {
         return new PolicyManager(serverPolicy, pluginPolicies, EntitlementBootstrap.bootstrapArgs().pluginResolver());
     }
 
-    private static Map<String, Policy> createPluginPolicies(Collection<Tuple<Path, Boolean>> pluginData) throws IOException {
+    private static Map<String, Policy> createPluginPolicies(Collection<EntitlementBootstrap.PluginData> pluginData) throws IOException {
         Map<String, Policy> pluginPolicies = new HashMap<>(pluginData.size());
-        for (Tuple<Path, Boolean> entry : pluginData) {
+        for (var entry : pluginData) {
-            Path pluginRoot = entry.v1();
+            Path pluginRoot = entry.pluginPath();
-            boolean isModular = entry.v2();
-
             String pluginName = pluginRoot.getFileName().toString();
-            final Policy policy = loadPluginPolicy(pluginRoot, isModular, pluginName);
+
+            final Policy policy = loadPluginPolicy(pluginRoot, entry.isModular(), pluginName, entry.isExternalPlugin());
 
             pluginPolicies.put(pluginName, policy);
         }
         return pluginPolicies;
     }
 
-    private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, String pluginName) throws IOException {
+    private static Policy loadPluginPolicy(Path pluginRoot, boolean isModular, String pluginName, boolean isExternalPlugin)
+        throws IOException {
         Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME);
 
         final Set<String> moduleNames = getModuleNames(pluginRoot, isModular);
-        final Policy policy = parsePolicyIfExists(pluginName, policyFile);
+        final Policy policy = parsePolicyIfExists(pluginName, policyFile, isExternalPlugin);
 
         // TODO: should this check actually be part of the parser?
         for (Scope scope : policy.scopes) {
@@ -125,9 +124,9 @@ public class EntitlementInitialization {
         return policy;
     }
 
-    private static Policy parsePolicyIfExists(String pluginName, Path policyFile) throws IOException {
+    private static Policy parsePolicyIfExists(String pluginName, Path policyFile, boolean isExternalPlugin) throws IOException {
         if (Files.exists(policyFile)) {
-            return new PolicyParser(Files.newInputStream(policyFile, StandardOpenOption.READ), pluginName).parsePolicy();
+            return new PolicyParser(Files.newInputStream(policyFile, StandardOpenOption.READ), pluginName, isExternalPlugin).parsePolicy();
         }
         return new Policy(pluginName, List.of());
     }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java

@@ -33,4 +33,12 @@ public @interface ExternalEntitlement {
      * have to match the parameter names of the constructor.
      */
     String[] parameterNames() default {};
+
+    /**
+     * This flag indicates if this Entitlement can be used in external plugins,
+     * or if it can be used only in Elasticsearch modules ("internal" plugins).
+     * Using an entitlement that is not {@code pluginsAccessible} in an external
+     * plugin policy will throw in exception while parsing.
+     */
+    boolean esModulesOnly() default true;
 }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileEntitlement.java

@@ -26,7 +26,7 @@ public class FileEntitlement implements Entitlement {
     private final String path;
     private final int actions;
 
-    @ExternalEntitlement(parameterNames = { "path", "actions" })
+    @ExternalEntitlement(parameterNames = { "path", "actions" }, esModulesOnly = false)
     public FileEntitlement(String path, List<String> actionsList) {
         this.path = path;
         int actionsInt = 0;

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -18,7 +18,6 @@ import org.elasticsearch.logging.Logger;
 import java.lang.module.ModuleFinder;
 import java.lang.module.ModuleReference;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
 import java.util.List;
@@ -56,8 +55,8 @@ public class PolicyManager {
 
     final Map<Module, ModuleEntitlements> moduleEntitlementsMap = new HashMap<>();
 
-    protected final Policy serverPolicy;
+    protected final Map<String, List<Entitlement>> serverEntitlements;
-    protected final Map<String, Policy> pluginPolicies;
+    protected final Map<String, Map<String, List<Entitlement>>> pluginsEntitlements;
     private final Function<Class<?>, String> pluginResolver;
 
     public static final String ALL_UNNAMED = "ALL-UNNAMED";
@@ -79,19 +78,16 @@ public class PolicyManager {
     }
 
     public PolicyManager(Policy defaultPolicy, Map<String, Policy> pluginPolicies, Function<Class<?>, String> pluginResolver) {
-        this.serverPolicy = Objects.requireNonNull(defaultPolicy);
+        this.serverEntitlements = buildScopeEntitlementsMap(Objects.requireNonNull(defaultPolicy));
-        this.pluginPolicies = Collections.unmodifiableMap(Objects.requireNonNull(pluginPolicies));
+        this.pluginsEntitlements = Objects.requireNonNull(pluginPolicies)
+            .entrySet()
+            .stream()
+            .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> buildScopeEntitlementsMap(e.getValue())));
         this.pluginResolver = pluginResolver;
     }
 
-    private static List<Entitlement> lookupEntitlementsForModule(Policy policy, String moduleName) {
+    private static Map<String, List<Entitlement>> buildScopeEntitlementsMap(Policy policy) {
-        for (int i = 0; i < policy.scopes.size(); ++i) {
+        return policy.scopes.stream().collect(Collectors.toUnmodifiableMap(scope -> scope.name, scope -> scope.entitlements));
-            var scope = policy.scopes.get(i);
-            if (scope.name.equals(moduleName)) {
-                return scope.entitlements;
-            }
-        }
-        return null;
     }
 
     public void checkExitVM(Class<?> callerClass) {
@@ -141,21 +137,21 @@ public class PolicyManager {
 
         if (isServerModule(requestingModule)) {
             var scopeName = requestingModule.getName();
-            return getModuleEntitlementsOrThrow(callerClass, requestingModule, serverPolicy, scopeName);
+            return getModuleEntitlementsOrThrow(callerClass, requestingModule, serverEntitlements, scopeName);
         }
 
         // plugins
         var pluginName = pluginResolver.apply(callerClass);
         if (pluginName != null) {
-            var pluginPolicy = pluginPolicies.get(pluginName);
+            var pluginEntitlements = pluginsEntitlements.get(pluginName);
-            if (pluginPolicy != null) {
+            if (pluginEntitlements != null) {
                 final String scopeName;
                 if (requestingModule.isNamed() == false) {
                     scopeName = ALL_UNNAMED;
                 } else {
                     scopeName = requestingModule.getName();
                 }
-                return getModuleEntitlementsOrThrow(callerClass, requestingModule, pluginPolicy, scopeName);
+                return getModuleEntitlementsOrThrow(callerClass, requestingModule, pluginEntitlements, scopeName);
             }
         }
 
@@ -167,15 +163,20 @@ public class PolicyManager {
         return Strings.format("Missing entitlement policy: caller [%s], module [%s]", callerClass, requestingModule.getName());
     }
 
-    private ModuleEntitlements getModuleEntitlementsOrThrow(Class<?> callerClass, Module module, Policy policy, String moduleName) {
+    private ModuleEntitlements getModuleEntitlementsOrThrow(
-        var entitlements = lookupEntitlementsForModule(policy, moduleName);
+        Class<?> callerClass,
+        Module module,
+        Map<String, List<Entitlement>> scopeEntitlements,
+        String moduleName
+    ) {
+        var entitlements = scopeEntitlements.get(moduleName);
         if (entitlements == null) {
             // Module without entitlements - remember we don't have any
             moduleEntitlementsMap.put(module, ModuleEntitlements.NONE);
             throw new NotEntitledException(buildModuleNoPolicyMessage(callerClass, module));
         }
         // We have a policy for this module
-        var classEntitlements = createClassEntitlements(entitlements);
+        var classEntitlements = new ModuleEntitlements(entitlements);
         moduleEntitlementsMap.put(module, classEntitlements);
         return classEntitlements;
     }
@@ -184,10 +185,6 @@ public class PolicyManager {
         return requestingModule.isNamed() && requestingModule.getLayer() == ModuleLayer.boot();
     }
 
-    private ModuleEntitlements createClassEntitlements(List<Entitlement> entitlements) {
-        return new ModuleEntitlements(entitlements);
-    }
-
     private static Module requestingModule(Class<?> callerClass) {
         if (callerClass != null) {
             Module callerModule = callerClass.getModule();
@@ -222,6 +219,6 @@ public class PolicyManager {
 
     @Override
     public String toString() {
-        return "PolicyManager{" + "serverPolicy=" + serverPolicy + ", pluginPolicies=" + pluginPolicies + '}';
+        return "PolicyManager{" + "serverEntitlements=" + serverEntitlements + ", pluginsEntitlements=" + pluginsEntitlements + '}';
     }
 }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java

@@ -39,6 +39,7 @@ public class PolicyParser {
 
     protected final XContentParser policyParser;
     protected final String policyName;
+    private final boolean isExternalPlugin;
 
     static String getEntitlementTypeName(Class<? extends Entitlement> entitlementClass) {
         var entitlementClassName = entitlementClass.getSimpleName();
@@ -56,9 +57,10 @@ public class PolicyParser {
             .collect(Collectors.joining("_"));
     }
 
-    public PolicyParser(InputStream inputStream, String policyName) throws IOException {
+    public PolicyParser(InputStream inputStream, String policyName, boolean isExternalPlugin) throws IOException {
         this.policyParser = YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, Objects.requireNonNull(inputStream));
         this.policyName = policyName;
+        this.isExternalPlugin = isExternalPlugin;
     }
 
     public Policy parsePolicy() {
@@ -125,6 +127,10 @@ public class PolicyParser {
             throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]");
         }
 
+        if (entitlementMetadata.esModulesOnly() && isExternalPlugin) {
+            throw newPolicyParserException("entitlement type [" + entitlementType + "] is allowed only on modules");
+        }
+
         Class<?>[] parameterTypes = entitlementConstructor.getParameterTypes();
         String[] parametersNames = entitlementMetadata.parameterNames();
 

libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java

@@ -19,7 +19,7 @@ public class PolicyParserFailureTests extends ESTestCase {
     public void testParserSyntaxFailures() {
         PolicyParserException ppe = expectThrows(
             PolicyParserException.class,
-            () -> new PolicyParser(new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml")
+            () -> new PolicyParser(new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false)
                 .parsePolicy()
         );
         assertEquals("[1:1] policy parsing error for [test-failure-policy.yaml]: expected object <scope name>", ppe.getMessage());
@@ -29,7 +29,7 @@ public class PolicyParserFailureTests extends ESTestCase {
         PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
             entitlement-module-name:
               - does_not_exist: {}
-            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
         assertEquals(
             "[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name]: "
                 + "unknown entitlement type [does_not_exist]",
@@ -41,7 +41,7 @@ public class PolicyParserFailureTests extends ESTestCase {
         PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
             entitlement-module-name:
               - file: {}
-            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
         assertEquals(
             "[2:12] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
                 + "for entitlement type [file]: missing entitlement parameter [path]",
@@ -52,7 +52,7 @@ public class PolicyParserFailureTests extends ESTestCase {
             entitlement-module-name:
               - file:
                   path: test-path
-            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
         assertEquals(
             "[4:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
                 + "for entitlement type [file]: missing entitlement parameter [actions]",
@@ -68,11 +68,22 @@ public class PolicyParserFailureTests extends ESTestCase {
                   actions:
                     - read
                   extra: test
-            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml").parsePolicy());
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
         assertEquals(
             "[7:1] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
                 + "for entitlement type [file]: extraneous entitlement parameter(s) {extra=test}",
             ppe.getMessage()
         );
     }
+
+    public void testEntitlementIsNotForExternalPlugins() {
+        PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
+            entitlement-module-name:
+              - create_class_loader
+            """.getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", true).parsePolicy());
+        assertEquals(
+            "[2:5] policy parsing error for [test-failure-policy.yaml]: entitlement type [create_class_loader] is allowed only on modules",
+            ppe.getMessage()
+        );
+    }
 }

libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java

@@ -37,7 +37,17 @@ public class PolicyParserTests extends ESTestCase {
     }
 
     public void testPolicyBuilder() throws IOException {
-        Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml")
+        Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", false)
+            .parsePolicy();
+        Policy builtPolicy = new Policy(
+            "test-policy.yaml",
+            List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", List.of("read", "write")))))
+        );
+        assertEquals(parsedPolicy, builtPolicy);
+    }
+
+    public void testPolicyBuilderOnExternalPlugin() throws IOException {
+        Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", true)
             .parsePolicy();
         Policy builtPolicy = new Policy(
             "test-policy.yaml",
@@ -50,7 +60,7 @@ public class PolicyParserTests extends ESTestCase {
         Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream("""
             entitlement-module-name:
               - create_class_loader
-            """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml").parsePolicy();
+            """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy();
         Policy builtPolicy = new Policy(
             "test-policy.yaml",
             List.of(new Scope("entitlement-module-name", List.of(new CreateClassLoaderEntitlement())))

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java

@@ -9,7 +9,6 @@
 
 package org.elasticsearch.index.rankeval;
 
-import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
@@ -46,9 +45,7 @@ public final class RankEvalRequest extends ActionRequest implements IndicesReque
         rankingEvaluationSpec = new RankEvalSpec(in);
         indices = in.readStringArray();
         indicesOptions = IndicesOptions.readIndicesOptions(in);
-        if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_6_0)) {
+        searchType = SearchType.fromId(in.readByte());
-            searchType = SearchType.fromId(in.readByte());
-        }
     }
 
     RankEvalRequest() {}
@@ -127,9 +124,7 @@ public final class RankEvalRequest extends ActionRequest implements IndicesReque
         rankingEvaluationSpec.writeTo(out);
         out.writeStringArray(indices);
         indicesOptions.writeIndicesOptions(out);
-        if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_6_0)) {
+        out.writeByte(searchType.id());
-            out.writeByte(searchType.id());
-        }
     }
 
     @Override

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsCredentialsRestIT.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.repositories.s3;
 
 import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.aws.imds.Ec2ImdsServiceBuilder;
 import fixture.aws.imds.Ec2ImdsVersion;
 import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
@@ -37,9 +38,8 @@ public class RepositoryS3EcsCredentialsRestIT extends AbstractRepositoryS3RestTe
     private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
 
     private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
-        Ec2ImdsVersion.V1,
+        new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).newCredentialsConsumer(dynamicS3Credentials::addValidCredentials)
-        dynamicS3Credentials::addValidCredentials,
+            .alternativeCredentialsEndpoints(Set.of("/ecs_credentials_endpoint"))
-        Set.of("/ecs_credentials_endpoint")
     );
 
     private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicS3Credentials::isAuthorized);

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV1CredentialsRestIT.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.repositories.s3;
 
 import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.aws.imds.Ec2ImdsServiceBuilder;
 import fixture.aws.imds.Ec2ImdsVersion;
 import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
@@ -23,8 +24,6 @@ import org.junit.ClassRule;
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
-import java.util.Set;
-
 @ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
 @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482
 public class RepositoryS3ImdsV1CredentialsRestIT extends AbstractRepositoryS3RestTestCase {
@@ -37,9 +36,7 @@ public class RepositoryS3ImdsV1CredentialsRestIT extends AbstractRepositoryS3Res
     private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
 
     private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
-        Ec2ImdsVersion.V1,
+        new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).newCredentialsConsumer(dynamicS3Credentials::addValidCredentials)
-        dynamicS3Credentials::addValidCredentials,
-        Set.of()
     );
 
     private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicS3Credentials::isAuthorized);
@@ -47,7 +44,7 @@ public class RepositoryS3ImdsV1CredentialsRestIT extends AbstractRepositoryS3Res
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
         .setting("s3.client." + CLIENT + ".endpoint", s3Fixture::getAddress)
-        .systemProperty("com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", ec2ImdsHttpFixture::getAddress)
+        .systemProperty(Ec2ImdsHttpFixture.ENDPOINT_OVERRIDE_SYSPROP_NAME, ec2ImdsHttpFixture::getAddress)
         .build();
 
     @ClassRule

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ImdsV2CredentialsRestIT.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.repositories.s3;
 
 import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.aws.imds.Ec2ImdsServiceBuilder;
 import fixture.aws.imds.Ec2ImdsVersion;
 import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
@@ -23,8 +24,6 @@ import org.junit.ClassRule;
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
-import java.util.Set;
-
 @ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
 @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482
 public class RepositoryS3ImdsV2CredentialsRestIT extends AbstractRepositoryS3RestTestCase {
@@ -37,9 +36,7 @@ public class RepositoryS3ImdsV2CredentialsRestIT extends AbstractRepositoryS3Res
     private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
 
     private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
-        Ec2ImdsVersion.V2,
+        new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V2).newCredentialsConsumer(dynamicS3Credentials::addValidCredentials)
-        dynamicS3Credentials::addValidCredentials,
-        Set.of()
     );
 
     private static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, dynamicS3Credentials::isAuthorized);
@@ -47,7 +44,7 @@ public class RepositoryS3ImdsV2CredentialsRestIT extends AbstractRepositoryS3Res
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
         .setting("s3.client." + CLIENT + ".endpoint", s3Fixture::getAddress)
-        .systemProperty("com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", ec2ImdsHttpFixture::getAddress)
+        .systemProperty(Ec2ImdsHttpFixture.ENDPOINT_OVERRIDE_SYSPROP_NAME, ec2ImdsHttpFixture::getAddress)
         .build();
 
     @ClassRule

muted-tests.yml

@@ -52,12 +52,6 @@ tests:
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=mtermvectors/10_basic/Tests catching other exceptions per item}
   issue: https://github.com/elastic/elasticsearch/issues/113325
-- class: org.elasticsearch.integration.KibanaUserRoleIntegTests
-  method: testFieldMappings
-  issue: https://github.com/elastic/elasticsearch/issues/113592
-- class: org.elasticsearch.integration.KibanaUserRoleIntegTests
-  method: testSearchAndMSearch
-  issue: https://github.com/elastic/elasticsearch/issues/113593
 - class: org.elasticsearch.xpack.transform.integration.TransformIT
   method: testStopWaitForCheckpoint
   issue: https://github.com/elastic/elasticsearch/issues/106113
@@ -180,6 +174,12 @@ tests:
 - class: "org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT"
   method: "test {scoring.*}"
   issue: https://github.com/elastic/elasticsearch/issues/117641
+- class: "org.elasticsearch.xpack.esql.qa.mixed.MultilusterEsqlSpecIT"
+  method: "test {scoring.*}"
+  issue: https://github.com/elastic/elasticsearch/issues/118460
+- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT"
+  method: "test {scoring.*}"
+  issue: https://github.com/elastic/elasticsearch/issues/118460
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {scoring.QstrWithFieldAndScoringSortedEval}
   issue: https://github.com/elastic/elasticsearch/issues/117751
@@ -276,9 +276,6 @@ tests:
 - class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests
   method: testBottomFieldSort
   issue: https://github.com/elastic/elasticsearch/issues/118214
-- class: org.elasticsearch.xpack.esql.action.CrossClustersEnrichIT
-  method: testTopNThenEnrichRemote
-  issue: https://github.com/elastic/elasticsearch/issues/118307
 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS1UnavailableRemotesIT
   method: testEsqlRcs1UnavailableRemoteScenarios
   issue: https://github.com/elastic/elasticsearch/issues/118350
@@ -317,6 +314,9 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test011SecurityEnabledStatus
   issue: https://github.com/elastic/elasticsearch/issues/118517
+- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
+  method: testInvalidJSON
+  issue: https://github.com/elastic/elasticsearch/issues/116521
 
 # Examples:
 #

plugins/discovery-ec2/build.gradle

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 apply plugin: 'elasticsearch.internal-java-rest-test'
+apply plugin: 'elasticsearch.internal-cluster-test'
 
 esplugin {
   description 'The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.'
@@ -26,6 +27,11 @@ dependencies {
   api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
   api "joda-time:joda-time:2.10.10"
+
+  javaRestTestImplementation project(':plugins:discovery-ec2')
+  javaRestTestImplementation project(':test:fixtures:ec2-imds-fixture')
+
+  internalClusterTestImplementation project(':test:fixtures:ec2-imds-fixture')
 }
 
 tasks.named("dependencyLicenses").configure {
@@ -79,7 +85,7 @@ tasks.register("writeTestJavaPolicy") {
   }
 }
 
-tasks.named("test").configure {
+tasks.withType(Test).configureEach {
   dependsOn "writeTestJavaPolicy"
   // this is needed for insecure plugins, remove if possible!
   systemProperty 'tests.artifact', project.name

plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2NetworkAddressesTestCase.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.http.HttpServerTransport;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.io.IOException;
+import java.util.Collection;
+
+@ESIntegTestCase.ClusterScope(numDataNodes = 0)
+public abstract class DiscoveryEc2NetworkAddressesTestCase extends ESIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return CollectionUtils.appendToCopyNoNullElements(super.nodePlugins(), Ec2DiscoveryPlugin.class);
+    }
+
+    @Override
+    protected boolean addMockHttpTransport() {
+        return false;
+    }
+
+    void verifyPublishAddress(String publishAddressSetting, String expectedAddress) throws IOException {
+        final var node = internalCluster().startNode(Settings.builder().put("http.publish_host", publishAddressSetting));
+        assertEquals(
+            expectedAddress,
+            internalCluster().getInstance(HttpServerTransport.class, node).boundAddress().publishAddress().getAddress()
+        );
+        internalCluster().stopNode(node);
+    }
+}

plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2RegularNetworkAddressesIT.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.aws.imds.Ec2ImdsServiceBuilder;
+import fixture.aws.imds.Ec2ImdsVersion;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.transport.BindTransportException;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class DiscoveryEc2RegularNetworkAddressesIT extends DiscoveryEc2NetworkAddressesTestCase {
+    public void testLocalIgnoresImds() {
+        Ec2ImdsHttpFixture.runWithFixture(new Ec2ImdsServiceBuilder(randomFrom(Ec2ImdsVersion.values())), imdsFixture -> {
+            try (var ignored = Ec2ImdsHttpFixture.withEc2MetadataServiceEndpointOverride(imdsFixture.getAddress())) {
+                verifyPublishAddress("_local_", "127.0.0.1");
+            }
+        });
+    }
+
+    public void testImdsNotAvailable() throws IOException {
+        try (var ignored = Ec2ImdsHttpFixture.withEc2MetadataServiceEndpointOverride("http://127.0.0.1")) {
+            // if IMDS is not running, regular values like `_local_` should still work
+            verifyPublishAddress("_local_", "127.0.0.1");
+
+            // but EC2 addresses will cause the node to fail to start
+            final var assertionError = expectThrows(
+                AssertionError.class,
+                () -> internalCluster().startNode(Settings.builder().put("http.publish_host", "_ec2_"))
+            );
+            final var executionException = asInstanceOf(ExecutionException.class, assertionError.getCause());
+            final var bindTransportException = asInstanceOf(BindTransportException.class, executionException.getCause());
+            assertEquals("Failed to resolve publish address", bindTransportException.getMessage());
+            final var ioException = asInstanceOf(IOException.class, bindTransportException.getCause());
+            assertThat(ioException.getMessage(), containsString("/latest/meta-data/local-ipv4"));
+        }
+    }
+}

plugins/discovery-ec2/src/internalClusterTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2SpecialNetworkAddressesIT.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.aws.imds.Ec2ImdsServiceBuilder;
+import fixture.aws.imds.Ec2ImdsVersion;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import java.util.Map;
+import java.util.stream.Stream;
+
+public class DiscoveryEc2SpecialNetworkAddressesIT extends DiscoveryEc2NetworkAddressesTestCase {
+
+    private final String imdsAddressName;
+    private final String elasticsearchAddressName;
+    private final Ec2ImdsVersion imdsVersion;
+
+    public DiscoveryEc2SpecialNetworkAddressesIT(
+        @Name("imdsAddressName") String imdsAddressName,
+        @Name("elasticsearchAddressName") String elasticsearchAddressName,
+        @Name("imdsVersion") Ec2ImdsVersion imdsVersion
+    ) {
+        this.imdsAddressName = imdsAddressName;
+        this.elasticsearchAddressName = elasticsearchAddressName;
+        this.imdsVersion = imdsVersion;
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() {
+        return Map.of(
+            "_ec2:privateIpv4_",
+            "local-ipv4",
+            "_ec2:privateDns_",
+            "local-hostname",
+            "_ec2:publicIpv4_",
+            "public-ipv4",
+            "_ec2:publicDns_",
+            "public-hostname",
+            "_ec2:publicIp_",
+            "public-ipv4",
+            "_ec2:privateIp_",
+            "local-ipv4",
+            "_ec2_",
+            "local-ipv4"
+        )
+            .entrySet()
+            .stream()
+            .flatMap(
+                addresses -> Stream.of(Ec2ImdsVersion.values())
+                    .map(ec2ImdsVersion -> new Object[] { addresses.getValue(), addresses.getKey(), ec2ImdsVersion })
+            )
+            .toList();
+    }
+
+    public void testSpecialNetworkAddresses() {
+        final var publishAddress = "10.0." + between(0, 255) + "." + between(0, 255);
+        Ec2ImdsHttpFixture.runWithFixture(
+            new Ec2ImdsServiceBuilder(imdsVersion).addInstanceAddress(imdsAddressName, publishAddress),
+            imdsFixture -> {
+                try (var ignored = Ec2ImdsHttpFixture.withEc2MetadataServiceEndpointOverride(imdsFixture.getAddress())) {
+                    verifyPublishAddress(elasticsearchAddressName, publishAddress);
+                }
+            }
+        );
+    }
+
+}

plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV1IT.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.aws.imds.Ec2ImdsServiceBuilder;
+import fixture.aws.imds.Ec2ImdsVersion;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+public class DiscoveryEc2AvailabilityZoneAttributeImdsV1IT extends DiscoveryEc2AvailabilityZoneAttributeTestCase {
+    private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
+        new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V1).availabilityZoneSupplier(
+            DiscoveryEc2AvailabilityZoneAttributeTestCase::getAvailabilityZone
+        )
+    );
+
+    public static ElasticsearchCluster cluster = buildCluster(ec2ImdsHttpFixture::getAddress);
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(ec2ImdsHttpFixture).around(cluster);
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}

plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeImdsV2IT.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.aws.imds.Ec2ImdsServiceBuilder;
+import fixture.aws.imds.Ec2ImdsVersion;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+public class DiscoveryEc2AvailabilityZoneAttributeImdsV2IT extends DiscoveryEc2AvailabilityZoneAttributeTestCase {
+    private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
+        new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V2).availabilityZoneSupplier(
+            DiscoveryEc2AvailabilityZoneAttributeTestCase::getAvailabilityZone
+        )
+    );
+
+    public static ElasticsearchCluster cluster = buildCluster(ec2ImdsHttpFixture::getAddress);
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(ec2ImdsHttpFixture).around(cluster);
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}

plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeNoImdsIT.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+
+public class DiscoveryEc2AvailabilityZoneAttributeNoImdsIT extends ESRestTestCase {
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .plugin("discovery-ec2")
+        .setting(AwsEc2Service.AUTO_ATTRIBUTE_SETTING.getKey(), "true")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    public void testAvailabilityZoneAttribute() throws IOException {
+        final var nodesInfoResponse = assertOKAndCreateObjectPath(client().performRequest(new Request("GET", "/_nodes/_all/_none")));
+        for (final var nodeId : nodesInfoResponse.evaluateMapKeys("nodes")) {
+            assertNull(nodesInfoResponse.evaluateExact("nodes", nodeId, "attributes", "aws_availability_zone"));
+        }
+    }
+}

plugins/discovery-ec2/src/javaRestTest/java/org/elasticsearch/discovery/ec2/DiscoveryEc2AvailabilityZoneAttributeTestCase.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import fixture.aws.imds.Ec2ImdsHttpFixture;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.hamcrest.Matchers;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Supplier;
+
+public abstract class DiscoveryEc2AvailabilityZoneAttributeTestCase extends ESRestTestCase {
+
+    private static final Set<String> createdAvailabilityZones = ConcurrentCollections.newConcurrentSet();
+
+    protected static String getAvailabilityZone() {
+        final var zoneName = randomIdentifier();
+        createdAvailabilityZones.add(zoneName);
+        return zoneName;
+    }
+
+    protected static ElasticsearchCluster buildCluster(Supplier<String> imdsFixtureAddressSupplier) {
+        return ElasticsearchCluster.local()
+            .plugin("discovery-ec2")
+            .setting(AwsEc2Service.AUTO_ATTRIBUTE_SETTING.getKey(), "true")
+            .systemProperty(Ec2ImdsHttpFixture.ENDPOINT_OVERRIDE_SYSPROP_NAME, imdsFixtureAddressSupplier)
+            .build();
+    }
+
+    public void testAvailabilityZoneAttribute() throws IOException {
+        final var nodesInfoResponse = assertOKAndCreateObjectPath(client().performRequest(new Request("GET", "/_nodes/_all/_none")));
+        for (final var nodeId : nodesInfoResponse.evaluateMapKeys("nodes")) {
+            assertThat(
+                createdAvailabilityZones,
+                Matchers.hasItem(
+                    Objects.requireNonNull(nodesInfoResponse.<String>evaluateExact("nodes", nodeId, "attributes", "aws_availability_zone"))
+                )
+            );
+        }
+    }
+}

plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2NetworkTests.java

@@ -1,181 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.discovery.ec2;
-
-import com.sun.net.httpserver.HttpServer;
-
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.network.NetworkService;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.mocksocket.MockHttpServer;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.test.ESTestCase;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.function.BiConsumer;
-
-import static com.amazonaws.SDKGlobalConfiguration.EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.hamcrest.Matchers.arrayContaining;
-import static org.hamcrest.Matchers.equalTo;
-
-/**
- * Test for EC2 network.host settings.
- * <p>
- * Warning: This test doesn't assert that the exceptions are thrown.
- * They aren't.
- */
-@SuppressForbidden(reason = "use http server")
-public class Ec2NetworkTests extends ESTestCase {
-
-    private static HttpServer httpServer;
-
-    @BeforeClass
-    public static void startHttp() throws Exception {
-        httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0);
-
-        BiConsumer<String, String> registerContext = (path, v) -> {
-            final byte[] message = v.getBytes(UTF_8);
-            httpServer.createContext(path, (s) -> {
-                s.sendResponseHeaders(RestStatus.OK.getStatus(), message.length);
-                OutputStream responseBody = s.getResponseBody();
-                responseBody.write(message);
-                responseBody.close();
-            });
-        };
-        registerContext.accept("/latest/meta-data/local-ipv4", "127.0.0.1");
-        registerContext.accept("/latest/meta-data/public-ipv4", "165.168.10.2");
-        registerContext.accept("/latest/meta-data/public-hostname", "165.168.10.3");
-        registerContext.accept("/latest/meta-data/local-hostname", "10.10.10.5");
-
-        httpServer.start();
-    }
-
-    @Before
-    public void setup() {
-        // redirect EC2 metadata service to httpServer
-        AccessController.doPrivileged(
-            (PrivilegedAction<String>) () -> System.setProperty(
-                EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY,
-                "http://" + httpServer.getAddress().getHostName() + ":" + httpServer.getAddress().getPort()
-            )
-        );
-    }
-
-    @AfterClass
-    public static void stopHttp() {
-        httpServer.stop(0);
-        httpServer = null;
-    }
-
-    /**
-     * Test for network.host: _ec2_
-     */
-    public void testNetworkHostEc2() throws IOException {
-        resolveEc2("_ec2_", InetAddress.getByName("127.0.0.1"));
-    }
-
-    /**
-     * Test for network.host: _ec2_
-     */
-    public void testNetworkHostUnableToResolveEc2() {
-        // redirect EC2 metadata service to unknown location
-        AccessController.doPrivileged(
-            (PrivilegedAction<String>) () -> System.setProperty(EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY, "http://127.0.0.1/")
-        );
-
-        try {
-            resolveEc2("_ec2_", (InetAddress[]) null);
-        } catch (IOException e) {
-            assertThat(
-                e.getMessage(),
-                equalTo("IOException caught when fetching InetAddress from [http://127.0.0.1//latest/meta-data/local-ipv4]")
-            );
-        }
-    }
-
-    /**
-     * Test for network.host: _ec2:publicIp_
-     */
-    public void testNetworkHostEc2PublicIp() throws IOException {
-        resolveEc2("_ec2:publicIp_", InetAddress.getByName("165.168.10.2"));
-    }
-
-    /**
-     * Test for network.host: _ec2:privateIp_
-     */
-    public void testNetworkHostEc2PrivateIp() throws IOException {
-        resolveEc2("_ec2:privateIp_", InetAddress.getByName("127.0.0.1"));
-    }
-
-    /**
-     * Test for network.host: _ec2:privateIpv4_
-     */
-    public void testNetworkHostEc2PrivateIpv4() throws IOException {
-        resolveEc2("_ec2:privateIpv4_", InetAddress.getByName("127.0.0.1"));
-    }
-
-    /**
-     * Test for network.host: _ec2:privateDns_
-     */
-    public void testNetworkHostEc2PrivateDns() throws IOException {
-        resolveEc2("_ec2:privateDns_", InetAddress.getByName("10.10.10.5"));
-    }
-
-    /**
-     * Test for network.host: _ec2:publicIpv4_
-     */
-    public void testNetworkHostEc2PublicIpv4() throws IOException {
-        resolveEc2("_ec2:publicIpv4_", InetAddress.getByName("165.168.10.2"));
-    }
-
-    /**
-     * Test for network.host: _ec2:publicDns_
-     */
-    public void testNetworkHostEc2PublicDns() throws IOException {
-        resolveEc2("_ec2:publicDns_", InetAddress.getByName("165.168.10.3"));
-    }
-
-    private InetAddress[] resolveEc2(String host, InetAddress... expected) throws IOException {
-        Settings nodeSettings = Settings.builder().put("network.host", host).build();
-
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver()));
-
-        InetAddress[] addresses = networkService.resolveBindHostAddresses(
-            NetworkService.GLOBAL_NETWORK_BIND_HOST_SETTING.get(nodeSettings).toArray(Strings.EMPTY_ARRAY)
-        );
-        if (expected == null) {
-            fail("We should get an IOException, resolved addressed:" + Arrays.toString(addresses));
-        }
-        assertThat(addresses, arrayContaining(expected));
-        return addresses;
-    }
-
-    /**
-     * Test that we don't have any regression with network host core settings such as
-     * network.host: _local_
-     */
-    public void testNetworkHostCoreLocal() throws IOException {
-        NetworkService networkService = new NetworkService(Collections.singletonList(new Ec2NameResolver()));
-        InetAddress[] addresses = networkService.resolveBindHostAddresses(null);
-        assertThat(addresses, arrayContaining(networkService.resolveBindHostAddresses(new String[] { "_local_" })));
-    }
-}

rest-api-spec/src/main/resources/rest-api-spec/api/migrate.cancel_reindex.json

@@ -0,0 +1,30 @@
+{
+  "migrate.cancel_reindex":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-stream-reindex.html",
+      "description":"This API returns the status of a migration reindex attempt for a data stream or index"
+    },
+    "stability":"experimental",
+    "visibility":"private",
+    "headers":{
+      "accept": [ "application/json"],
+      "content_type": ["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/_migration/reindex/{index}/_cancel",
+          "methods":[
+            "POST"
+          ],
+          "parts":{
+            "index":{
+              "type":"string",
+              "description":"The index or data stream name"
+            }
+          }
+        }
+      ]
+    }
+  }
+}

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml

@@ -84,32 +84,6 @@
   - match: { error.type: "illegal_argument_exception" }
   - match: { error.reason: "Types cannot be provided in put mapping requests" }
 
----
-"Put mappings with explicit _doc type bwc":
-  - skip:
-          cluster_features: [ "gte_v8.0.0"]
-          reason:      "old deprecation message for pre 8.0"
-  - requires:
-          test_runner_features: ["node_selector"]
-  - do:
-      indices.create:
-        index: test_index
-
-  - do:
-      node_selector:
-        version: "original"
-      catch: bad_request
-      indices.put_mapping:
-        index: test_index
-        body:
-          _doc:
-            properties:
-              field:
-                type: keyword
-
-  - match: { error.type: "illegal_argument_exception" }
-  - match: { error.reason: "Types cannot be provided in put mapping requests, unless the include_type_name parameter is set to true." }
-
 ---
 "Update per-field metadata":
 

server/src/main/java/module-info.java

@@ -480,5 +480,5 @@ module org.elasticsearch.server {
     exports org.elasticsearch.lucene.spatial;
     exports org.elasticsearch.inference.configuration;
     exports org.elasticsearch.monitor.metrics;
-
+    exports org.elasticsearch.plugins.internal.rewriter to org.elasticsearch.inference;
 }

server/src/main/java/org/elasticsearch/ElasticsearchException.java

@@ -40,7 +40,6 @@ import org.elasticsearch.persistent.NotPersistentTaskNodeException;
 import org.elasticsearch.persistent.PersistentTaskNodeNotAssignedException;
 import org.elasticsearch.rest.ApiNotAvailableException;
 import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.SearchException;
 import org.elasticsearch.search.TooManyScrollContextsException;
 import org.elasticsearch.search.aggregations.AggregationExecutionException;
 import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
@@ -319,10 +318,6 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
     public static ElasticsearchException readException(StreamInput input, int id) throws IOException {
         CheckedFunction<StreamInput, ? extends ElasticsearchException, IOException> elasticsearchException = ID_TO_SUPPLIER.get(id);
         if (elasticsearchException == null) {
-            if (id == 127 && input.getTransportVersion().before(TransportVersions.V_7_5_0)) {
-                // was SearchContextException
-                return new SearchException(input);
-            }
             throw new IllegalStateException("unknown exception for id: " + id);
         }
         return elasticsearchException.apply(input);
@@ -1817,13 +1812,13 @@ public class ElasticsearchException extends RuntimeException implements ToXConte
             org.elasticsearch.index.seqno.RetentionLeaseInvalidRetainingSeqNoException.class,
             org.elasticsearch.index.seqno.RetentionLeaseInvalidRetainingSeqNoException::new,
             156,
-            TransportVersions.V_7_5_0
+            UNKNOWN_VERSION_ADDED
         ),
         INGEST_PROCESSOR_EXCEPTION(
             org.elasticsearch.ingest.IngestProcessorException.class,
             org.elasticsearch.ingest.IngestProcessorException::new,
             157,
-            TransportVersions.V_7_5_0
+            UNKNOWN_VERSION_ADDED
         ),
         PEER_RECOVERY_NOT_FOUND_EXCEPTION(
             org.elasticsearch.indices.recovery.PeerRecoveryNotFound.class,

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -55,7 +55,6 @@ public class TransportVersions {
     public static final TransportVersion V_7_0_0 = def(7_00_00_99);
     public static final TransportVersion V_7_3_0 = def(7_03_00_99);
     public static final TransportVersion V_7_4_0 = def(7_04_00_99);
-    public static final TransportVersion V_7_5_0 = def(7_05_00_99);
     public static final TransportVersion V_7_6_0 = def(7_06_00_99);
     public static final TransportVersion V_7_7_0 = def(7_07_00_99);
     public static final TransportVersion V_7_8_0 = def(7_08_00_99);
@@ -139,6 +138,7 @@ public class TransportVersions {
     public static final TransportVersion RETRIES_AND_OPERATIONS_IN_BLOBSTORE_STATS = def(8_804_00_0);
     public static final TransportVersion ADD_DATA_STREAM_OPTIONS_TO_TEMPLATES = def(8_805_00_0);
     public static final TransportVersion KNN_QUERY_RESCORE_OVERSAMPLE = def(8_806_00_0);
+    public static final TransportVersion SEMANTIC_QUERY_LENIENT = def(8_807_00_0);
 
     /*
      * WARNING: DO NOT MERGE INTO MAIN!

server/src/main/java/org/elasticsearch/Version.java

@@ -187,7 +187,7 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_15_2 = new Version(8_15_02_99);
     public static final Version V_8_15_3 = new Version(8_15_03_99);
     public static final Version V_8_15_4 = new Version(8_15_04_99);
-    public static final Version V_8_15_6 = new Version(8_15_06_99);
+    public static final Version V_8_15_5 = new Version(8_15_05_99);
     public static final Version V_8_16_0 = new Version(8_16_00_99);
     public static final Version V_8_16_1 = new Version(8_16_01_99);
     public static final Version V_8_16_2 = new Version(8_16_02_99);

server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

@@ -30,7 +30,6 @@ import org.elasticsearch.common.util.concurrent.RunOnce;
 import org.elasticsearch.core.AbstractRefCounted;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.entitlement.bootstrap.EntitlementBootstrap;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexVersion;
@@ -59,6 +58,7 @@ import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.bootstrap.BootstrapSettings.SECURITY_FILTER_BAD_DEFAULTS_SETTING;
 import static org.elasticsearch.nativeaccess.WindowsFunctions.ConsoleCtrlHandler.CTRL_CLOSE_EVENT;
@@ -218,10 +218,14 @@ class Elasticsearch {
         if (Boolean.parseBoolean(System.getProperty("es.entitlements.enabled"))) {
             LogManager.getLogger(Elasticsearch.class).info("Bootstrapping Entitlements");
 
-            List<Tuple<Path, Boolean>> pluginData = pluginsLoader.allBundles()
+            List<EntitlementBootstrap.PluginData> pluginData = Stream.concat(
-                .stream()
+                pluginsLoader.moduleBundles()
-                .map(bundle -> Tuple.tuple(bundle.getDir(), bundle.pluginDescriptor().isModular()))
+                    .stream()
-                .toList();
+                    .map(bundle -> new EntitlementBootstrap.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), false)),
+                pluginsLoader.pluginBundles()
+                    .stream()
+                    .map(bundle -> new EntitlementBootstrap.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), true))
+            ).toList();
 
             EntitlementBootstrap.bootstrap(pluginData, pluginsResolver::resolveClassToPluginName);
         } else if (RuntimeVersionFeature.isSecurityManagerAvailable()) {

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -1116,10 +1116,10 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
 
     /**
      * Return the {@link IndexVersion} that this index provides compatibility for.
-     * This is typically compared to the {@link IndexVersions#MINIMUM_COMPATIBLE} to figure out whether the index can be handled
+     * This is typically compared to the {@link IndexVersions#MINIMUM_COMPATIBLE} or {@link IndexVersions#MINIMUM_READONLY_COMPATIBLE}
-     * by the cluster.
+     * to figure out whether the index can be handled by the cluster.
-     * By default, this is equal to the {@link #getCreationVersion()}, but can also be a newer version if the index has been imported as
+     * By default, this is equal to the {@link #getCreationVersion()}, but can also be a newer version if the index has been created by
-     * a legacy index from an older snapshot, and its metadata has been converted to be handled by newer version nodes.
+     * a legacy version, and imported archive, in which case its metadata has been converted to be handled by newer version nodes.
      */
     public IndexVersion getCompatibilityVersion() {
         return indexCompatibilityVersion;

server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java

@@ -67,6 +67,7 @@ import java.util.function.Predicate;
  * Note: This class is performance sensitive, so we pay extra attention on the data structure usage and we avoid streams and iterators
  * when possible in favor of the classic for-i loops.
  */
+@SuppressWarnings("ForLoopReplaceableByForEach")
 public class IndexNameExpressionResolver {
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(IndexNameExpressionResolver.class);
 

server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodes.java

@@ -65,7 +65,6 @@ public class DiscoveryNodes implements Iterable<DiscoveryNode>, SimpleDiffable<D
     private final String localNodeId;
     @Nullable
     private final DiscoveryNode localNode;
-    private final Version minNonClientNodeVersion;
     private final Version maxNodeVersion;
     private final Version minNodeVersion;
     private final IndexVersion maxDataNodeCompatibleIndexVersion;
@@ -81,7 +80,6 @@ public class DiscoveryNodes implements Iterable<DiscoveryNode>, SimpleDiffable<D
         Map<String, DiscoveryNode> ingestNodes,
         @Nullable String masterNodeId,
         @Nullable String localNodeId,
-        Version minNonClientNodeVersion,
         Version maxNodeVersion,
         Version minNodeVersion,
         IndexVersion maxDataNodeCompatibleIndexVersion,
@@ -98,7 +96,6 @@ public class DiscoveryNodes implements Iterable<DiscoveryNode>, SimpleDiffable<D
         assert (masterNodeId == null) == (masterNode == null);
         this.localNodeId = localNodeId;
         this.localNode = localNodeId == null ? null : nodes.get(localNodeId);
-        this.minNonClientNodeVersion = minNonClientNodeVersion;
         this.minNodeVersion = minNodeVersion;
         this.maxNodeVersion = maxNodeVersion;
         this.maxDataNodeCompatibleIndexVersion = maxDataNodeCompatibleIndexVersion;
@@ -117,7 +114,6 @@ public class DiscoveryNodes implements Iterable<DiscoveryNode>, SimpleDiffable<D
             ingestNodes,
             masterNodeId,
             localNodeId,
-            minNonClientNodeVersion,
             maxNodeVersion,
             minNodeVersion,
             maxDataNodeCompatibleIndexVersion,
@@ -346,17 +342,6 @@ public class DiscoveryNodes implements Iterable<DiscoveryNode>, SimpleDiffable<D
         return minNodeVersion.equals(maxNodeVersion) == false;
     }
 
-    /**
-     * Returns the version of the node with the oldest version in the cluster that is not a client node
-     *
-     * If there are no non-client nodes, Version.CURRENT will be returned.
-     *
-     * @return the oldest version in the cluster
-     */
-    public Version getSmallestNonClientNodeVersion() {
-        return minNonClientNodeVersion;
-    }
-
     /**
      * Returns the highest index version supported by all data nodes in the cluster
      */
@@ -853,14 +838,12 @@ public class DiscoveryNodes implements Iterable<DiscoveryNode>, SimpleDiffable<D
              */
             Version minNodeVersion = null;
             Version maxNodeVersion = null;
-            Version minNonClientNodeVersion = null;
             IndexVersion maxDataNodeCompatibleIndexVersion = null;
             IndexVersion minSupportedIndexVersion = null;
             for (Map.Entry<String, DiscoveryNode> nodeEntry : nodes.entrySet()) {
                 DiscoveryNode discoNode = nodeEntry.getValue();
                 Version version = discoNode.getVersion();
                 if (discoNode.canContainData() || discoNode.isMasterNode()) {
-                    minNonClientNodeVersion = min(minNonClientNodeVersion, version);
                     maxDataNodeCompatibleIndexVersion = min(maxDataNodeCompatibleIndexVersion, discoNode.getMaxIndexVersion());
                 }
                 minNodeVersion = min(minNodeVersion, version);
@@ -894,7 +877,6 @@ public class DiscoveryNodes implements Iterable<DiscoveryNode>, SimpleDiffable<D
                 filteredNodes(nodes, DiscoveryNode::isIngestNode),
                 masterNodeId,
                 localNodeId,
-                Objects.requireNonNullElse(minNonClientNodeVersion, Version.CURRENT),
                 Objects.requireNonNullElse(maxNodeVersion, Version.CURRENT),
                 Objects.requireNonNullElse(minNodeVersion, Version.CURRENT.minimumCompatibilityVersion()),
                 Objects.requireNonNullElse(maxDataNodeCompatibleIndexVersion, IndexVersion.current()),

server/src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -524,8 +524,7 @@ public final class NodeEnvironment implements Closeable {
 
         logger.info("oldest index version recorded in NodeMetadata {}", metadata.oldestIndexVersion());
 
-        if (metadata.oldestIndexVersion().isLegacyIndexVersion()) {
+        if (metadata.oldestIndexVersion().before(IndexVersions.MINIMUM_COMPATIBLE)) {
-
             String bestDowngradeVersion = getBestDowngradeVersion(metadata.previousNodeVersion().toString());
             throw new IllegalStateException(
                 "Cannot start this node because it holds metadata for indices with version ["

server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java

@@ -12,7 +12,6 @@ package org.elasticsearch.gateway;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
@@ -239,11 +238,7 @@ public class TransportNodesListGatewayStartedShards extends TransportNodesAction
         public NodeRequest(StreamInput in) throws IOException {
             super(in);
             shardId = new ShardId(in);
-            if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_6_0)) {
+            customDataPath = in.readString();
-                customDataPath = in.readString();
-            } else {
-                customDataPath = null;
-            }
         }
 
         public NodeRequest(Request request) {
@@ -255,10 +250,7 @@ public class TransportNodesListGatewayStartedShards extends TransportNodesAction
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             shardId.writeTo(out);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_6_0)) {
+            out.writeString(customDataPath);
-                assert customDataPath != null;
-                out.writeString(customDataPath);
-            }
         }
 
         public ShardId getShardId() {

server/src/main/java/org/elasticsearch/index/IndexModule.java

@@ -58,6 +58,7 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.plugins.IndexStorePlugin;
+import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -478,7 +479,8 @@ public final class IndexModule {
         IdFieldMapper idFieldMapper,
         ValuesSourceRegistry valuesSourceRegistry,
         IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener,
-        Map<String, IndexStorePlugin.SnapshotCommitSupplier> snapshotCommitSuppliers
+        Map<String, IndexStorePlugin.SnapshotCommitSupplier> snapshotCommitSuppliers,
+        QueryRewriteInterceptor queryRewriteInterceptor
     ) throws IOException {
         final IndexEventListener eventListener = freeze();
         Function<IndexService, CheckedFunction<DirectoryReader, DirectoryReader, IOException>> readerWrapperFactory = indexReaderWrapper
@@ -540,7 +542,8 @@ public final class IndexModule {
                 indexFoldersDeletionListener,
                 snapshotCommitSupplier,
                 indexCommitListener.get(),
-                mapperMetrics
+                mapperMetrics,
+                queryRewriteInterceptor
             );
             success = true;
             return indexService;

server/src/main/java/org/elasticsearch/index/IndexService.java

@@ -85,6 +85,7 @@ import org.elasticsearch.indices.cluster.IndicesClusterStateService;
 import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.elasticsearch.indices.recovery.RecoveryState;
 import org.elasticsearch.plugins.IndexStorePlugin;
+import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -162,6 +163,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final Supplier<Sort> indexSortSupplier;
     private final ValuesSourceRegistry valuesSourceRegistry;
     private final MapperMetrics mapperMetrics;
+    private final QueryRewriteInterceptor queryRewriteInterceptor;
 
     @SuppressWarnings("this-escape")
     public IndexService(
@@ -196,7 +198,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
         IndexStorePlugin.IndexFoldersDeletionListener indexFoldersDeletionListener,
         IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier,
         Engine.IndexCommitListener indexCommitListener,
-        MapperMetrics mapperMetrics
+        MapperMetrics mapperMetrics,
+        QueryRewriteInterceptor queryRewriteInterceptor
     ) {
         super(indexSettings);
         assert indexCreationContext != IndexCreationContext.RELOAD_ANALYZERS
@@ -271,6 +274,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
         this.indexingOperationListeners = Collections.unmodifiableList(indexingOperationListeners);
         this.indexCommitListener = indexCommitListener;
         this.mapperMetrics = mapperMetrics;
+        this.queryRewriteInterceptor = queryRewriteInterceptor;
         try (var ignored = threadPool.getThreadContext().clearTraceContext()) {
             // kick off async ops for the first shard in this index
             this.refreshTask = new AsyncRefreshTask(this);
@@ -802,6 +806,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
             allowExpensiveQueries,
             scriptService,
             null,
+            null,
             null
         );
     }

server/src/main/java/org/elasticsearch/index/IndexVersion.java

@@ -123,8 +123,13 @@ public record IndexVersion(int id, Version luceneVersion) implements VersionId<I
         return CurrentHolder.CURRENT;
     }
 
+    /**
+     * Returns whether this index version is supported by this node version out-of-the-box.
+     * This is used to distinguish between ordinary indices and archive indices that may be
+     * imported into the cluster in read-only mode, and with limited functionality.
+     */
     public boolean isLegacyIndexVersion() {
-        return before(IndexVersions.MINIMUM_COMPATIBLE);
+        return before(IndexVersions.MINIMUM_READONLY_COMPATIBLE);
     }
 
     public static IndexVersion getMinimumCompatibleIndexVersion(int versionId) {

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -192,6 +192,7 @@ public class IndexVersions {
      */
 
     public static final IndexVersion MINIMUM_COMPATIBLE = V_8_0_0;
+    public static final IndexVersion MINIMUM_READONLY_COMPATIBLE = V_7_0_0;
 
     static final NavigableMap<Integer, IndexVersion> VERSION_IDS = getAllVersionIds(IndexVersions.class);
     static final IndexVersion LATEST_DEFINED;
@@ -207,7 +208,7 @@ public class IndexVersions {
         Map<Integer, String> versionIdFields = new HashMap<>();
         NavigableMap<Integer, IndexVersion> builder = new TreeMap<>();
 
-        Set<String> ignore = Set.of("ZERO", "MINIMUM_COMPATIBLE");
+        Set<String> ignore = Set.of("ZERO", "MINIMUM_COMPATIBLE", "MINIMUM_READONLY_COMPATIBLE");
 
         for (Field declaredField : cls.getFields()) {
             if (declaredField.getType().equals(IndexVersion.class)) {

server/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java

@@ -21,6 +21,7 @@ import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.SuggestingErrorOnUnknown;
+import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor;
 import org.elasticsearch.xcontent.AbstractObjectParser;
 import org.elasticsearch.xcontent.FilterXContentParser;
 import org.elasticsearch.xcontent.FilterXContentParserWrapper;
@@ -278,6 +279,14 @@ public abstract class AbstractQueryBuilder<QB extends AbstractQueryBuilder<QB>>
 
     @Override
     public final QueryBuilder rewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        QueryRewriteInterceptor queryRewriteInterceptor = queryRewriteContext.getQueryRewriteInterceptor();
+        if (queryRewriteInterceptor != null) {
+            var rewritten = queryRewriteInterceptor.interceptAndRewrite(queryRewriteContext, this);
+            if (rewritten != this) {
+                return new InterceptedQueryBuilderWrapper(rewritten);
+            }
+        }
+
         QueryBuilder rewritten = doRewrite(queryRewriteContext);
         if (rewritten == this) {
             return rewritten;

server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContext.java

@@ -104,6 +104,7 @@ public class CoordinatorRewriteContext extends QueryRewriteContext {
             null,
             null,
             null,
+            null,
             null
         );
         this.dateFieldRangeInfo = dateFieldRangeInfo;

server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java

@@ -66,6 +66,9 @@ public abstract class InnerHitContextBuilder {
     public static void extractInnerHits(QueryBuilder query, Map<String, InnerHitContextBuilder> innerHitBuilders) {
         if (query instanceof AbstractQueryBuilder) {
             ((AbstractQueryBuilder<?>) query).extractInnerHitBuilders(innerHitBuilders);
+        } else if (query instanceof InterceptedQueryBuilderWrapper interceptedQuery) {
+            // Unwrap an intercepted query here
+            extractInnerHits(interceptedQuery.queryBuilder, innerHitBuilders);
         } else {
             throw new IllegalStateException(
                 "provided query builder [" + query.getClass() + "] class should inherit from AbstractQueryBuilder, but it doesn't"

server/src/main/java/org/elasticsearch/index/query/InterceptedQueryBuilderWrapper.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.query;
+
+import org.apache.lucene.search.Query;
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor;
+import org.elasticsearch.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Wrapper for instances of {@link QueryBuilder} that have been intercepted using the {@link QueryRewriteInterceptor} to
+ * break out of the rewrite phase. These instances are unwrapped on serialization.
+ */
+class InterceptedQueryBuilderWrapper implements QueryBuilder {
+
+    protected final QueryBuilder queryBuilder;
+
+    InterceptedQueryBuilderWrapper(QueryBuilder queryBuilder) {
+        super();
+        this.queryBuilder = queryBuilder;
+    }
+
+    @Override
+    public QueryBuilder rewrite(QueryRewriteContext queryRewriteContext) throws IOException {
+        QueryRewriteInterceptor queryRewriteInterceptor = queryRewriteContext.getQueryRewriteInterceptor();
+        try {
+            queryRewriteContext.setQueryRewriteInterceptor(null);
+            QueryBuilder rewritten = queryBuilder.rewrite(queryRewriteContext);
+            return rewritten != queryBuilder ? new InterceptedQueryBuilderWrapper(rewritten) : this;
+        } finally {
+            queryRewriteContext.setQueryRewriteInterceptor(queryRewriteInterceptor);
+        }
+    }
+
+    @Override
+    public String getWriteableName() {
+        return queryBuilder.getWriteableName();
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return queryBuilder.getMinimalSupportedVersion();
+    }
+
+    @Override
+    public Query toQuery(SearchExecutionContext context) throws IOException {
+        return queryBuilder.toQuery(context);
+    }
+
+    @Override
+    public QueryBuilder queryName(String queryName) {
+        queryBuilder.queryName(queryName);
+        return this;
+    }
+
+    @Override
+    public String queryName() {
+        return queryBuilder.queryName();
+    }
+
+    @Override
+    public float boost() {
+        return queryBuilder.boost();
+    }
+
+    @Override
+    public QueryBuilder boost(float boost) {
+        queryBuilder.boost(boost);
+        return this;
+    }
+
+    @Override
+    public String getName() {
+        return queryBuilder.getName();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        queryBuilder.writeTo(out);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        return queryBuilder.toXContent(builder, params);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o instanceof InterceptedQueryBuilderWrapper == false) return false;
+        return Objects.equals(queryBuilder, ((InterceptedQueryBuilderWrapper) o).queryBuilder);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(queryBuilder);
+    }
+}

server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java

@@ -28,6 +28,7 @@ import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
+import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
 import org.elasticsearch.search.builder.PointInTimeBuilder;
@@ -70,6 +71,7 @@ public class QueryRewriteContext {
     protected Predicate<String> allowedFields;
     private final ResolvedIndices resolvedIndices;
     private final PointInTimeBuilder pit;
+    private QueryRewriteInterceptor queryRewriteInterceptor;
 
     public QueryRewriteContext(
         final XContentParserConfiguration parserConfiguration,
@@ -86,7 +88,8 @@ public class QueryRewriteContext {
         final BooleanSupplier allowExpensiveQueries,
         final ScriptCompiler scriptService,
         final ResolvedIndices resolvedIndices,
-        final PointInTimeBuilder pit
+        final PointInTimeBuilder pit,
+        final QueryRewriteInterceptor queryRewriteInterceptor
     ) {
 
         this.parserConfiguration = parserConfiguration;
@@ -105,6 +108,7 @@ public class QueryRewriteContext {
         this.scriptService = scriptService;
         this.resolvedIndices = resolvedIndices;
         this.pit = pit;
+        this.queryRewriteInterceptor = queryRewriteInterceptor;
     }
 
     public QueryRewriteContext(final XContentParserConfiguration parserConfiguration, final Client client, final LongSupplier nowInMillis) {
@@ -123,6 +127,7 @@ public class QueryRewriteContext {
             null,
             null,
             null,
+            null,
             null
         );
     }
@@ -132,7 +137,8 @@ public class QueryRewriteContext {
         final Client client,
         final LongSupplier nowInMillis,
         final ResolvedIndices resolvedIndices,
-        final PointInTimeBuilder pit
+        final PointInTimeBuilder pit,
+        final QueryRewriteInterceptor queryRewriteInterceptor
     ) {
         this(
             parserConfiguration,
@@ -149,7 +155,8 @@ public class QueryRewriteContext {
             null,
             null,
             resolvedIndices,
-            pit
+            pit,
+            queryRewriteInterceptor
         );
     }
 
@@ -428,4 +435,13 @@ public class QueryRewriteContext {
         // It was decided we should only test the first of these potentially multiple preferences.
         return value.split(",")[0].trim();
     }
+
+    public QueryRewriteInterceptor getQueryRewriteInterceptor() {
+        return queryRewriteInterceptor;
+    }
+
+    public void setQueryRewriteInterceptor(QueryRewriteInterceptor queryRewriteInterceptor) {
+        this.queryRewriteInterceptor = queryRewriteInterceptor;
+    }
+
 }

server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java

@@ -271,6 +271,7 @@ public class SearchExecutionContext extends QueryRewriteContext {
             allowExpensiveQueries,
             scriptService,
             null,
+            null,
             null
         );
         this.shardId = shardId;

server/src/main/java/org/elasticsearch/indices/IndicesService.java

@@ -137,6 +137,7 @@ import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.FieldPredicate;
 import org.elasticsearch.plugins.IndexStorePlugin;
 import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
@@ -265,6 +266,7 @@ public class IndicesService extends AbstractLifecycleComponent
     private final MapperMetrics mapperMetrics;
     private final PostRecoveryMerger postRecoveryMerger;
     private final List<SearchOperationListener> searchOperationListeners;
+    private final QueryRewriteInterceptor queryRewriteInterceptor;
 
     @Override
     protected void doStart() {
@@ -334,6 +336,7 @@ public class IndicesService extends AbstractLifecycleComponent
         this.indexFoldersDeletionListeners = new CompositeIndexFoldersDeletionListener(builder.indexFoldersDeletionListeners);
         this.snapshotCommitSuppliers = builder.snapshotCommitSuppliers;
         this.requestCacheKeyDifferentiator = builder.requestCacheKeyDifferentiator;
+        this.queryRewriteInterceptor = builder.queryRewriteInterceptor;
         this.mapperMetrics = builder.mapperMetrics;
         // doClose() is called when shutting down a node, yet there might still be ongoing requests
         // that we need to wait for before closing some resources such as the caches. In order to
@@ -783,7 +786,8 @@ public class IndicesService extends AbstractLifecycleComponent
             idFieldMappers.apply(idxSettings.getMode()),
             valuesSourceRegistry,
             indexFoldersDeletionListeners,
-            snapshotCommitSuppliers
+            snapshotCommitSuppliers,
+            queryRewriteInterceptor
         );
     }
 
@@ -1771,7 +1775,7 @@ public class IndicesService extends AbstractLifecycleComponent
      * Returns a new {@link QueryRewriteContext} with the given {@code now} provider
      */
     public QueryRewriteContext getRewriteContext(LongSupplier nowInMillis, ResolvedIndices resolvedIndices, PointInTimeBuilder pit) {
-        return new QueryRewriteContext(parserConfig, client, nowInMillis, resolvedIndices, pit);
+        return new QueryRewriteContext(parserConfig, client, nowInMillis, resolvedIndices, pit, queryRewriteInterceptor);
     }
 
     public DataRewriteContext getDataRewriteContext(LongSupplier nowInMillis) {

server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java

@@ -33,6 +33,8 @@ import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.plugins.EnginePlugin;
 import org.elasticsearch.plugins.IndexStorePlugin;
 import org.elasticsearch.plugins.PluginsService;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
 import org.elasticsearch.search.internal.ShardSearchRequest;
@@ -78,6 +80,7 @@ public class IndicesServiceBuilder {
     CheckedBiConsumer<ShardSearchRequest, StreamOutput, IOException> requestCacheKeyDifferentiator;
     MapperMetrics mapperMetrics;
     List<SearchOperationListener> searchOperationListener = List.of();
+    QueryRewriteInterceptor queryRewriteInterceptor = null;
 
     public IndicesServiceBuilder settings(Settings settings) {
         this.settings = settings;
@@ -247,6 +250,27 @@ public class IndicesServiceBuilder {
             .flatMap(m -> m.entrySet().stream())
             .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
 
+        var queryRewriteInterceptors = pluginsService.filterPlugins(SearchPlugin.class)
+            .map(SearchPlugin::getQueryRewriteInterceptors)
+            .flatMap(List::stream)
+            .collect(Collectors.toMap(QueryRewriteInterceptor::getQueryName, interceptor -> {
+                if (interceptor.getQueryName() == null) {
+                    throw new IllegalArgumentException("QueryRewriteInterceptor [" + interceptor.getClass().getName() + "] requires name");
+                }
+                return interceptor;
+            }, (a, b) -> {
+                throw new IllegalStateException(
+                    "Conflicting rewrite interceptors ["
+                        + a.getQueryName()
+                        + "] found in ["
+                        + a.getClass().getName()
+                        + "] and ["
+                        + b.getClass().getName()
+                        + "]"
+                );
+            }));
+        queryRewriteInterceptor = QueryRewriteInterceptor.multi(queryRewriteInterceptors);
+
         return new IndicesService(this);
     }
 }

server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java

@@ -99,7 +99,7 @@ public class SystemIndexMappingUpdateService implements ClusterStateListener {
         }
 
         // if we're in a mixed-version cluster, exit
-        if (state.nodes().getMaxNodeVersion().after(state.nodes().getSmallestNonClientNodeVersion())) {
+        if (state.nodes().isMixedVersionCluster()) {
             logger.debug("Skipping system indices up-to-date check as cluster has mixed versions");
             return;
         }

server/src/main/java/org/elasticsearch/injection/guice/Binder.java

@@ -65,9 +65,7 @@ import org.elasticsearch.injection.guice.spi.Message;
  *
  * <p>The {@link Provider} you use here does not have to be a "factory"; that
  * is, a provider which always <i>creates</i> each instance it provides.
- * However, this is generally a good practice to follow.  You can then use
+ * However, this is generally a good practice to follow.
- * Guice's concept of {@link Scope scopes} to guide when creation should happen
- * -- "letting Guice work for you".
  *
  * <pre>
  *     bind(Service.class).annotatedWith(Red.class).to(ServiceImpl.class);</pre>

server/src/main/java/org/elasticsearch/injection/guice/BindingProcessor.java

@@ -218,7 +218,6 @@ class BindingProcessor extends AbstractProcessor {
         MembersInjector.class,
         Module.class,
         Provider.class,
-        Scope.class,
         TypeLiteral.class
     );
     // TODO(jessewilson): fix BuiltInModule, then add Stage

server/src/main/java/org/elasticsearch/injection/guice/InjectorBuilder.java

@@ -20,6 +20,7 @@ import org.elasticsearch.injection.guice.internal.BindingImpl;
 import org.elasticsearch.injection.guice.internal.Errors;
 import org.elasticsearch.injection.guice.internal.ErrorsException;
 import org.elasticsearch.injection.guice.internal.InternalContext;
+import org.elasticsearch.injection.guice.internal.Scoping;
 import org.elasticsearch.injection.guice.internal.Stopwatch;
 import org.elasticsearch.injection.guice.spi.Dependency;
 
@@ -154,7 +155,7 @@ class InjectorBuilder {
     }
 
     private static void loadEagerSingletons(InjectorImpl injector, final Errors errors, BindingImpl<?> binding) {
-        if (binding.getScoping().isEagerSingleton()) {
+        if (binding.getScoping() == Scoping.EAGER_SINGLETON) {
             try {
                 injector.callInContext(new ContextualCallable<Void>() {
                     final Dependency<?> dependency = Dependency.get(binding.getKey());

server/src/main/java/org/elasticsearch/injection/guice/Provider.java

@@ -28,8 +28,6 @@ package org.elasticsearch.injection.guice;
  * instances, instances you wish to safely mutate and discard, instances which are out of scope
  * (e.g. using a {@code @RequestScoped} object from within a {@code @SessionScoped} object), or
  * instances that will be initialized lazily.
- * <li>A custom {@link Scope} is implemented as a decorator of {@code Provider<T>}, which decides
- * when to delegate to the backing provider and when to provide the instance some other way.
  * <li>The {@link Injector} offers access to the {@code Provider<T>} it uses to fulfill requests
  * for a given key, via the {@link Injector#getProvider} methods.
  * </ul>

server/src/main/java/org/elasticsearch/injection/guice/Scope.java

@@ -1,59 +0,0 @@
-/*
- * Copyright (C) 2006 Google Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.elasticsearch.injection.guice;
-
-/**
- * A scope is a level of visibility that instances provided by Guice may have.
- * By default, an instance created by the {@link Injector} has <i>no scope</i>,
- * meaning it has no state from the framework's perspective -- the
- * {@code Injector} creates it, injects it once into the class that required it,
- * and then immediately forgets it. Associating a scope with a particular
- * binding allows the created instance to be "remembered" and possibly used
- * again for other injections.
- * <p>
- * An example of a scope is {@link Scopes#SINGLETON}.
- *
- * @author crazybob@google.com (Bob Lee)
- */
-public interface Scope {
-
-    /**
-     * Scopes a provider. The returned provider returns objects from this scope.
-     * If an object does not exist in this scope, the provider can use the given
-     * unscoped provider to retrieve one.
-     * <p>
-     * Scope implementations are strongly encouraged to override
-     * {@link Object#toString} in the returned provider and include the backing
-     * provider's {@code toString()} output.
-     *
-     * @param unscoped locates an instance when one doesn't already exist in this
-     *                 scope.
-     * @return a new provider which only delegates to the given unscoped provider
-     *         when an instance of the requested object doesn't already exist in this
-     *         scope
-     */
-    <T> Provider<T> scope(Provider<T> unscoped);
-
-    /**
-     * A short but useful description of this scope.  For comparison, the standard
-     * scopes that ship with guice use the descriptions
-     * {@code "Scopes.SINGLETON"}, {@code "ServletScopes.SESSION"} and
-     * {@code "ServletScopes.REQUEST"}.
-     */
-    @Override
-    String toString();
-}

server/src/main/java/org/elasticsearch/injection/guice/Scopes.java

@@ -19,8 +19,6 @@ package org.elasticsearch.injection.guice;
 import org.elasticsearch.injection.guice.internal.InternalFactory;
 import org.elasticsearch.injection.guice.internal.Scoping;
 
-import java.util.Locale;
-
 /**
  * Built-in scope implementations.
  *
@@ -31,29 +29,27 @@ public class Scopes {
     private Scopes() {}
 
     /**
-     * One instance per {@link Injector}.
+     * Scopes an internal factory.
      */
-    public static final Scope SINGLETON = new Scope() {
+    static <T> InternalFactory<? extends T> scope(InjectorImpl injector, InternalFactory<? extends T> creator, Scoping scoping) {
-        @Override
+        return switch (scoping) {
-        public <T> Provider<T> scope(final Provider<T> creator) {
+            case UNSCOPED -> creator;
-            return new Provider<T>() {
+            case EAGER_SINGLETON -> new InternalFactoryToProviderAdapter<>(Initializables.of(new Provider<>() {
 
                 private volatile T instance;
 
-                // DCL on a volatile is safe as of Java 5, which we obviously require.
                 @Override
-                @SuppressWarnings("DoubleCheckedLocking")
                 public T get() {
                     if (instance == null) {
                         /*
-                        * Use a pretty coarse lock. We don't want to run into deadlocks
+                         * Use a pretty coarse lock. We don't want to run into deadlocks
-                        * when two threads try to load circularly-dependent objects.
+                         * when two threads try to load circularly-dependent objects.
-                        * Maybe one of these days we will identify independent graphs of
+                         * Maybe one of these days we will identify independent graphs of
-                        * objects and offer to load them in parallel.
+                         * objects and offer to load them in parallel.
-                        */
+                         */
-                        synchronized (InjectorImpl.class) {
+                        synchronized (injector) {
                             if (instance == null) {
-                                instance = creator.get();
+                                instance = new ProviderToInternalFactoryAdapter<>(injector, creator).get();
                             }
                         }
                     }
@@ -62,54 +58,10 @@ public class Scopes {
 
                 @Override
                 public String toString() {
-                    return String.format(Locale.ROOT, "%s[%s]", creator, SINGLETON);
+                    return creator + "[SINGLETON]";
                 }
-            };
+            }));
-        }
+        };
-
-        @Override
-        public String toString() {
-            return "Scopes.SINGLETON";
-        }
-    };
-
-    /**
-     * No scope; the same as not applying any scope at all.  Each time the
-     * Injector obtains an instance of an object with "no scope", it injects this
-     * instance then immediately forgets it.  When the next request for the same
-     * binding arrives it will need to obtain the instance over again.
-     * <p>
-     * This exists only in case a class has been annotated with a scope
-     * annotation and you need to override this to "no scope" in your binding.
-     *
-     * @since 2.0
-     */
-    public static final Scope NO_SCOPE = new Scope() {
-        @Override
-        public <T> Provider<T> scope(Provider<T> unscoped) {
-            return unscoped;
-        }
-
-        @Override
-        public String toString() {
-            return "Scopes.NO_SCOPE";
-        }
-    };
-
-    /**
-     * Scopes an internal factory.
-     */
-    static <T> InternalFactory<? extends T> scope(InjectorImpl injector, InternalFactory<? extends T> creator, Scoping scoping) {
-
-        if (scoping.isNoScope()) {
-            return creator;
-        }
-
-        Scope scope = scoping.getScopeInstance();
-
-        // TODO: use diamond operator once JI-9019884 is fixed
-        Provider<T> scoped = scope.scope(new ProviderToInternalFactoryAdapter<T>(injector, creator));
-        return new InternalFactoryToProviderAdapter<>(Initializables.of(scoped));
     }
 
 }

server/src/main/java/org/elasticsearch/injection/guice/internal/AbstractBindingBuilder.java

@@ -77,7 +77,7 @@ public abstract class AbstractBindingBuilder<T> {
             return;
         }
 
-        if (binding.getScoping().isExplicitlyScoped()) {
+        if (binding.getScoping() != Scoping.UNSCOPED) {
             binder.addError(SCOPE_ALREADY_SET);
         }
     }

server/src/main/java/org/elasticsearch/injection/guice/internal/Scoping.java

@@ -16,8 +16,7 @@
 
 package org.elasticsearch.injection.guice.internal;
 
-import org.elasticsearch.injection.guice.Scope;
+import org.elasticsearch.injection.guice.Injector;
-import org.elasticsearch.injection.guice.Scopes;
 
 /**
  * References a scope, either directly (as a scope instance), or indirectly (as a scope annotation).
@@ -25,69 +24,14 @@ import org.elasticsearch.injection.guice.Scopes;
  *
  * @author jessewilson@google.com (Jesse Wilson)
  */
-public abstract class Scoping {
+public enum Scoping {
-
     /**
      * No scoping annotation has been applied. Note that this is different from {@code
      * in(Scopes.NO_SCOPE)}, where the 'NO_SCOPE' has been explicitly applied.
      */
-    public static final Scoping UNSCOPED = new Scoping() {
+    UNSCOPED,
-
-        @Override
-        public Scope getScopeInstance() {
-            return Scopes.NO_SCOPE;
-        }
-
-        @Override
-        public String toString() {
-            return Scopes.NO_SCOPE.toString();
-        }
-
-    };
-
-    public static final Scoping EAGER_SINGLETON = new Scoping() {
-
-        @Override
-        public Scope getScopeInstance() {
-            return Scopes.SINGLETON;
-        }
-
-        @Override
-        public String toString() {
-            return "eager singleton";
-        }
-
-    };
-
     /**
-     * Returns true if this scope was explicitly applied. If no scope was explicitly applied then the
+     * One instance per {@link Injector}.
-     * scoping annotation will be used.
      */
-    public boolean isExplicitlyScoped() {
+    EAGER_SINGLETON
-        return this != UNSCOPED;
-    }
-
-    /**
-     * Returns true if this is the default scope. In this case a new instance will be provided for
-     * each injection.
-     */
-    public boolean isNoScope() {
-        return getScopeInstance() == Scopes.NO_SCOPE;
-    }
-
-    /**
-     * Returns true if this scope is a singleton that should be loaded eagerly in {@code stage}.
-     */
-    public boolean isEagerSingleton() {
-        return this == EAGER_SINGLETON;
-    }
-
-    /**
-     * Returns the scope instance, or {@code null} if that isn't known for this instance.
-     */
-    public Scope getScopeInstance() {
-        return null;
-    }
-
-    private Scoping() {}
 }

server/src/main/java/org/elasticsearch/plugins/PluginsLoader.java

@@ -122,7 +122,8 @@ public class PluginsLoader {
     private final List<PluginDescriptor> moduleDescriptors;
     private final List<PluginDescriptor> pluginDescriptors;
     private final Map<String, LoadedPluginLayer> loadedPluginLayers;
-    private final Set<PluginBundle> allBundles;
+    private final Set<PluginBundle> moduleBundles;
+    private final Set<PluginBundle> pluginBundles;
 
     /**
      * Constructs a new PluginsLoader
@@ -153,37 +154,36 @@ public class PluginsLoader {
         Set<PluginBundle> seenBundles = new LinkedHashSet<>();
 
         // load (elasticsearch) module layers
-        List<PluginDescriptor> moduleDescriptors;
+        final Set<PluginBundle> modules;
         if (modulesDirectory != null) {
             try {
-                Set<PluginBundle> modules = PluginsUtils.getModuleBundles(modulesDirectory);
+                modules = PluginsUtils.getModuleBundles(modulesDirectory);
-                moduleDescriptors = modules.stream().map(PluginBundle::pluginDescriptor).toList();
                 seenBundles.addAll(modules);
             } catch (IOException ex) {
                 throw new IllegalStateException("Unable to initialize modules", ex);
             }
         } else {
-            moduleDescriptors = Collections.emptyList();
+            modules = Collections.emptySet();
         }
 
         // load plugin layers
-        List<PluginDescriptor> pluginDescriptors;
+        final Set<PluginBundle> plugins;
         if (pluginsDirectory != null) {
             try {
                 // TODO: remove this leniency, but tests bogusly rely on it
                 if (isAccessibleDirectory(pluginsDirectory, logger)) {
                     PluginsUtils.checkForFailedPluginRemovals(pluginsDirectory);
-                    Set<PluginBundle> plugins = PluginsUtils.getPluginBundles(pluginsDirectory);
+                    plugins = PluginsUtils.getPluginBundles(pluginsDirectory);
-                    pluginDescriptors = plugins.stream().map(PluginBundle::pluginDescriptor).toList();
+
                     seenBundles.addAll(plugins);
                 } else {
-                    pluginDescriptors = Collections.emptyList();
+                    plugins = Collections.emptySet();
                 }
             } catch (IOException ex) {
                 throw new IllegalStateException("Unable to initialize plugins", ex);
             }
         } else {
-            pluginDescriptors = Collections.emptyList();
+            plugins = Collections.emptySet();
         }
 
         Map<String, LoadedPluginLayer> loadedPluginLayers = new LinkedHashMap<>();
@@ -197,19 +197,15 @@ public class PluginsLoader {
             }
         }
 
-        return new PluginsLoader(moduleDescriptors, pluginDescriptors, loadedPluginLayers, Set.copyOf(seenBundles));
+        return new PluginsLoader(modules, plugins, loadedPluginLayers);
     }
 
-    PluginsLoader(
+    PluginsLoader(Set<PluginBundle> modules, Set<PluginBundle> plugins, Map<String, LoadedPluginLayer> loadedPluginLayers) {
-        List<PluginDescriptor> moduleDescriptors,
+        this.moduleBundles = modules;
-        List<PluginDescriptor> pluginDescriptors,
+        this.pluginBundles = plugins;
-        Map<String, LoadedPluginLayer> loadedPluginLayers,
+        this.moduleDescriptors = modules.stream().map(PluginBundle::pluginDescriptor).toList();
-        Set<PluginBundle> allBundles
+        this.pluginDescriptors = plugins.stream().map(PluginBundle::pluginDescriptor).toList();
-    ) {
-        this.moduleDescriptors = moduleDescriptors;
-        this.pluginDescriptors = pluginDescriptors;
         this.loadedPluginLayers = loadedPluginLayers;
-        this.allBundles = allBundles;
     }
 
     public List<PluginDescriptor> moduleDescriptors() {
@@ -224,8 +220,12 @@ public class PluginsLoader {
         return loadedPluginLayers.values().stream().map(Function.identity());
     }
 
-    public Set<PluginBundle> allBundles() {
+    public Set<PluginBundle> moduleBundles() {
-        return allBundles;
+        return moduleBundles;
+    }
+
+    public Set<PluginBundle> pluginBundles() {
+        return pluginBundles;
     }
 
     private static void loadPluginLayer(
@@ -416,7 +416,7 @@ public class PluginsLoader {
         return result;
     }
 
-    static final String toPackageName(String className) {
+    static String toPackageName(String className) {
         assert className.endsWith(".") == false;
         int index = className.lastIndexOf('.');
         if (index == -1) {
@@ -426,11 +426,11 @@ public class PluginsLoader {
     }
 
     @SuppressForbidden(reason = "I need to convert URL's to Paths")
-    static final Path[] urlsToPaths(Set<URL> urls) {
+    static Path[] urlsToPaths(Set<URL> urls) {
         return urls.stream().map(PluginsLoader::uncheckedToURI).map(PathUtils::get).toArray(Path[]::new);
     }
 
-    static final URI uncheckedToURI(URL url) {
+    static URI uncheckedToURI(URL url) {
         try {
             return url.toURI();
         } catch (URISyntaxException e) {

server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java

@@ -23,6 +23,7 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryParser;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
 import org.elasticsearch.index.query.functionscore.ScoreFunctionParser;
+import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor;
 import org.elasticsearch.search.SearchExtBuilder;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
@@ -128,6 +129,14 @@ public interface SearchPlugin {
         return emptyList();
     }
 
+    /**
+     * @return Applicable {@link QueryRewriteInterceptor}s configured for this plugin.
+     * Note: This is internal to Elasticsearch's API and not extensible by external plugins.
+     */
+    default List<QueryRewriteInterceptor> getQueryRewriteInterceptors() {
+        return emptyList();
+    }
+
     /**
      * The new {@link Aggregation}s added by this plugin.
      */

server/src/main/java/org/elasticsearch/plugins/internal/rewriter/QueryRewriteInterceptor.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.plugins.internal.rewriter;
+
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryRewriteContext;
+
+import java.util.Map;
+
+/**
+ * Enables modules and plugins to intercept and rewrite queries during the query rewrite phase on the coordinator node.
+ */
+public interface QueryRewriteInterceptor {
+
+    /**
+     * Intercepts and returns a rewritten query if modifications are required; otherwise,
+     * returns the same provided {@link QueryBuilder} instance unchanged.
+     *
+     * @param context the {@link QueryRewriteContext} providing the context for the rewrite operation
+     * @param queryBuilder the original {@link QueryBuilder} to potentially rewrite
+     * @return the rewritten {@link QueryBuilder}, or the original instance if no rewrite was needed
+     */
+    QueryBuilder interceptAndRewrite(QueryRewriteContext context, QueryBuilder queryBuilder);
+
+    /**
+     * Name of the query to be intercepted and rewritten.
+     */
+    String getQueryName();
+
+    static QueryRewriteInterceptor multi(Map<String, QueryRewriteInterceptor> interceptors) {
+        return interceptors.isEmpty() ? new NoOpQueryRewriteInterceptor() : new CompositeQueryRewriteInterceptor(interceptors);
+    }
+
+    class CompositeQueryRewriteInterceptor implements QueryRewriteInterceptor {
+        final String NAME = "composite";
+        private final Map<String, QueryRewriteInterceptor> interceptors;
+
+        private CompositeQueryRewriteInterceptor(Map<String, QueryRewriteInterceptor> interceptors) {
+            this.interceptors = interceptors;
+        }
+
+        @Override
+        public String getQueryName() {
+            return NAME;
+        }
+
+        @Override
+        public QueryBuilder interceptAndRewrite(QueryRewriteContext context, QueryBuilder queryBuilder) {
+            QueryRewriteInterceptor interceptor = interceptors.get(queryBuilder.getName());
+            if (interceptor != null) {
+                return interceptor.interceptAndRewrite(context, queryBuilder);
+            }
+            return queryBuilder;
+        }
+    }
+
+    class NoOpQueryRewriteInterceptor implements QueryRewriteInterceptor {
+        @Override
+        public QueryBuilder interceptAndRewrite(QueryRewriteContext context, QueryBuilder queryBuilder) {
+            return queryBuilder;
+        }
+
+        @Override
+        public String getQueryName() {
+            return null;
+        }
+    }
+}

server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java

@@ -102,7 +102,9 @@ public final class RepositoriesModule {
         }
         if (preRestoreChecks.isEmpty()) {
             preRestoreChecks.add((snapshot, version) -> {
-                if (version.isLegacyIndexVersion()) {
+                // pre-restore checks will be run against the version in which the snapshot was created as well as
+                // the version in which the restored index was created
+                if (version.before(IndexVersions.MINIMUM_COMPATIBLE)) {
                     throw new SnapshotRestoreException(
                         snapshot,
                         "the snapshot was created with Elasticsearch version ["

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java

@@ -78,10 +78,7 @@ public class RestCreateIndexAction extends BaseRestHandler {
         Map<String, Object> mappings = (Map<String, Object>) source.get("mappings");
         if (MapperService.isMappingSourceTyped(MapperService.SINGLE_MAPPING_NAME, mappings)) {
             throw new IllegalArgumentException(
-                "The mapping definition cannot be nested under a type "
+                "The mapping definition cannot be nested under a type [" + MapperService.SINGLE_MAPPING_NAME + "]."
-                    + "["
-                    + MapperService.SINGLE_MAPPING_NAME
-                    + "] unless include_type_name is set to true."
             );
         }
 

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java

@@ -9,15 +9,12 @@
 
 package org.elasticsearch.rest.action.admin.indices;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.RestResponse;
@@ -35,13 +32,6 @@ import static org.elasticsearch.rest.RestStatus.OK;
 
 public class RestGetFieldMappingAction extends BaseRestHandler {
 
-    private static final Logger logger = LogManager.getLogger(RestGetFieldMappingAction.class);
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(logger.getName());
-    public static final String INCLUDE_TYPE_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in get "
-        + "field mapping requests is deprecated. The parameter will be removed in the next major version.";
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in get field mapping request is deprecated. "
-        + "Use typeless api instead";
-
     @Override
     public List<Route> routes() {
         return List.of(new Route(GET, "/_mapping/field/{fields}"), new Route(GET, "/{index}/_mapping/field/{fields}"));

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetIndicesAction.java

@@ -13,7 +13,6 @@ import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -35,9 +34,6 @@ import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
  */
 @ServerlessScope(Scope.PUBLIC)
 public class RestGetIndicesAction extends BaseRestHandler {
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetIndicesAction.class);
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using `include_type_name` in get indices requests"
-        + " is deprecated. The parameter will be removed in the next major version.";
 
     @Override
     public List<Route> routes() {

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java

@@ -13,7 +13,6 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.http.HttpChannel;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -31,11 +30,6 @@ import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 
 @ServerlessScope(Scope.PUBLIC)
 public class RestGetMappingAction extends BaseRestHandler {
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestGetMappingAction.class);
-    public static final String INCLUDE_TYPE_DEPRECATION_MSG = "[types removal] Using include_type_name in get"
-        + " mapping requests is deprecated. The parameter will be removed in the next major version.";
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Specifying types in get mapping request is deprecated. "
-        + "Use typeless api instead";
 
     public RestGetMappingAction() {}
 

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestResizeHandler.java

@@ -13,7 +13,6 @@ import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.client.internal.node.NodeClient;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -27,7 +26,6 @@ import static org.elasticsearch.rest.RestUtils.getAckTimeout;
 import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 
 public abstract class RestResizeHandler extends BaseRestHandler {
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestResizeHandler.class);
 
     RestResizeHandler() {}
 

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestRolloverIndexAction.java

@@ -14,7 +14,6 @@ import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.cluster.metadata.DataStream;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.Scope;
@@ -33,10 +32,6 @@ import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout;
 @ServerlessScope(Scope.PUBLIC)
 public class RestRolloverIndexAction extends BaseRestHandler {
 
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestRolloverIndexAction.class);
-    public static final String TYPES_DEPRECATION_MESSAGE = "[types removal] Using include_type_name in rollover "
-        + "index requests is deprecated. The parameter will be removed in the next major version.";
-
     @Override
     public List<Route> routes() {
         return List.of(new Route(POST, "/{index}/_rollover"), new Route(POST, "/{index}/_rollover/{new_index}"));

server/src/main/java/org/elasticsearch/transport/RemoteConnectionInfo.java

@@ -18,8 +18,6 @@ import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
 import java.util.Objects;
 
 /**
@@ -49,25 +47,14 @@ public final class RemoteConnectionInfo implements ToXContentFragment, Writeable
     }
 
     public RemoteConnectionInfo(StreamInput input) throws IOException {
-        if (input.getTransportVersion().onOrAfter(TransportVersions.V_7_6_0)) {
+        RemoteConnectionStrategy.ConnectionStrategy mode = input.readEnum(RemoteConnectionStrategy.ConnectionStrategy.class);
-            RemoteConnectionStrategy.ConnectionStrategy mode = input.readEnum(RemoteConnectionStrategy.ConnectionStrategy.class);
+        modeInfo = mode.getReader().read(input);
-            modeInfo = mode.getReader().read(input);
+        initialConnectionTimeout = input.readTimeValue();
-            initialConnectionTimeout = input.readTimeValue();
+        clusterAlias = input.readString();
-            clusterAlias = input.readString();
+        skipUnavailable = input.readBoolean();
-            skipUnavailable = input.readBoolean();
+        if (input.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) {
-            if (input.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) {
+            hasClusterCredentials = input.readBoolean();
-                hasClusterCredentials = input.readBoolean();
-            } else {
-                hasClusterCredentials = false;
-            }
         } else {
-            List<String> seedNodes = Arrays.asList(input.readStringArray());
-            int connectionsPerCluster = input.readVInt();
-            initialConnectionTimeout = input.readTimeValue();
-            int numNodesConnected = input.readVInt();
-            clusterAlias = input.readString();
-            skipUnavailable = input.readBoolean();
-            modeInfo = new SniffConnectionStrategy.SniffModeInfo(seedNodes, connectionsPerCluster, numNodesConnected);
             hasClusterCredentials = false;
         }
     }
@@ -90,24 +77,9 @@ public final class RemoteConnectionInfo implements ToXContentFragment, Writeable
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_6_0)) {
+        out.writeEnum(modeInfo.modeType());
-            out.writeEnum(modeInfo.modeType());
+        modeInfo.writeTo(out);
-            modeInfo.writeTo(out);
+        out.writeTimeValue(initialConnectionTimeout);
-            out.writeTimeValue(initialConnectionTimeout);
-        } else {
-            if (modeInfo.modeType() == RemoteConnectionStrategy.ConnectionStrategy.SNIFF) {
-                SniffConnectionStrategy.SniffModeInfo sniffInfo = (SniffConnectionStrategy.SniffModeInfo) this.modeInfo;
-                out.writeStringCollection(sniffInfo.seedNodes);
-                out.writeVInt(sniffInfo.maxConnectionsPerCluster);
-                out.writeTimeValue(initialConnectionTimeout);
-                out.writeVInt(sniffInfo.numNodesConnected);
-            } else {
-                out.writeStringArray(new String[0]);
-                out.writeVInt(0);
-                out.writeTimeValue(initialConnectionTimeout);
-                out.writeVInt(0);
-            }
-        }
         out.writeString(clusterAlias);
         out.writeBoolean(skipUnavailable);
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_8_0)) {

server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java

@@ -12,24 +12,16 @@ package org.elasticsearch.action.admin.indices.mapping.get;
 import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
-import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.Writeable;
-import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
-import org.elasticsearch.xcontent.ToXContent;
-import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xcontent.json.JsonXContent;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.Matchers.hasKey;
-
 public class GetFieldMappingsResponseTests extends AbstractWireSerializingTestCase<GetFieldMappingsResponse> {
 
     public void testManualSerialization() throws IOException {
@@ -56,36 +48,6 @@ public class GetFieldMappingsResponseTests extends AbstractWireSerializingTestCa
         assertEquals("{\"index\":{\"mappings\":{}}}", Strings.toString(response));
     }
 
-    public void testToXContentIncludesType() throws Exception {
-        Map<String, Map<String, FieldMappingMetadata>> mappings = new HashMap<>();
-        FieldMappingMetadata fieldMappingMetadata = new FieldMappingMetadata("my field", new BytesArray("{}"));
-        mappings.put("index", Collections.singletonMap("field", fieldMappingMetadata));
-        GetFieldMappingsResponse response = new GetFieldMappingsResponse(mappings);
-        ToXContent.Params params = new ToXContent.MapParams(Collections.singletonMap("include_type_name", "true"));
-
-        // v8 does not have _doc, even when include_type_name is present
-        // (although this throws unconsumed parameter exception in RestGetFieldMappingsAction)
-        try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_8)) {
-            response.toXContent(builder, params);
-
-            try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
-                @SuppressWarnings("unchecked")
-                Map<String, Map<String, Object>> index = (Map<String, Map<String, Object>>) parser.map().get("index");
-                assertThat(index.get("mappings"), hasKey("field"));
-            }
-        }
-
-        try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent, RestApiVersion.V_8)) {
-            response.toXContent(builder, ToXContent.EMPTY_PARAMS);
-
-            try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) {
-                @SuppressWarnings("unchecked")
-                Map<String, Map<String, Object>> index = (Map<String, Map<String, Object>>) parser.map().get("index");
-                assertThat(index.get("mappings"), hasKey("field"));
-            }
-        }
-    }
-
     @Override
     protected GetFieldMappingsResponse createTestInstance() {
         return new GetFieldMappingsResponse(randomMapping());

server/src/test/java/org/elasticsearch/action/search/TransportSearchActionTests.java

@@ -1758,7 +1758,9 @@ public class TransportSearchActionTests extends ESTestCase {
             NodeClient client = new NodeClient(settings, threadPool);
 
             SearchService searchService = mock(SearchService.class);
-            when(searchService.getRewriteContext(any(), any(), any())).thenReturn(new QueryRewriteContext(null, null, null, null, null));
+            when(searchService.getRewriteContext(any(), any(), any())).thenReturn(
+                new QueryRewriteContext(null, null, null, null, null, null)
+            );
             ClusterService clusterService = new ClusterService(
                 settings,
                 new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),