Merge main into multi-project

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommand.java

@@ -74,7 +74,7 @@ class AddFileKeyStoreCommand extends BaseKeyStoreCommand {
             keyStore.setFile(setting, Files.readAllBytes(file));
         }
 
-        keyStore.save(env.configFile(), getKeyStorePassword().getChars());
+        keyStore.save(env.configDir(), getKeyStorePassword().getChars());
     }
 
     @SuppressForbidden(reason = "file arg for cli")

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommand.java

@@ -100,7 +100,7 @@ class AddStringKeyStoreCommand extends BaseKeyStoreCommand {
             }
         }
 
-        keyStore.save(env.configFile(), getKeyStorePassword().getChars());
+        keyStore.save(env.configDir(), getKeyStorePassword().getChars());
     }
 
 }

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/BaseKeyStoreCommand.java

@@ -39,14 +39,14 @@ public abstract class BaseKeyStoreCommand extends KeyStoreAwareCommand {
     @Override
     public final void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception {
         try {
-            final Path configFile = env.configFile();
+            final Path configFile = env.configDir();
             keyStore = KeyStoreWrapper.load(configFile);
             if (keyStore == null) {
                 if (keyStoreMustExist) {
                     throw new UserException(
                         ExitCodes.DATA_ERROR,
                         "Elasticsearch keystore not found at ["
-                            + KeyStoreWrapper.keystorePath(env.configFile())
+                            + KeyStoreWrapper.keystorePath(env.configDir())
                             + "]. Use 'create' command to create one."
                     );
                 } else if (options.has(forceOption) == false) {

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/ChangeKeyStorePasswordCommand.java

@@ -31,7 +31,7 @@ class ChangeKeyStorePasswordCommand extends BaseKeyStoreCommand {
     protected void executeCommand(Terminal terminal, OptionSet options, Environment env) throws Exception {
         try (SecureString newPassword = readPassword(terminal, true)) {
             final KeyStoreWrapper keyStore = getKeyStore();
-            keyStore.save(env.configFile(), newPassword.getChars());
+            keyStore.save(env.configDir(), newPassword.getChars());
             terminal.println("Elasticsearch keystore password changed successfully.");
         } catch (SecurityException e) {
             throw new UserException(ExitCodes.DATA_ERROR, e.getMessage());

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommand.java

@@ -40,7 +40,7 @@ class CreateKeyStoreCommand extends KeyStoreAwareCommand {
     @Override
     public void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception {
         try (SecureString password = options.has(passwordOption) ? readPassword(terminal, true) : new SecureString(new char[0])) {
-            Path keystoreFile = KeyStoreWrapper.keystorePath(env.configFile());
+            Path keystoreFile = KeyStoreWrapper.keystorePath(env.configDir());
             if (Files.exists(keystoreFile)) {
                 if (terminal.promptYesNo("An elasticsearch keystore already exists. Overwrite?", false) == false) {
                     terminal.println("Exiting without creating keystore.");
@@ -48,8 +48,8 @@ class CreateKeyStoreCommand extends KeyStoreAwareCommand {
                 }
             }
             KeyStoreWrapper keystore = KeyStoreWrapper.create();
-            keystore.save(env.configFile(), password.getChars());
+            keystore.save(env.configDir(), password.getChars());
-            terminal.println("Created elasticsearch keystore in " + KeyStoreWrapper.keystorePath(env.configFile()));
+            terminal.println("Created elasticsearch keystore in " + KeyStoreWrapper.keystorePath(env.configDir()));
         } catch (SecurityException e) {
             throw new UserException(ExitCodes.IO_ERROR, "Error creating the elasticsearch keystore.");
         }

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/HasPasswordKeyStoreCommand.java

@@ -32,7 +32,7 @@ public class HasPasswordKeyStoreCommand extends KeyStoreAwareCommand {
 
     @Override
     public void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception {
-        final Path configFile = env.configFile();
+        final Path configFile = env.configDir();
         final KeyStoreWrapper keyStore = KeyStoreWrapper.load(configFile);
 
         // We handle error printing here so we can respect the "--silent" flag

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/RemoveSettingKeyStoreCommand.java

@@ -45,6 +45,6 @@ class RemoveSettingKeyStoreCommand extends BaseKeyStoreCommand {
             }
             keyStore.remove(setting);
         }
-        keyStore.save(env.configFile(), getKeyStorePassword().getChars());
+        keyStore.save(env.configDir(), getKeyStorePassword().getChars());
     }
 }

distribution/tools/keystore-cli/src/main/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommand.java

@@ -26,7 +26,7 @@ public class UpgradeKeyStoreCommand extends BaseKeyStoreCommand {
 
     @Override
     protected void executeCommand(final Terminal terminal, final OptionSet options, final Environment env) throws Exception {
-        KeyStoreWrapper.upgrade(getKeyStore(), env.configFile(), getKeyStorePassword().getChars());
+        KeyStoreWrapper.upgrade(getKeyStore(), env.configDir(), getKeyStorePassword().getChars());
     }
 
 }

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddFileKeyStoreCommandTests.java

@@ -46,14 +46,14 @@ public class AddFileKeyStoreCommandTests extends KeyStoreCommandTestCase {
         for (int i = 0; i < length; ++i) {
             bytes[i] = randomByte();
         }
-        Path file = env.configFile().resolve(randomAlphaOfLength(16));
+        Path file = env.configDir().resolve(randomAlphaOfLength(16));
         Files.write(file, bytes);
         return file;
     }
 
     private void addFile(KeyStoreWrapper keystore, String setting, Path file, String password) throws Exception {
         keystore.setFile(setting, Files.readAllBytes(file));
-        keystore.save(env.configFile(), password.toCharArray());
+        keystore.save(env.configDir(), password.toCharArray());
     }
 
     public void testMissingCreateWithEmptyPasswordWhenPrompted() throws Exception {
@@ -77,7 +77,7 @@ public class AddFileKeyStoreCommandTests extends KeyStoreCommandTestCase {
         terminal.addSecretInput(randomFrom("", "keystorepassword"));
         terminal.addTextInput("n"); // explicit no
         execute("foo");
-        assertNull(KeyStoreWrapper.load(env.configFile()));
+        assertNull(KeyStoreWrapper.load(env.configDir()));
     }
 
     public void testOverwritePromptDefault() throws Exception {

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/AddStringKeyStoreCommandTests.java

@@ -83,7 +83,7 @@ public class AddStringKeyStoreCommandTests extends KeyStoreCommandTestCase {
     public void testMissingNoCreate() throws Exception {
         terminal.addTextInput("n"); // explicit no
         execute("foo");
-        assertNull(KeyStoreWrapper.load(env.configFile()));
+        assertNull(KeyStoreWrapper.load(env.configDir()));
     }
 
     public void testOverwritePromptDefault() throws Exception {
@@ -143,7 +143,7 @@ public class AddStringKeyStoreCommandTests extends KeyStoreCommandTestCase {
 
     public void testPromptForValue() throws Exception {
         String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         terminal.addSecretInput("secret value");
         execute("foo");
@@ -152,7 +152,7 @@ public class AddStringKeyStoreCommandTests extends KeyStoreCommandTestCase {
 
     public void testPromptForMultipleValues() throws Exception {
         final String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         terminal.addSecretInput("bar1");
         terminal.addSecretInput("bar2");
@@ -165,7 +165,7 @@ public class AddStringKeyStoreCommandTests extends KeyStoreCommandTestCase {
 
     public void testStdinShort() throws Exception {
         String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         setInput("secret value 1");
         execute("-x", "foo");
@@ -174,7 +174,7 @@ public class AddStringKeyStoreCommandTests extends KeyStoreCommandTestCase {
 
     public void testStdinLong() throws Exception {
         String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         setInput("secret value 2");
         execute("--stdin", "foo");
@@ -183,7 +183,7 @@ public class AddStringKeyStoreCommandTests extends KeyStoreCommandTestCase {
 
     public void testStdinNoInput() throws Exception {
         String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         setInput("");
         execute("-x", "foo");
@@ -192,7 +192,7 @@ public class AddStringKeyStoreCommandTests extends KeyStoreCommandTestCase {
 
     public void testStdinInputWithLineBreaks() throws Exception {
         String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         setInput("Typedthisandhitenter\n");
         execute("-x", "foo");
@@ -201,7 +201,7 @@ public class AddStringKeyStoreCommandTests extends KeyStoreCommandTestCase {
 
     public void testStdinInputWithCarriageReturn() throws Exception {
         String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         setInput("Typedthisandhitenter\r");
         execute("-x", "foo");
@@ -210,7 +210,7 @@ public class AddStringKeyStoreCommandTests extends KeyStoreCommandTestCase {
 
     public void testStdinWithMultipleValues() throws Exception {
         final String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         setInput("bar1\nbar2\nbar3");
         execute(randomFrom("-x", "--stdin"), "foo1", "foo2", "foo3");
@@ -221,7 +221,7 @@ public class AddStringKeyStoreCommandTests extends KeyStoreCommandTestCase {
 
     public void testAddUtf8String() throws Exception {
         String password = "keystorepassword";
-        KeyStoreWrapper.create().save(env.configFile(), password.toCharArray());
+        KeyStoreWrapper.create().save(env.configDir(), password.toCharArray());
         terminal.addSecretInput(password);
         final int stringSize = randomIntBetween(8, 16);
         try (CharArrayWriter secretChars = new CharArrayWriter(stringSize)) {

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/BootstrapTests.java

@@ -42,7 +42,7 @@ public class BootstrapTests extends ESTestCase {
 
     public void testLoadSecureSettings() throws Exception {
         final char[] password = KeyStoreWrapperTests.getPossibleKeystorePassword();
-        final Path configPath = env.configFile();
+        final Path configPath = env.configDir();
         final SecureString seed;
         try (KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create()) {
             seed = KeyStoreWrapper.SEED_SETTING.get(Settings.builder().setSecureSettings(keyStoreWrapper).build());

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/CreateKeyStoreCommandTests.java

@@ -48,7 +48,7 @@ public class CreateKeyStoreCommandTests extends KeyStoreCommandTestCase {
     public void testDefaultNotPromptForPassword() throws Exception {
         assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm());
         execute();
-        Path configDir = env.configFile();
+        Path configDir = env.configDir();
         assertNotNull(KeyStoreWrapper.load(configDir));
     }
 
@@ -63,7 +63,7 @@ public class CreateKeyStoreCommandTests extends KeyStoreCommandTestCase {
         } else {
             execute();
         }
-        Path configDir = env.configFile();
+        Path configDir = env.configDir();
         assertNotNull(KeyStoreWrapper.load(configDir));
     }
 
@@ -79,13 +79,13 @@ public class CreateKeyStoreCommandTests extends KeyStoreCommandTestCase {
         } else {
             execute();
         }
-        Path configDir = env.configFile();
+        Path configDir = env.configDir();
         assertNotNull(KeyStoreWrapper.load(configDir));
     }
 
     public void testOverwrite() throws Exception {
         String password = getPossibleKeystorePassword();
-        Path keystoreFile = KeyStoreWrapper.keystorePath(env.configFile());
+        Path keystoreFile = KeyStoreWrapper.keystorePath(env.configDir());
         byte[] content = "not a keystore".getBytes(StandardCharsets.UTF_8);
         Files.write(keystoreFile, content);
 
@@ -110,6 +110,6 @@ public class CreateKeyStoreCommandTests extends KeyStoreCommandTestCase {
         } else {
             execute();
         }
-        assertNotNull(KeyStoreWrapper.load(env.configFile()));
+        assertNotNull(KeyStoreWrapper.load(env.configDir()));
     }
 }

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreCommandTestCase.java

@@ -77,11 +77,11 @@ public abstract class KeyStoreCommandTestCase extends CommandTestCase {
     }
 
     void saveKeystore(KeyStoreWrapper keystore, String password) throws Exception {
-        keystore.save(env.configFile(), password.toCharArray());
+        keystore.save(env.configDir(), password.toCharArray());
     }
 
     KeyStoreWrapper loadKeystore(String password) throws Exception {
-        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configFile());
+        KeyStoreWrapper keystore = KeyStoreWrapper.load(env.configDir());
         keystore.decrypt(password.toCharArray());
         return keystore;
     }

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java

@@ -84,8 +84,8 @@ public class KeyStoreWrapperTests extends ESTestCase {
             bytes[i] = (byte) i;
         }
         keystore.setFile("foo", bytes);
-        keystore.save(env.configFile(), password);
+        keystore.save(env.configDir(), password);
-        keystore = KeyStoreWrapper.load(env.configFile());
+        keystore = KeyStoreWrapper.load(env.configDir());
         keystore.decrypt(password);
         try (InputStream stream = keystore.getFile("foo")) {
             for (int i = 0; i < 256; ++i) {
@@ -114,8 +114,8 @@ public class KeyStoreWrapperTests extends ESTestCase {
             invalidPassword[realPassword.length] = '#';
         }
         KeyStoreWrapper keystore = KeyStoreWrapper.create();
-        keystore.save(env.configFile(), realPassword);
+        keystore.save(env.configDir(), realPassword);
-        final KeyStoreWrapper loadedkeystore = KeyStoreWrapper.load(env.configFile());
+        final KeyStoreWrapper loadedkeystore = KeyStoreWrapper.load(env.configDir());
         final SecurityException exception = expectThrows(SecurityException.class, () -> loadedkeystore.decrypt(invalidPassword));
         if (inFipsJvm()) {
             assertThat(
@@ -133,8 +133,8 @@ public class KeyStoreWrapperTests extends ESTestCase {
     public void testDecryptKeyStoreWithShortPasswordInFips() throws Exception {
         assumeTrue("This should run only in FIPS mode", inFipsJvm());
         KeyStoreWrapper keystore = KeyStoreWrapper.create();
-        keystore.save(env.configFile(), "alongenoughpassword".toCharArray());
+        keystore.save(env.configDir(), "alongenoughpassword".toCharArray());
-        final KeyStoreWrapper loadedkeystore = KeyStoreWrapper.load(env.configFile());
+        final KeyStoreWrapper loadedkeystore = KeyStoreWrapper.load(env.configDir());
         final GeneralSecurityException exception = expectThrows(
             GeneralSecurityException.class,
             () -> loadedkeystore.decrypt("shortpwd".toCharArray()) // shorter than 14 characters
@@ -147,7 +147,7 @@ public class KeyStoreWrapperTests extends ESTestCase {
         KeyStoreWrapper keystore = KeyStoreWrapper.create();
         final GeneralSecurityException exception = expectThrows(
             GeneralSecurityException.class,
-            () -> keystore.save(env.configFile(), "shortpwd".toCharArray()) // shorter than 14 characters
+            () -> keystore.save(env.configDir(), "shortpwd".toCharArray()) // shorter than 14 characters
         );
         assertThat(exception.getMessage(), containsString("Error generating an encryption key from the provided password"));
     }
@@ -192,18 +192,18 @@ public class KeyStoreWrapperTests extends ESTestCase {
         final char[] password = getPossibleKeystorePassword();
         KeyStoreWrapper keystore = KeyStoreWrapper.create();
         SecureString seed = keystore.getString(KeyStoreWrapper.SEED_SETTING.getKey());
-        keystore.save(env.configFile(), password);
+        keystore.save(env.configDir(), password);
         // upgrade does not overwrite seed
-        KeyStoreWrapper.upgrade(keystore, env.configFile(), password);
+        KeyStoreWrapper.upgrade(keystore, env.configDir(), password);
         assertEquals(seed.toString(), keystore.getString(KeyStoreWrapper.SEED_SETTING.getKey()).toString());
-        keystore = KeyStoreWrapper.load(env.configFile());
+        keystore = KeyStoreWrapper.load(env.configDir());
         keystore.decrypt(password);
         assertEquals(seed.toString(), keystore.getString(KeyStoreWrapper.SEED_SETTING.getKey()).toString());
     }
 
     public void testFailWhenCannotConsumeSecretStream() throws Exception {
         assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm());
-        Path configDir = env.configFile();
+        Path configDir = env.configDir();
         try (
             Directory directory = newFSDirectory(configDir);
             IndexOutput indexOutput = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT)
@@ -234,7 +234,7 @@ public class KeyStoreWrapperTests extends ESTestCase {
 
     public void testFailWhenCannotConsumeEncryptedBytesStream() throws Exception {
         assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm());
-        Path configDir = env.configFile();
+        Path configDir = env.configDir();
         try (
             Directory directory = newFSDirectory(configDir);
             IndexOutput indexOutput = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT)
@@ -266,7 +266,7 @@ public class KeyStoreWrapperTests extends ESTestCase {
 
     public void testFailWhenSecretStreamNotConsumed() throws Exception {
         assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm());
-        Path configDir = env.configFile();
+        Path configDir = env.configDir();
         try (
             Directory directory = newFSDirectory(configDir);
             IndexOutput indexOutput = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT)
@@ -296,7 +296,7 @@ public class KeyStoreWrapperTests extends ESTestCase {
 
     public void testFailWhenEncryptedBytesStreamIsNotConsumed() throws Exception {
         assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm());
-        Path configDir = env.configFile();
+        Path configDir = env.configDir();
         try (
             Directory directory = newFSDirectory(configDir);
             IndexOutput indexOutput = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT)
@@ -359,11 +359,11 @@ public class KeyStoreWrapperTests extends ESTestCase {
         final char[] password = getPossibleKeystorePassword();
         KeyStoreWrapper keystore = KeyStoreWrapper.create();
         keystore.remove(KeyStoreWrapper.SEED_SETTING.getKey());
-        keystore.save(env.configFile(), password);
+        keystore.save(env.configDir(), password);
-        KeyStoreWrapper.upgrade(keystore, env.configFile(), password);
+        KeyStoreWrapper.upgrade(keystore, env.configDir(), password);
         SecureString seed = keystore.getString(KeyStoreWrapper.SEED_SETTING.getKey());
         assertNotNull(seed);
-        keystore = KeyStoreWrapper.load(env.configFile());
+        keystore = KeyStoreWrapper.load(env.configDir());
         keystore.decrypt(password);
         assertEquals(seed.toString(), keystore.getString(KeyStoreWrapper.SEED_SETTING.getKey()).toString());
     }
@@ -380,7 +380,7 @@ public class KeyStoreWrapperTests extends ESTestCase {
 
     public void testBackcompatV4() throws Exception {
         assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
-        Path configDir = env.configFile();
+        Path configDir = env.configDir();
         try (
             Directory directory = newFSDirectory(configDir);
             IndexOutput indexOutput = EndiannessReverserUtil.createOutput(directory, "elasticsearch.keystore", IOContext.DEFAULT)
@@ -421,10 +421,10 @@ public class KeyStoreWrapperTests extends ESTestCase {
         final Path temp = createTempDir();
         Files.writeString(temp.resolve("file_setting"), "file_value", StandardCharsets.UTF_8);
         wrapper.setFile("file_setting", Files.readAllBytes(temp.resolve("file_setting")));
-        wrapper.save(env.configFile(), password);
+        wrapper.save(env.configDir(), password);
         wrapper.close();
 
-        final KeyStoreWrapper afterSave = KeyStoreWrapper.load(env.configFile());
+        final KeyStoreWrapper afterSave = KeyStoreWrapper.load(env.configDir());
         assertNotNull(afterSave);
         afterSave.decrypt(password);
         assertThat(afterSave.getSettingNames(), equalTo(Set.of("keystore.seed", "string_setting", "file_setting")));
@@ -510,8 +510,8 @@ public class KeyStoreWrapperTests extends ESTestCase {
 
         // testing with password and raw dataBytes[]
         final char[] password = getPossibleKeystorePassword();
-        wrapper.save(env.configFile(), password);
+        wrapper.save(env.configDir(), password);
-        final KeyStoreWrapper fromFile = KeyStoreWrapper.load(env.configFile());
+        final KeyStoreWrapper fromFile = KeyStoreWrapper.load(env.configDir());
         fromFile.decrypt(password);
 
         assertThat(fromFile.getSettingNames(), hasSize(2));

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/UpgradeKeyStoreCommandTests.java

@@ -62,11 +62,11 @@ public class UpgradeKeyStoreCommandTests extends KeyStoreCommandTestCase {
     }
 
     private void assertKeystoreUpgrade(String file, int version, @Nullable String password) throws Exception {
-        final Path keystore = KeyStoreWrapper.keystorePath(env.configFile());
+        final Path keystore = KeyStoreWrapper.keystorePath(env.configDir());
         try (InputStream is = KeyStoreWrapperTests.class.getResourceAsStream(file); OutputStream os = Files.newOutputStream(keystore)) {
             is.transferTo(os);
         }
-        try (KeyStoreWrapper beforeUpgrade = KeyStoreWrapper.load(env.configFile())) {
+        try (KeyStoreWrapper beforeUpgrade = KeyStoreWrapper.load(env.configDir())) {
             assertNotNull(beforeUpgrade);
             assertThat(beforeUpgrade.getFormatVersion(), equalTo(version));
         }
@@ -77,7 +77,7 @@ public class UpgradeKeyStoreCommandTests extends KeyStoreCommandTestCase {
         execute();
         terminal.reset();
 
-        try (KeyStoreWrapper afterUpgrade = KeyStoreWrapper.load(env.configFile())) {
+        try (KeyStoreWrapper afterUpgrade = KeyStoreWrapper.load(env.configDir())) {
             assertNotNull(afterUpgrade);
             assertThat(afterUpgrade.getFormatVersion(), equalTo(KeyStoreWrapper.CURRENT_VERSION));
             afterUpgrade.decrypt(password != null ? password.toCharArray() : new char[0]);
@@ -87,6 +87,6 @@ public class UpgradeKeyStoreCommandTests extends KeyStoreCommandTestCase {
 
     public void testKeystoreDoesNotExist() {
         final UserException e = expectThrows(UserException.class, this::execute);
-        assertThat(e, hasToString(containsString("keystore not found at [" + KeyStoreWrapper.keystorePath(env.configFile()) + "]")));
+        assertThat(e, hasToString(containsString("keystore not found at [" + KeyStoreWrapper.keystorePath(env.configDir()) + "]")));
     }
 }

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java

@@ -249,8 +249,8 @@ public class InstallPluginAction implements Closeable {
                 final List<Path> deleteOnFailure = new ArrayList<>();
                 deleteOnFailures.put(pluginId, deleteOnFailure);
 
-                final Path pluginZip = download(plugin, env.tmpFile());
+                final Path pluginZip = download(plugin, env.tmpDir());
-                final Path extractedZip = unzip(pluginZip, env.pluginsFile());
+                final Path extractedZip = unzip(pluginZip, env.pluginsDir());
                 deleteOnFailure.add(extractedZip);
                 final PluginDescriptor pluginDescriptor = installPlugin(plugin, extractedZip, deleteOnFailure);
                 terminal.println(logPrefix + "Installed " + pluginDescriptor.getName());
@@ -868,14 +868,14 @@ public class InstallPluginAction implements Closeable {
         PluginsUtils.verifyCompatibility(info);
 
         // checking for existing version of the plugin
-        verifyPluginName(env.pluginsFile(), info.getName());
+        verifyPluginName(env.pluginsDir(), info.getName());
 
-        PluginsUtils.checkForFailedPluginRemovals(env.pluginsFile());
+        PluginsUtils.checkForFailedPluginRemovals(env.pluginsDir());
 
         terminal.println(VERBOSE, info.toString());
 
         // check for jar hell before any copying
-        jarHellCheck(info, pluginRoot, env.pluginsFile(), env.modulesFile());
+        jarHellCheck(info, pluginRoot, env.pluginsDir(), env.modulesDir());
 
         if (info.isStable() && hasNamedComponentFile(pluginRoot) == false) {
             generateNameComponentFile(pluginRoot);
@@ -922,9 +922,9 @@ public class InstallPluginAction implements Closeable {
      */
     private PluginDescriptor installPlugin(InstallablePlugin descriptor, Path tmpRoot, List<Path> deleteOnFailure) throws Exception {
         final PluginDescriptor info = loadPluginInfo(tmpRoot);
-        PluginPolicyInfo pluginPolicy = PolicyUtil.getPluginPolicyInfo(tmpRoot, env.tmpFile());
+        PluginPolicyInfo pluginPolicy = PolicyUtil.getPluginPolicyInfo(tmpRoot, env.tmpDir());
         if (pluginPolicy != null) {
-            Set<String> permissions = PluginSecurity.getPermissionDescriptions(pluginPolicy, env.tmpFile());
+            Set<String> permissions = PluginSecurity.getPermissionDescriptions(pluginPolicy, env.tmpDir());
             PluginSecurity.confirmPolicyExceptions(terminal, permissions, batch);
         }
 
@@ -938,14 +938,14 @@ public class InstallPluginAction implements Closeable {
             );
         }
 
-        final Path destination = env.pluginsFile().resolve(info.getName());
+        final Path destination = env.pluginsDir().resolve(info.getName());
         deleteOnFailure.add(destination);
 
         installPluginSupportFiles(
             info,
             tmpRoot,
-            env.binFile().resolve(info.getName()),
+            env.binDir().resolve(info.getName()),
-            env.configFile().resolve(info.getName()),
+            env.configDir().resolve(info.getName()),
             deleteOnFailure
         );
         movePlugin(tmpRoot, destination);

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/ListPluginsCommand.java

@@ -40,13 +40,13 @@ class ListPluginsCommand extends EnvironmentAwareCommand {
 
     @Override
     public void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception {
-        if (Files.exists(env.pluginsFile()) == false) {
+        if (Files.exists(env.pluginsDir()) == false) {
-            throw new IOException("Plugins directory missing: " + env.pluginsFile());
+            throw new IOException("Plugins directory missing: " + env.pluginsDir());
         }
 
-        terminal.println(Terminal.Verbosity.VERBOSE, "Plugins directory: " + env.pluginsFile());
+        terminal.println(Terminal.Verbosity.VERBOSE, "Plugins directory: " + env.pluginsDir());
         final List<Path> plugins = new ArrayList<>();
-        try (DirectoryStream<Path> paths = Files.newDirectoryStream(env.pluginsFile())) {
+        try (DirectoryStream<Path> paths = Files.newDirectoryStream(env.pluginsDir())) {
             for (Path path : paths) {
                 if (path.getFileName().toString().equals(ELASTICSEARCH_PLUGINS_YML_CACHE) == false) {
                     plugins.add(path);
@@ -61,7 +61,7 @@ class ListPluginsCommand extends EnvironmentAwareCommand {
 
     private static void printPlugin(Environment env, Terminal terminal, Path plugin, String prefix) throws IOException {
         terminal.println(Terminal.Verbosity.SILENT, prefix + plugin.getFileName().toString());
-        PluginDescriptor info = PluginDescriptor.readFromProperties(env.pluginsFile().resolve(plugin));
+        PluginDescriptor info = PluginDescriptor.readFromProperties(env.pluginsDir().resolve(plugin));
         terminal.println(Terminal.Verbosity.VERBOSE, info.toString(prefix));
 
         // When PluginDescriptor#getElasticsearchVersion returns a string, we can revisit the need

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/RemovePluginAction.java

@@ -93,7 +93,7 @@ public class RemovePluginAction {
         // We build a new map where the keys are plugins that extend plugins
         // we want to remove and the values are the plugins we can't remove
         // because of this dependency
-        Map<String, List<String>> pluginDependencyMap = PluginsUtils.getDependencyMapView(env.pluginsFile());
+        Map<String, List<String>> pluginDependencyMap = PluginsUtils.getDependencyMapView(env.pluginsDir());
         for (Map.Entry<String, List<String>> entry : pluginDependencyMap.entrySet()) {
             for (String extendedPlugin : entry.getValue()) {
                 for (InstallablePlugin plugin : plugins) {
@@ -121,9 +121,9 @@ public class RemovePluginAction {
     private void checkCanRemove(InstallablePlugin plugin) throws UserException {
         String pluginId = plugin.getId();
 
-        final Path pluginDir = env.pluginsFile().resolve(pluginId);
+        final Path pluginDir = env.pluginsDir().resolve(pluginId);
-        final Path pluginConfigDir = env.configFile().resolve(pluginId);
+        final Path pluginConfigDir = env.configDir().resolve(pluginId);
-        final Path removing = env.pluginsFile().resolve(".removing-" + pluginId);
+        final Path removing = env.pluginsDir().resolve(".removing-" + pluginId);
 
         /*
          * If the plugin does not exist and the plugin config does not exist, fail to the user that the plugin is not found, unless there's
@@ -147,7 +147,7 @@ public class RemovePluginAction {
             }
         }
 
-        final Path pluginBinDir = env.binFile().resolve(pluginId);
+        final Path pluginBinDir = env.binDir().resolve(pluginId);
         if (Files.exists(pluginBinDir)) {
             if (Files.isDirectory(pluginBinDir) == false) {
                 throw new UserException(ExitCodes.IO_ERROR, "bin dir for " + pluginId + " is not a directory");
@@ -157,9 +157,9 @@ public class RemovePluginAction {
 
     private void removePlugin(InstallablePlugin plugin) throws IOException {
         final String pluginId = plugin.getId();
-        final Path pluginDir = env.pluginsFile().resolve(pluginId);
+        final Path pluginDir = env.pluginsDir().resolve(pluginId);
-        final Path pluginConfigDir = env.configFile().resolve(pluginId);
+        final Path pluginConfigDir = env.configDir().resolve(pluginId);
-        final Path removing = env.pluginsFile().resolve(".removing-" + pluginId);
+        final Path removing = env.pluginsDir().resolve(".removing-" + pluginId);
 
         terminal.println("-> removing [" + pluginId + "]...");
 
@@ -176,7 +176,7 @@ public class RemovePluginAction {
             terminal.println(VERBOSE, "removing [" + pluginDir + "]");
         }
 
-        final Path pluginBinDir = env.binFile().resolve(pluginId);
+        final Path pluginBinDir = env.binDir().resolve(pluginId);
         if (Files.exists(pluginBinDir)) {
             try (Stream<Path> paths = Files.list(pluginBinDir)) {
                 pluginPaths.addAll(paths.toList());

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsAction.java

@@ -61,7 +61,7 @@ public class SyncPluginsAction {
      * @throws UserException if a plugins config file is found.
      */
     public static void ensureNoConfigFile(Environment env) throws UserException {
-        final Path pluginsConfig = env.configFile().resolve(ELASTICSEARCH_PLUGINS_YML);
+        final Path pluginsConfig = env.configDir().resolve(ELASTICSEARCH_PLUGINS_YML);
         if (Files.exists(pluginsConfig)) {
             throw new UserException(
                 ExitCodes.USAGE,
@@ -79,16 +79,16 @@ public class SyncPluginsAction {
      * @throws Exception if anything goes wrong
      */
     public void execute() throws Exception {
-        final Path configPath = this.env.configFile().resolve(ELASTICSEARCH_PLUGINS_YML);
+        final Path configPath = this.env.configDir().resolve(ELASTICSEARCH_PLUGINS_YML);
-        final Path previousConfigPath = this.env.pluginsFile().resolve(ELASTICSEARCH_PLUGINS_YML_CACHE);
+        final Path previousConfigPath = this.env.pluginsDir().resolve(ELASTICSEARCH_PLUGINS_YML_CACHE);
 
         if (Files.exists(configPath) == false) {
             // The `PluginsManager` will have checked that this file exists before invoking the action.
             throw new PluginSyncException("Plugins config does not exist: " + configPath.toAbsolutePath());
         }
 
-        if (Files.exists(env.pluginsFile()) == false) {
+        if (Files.exists(env.pluginsDir()) == false) {
-            throw new PluginSyncException("Plugins directory missing: " + env.pluginsFile());
+            throw new PluginSyncException("Plugins directory missing: " + env.pluginsDir());
         }
 
         // Parse descriptor file
@@ -267,14 +267,14 @@ public class SyncPluginsAction {
         final List<PluginDescriptor> plugins = new ArrayList<>();
 
         try {
-            try (DirectoryStream<Path> paths = Files.newDirectoryStream(env.pluginsFile())) {
+            try (DirectoryStream<Path> paths = Files.newDirectoryStream(env.pluginsDir())) {
                 for (Path pluginPath : paths) {
                     String filename = pluginPath.getFileName().toString();
                     if (filename.startsWith(".")) {
                         continue;
                     }
 
-                    PluginDescriptor info = PluginDescriptor.readFromProperties(env.pluginsFile().resolve(pluginPath));
+                    PluginDescriptor info = PluginDescriptor.readFromProperties(env.pluginsDir().resolve(pluginPath));
                     plugins.add(info);
 
                     // Check for a version mismatch, unless it's an official plugin since we can upgrade them.

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/SyncPluginsCliProvider.java

@@ -37,7 +37,7 @@ public class SyncPluginsCliProvider implements CliToolProvider {
             @Override
             public void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception {
                 var action = new SyncPluginsAction(terminal, env);
-                if (Files.exists(env.configFile().resolve(ELASTICSEARCH_PLUGINS_YML)) == false) {
+                if (Files.exists(env.configDir().resolve(ELASTICSEARCH_PLUGINS_YML)) == false) {
                     return;
                 }
                 if (Build.current().type() != Build.Type.DOCKER) {

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java

@@ -354,7 +354,7 @@ public class InstallPluginActionTests extends ESTestCase {
     }
 
     void assertPlugin(String name, Path original, Environment environment) throws IOException {
-        assertPluginInternal(name, environment.pluginsFile(), original);
+        assertPluginInternal(name, environment.pluginsDir(), original);
         assertConfigAndBin(name, original, environment);
         assertInstallCleaned(environment);
     }
@@ -395,7 +395,7 @@ public class InstallPluginActionTests extends ESTestCase {
 
     void assertConfigAndBin(String name, Path original, Environment environment) throws IOException {
         if (Files.exists(original.resolve("bin"))) {
-            Path binDir = environment.binFile().resolve(name);
+            Path binDir = environment.binDir().resolve(name);
             assertTrue("bin dir exists", Files.exists(binDir));
             assertTrue("bin is a dir", Files.isDirectory(binDir));
             try (DirectoryStream<Path> stream = Files.newDirectoryStream(binDir)) {
@@ -409,7 +409,7 @@ public class InstallPluginActionTests extends ESTestCase {
             }
         }
         if (Files.exists(original.resolve("config"))) {
-            Path configDir = environment.configFile().resolve(name);
+            Path configDir = environment.configDir().resolve(name);
             assertTrue("config dir exists", Files.exists(configDir));
             assertTrue("config is a dir", Files.isDirectory(configDir));
 
@@ -417,7 +417,7 @@ public class InstallPluginActionTests extends ESTestCase {
             GroupPrincipal group = null;
 
             if (isPosix) {
-                PosixFileAttributes configAttributes = Files.getFileAttributeView(environment.configFile(), PosixFileAttributeView.class)
+                PosixFileAttributes configAttributes = Files.getFileAttributeView(environment.configDir(), PosixFileAttributeView.class)
                     .readAttributes();
                 user = configAttributes.owner();
                 group = configAttributes.group();
@@ -446,7 +446,7 @@ public class InstallPluginActionTests extends ESTestCase {
     }
 
     void assertInstallCleaned(Environment environment) throws IOException {
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(environment.pluginsFile())) {
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(environment.pluginsDir())) {
             for (Path file : stream) {
                 if (file.getFileName().toString().startsWith(".installing")) {
                     fail("Installation dir still exists, " + file);
@@ -549,7 +549,7 @@ public class InstallPluginActionTests extends ESTestCase {
             () -> installPlugins(List.of(pluginZip, nonexistentPluginZip), env.v1())
         );
         assertThat(e.getMessage(), containsString("does-not-exist"));
-        final Path fakeInstallPath = env.v2().pluginsFile().resolve("fake");
+        final Path fakeInstallPath = env.v2().pluginsDir().resolve("fake");
         // fake should have been removed when the file not found exception occurred
         assertFalse(Files.exists(fakeInstallPath));
         assertInstallCleaned(env.v2());
@@ -557,7 +557,7 @@ public class InstallPluginActionTests extends ESTestCase {
 
     public void testInstallFailsIfPreviouslyRemovedPluginFailed() throws Exception {
         InstallablePlugin pluginZip = createPluginZip("fake", pluginDir);
-        final Path removing = env.v2().pluginsFile().resolve(".removing-failed");
+        final Path removing = env.v2().pluginsDir().resolve(".removing-failed");
         Files.createDirectory(removing);
         final IllegalStateException e = expectThrows(IllegalStateException.class, () -> installPlugin(pluginZip));
         final String expected = Strings.format(
@@ -603,11 +603,11 @@ public class InstallPluginActionTests extends ESTestCase {
 
     public void testPluginsDirReadOnly() throws Exception {
         assumeTrue("posix and filesystem", isPosix && isReal);
-        try (PosixPermissionsResetter pluginsAttrs = new PosixPermissionsResetter(env.v2().pluginsFile())) {
+        try (PosixPermissionsResetter pluginsAttrs = new PosixPermissionsResetter(env.v2().pluginsDir())) {
             pluginsAttrs.setPermissions(new HashSet<>());
             InstallablePlugin pluginZip = createPluginZip("fake", pluginDir);
             IOException e = expectThrows(IOException.class, () -> installPlugin(pluginZip));
-            assertThat(e.getMessage(), containsString(env.v2().pluginsFile().toString()));
+            assertThat(e.getMessage(), containsString(env.v2().pluginsDir().toString()));
         }
         assertInstallCleaned(env.v2());
     }
@@ -694,7 +694,7 @@ public class InstallPluginActionTests extends ESTestCase {
         Files.createFile(binDir.resolve("somescript"));
         InstallablePlugin pluginZip = createPluginZip("elasticsearch", pluginDir);
         FileAlreadyExistsException e = expectThrows(FileAlreadyExistsException.class, () -> installPlugin(pluginZip));
-        assertThat(e.getMessage(), containsString(env.v2().binFile().resolve("elasticsearch").toString()));
+        assertThat(e.getMessage(), containsString(env.v2().binDir().resolve("elasticsearch").toString()));
         assertInstallCleaned(env.v2());
     }
 
@@ -704,7 +704,7 @@ public class InstallPluginActionTests extends ESTestCase {
         Files.createDirectory(binDir);
         Files.createFile(binDir.resolve("somescript"));
         InstallablePlugin pluginZip = createPluginZip("fake", pluginDir);
-        try (PosixPermissionsResetter binAttrs = new PosixPermissionsResetter(env.v2().binFile())) {
+        try (PosixPermissionsResetter binAttrs = new PosixPermissionsResetter(env.v2().binDir())) {
             Set<PosixFilePermission> perms = binAttrs.getCopyPermissions();
             // make sure at least one execute perm is missing, so we know we forced it during installation
             perms.remove(PosixFilePermission.GROUP_EXECUTE);
@@ -734,7 +734,7 @@ public class InstallPluginActionTests extends ESTestCase {
         installPlugin(pluginZip);
         assertPlugin("fake", tempPluginDir, env.v2());
 
-        final Path fake = env.v2().pluginsFile().resolve("fake");
+        final Path fake = env.v2().pluginsDir().resolve("fake");
         final Path resources = fake.resolve("resources");
         final Path platform = fake.resolve("platform");
         final Path platformName = platform.resolve("linux-x86_64");
@@ -784,7 +784,7 @@ public class InstallPluginActionTests extends ESTestCase {
     }
 
     public void testExistingConfig() throws Exception {
-        Path envConfigDir = env.v2().configFile().resolve("fake");
+        Path envConfigDir = env.v2().configDir().resolve("fake");
         Files.createDirectories(envConfigDir);
         Files.write(envConfigDir.resolve("custom.yml"), "existing config".getBytes(StandardCharsets.UTF_8));
         Path configDir = pluginDir.resolve("config");
@@ -921,7 +921,7 @@ public class InstallPluginActionTests extends ESTestCase {
             e.getMessage(),
             equalTo(
                 "plugin directory ["
-                    + env.v2().pluginsFile().resolve("fake")
+                    + env.v2().pluginsDir().resolve("fake")
                     + "] already exists; "
                     + "if you need to update the plugin, uninstall it first using command 'remove fake'"
             )
@@ -1499,7 +1499,7 @@ public class InstallPluginActionTests extends ESTestCase {
             assertThat(e.getMessage(), containsString("installation aborted by user"));
 
             assertThat(terminal.getErrorOutput(), containsString("WARNING: " + warning));
-            try (Stream<Path> fileStream = Files.list(pathEnvironmentTuple.v2().pluginsFile())) {
+            try (Stream<Path> fileStream = Files.list(pathEnvironmentTuple.v2().pluginsDir())) {
                 assertThat(fileStream.collect(Collectors.toList()), empty());
             }
 
@@ -1512,7 +1512,7 @@ public class InstallPluginActionTests extends ESTestCase {
             e = expectThrows(UserException.class, () -> installPlugin(pluginZip));
             assertThat(e.getMessage(), containsString("installation aborted by user"));
             assertThat(terminal.getErrorOutput(), containsString("WARNING: " + warning));
-            try (Stream<Path> fileStream = Files.list(pathEnvironmentTuple.v2().pluginsFile())) {
+            try (Stream<Path> fileStream = Files.list(pathEnvironmentTuple.v2().pluginsDir())) {
                 assertThat(fileStream.collect(Collectors.toList()), empty());
             }
         }
@@ -1566,7 +1566,7 @@ public class InstallPluginActionTests extends ESTestCase {
         InstallablePlugin stablePluginZip = createStablePlugin("stable1", pluginDir, true);
         installPlugins(List.of(stablePluginZip), env.v1());
         assertPlugin("stable1", pluginDir, env.v2());
-        assertNamedComponentFile("stable1", env.v2().pluginsFile(), namedComponentsJSON());
+        assertNamedComponentFile("stable1", env.v2().pluginsDir(), namedComponentsJSON());
     }
 
     @SuppressWarnings("unchecked")
@@ -1577,7 +1577,7 @@ public class InstallPluginActionTests extends ESTestCase {
         installPlugins(List.of(stablePluginZip), env.v1());
 
         assertPlugin("stable1", pluginDir, env.v2());
-        assertNamedComponentFile("stable1", env.v2().pluginsFile(), namedComponentsJSON());
+        assertNamedComponentFile("stable1", env.v2().pluginsDir(), namedComponentsJSON());
     }
 
     public void testGetSemanticVersion() {

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/ListPluginsCommandTests.java

@@ -65,7 +65,7 @@ public class ListPluginsCommandTests extends CommandTestCase {
         final boolean hasNativeController
     ) throws IOException {
         PluginTestUtil.writePluginProperties(
-            env.pluginsFile().resolve(name),
+            env.pluginsDir().resolve(name),
             "description",
             description,
             "name",
@@ -84,9 +84,9 @@ public class ListPluginsCommandTests extends CommandTestCase {
     }
 
     public void testPluginsDirMissing() throws Exception {
-        Files.delete(env.pluginsFile());
+        Files.delete(env.pluginsDir());
         IOException e = expectThrows(IOException.class, () -> execute());
-        assertEquals("Plugins directory missing: " + env.pluginsFile(), e.getMessage());
+        assertEquals("Plugins directory missing: " + env.pluginsDir(), e.getMessage());
     }
 
     public void testNoPlugins() throws Exception {
@@ -112,7 +112,7 @@ public class ListPluginsCommandTests extends CommandTestCase {
         execute("-v");
         assertEquals(
             buildMultiline(
-                "Plugins directory: " + env.pluginsFile(),
+                "Plugins directory: " + env.pluginsDir(),
                 "fake_plugin",
                 "- Plugin information:",
                 "Name: fake_plugin",
@@ -134,7 +134,7 @@ public class ListPluginsCommandTests extends CommandTestCase {
         execute("-v");
         assertEquals(
             buildMultiline(
-                "Plugins directory: " + env.pluginsFile(),
+                "Plugins directory: " + env.pluginsDir(),
                 "fake_plugin1",
                 "- Plugin information:",
                 "Name: fake_plugin1",
@@ -157,7 +157,7 @@ public class ListPluginsCommandTests extends CommandTestCase {
         execute("-v");
         assertEquals(
             buildMultiline(
-                "Plugins directory: " + env.pluginsFile(),
+                "Plugins directory: " + env.pluginsDir(),
                 "fake_plugin1",
                 "- Plugin information:",
                 "Name: fake_plugin1",
@@ -193,14 +193,14 @@ public class ListPluginsCommandTests extends CommandTestCase {
     }
 
     public void testPluginWithoutDescriptorFile() throws Exception {
-        final Path pluginDir = env.pluginsFile().resolve("fake1");
+        final Path pluginDir = env.pluginsDir().resolve("fake1");
         Files.createDirectories(pluginDir);
         var e = expectThrows(IllegalStateException.class, () -> execute());
         assertThat(e.getMessage(), equalTo("Plugin [fake1] is missing a descriptor properties file."));
     }
 
     public void testPluginWithWrongDescriptorFile() throws Exception {
-        final Path pluginDir = env.pluginsFile().resolve("fake1");
+        final Path pluginDir = env.pluginsDir().resolve("fake1");
         PluginTestUtil.writePluginProperties(pluginDir, "description", "fake desc");
         var e = expectThrows(IllegalArgumentException.class, () -> execute());
         assertThat(e.getMessage(), startsWith("property [name] is missing for plugin"));
@@ -208,7 +208,7 @@ public class ListPluginsCommandTests extends CommandTestCase {
 
     public void testExistingIncompatiblePlugin() throws Exception {
         PluginTestUtil.writePluginProperties(
-            env.pluginsFile().resolve("fake_plugin1"),
+            env.pluginsDir().resolve("fake_plugin1"),
             "description",
             "fake desc 1",
             "name",

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/RemovePluginActionTests.java

@@ -58,11 +58,11 @@ public class RemovePluginActionTests extends ESTestCase {
     }
 
     void createPlugin(String name) throws IOException {
-        createPlugin(env.pluginsFile(), name, Version.CURRENT);
+        createPlugin(env.pluginsDir(), name, Version.CURRENT);
     }
 
     void createPlugin(String name, Version version) throws IOException {
-        createPlugin(env.pluginsFile(), name, version);
+        createPlugin(env.pluginsDir(), name, version);
     }
 
     void createPlugin(Path path, String name, Version version) throws IOException {
@@ -98,7 +98,7 @@ public class RemovePluginActionTests extends ESTestCase {
     }
 
     static void assertRemoveCleaned(Environment env) throws IOException {
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(env.pluginsFile())) {
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(env.pluginsDir())) {
             for (Path file : stream) {
                 if (file.getFileName().toString().startsWith(".removing")) {
                     fail("Removal dir still exists, " + file);
@@ -115,84 +115,84 @@ public class RemovePluginActionTests extends ESTestCase {
 
     public void testBasic() throws Exception {
         createPlugin("fake");
-        Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar"));
+        Files.createFile(env.pluginsDir().resolve("fake").resolve("plugin.jar"));
-        Files.createDirectory(env.pluginsFile().resolve("fake").resolve("subdir"));
+        Files.createDirectory(env.pluginsDir().resolve("fake").resolve("subdir"));
         createPlugin("other");
         removePlugin("fake", home, randomBoolean());
-        assertFalse(Files.exists(env.pluginsFile().resolve("fake")));
+        assertFalse(Files.exists(env.pluginsDir().resolve("fake")));
-        assertTrue(Files.exists(env.pluginsFile().resolve("other")));
+        assertTrue(Files.exists(env.pluginsDir().resolve("other")));
         assertRemoveCleaned(env);
     }
 
     /** Check that multiple plugins can be removed at the same time. */
     public void testRemoveMultiple() throws Exception {
         createPlugin("fake");
-        Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar"));
+        Files.createFile(env.pluginsDir().resolve("fake").resolve("plugin.jar"));
-        Files.createDirectory(env.pluginsFile().resolve("fake").resolve("subdir"));
+        Files.createDirectory(env.pluginsDir().resolve("fake").resolve("subdir"));
 
         createPlugin("other");
-        Files.createFile(env.pluginsFile().resolve("other").resolve("plugin.jar"));
+        Files.createFile(env.pluginsDir().resolve("other").resolve("plugin.jar"));
-        Files.createDirectory(env.pluginsFile().resolve("other").resolve("subdir"));
+        Files.createDirectory(env.pluginsDir().resolve("other").resolve("subdir"));
 
         removePlugin("fake", home, randomBoolean());
         removePlugin("other", home, randomBoolean());
-        assertFalse(Files.exists(env.pluginsFile().resolve("fake")));
+        assertFalse(Files.exists(env.pluginsDir().resolve("fake")));
-        assertFalse(Files.exists(env.pluginsFile().resolve("other")));
+        assertFalse(Files.exists(env.pluginsDir().resolve("other")));
         assertRemoveCleaned(env);
     }
 
     public void testBin() throws Exception {
         createPlugin("fake");
-        Path binDir = env.binFile().resolve("fake");
+        Path binDir = env.binDir().resolve("fake");
         Files.createDirectories(binDir);
         Files.createFile(binDir.resolve("somescript"));
         removePlugin("fake", home, randomBoolean());
-        assertFalse(Files.exists(env.pluginsFile().resolve("fake")));
+        assertFalse(Files.exists(env.pluginsDir().resolve("fake")));
-        assertTrue(Files.exists(env.binFile().resolve("elasticsearch")));
+        assertTrue(Files.exists(env.binDir().resolve("elasticsearch")));
         assertFalse(Files.exists(binDir));
         assertRemoveCleaned(env);
     }
 
     public void testBinNotDir() throws Exception {
         createPlugin("fake");
-        Files.createFile(env.binFile().resolve("fake"));
+        Files.createFile(env.binDir().resolve("fake"));
         UserException e = expectThrows(UserException.class, () -> removePlugin("fake", home, randomBoolean()));
         assertThat(e.getMessage(), containsString("not a directory"));
-        assertTrue(Files.exists(env.pluginsFile().resolve("fake"))); // did not remove
+        assertTrue(Files.exists(env.pluginsDir().resolve("fake"))); // did not remove
-        assertTrue(Files.exists(env.binFile().resolve("fake")));
+        assertTrue(Files.exists(env.binDir().resolve("fake")));
         assertRemoveCleaned(env);
     }
 
     public void testConfigDirPreserved() throws Exception {
         createPlugin("fake");
-        final Path configDir = env.configFile().resolve("fake");
+        final Path configDir = env.configDir().resolve("fake");
         Files.createDirectories(configDir);
         Files.createFile(configDir.resolve("fake.yml"));
         final MockTerminal terminal = removePlugin("fake", home, false);
-        assertTrue(Files.exists(env.configFile().resolve("fake")));
+        assertTrue(Files.exists(env.configDir().resolve("fake")));
         assertThat(terminal.getOutput(), containsString(expectedConfigDirPreservedMessage(configDir)));
         assertRemoveCleaned(env);
     }
 
     public void testPurgePluginExists() throws Exception {
         createPlugin("fake");
-        final Path configDir = env.configFile().resolve("fake");
+        final Path configDir = env.configDir().resolve("fake");
         if (randomBoolean()) {
             Files.createDirectories(configDir);
             Files.createFile(configDir.resolve("fake.yml"));
         }
         final MockTerminal terminal = removePlugin("fake", home, true);
-        assertFalse(Files.exists(env.configFile().resolve("fake")));
+        assertFalse(Files.exists(env.configDir().resolve("fake")));
         assertThat(terminal.getOutput(), not(containsString(expectedConfigDirPreservedMessage(configDir))));
         assertRemoveCleaned(env);
     }
 
     public void testPurgePluginDoesNotExist() throws Exception {
-        final Path configDir = env.configFile().resolve("fake");
+        final Path configDir = env.configDir().resolve("fake");
         Files.createDirectories(configDir);
         Files.createFile(configDir.resolve("fake.yml"));
         final MockTerminal terminal = removePlugin("fake", home, true);
-        assertFalse(Files.exists(env.configFile().resolve("fake")));
+        assertFalse(Files.exists(env.configDir().resolve("fake")));
         assertThat(terminal.getOutput(), not(containsString(expectedConfigDirPreservedMessage(configDir))));
         assertRemoveCleaned(env);
     }
@@ -203,8 +203,8 @@ public class RemovePluginActionTests extends ESTestCase {
     }
 
     public void testPurgeOnlyMarkerFileExists() throws Exception {
-        final Path configDir = env.configFile().resolve("fake");
+        final Path configDir = env.configDir().resolve("fake");
-        final Path removing = env.pluginsFile().resolve(".removing-fake");
+        final Path removing = env.pluginsDir().resolve(".removing-fake");
         Files.createFile(removing);
         final MockTerminal terminal = removePlugin("fake", home, randomBoolean());
         assertFalse(Files.exists(removing));
@@ -213,7 +213,7 @@ public class RemovePluginActionTests extends ESTestCase {
 
     public void testNoConfigDirPreserved() throws Exception {
         createPlugin("fake");
-        final Path configDir = env.configFile().resolve("fake");
+        final Path configDir = env.configDir().resolve("fake");
         final MockTerminal terminal = removePlugin("fake", home, randomBoolean());
         assertThat(terminal.getOutput(), not(containsString(expectedConfigDirPreservedMessage(configDir))));
     }
@@ -250,8 +250,8 @@ public class RemovePluginActionTests extends ESTestCase {
 
     public void testRemoveWhenRemovingMarker() throws Exception {
         createPlugin("fake");
-        Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar"));
+        Files.createFile(env.pluginsDir().resolve("fake").resolve("plugin.jar"));
-        Files.createFile(env.pluginsFile().resolve(".removing-fake"));
+        Files.createFile(env.pluginsDir().resolve(".removing-fake"));
         removePlugin("fake", home, randomBoolean());
     }
 
@@ -262,10 +262,10 @@ public class RemovePluginActionTests extends ESTestCase {
     public void testRemoveMigratedPluginsWhenInstalled() throws Exception {
         for (String id : List.of("repository-azure", "repository-gcs", "repository-s3")) {
             createPlugin(id);
-            Files.createFile(env.pluginsFile().resolve(id).resolve("plugin.jar"));
+            Files.createFile(env.pluginsDir().resolve(id).resolve("plugin.jar"));
             final MockTerminal terminal = removePlugin(id, home, randomBoolean());
 
-            assertThat(Files.exists(env.pluginsFile().resolve(id)), is(false));
+            assertThat(Files.exists(env.pluginsDir().resolve(id)), is(false));
             // This message shouldn't be printed if plugin was actually installed.
             assertThat(terminal.getErrorOutput(), not(containsString("plugin [" + id + "] is no longer a plugin")));
         }
@@ -288,11 +288,11 @@ public class RemovePluginActionTests extends ESTestCase {
      */
     public void testRemoveRegularInstalledPluginAndMigratedUninstalledPlugin() throws Exception {
         createPlugin("fake");
-        Files.createFile(env.pluginsFile().resolve("fake").resolve("plugin.jar"));
+        Files.createFile(env.pluginsDir().resolve("fake").resolve("plugin.jar"));
 
         final MockTerminal terminal = removePlugin(List.of("fake", "repository-s3"), home, randomBoolean());
 
-        assertThat(Files.exists(env.pluginsFile().resolve("fake")), is(false));
+        assertThat(Files.exists(env.pluginsDir().resolve("fake")), is(false));
         assertThat(terminal.getErrorOutput(), containsString("plugin [repository-s3] is no longer a plugin"));
     }
 

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/SyncPluginsActionTests.java

@@ -55,10 +55,10 @@ public class SyncPluginsActionTests extends ESTestCase {
         Path home = createTempDir();
         Settings settings = Settings.builder().put("path.home", home).build();
         env = TestEnvironment.newEnvironment(settings);
-        Files.createDirectories(env.binFile());
+        Files.createDirectories(env.binDir());
-        Files.createFile(env.binFile().resolve("elasticsearch"));
+        Files.createFile(env.binDir().resolve("elasticsearch"));
-        Files.createDirectories(env.configFile());
+        Files.createDirectories(env.configDir());
-        Files.createDirectories(env.pluginsFile());
+        Files.createDirectories(env.pluginsDir());
 
         terminal = MockTerminal.create();
         action = new SyncPluginsAction(terminal, env);
@@ -78,7 +78,7 @@ public class SyncPluginsActionTests extends ESTestCase {
      * then an exception is thrown.
      */
     public void test_ensureNoConfigFile_withConfig_throwsException() throws Exception {
-        Files.createFile(env.configFile().resolve("elasticsearch-plugins.yml"));
+        Files.createFile(env.configDir().resolve("elasticsearch-plugins.yml"));
         final UserException e = expectThrows(UserException.class, () -> SyncPluginsAction.ensureNoConfigFile(env));
 
         assertThat(e.getMessage(), Matchers.matchesPattern("^Plugins config \\[.*] exists.*$"));
@@ -354,7 +354,7 @@ public class SyncPluginsActionTests extends ESTestCase {
 
     private void createPlugin(String name, String version) throws IOException {
         PluginTestUtil.writePluginProperties(
-            env.pluginsFile().resolve(name),
+            env.pluginsDir().resolve(name),
             "description",
             "dummy",
             "name",

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/KeyStoreLoader.java

@@ -24,7 +24,7 @@ public class KeyStoreLoader implements SecureSettingsLoader {
     @Override
     public LoadedSecrets load(Environment environment, Terminal terminal) throws Exception {
         // See if we have a keystore already present
-        KeyStoreWrapper secureSettings = KeyStoreWrapper.load(environment.configFile());
+        KeyStoreWrapper secureSettings = KeyStoreWrapper.load(environment.configDir());
         // If there's no keystore or the keystore has no password, set an empty password
         var password = (secureSettings == null || secureSettings.hasPassword() == false)
             ? new SecureString(new char[0])
@@ -35,7 +35,7 @@ public class KeyStoreLoader implements SecureSettingsLoader {
 
     @Override
     public SecureSettings bootstrap(Environment environment, SecureString password) throws Exception {
-        return KeyStoreWrapper.bootstrap(environment.configFile(), () -> password);
+        return KeyStoreWrapper.bootstrap(environment.configDir(), () -> password);
     }
 
     @Override

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java

@@ -150,7 +150,7 @@ class ServerCli extends EnvironmentAwareCommand {
             throw new UserException(ExitCodes.USAGE, "Multiple --enrollment-token parameters are not allowed");
         }
 
-        Path log4jConfig = env.configFile().resolve("log4j2.properties");
+        Path log4jConfig = env.configDir().resolve("log4j2.properties");
         if (Files.exists(log4jConfig) == false) {
             throw new UserException(ExitCodes.CONFIG, "Missing logging config file at " + log4jConfig);
         }
@@ -239,7 +239,7 @@ class ServerCli extends EnvironmentAwareCommand {
             }
             validatePidFile(pidFile);
         }
-        return new ServerArgs(daemonize, quiet, pidFile, secrets, env.settings(), env.configFile(), env.logsFile());
+        return new ServerArgs(daemonize, quiet, pidFile, secrets, env.settings(), env.configDir(), env.logsDir());
     }
 
     @Override

distribution/tools/windows-service-cli/src/main/java/org/elasticsearch/windows/service/WindowsServiceDaemon.java

@@ -43,8 +43,8 @@ class WindowsServiceDaemon extends EnvironmentAwareCommand {
     @Override
     public void execute(Terminal terminal, OptionSet options, Environment env, ProcessInfo processInfo) throws Exception {
         // the Windows service daemon doesn't support secure settings implementations other than the keystore
-        try (var loadedSecrets = KeyStoreWrapper.bootstrap(env.configFile(), () -> new SecureString(new char[0]))) {
+        try (var loadedSecrets = KeyStoreWrapper.bootstrap(env.configDir(), () -> new SecureString(new char[0]))) {
-            var args = new ServerArgs(false, true, null, loadedSecrets, env.settings(), env.configFile(), env.logsFile());
+            var args = new ServerArgs(false, true, null, loadedSecrets, env.settings(), env.configDir(), env.logsDir());
             var tempDir = ServerProcessUtils.setupTempDir(processInfo);
             var jvmOptions = JvmOptionsParser.determineJvmOptions(args, processInfo, tempDir, new MachineDependentHeap());
             var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal)

docs/changelog/119546.yaml

@@ -0,0 +1,5 @@
+pr: 119546
+summary: Introduce `FallbackSyntheticSourceBlockLoader` and apply it to keyword fields
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/121396.yaml

@@ -0,0 +1,5 @@
+pr: 121396
+summary: Change format for Unified Chat
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/121552.yaml

@@ -0,0 +1,5 @@
+pr: 121552
+summary: Fix a bug in TOP
+area: ES|QL
+type: bug
+issues: []

docs/changelog/121559.yaml

@@ -0,0 +1,6 @@
+pr: 121559
+summary: Skip Usage stats update when ML is disabled
+area: Machine Learning
+type: bug
+issues:
+ - 121532

docs/changelog/121568.yaml

@@ -0,0 +1,6 @@
+pr: 121568
+summary: Analyze API to return 400 for wrong custom analyzer
+area: Analysis
+type: bug
+issues:
+ - 121443

docs/changelog/121715.yaml

@@ -0,0 +1,5 @@
+pr: 121715
+summary: Fix synthetic source issue with deeply nested ignored source fields
+area: Mapping
+type: bug
+issues: []

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/EntitlementTest.java

@@ -20,6 +20,7 @@ public @interface EntitlementTest {
     enum ExpectedAccess {
         PLUGINS,
         ES_MODULES_ONLY,
+        SERVER_ONLY,
         ALWAYS_DENIED
     }
 

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java

@@ -13,18 +13,6 @@ import org.elasticsearch.client.internal.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyBreakIteratorProvider;
-import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyCalendarDataProvider;
-import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyCalendarNameProvider;
-import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyCollatorProvider;
-import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyCurrencyNameProvider;
-import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyDateFormatProvider;
-import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyDateFormatSymbolsProvider;
-import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyDecimalFormatSymbolsProvider;
-import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyLocaleNameProvider;
-import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyLocaleServiceProvider;
-import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyNumberFormatProvider;
-import org.elasticsearch.entitlement.qa.test.DummyImplementations.DummyTimeZoneNameProvider;
 import org.elasticsearch.logging.LogManager;
 import org.elasticsearch.logging.Logger;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -59,6 +47,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -75,7 +64,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET;
 @SuppressWarnings("unused")
 public class RestEntitlementsCheckAction extends BaseRestHandler {
     private static final Logger logger = LogManager.getLogger(RestEntitlementsCheckAction.class);
-    public static final Thread NO_OP_SHUTDOWN_HOOK = new Thread(() -> {}, "Shutdown hook for testing");
 
     record CheckAction(CheckedRunnable<Exception> action, boolean isAlwaysDeniedToPlugins, Integer fromJavaVersion) {
         /**
@@ -94,11 +82,8 @@ public class RestEntitlementsCheckAction extends BaseRestHandler {
         }
     }
 
-    private static final Map<String, CheckAction> checkActions = Stream.concat(
+    private static final Map<String, CheckAction> checkActions = Stream.of(
         Stream.<Entry<String, CheckAction>>of(
-            entry("runtime_exit", deniedToPlugins(RestEntitlementsCheckAction::runtimeExit)),
-            entry("runtime_halt", deniedToPlugins(RestEntitlementsCheckAction::runtimeHalt)),
-            entry("system_exit", deniedToPlugins(RestEntitlementsCheckAction::systemExit)),
             entry("create_classloader", forPlugins(RestEntitlementsCheckAction::createClassLoader)),
             entry("processBuilder_start", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_start)),
             entry("processBuilder_startPipeline", deniedToPlugins(RestEntitlementsCheckAction::processBuilder_startPipeline)),
@@ -106,27 +91,10 @@ public class RestEntitlementsCheckAction extends BaseRestHandler {
             entry("set_default_ssl_socket_factory", alwaysDenied(RestEntitlementsCheckAction::setDefaultSSLSocketFactory)),
             entry("set_default_hostname_verifier", alwaysDenied(RestEntitlementsCheckAction::setDefaultHostnameVerifier)),
             entry("set_default_ssl_context", alwaysDenied(RestEntitlementsCheckAction::setDefaultSSLContext)),
-            entry("system_setIn", alwaysDenied(RestEntitlementsCheckAction::system$$setIn)),
-            entry("system_setOut", alwaysDenied(RestEntitlementsCheckAction::system$$setOut)),
-            entry("system_setErr", alwaysDenied(RestEntitlementsCheckAction::system$$setErr)),
-            entry("runtime_addShutdownHook", alwaysDenied(RestEntitlementsCheckAction::runtime$addShutdownHook)),
-            entry("runtime_removeShutdownHook", alwaysDenied(RestEntitlementsCheckAction::runtime$$removeShutdownHook)),
             entry(
                 "thread_setDefaultUncaughtExceptionHandler",
                 alwaysDenied(RestEntitlementsCheckAction::thread$$setDefaultUncaughtExceptionHandler)
             ),
-            entry("localeServiceProvider", alwaysDenied(RestEntitlementsCheckAction::localeServiceProvider$)),
-            entry("breakIteratorProvider", alwaysDenied(RestEntitlementsCheckAction::breakIteratorProvider$)),
-            entry("collatorProvider", alwaysDenied(RestEntitlementsCheckAction::collatorProvider$)),
-            entry("dateFormatProvider", alwaysDenied(RestEntitlementsCheckAction::dateFormatProvider$)),
-            entry("dateFormatSymbolsProvider", alwaysDenied(RestEntitlementsCheckAction::dateFormatSymbolsProvider$)),
-            entry("decimalFormatSymbolsProvider", alwaysDenied(RestEntitlementsCheckAction::decimalFormatSymbolsProvider$)),
-            entry("numberFormatProvider", alwaysDenied(RestEntitlementsCheckAction::numberFormatProvider$)),
-            entry("calendarDataProvider", alwaysDenied(RestEntitlementsCheckAction::calendarDataProvider$)),
-            entry("calendarNameProvider", alwaysDenied(RestEntitlementsCheckAction::calendarNameProvider$)),
-            entry("currencyNameProvider", alwaysDenied(RestEntitlementsCheckAction::currencyNameProvider$)),
-            entry("localeNameProvider", alwaysDenied(RestEntitlementsCheckAction::localeNameProvider$)),
-            entry("timeZoneNameProvider", alwaysDenied(RestEntitlementsCheckAction::timeZoneNameProvider$)),
             entry("logManager", alwaysDenied(RestEntitlementsCheckAction::logManager$)),
 
             entry("locale_setDefault", alwaysDenied(WritePropertiesCheckActions::setDefaultLocale)),
@@ -230,8 +198,11 @@ public class RestEntitlementsCheckAction extends BaseRestHandler {
             entry("symbol_lookup_name", new CheckAction(VersionSpecificNativeChecks::symbolLookupWithName, false, 22)),
             entry("symbol_lookup_path", new CheckAction(VersionSpecificNativeChecks::symbolLookupWithPath, false, 22))
         ),
-        getTestEntries(FileCheckActions.class)
+        getTestEntries(FileCheckActions.class),
+        getTestEntries(SpiActions.class),
+        getTestEntries(SystemActions.class)
     )
+        .flatMap(Function.identity())
         .filter(entry -> entry.getValue().fromJavaVersion() == null || Runtime.version().feature() >= entry.getValue().fromJavaVersion())
         .collect(Collectors.toUnmodifiableMap(Entry::getKey, Entry::getValue));
 
@@ -267,7 +238,7 @@ public class RestEntitlementsCheckAction extends BaseRestHandler {
                     }
                 }
             };
-            boolean deniedToPlugins = testAnnotation.expectedAccess() == PLUGINS;
+            boolean deniedToPlugins = testAnnotation.expectedAccess() != PLUGINS;
             Integer fromJavaVersion = testAnnotation.fromJavaVersion() == -1 ? null : testAnnotation.fromJavaVersion();
             entries.add(entry(method.getName(), new CheckAction(runnable, deniedToPlugins, fromJavaVersion)));
         }
@@ -323,21 +294,6 @@ public class RestEntitlementsCheckAction extends BaseRestHandler {
         HttpsURLConnection.setDefaultSSLSocketFactory(new DummyImplementations.DummySSLSocketFactory());
     }
 
-    @SuppressForbidden(reason = "Specifically testing Runtime.exit")
-    private static void runtimeExit() {
-        Runtime.getRuntime().exit(123);
-    }
-
-    @SuppressForbidden(reason = "Specifically testing Runtime.halt")
-    private static void runtimeHalt() {
-        Runtime.getRuntime().halt(123);
-    }
-
-    @SuppressForbidden(reason = "Specifically testing System.exit")
-    private static void systemExit() {
-        System.exit(123);
-    }
-
     private static void createClassLoader() throws IOException {
         try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) {
             logger.info("Created URLClassLoader [{}]", classLoader.getName());
@@ -356,80 +312,10 @@ public class RestEntitlementsCheckAction extends BaseRestHandler {
         new DummyImplementations.DummyHttpsURLConnection().setSSLSocketFactory(new DummyImplementations.DummySSLSocketFactory());
     }
 
-    private static void system$$setIn() {
-        System.setIn(System.in);
-    }
-
-    @SuppressForbidden(reason = "This should be a no-op so we don't interfere with system streams")
-    private static void system$$setOut() {
-        System.setOut(System.out);
-    }
-
-    @SuppressForbidden(reason = "This should be a no-op so we don't interfere with system streams")
-    private static void system$$setErr() {
-        System.setErr(System.err);
-    }
-
-    private static void runtime$addShutdownHook() {
-        Runtime.getRuntime().addShutdownHook(NO_OP_SHUTDOWN_HOOK);
-    }
-
-    private static void runtime$$removeShutdownHook() {
-        Runtime.getRuntime().removeShutdownHook(NO_OP_SHUTDOWN_HOOK);
-    }
-
     private static void thread$$setDefaultUncaughtExceptionHandler() {
         Thread.setDefaultUncaughtExceptionHandler(Thread.getDefaultUncaughtExceptionHandler());
     }
 
-    private static void localeServiceProvider$() {
-        new DummyLocaleServiceProvider();
-    }
-
-    private static void breakIteratorProvider$() {
-        new DummyBreakIteratorProvider();
-    }
-
-    private static void collatorProvider$() {
-        new DummyCollatorProvider();
-    }
-
-    private static void dateFormatProvider$() {
-        new DummyDateFormatProvider();
-    }
-
-    private static void dateFormatSymbolsProvider$() {
-        new DummyDateFormatSymbolsProvider();
-    }
-
-    private static void decimalFormatSymbolsProvider$() {
-        new DummyDecimalFormatSymbolsProvider();
-    }
-
-    private static void numberFormatProvider$() {
-        new DummyNumberFormatProvider();
-    }
-
-    private static void calendarDataProvider$() {
-        new DummyCalendarDataProvider();
-    }
-
-    private static void calendarNameProvider$() {
-        new DummyCalendarNameProvider();
-    }
-
-    private static void currencyNameProvider$() {
-        new DummyCurrencyNameProvider();
-    }
-
-    private static void localeNameProvider$() {
-        new DummyLocaleNameProvider();
-    }
-
-    private static void timeZoneNameProvider$() {
-        new DummyTimeZoneNameProvider();
-    }
-
     private static void logManager$() {
         new java.util.logging.LogManager() {
         };

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/SpiActions.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.qa.test;
+
+import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED;
+
+class SpiActions {
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void createBreakIteratorProvider() {
+        new DummyImplementations.DummyBreakIteratorProvider();
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void createCollatorProvider() {
+        new DummyImplementations.DummyCollatorProvider();
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void createDateFormatProvider() {
+        new DummyImplementations.DummyDateFormatProvider();
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void createDateFormatSymbolsProvider() {
+        new DummyImplementations.DummyDateFormatSymbolsProvider();
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void createDecimalFormatSymbolsProvider() {
+        new DummyImplementations.DummyDecimalFormatSymbolsProvider();
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void createNumberFormatProvider() {
+        new DummyImplementations.DummyNumberFormatProvider();
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void createCalendarDataProvider() {
+        new DummyImplementations.DummyCalendarDataProvider();
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void createCalendarNameProvider() {
+        new DummyImplementations.DummyCalendarNameProvider();
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void createCurrencyNameProvider() {
+        new DummyImplementations.DummyCurrencyNameProvider();
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void createLocaleNameProvider() {
+        new DummyImplementations.DummyLocaleNameProvider();
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void createTimeZoneNameProvider() {
+        new DummyImplementations.DummyTimeZoneNameProvider();
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void createLocaleServiceProvider() {
+        new DummyImplementations.DummyLocaleServiceProvider();
+    }
+
+    private SpiActions() {}
+}

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/SystemActions.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.entitlement.qa.test;
+
+import org.elasticsearch.core.SuppressForbidden;
+
+import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.ALWAYS_DENIED;
+import static org.elasticsearch.entitlement.qa.test.EntitlementTest.ExpectedAccess.SERVER_ONLY;
+
+class SystemActions {
+
+    @SuppressForbidden(reason = "Specifically testing Runtime.exit")
+    @EntitlementTest(expectedAccess = SERVER_ONLY)
+    static void runtimeExit() {
+        Runtime.getRuntime().exit(123);
+    }
+
+    @SuppressForbidden(reason = "Specifically testing Runtime.halt")
+    @EntitlementTest(expectedAccess = SERVER_ONLY)
+    static void runtimeHalt() {
+        Runtime.getRuntime().halt(123);
+    }
+
+    @SuppressForbidden(reason = "Specifically testing System.exit")
+    @EntitlementTest(expectedAccess = SERVER_ONLY)
+    static void systemExit() {
+        System.exit(123);
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void systemSetIn() {
+        System.setIn(System.in);
+    }
+
+    @SuppressForbidden(reason = "This should be a no-op so we don't interfere with system streams")
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void systemSetOut() {
+        System.setOut(System.out);
+    }
+
+    @SuppressForbidden(reason = "This should be a no-op so we don't interfere with system streams")
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void systemSetErr() {
+        System.setErr(System.err);
+    }
+
+    private static final Thread NO_OP_SHUTDOWN_HOOK = new Thread(() -> {}, "Shutdown hook for testing");
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void runtimeAddShutdownHook() {
+        Runtime.getRuntime().addShutdownHook(NO_OP_SHUTDOWN_HOOK);
+    }
+
+    @EntitlementTest(expectedAccess = ALWAYS_DENIED)
+    static void runtimeRemoveShutdownHook() {
+        Runtime.getRuntime().removeShutdownHook(NO_OP_SHUTDOWN_HOOK);
+    }
+
+    private SystemActions() {}
+}

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/ExternalEntitlement.java

@@ -22,7 +22,7 @@ import java.lang.annotation.Target;
  * using this annotation is considered parseable as part of a policy file
  * for entitlements.
  */
-@Target(ElementType.CONSTRUCTOR)
+@Target({ ElementType.CONSTRUCTOR, ElementType.METHOD })
 @Retention(RetentionPolicy.RUNTIME)
 public @interface ExternalEntitlement {
 

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java

@@ -17,8 +17,11 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 
+import static org.elasticsearch.core.PathUtils.getDefaultFileSystem;
+
 public final class FileAccessTree {
     public static final FileAccessTree EMPTY = new FileAccessTree(List.of());
+    private static final String FILE_SEPARATOR = getDefaultFileSystem().getSeparator();
 
     private final String[] readPaths;
     private final String[] writePaths;
@@ -27,11 +30,11 @@ public final class FileAccessTree {
         List<String> readPaths = new ArrayList<>();
         List<String> writePaths = new ArrayList<>();
         for (FileEntitlement fileEntitlement : fileEntitlements) {
-            var mode = fileEntitlement.mode();
+            String path = normalizePath(Path.of(fileEntitlement.path()));
-            if (mode == FileEntitlement.Mode.READ_WRITE) {
+            if (fileEntitlement.mode() == FileEntitlement.Mode.READ_WRITE) {
-                writePaths.add(fileEntitlement.path());
+                writePaths.add(path);
             }
-            readPaths.add(fileEntitlement.path());
+            readPaths.add(path);
         }
 
         readPaths.sort(String::compareTo);
@@ -46,14 +49,20 @@ public final class FileAccessTree {
     }
 
     boolean canRead(Path path) {
-        return checkPath(normalize(path), readPaths);
+        return checkPath(normalizePath(path), readPaths);
     }
 
     boolean canWrite(Path path) {
-        return checkPath(normalize(path), writePaths);
+        return checkPath(normalizePath(path), writePaths);
     }
 
-    private static String normalize(Path path) {
+    /**
+     * @return the "canonical" form of the given {@code path}, to be used for entitlement checks.
+     */
+    static String normalizePath(Path path) {
+        // Note that toAbsolutePath produces paths separated by the default file separator,
+        // so on Windows, if the given path uses forward slashes, this consistently
+        // converts it to backslashes.
         return path.toAbsolutePath().normalize().toString();
     }
 
@@ -64,7 +73,7 @@ public final class FileAccessTree {
         int ndx = Arrays.binarySearch(paths, path);
         if (ndx < -1) {
             String maybeParent = paths[-ndx - 2];
-            return path.startsWith(maybeParent);
+            return path.startsWith(maybeParent) && path.startsWith(FILE_SEPARATOR, maybeParent.length());
         }
         return ndx >= 0;
     }

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java

@@ -27,6 +27,8 @@ import java.io.InputStream;
 import java.io.UncheckedIOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -147,6 +149,7 @@ public class PolicyParser {
         }
 
         Constructor<?> entitlementConstructor = null;
+        Method entitlementMethod = null;
         ExternalEntitlement entitlementMetadata = null;
         for (var ctor : entitlementClass.getConstructors()) {
             var metadata = ctor.getAnnotation(ExternalEntitlement.class);
@@ -161,8 +164,27 @@ public class PolicyParser {
                 entitlementConstructor = ctor;
                 entitlementMetadata = metadata;
             }
-
         }
+        for (var method : entitlementClass.getMethods()) {
+            var metadata = method.getAnnotation(ExternalEntitlement.class);
+            if (metadata != null) {
+                if (Modifier.isStatic(method.getModifiers()) == false) {
+                    throw new IllegalStateException(
+                        "entitlement class [" + entitlementClass.getName() + "] has non-static method annotated with ExternalEntitlement"
+                    );
+                }
+                if (entitlementMetadata != null) {
+                    throw new IllegalStateException(
+                        "entitlement class ["
+                            + entitlementClass.getName()
+                            + "] has more than one constructor and/or method annotated with ExternalEntitlement"
+                    );
+                }
+                entitlementMethod = method;
+                entitlementMetadata = metadata;
+            }
+        }
+
         if (entitlementMetadata == null) {
             throw newPolicyParserException(scopeName, "unknown entitlement type [" + entitlementType + "]");
         }
@@ -171,7 +193,9 @@ public class PolicyParser {
             throw newPolicyParserException("entitlement type [" + entitlementType + "] is allowed only on modules");
         }
 
-        Class<?>[] parameterTypes = entitlementConstructor.getParameterTypes();
+        Class<?>[] parameterTypes = entitlementConstructor != null
+            ? entitlementConstructor.getParameterTypes()
+            : entitlementMethod.getParameterTypes();
         String[] parametersNames = entitlementMetadata.parameterNames();
 
         if (parameterTypes.length != 0 || parametersNames.length != 0) {
@@ -204,7 +228,11 @@ public class PolicyParser {
         }
 
         try {
-            return (Entitlement) entitlementConstructor.newInstance(parameterValues);
+            if (entitlementConstructor != null) {
+                return (Entitlement) entitlementConstructor.newInstance(parameterValues);
+            } else {
+                return (Entitlement) entitlementMethod.invoke(null, parameterValues);
+            }
         } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) {
             if (e.getCause() instanceof PolicyValidationException piae) {
                 throw newPolicyParserException(startLocation, scopeName, entitlementType, piae);

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FileEntitlement.java

@@ -12,10 +12,12 @@ package org.elasticsearch.entitlement.runtime.policy.entitlements;
 import org.elasticsearch.entitlement.runtime.policy.ExternalEntitlement;
 import org.elasticsearch.entitlement.runtime.policy.PolicyValidationException;
 
-import java.nio.file.Paths;
-
 /**
- * Describes a file entitlement with a path and mode.
+ * Describes entitlement to access files at a particular location.
+ *
+ * @param path the location of the files. For directories, implicitly includes access to
+ *            all contained files and (recursively) subdirectories.
+ * @param mode the type of operation
  */
 public record FileEntitlement(String path, Mode mode) implements Entitlement {
 
@@ -24,14 +26,6 @@ public record FileEntitlement(String path, Mode mode) implements Entitlement {
         READ_WRITE
     }
 
-    public FileEntitlement {
-        path = normalizePath(path);
-    }
-
-    private static String normalizePath(String path) {
-        return Paths.get(path).toAbsolutePath().normalize().toString();
-    }
-
     private static Mode parseMode(String mode) {
         if (mode.equals("read")) {
             return Mode.READ;
@@ -43,7 +37,7 @@ public record FileEntitlement(String path, Mode mode) implements Entitlement {
     }
 
     @ExternalEntitlement(parameterNames = { "path", "mode" }, esModulesOnly = false)
-    public FileEntitlement(String path, String mode) {
+    public static FileEntitlement create(String path, String mode) {
-        this(path, parseMode(mode));
+        return new FileEntitlement(path, parseMode(mode));
     }
 }

libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java

@@ -16,6 +16,7 @@ import org.junit.BeforeClass;
 import java.nio.file.Path;
 import java.util.List;
 
+import static org.elasticsearch.core.PathUtils.getDefaultFileSystem;
 import static org.hamcrest.Matchers.is;
 
 public class FileAccessTreeTests extends ESTestCase {
@@ -41,7 +42,9 @@ public class FileAccessTreeTests extends ESTestCase {
         var tree = FileAccessTree.of(List.of(entitlement("foo", "read")));
         assertThat(tree.canRead(path("foo")), is(true));
         assertThat(tree.canRead(path("foo/subdir")), is(true));
+        assertThat(tree.canRead(path("food")), is(false));
         assertThat(tree.canWrite(path("foo")), is(false));
+        assertThat(tree.canWrite(path("food")), is(false));
 
         assertThat(tree.canRead(path("before")), is(false));
         assertThat(tree.canRead(path("later")), is(false));
@@ -51,7 +54,9 @@ public class FileAccessTreeTests extends ESTestCase {
         var tree = FileAccessTree.of(List.of(entitlement("foo", "read_write")));
         assertThat(tree.canWrite(path("foo")), is(true));
         assertThat(tree.canWrite(path("foo/subdir")), is(true));
+        assertThat(tree.canWrite(path("food")), is(false));
         assertThat(tree.canRead(path("foo")), is(true));
+        assertThat(tree.canRead(path("food")), is(false));
 
         assertThat(tree.canWrite(path("before")), is(false));
         assertThat(tree.canWrite(path("later")), is(false));
@@ -83,8 +88,24 @@ public class FileAccessTreeTests extends ESTestCase {
         assertThat(tree.canRead(path("")), is(false));
     }
 
+    public void testForwardSlashes() {
+        String sep = getDefaultFileSystem().getSeparator();
+        var tree = FileAccessTree.of(List.of(entitlement("a/b", "read"), entitlement("m" + sep + "n", "read")));
+
+        // Native separators work
+        assertThat(tree.canRead(path("a" + sep + "b")), is(true));
+        assertThat(tree.canRead(path("m" + sep + "n")), is(true));
+
+        // Forward slashes also work
+        assertThat(tree.canRead(path("a/b")), is(true));
+        assertThat(tree.canRead(path("m/n")), is(true));
+
+        // In case the native separator is a backslash, don't treat that as an escape
+        assertThat(tree.canRead(path("m\n")), is(false));
+    }
+
     FileEntitlement entitlement(String path, String mode) {
         Path p = path(path);
-        return new FileEntitlement(p.toString(), mode);
+        return FileEntitlement.create(p.toString(), mode);
     }
 }

libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java

@@ -40,6 +40,35 @@ public class PolicyParserTests extends ESTestCase {
         public ManyConstructorsEntitlement(int i) {}
     }
 
+    public static class ManyMethodsEntitlement implements Entitlement {
+        @ExternalEntitlement
+        public static ManyMethodsEntitlement create(String s) {
+            return new ManyMethodsEntitlement();
+        }
+
+        @ExternalEntitlement
+        public static ManyMethodsEntitlement create(int i) {
+            return new ManyMethodsEntitlement();
+        }
+    }
+
+    public static class ConstructorAndMethodEntitlement implements Entitlement {
+        @ExternalEntitlement
+        public static ConstructorAndMethodEntitlement create(String s) {
+            return new ConstructorAndMethodEntitlement(s);
+        }
+
+        @ExternalEntitlement
+        public ConstructorAndMethodEntitlement(String s) {}
+    }
+
+    public static class NonStaticMethodEntitlement implements Entitlement {
+        @ExternalEntitlement
+        public NonStaticMethodEntitlement create() {
+            return new NonStaticMethodEntitlement();
+        }
+    }
+
     public void testGetEntitlementTypeName() {
         assertEquals("create_class_loader", PolicyParser.getEntitlementTypeName(CreateClassLoaderEntitlement.class));
 
@@ -55,7 +84,7 @@ public class PolicyParserTests extends ESTestCase {
             .parsePolicy();
         Policy expected = new Policy(
             "test-policy.yaml",
-            List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", "read_write"))))
+            List.of(new Scope("entitlement-module-name", List.of(FileEntitlement.create("test/path/to/file", "read_write"))))
         );
         assertEquals(expected, parsedPolicy);
     }
@@ -65,7 +94,7 @@ public class PolicyParserTests extends ESTestCase {
             .parsePolicy();
         Policy expected = new Policy(
             "test-policy.yaml",
-            List.of(new Scope("entitlement-module-name", List.of(new FileEntitlement("test/path/to/file", "read_write"))))
+            List.of(new Scope("entitlement-module-name", List.of(FileEntitlement.create("test/path/to/file", "read_write"))))
         );
         assertEquals(expected, parsedPolicy);
     }
@@ -174,4 +203,60 @@ public class PolicyParserTests extends ESTestCase {
             )
         );
     }
+
+    public void testMultipleMethodsAnnotated() throws IOException {
+        var parser = new PolicyParser(new ByteArrayInputStream("""
+            entitlement-module-name:
+              - many_methods
+            """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", true, Map.of("many_methods", ManyMethodsEntitlement.class));
+
+        var e = expectThrows(IllegalStateException.class, parser::parsePolicy);
+        assertThat(
+            e.getMessage(),
+            equalTo(
+                "entitlement class "
+                    + "[org.elasticsearch.entitlement.runtime.policy.PolicyParserTests$ManyMethodsEntitlement]"
+                    + " has more than one constructor and/or method annotated with ExternalEntitlement"
+            )
+        );
+    }
+
+    public void testConstructorAndMethodAnnotated() throws IOException {
+        var parser = new PolicyParser(
+            new ByteArrayInputStream("""
+                entitlement-module-name:
+                  - constructor_and_method
+                """.getBytes(StandardCharsets.UTF_8)),
+            "test-policy.yaml",
+            true,
+            Map.of("constructor_and_method", ConstructorAndMethodEntitlement.class)
+        );
+
+        var e = expectThrows(IllegalStateException.class, parser::parsePolicy);
+        assertThat(
+            e.getMessage(),
+            equalTo(
+                "entitlement class "
+                    + "[org.elasticsearch.entitlement.runtime.policy.PolicyParserTests$ConstructorAndMethodEntitlement]"
+                    + " has more than one constructor and/or method annotated with ExternalEntitlement"
+            )
+        );
+    }
+
+    public void testNonStaticMethodAnnotated() throws IOException {
+        var parser = new PolicyParser(new ByteArrayInputStream("""
+            entitlement-module-name:
+              - non_static
+            """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", true, Map.of("non_static", NonStaticMethodEntitlement.class));
+
+        var e = expectThrows(IllegalStateException.class, parser::parsePolicy);
+        assertThat(
+            e.getMessage(),
+            equalTo(
+                "entitlement class "
+                    + "[org.elasticsearch.entitlement.runtime.policy.PolicyParserTests$NonStaticMethodEntitlement]"
+                    + " has non-static method annotated with ExternalEntitlement"
+            )
+        );
+    }
 }

modules/analysis-common/build.gradle

@@ -20,7 +20,7 @@ esplugin {
 
 restResources {
   restApi {
-    include '_common', 'indices', 'index', 'cluster', 'search', 'nodes', 'bulk', 'termvectors', 'explain', 'count'
+    include '_common', 'indices', 'index', 'cluster', 'search', 'nodes', 'bulk', 'termvectors', 'explain', 'count', 'capabilities'
   }
 }
 

modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadAnalyzerTests.java

@@ -207,7 +207,7 @@ public class ReloadAnalyzerTests extends ESSingleNodeTestCase {
     public void testUpdateableSynonymsRejectedAtIndexTime() throws FileNotFoundException, IOException {
         String synonymsFileName = "synonyms.txt";
         setupResourceFile(synonymsFileName, "foo, baz");
-        Path configDir = node().getEnvironment().configFile();
+        Path configDir = node().getEnvironment().configDir();
         if (Files.exists(configDir) == false) {
             Files.createDirectory(configDir);
         }
@@ -319,7 +319,7 @@ public class ReloadAnalyzerTests extends ESSingleNodeTestCase {
     }
 
     private Path setupResourceFile(String fileName, String... content) throws IOException {
-        Path configDir = node().getEnvironment().configFile();
+        Path configDir = node().getEnvironment().configDir();
         if (Files.exists(configDir) == false) {
             Files.createDirectory(configDir);
         }

modules/analysis-common/src/internalClusterTest/java/org/elasticsearch/analysis/common/ReloadSynonymAnalyzerIT.java

@@ -57,7 +57,7 @@ public class ReloadSynonymAnalyzerIT extends ESIntegTestCase {
     }
 
     private void testSynonymsUpdate(boolean preview) throws FileNotFoundException, IOException, InterruptedException {
-        Path config = internalCluster().getInstance(Environment.class).configFile();
+        Path config = internalCluster().getInstance(Environment.class).configDir();
         String synonymsFileName = "synonyms.txt";
         Path synonymsFile = config.resolve(synonymsFileName);
         writeFile(synonymsFile, "foo, baz");
@@ -106,7 +106,7 @@ public class ReloadSynonymAnalyzerIT extends ESIntegTestCase {
         final String synonymsFileName = "synonyms.txt";
         final String fieldName = "field";
 
-        Path config = internalCluster().getInstance(Environment.class).configFile();
+        Path config = internalCluster().getInstance(Environment.class).configDir();
         Path synonymsFile = config.resolve(synonymsFileName);
         writeFile(synonymsFile, "foo, baz");
 

modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/HyphenationCompoundWordTokenFilterFactory.java

@@ -40,7 +40,7 @@ public class HyphenationCompoundWordTokenFilterFactory extends AbstractCompoundW
             throw new IllegalArgumentException("hyphenation_patterns_path is a required setting.");
         }
 
-        Path hyphenationPatternsFile = env.configFile().resolve(hyphenationPatternsPath);
+        Path hyphenationPatternsFile = env.configDir().resolve(hyphenationPatternsPath);
 
         try {
             InputStream in = Files.newInputStream(hyphenationPatternsFile);

modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/indices.analyze/15_analyze.yml

@@ -59,3 +59,28 @@
     - match:  { detail.tokenizer.tokens.0.token: ABc }
     - match:  { detail.tokenfilters.0.name: lowercase }
     - match:  { detail.tokenfilters.0.tokens.0.token: abc }
+
+---
+"Custom analyzer is not buildable":
+  - requires:
+      test_runner_features: [ capabilities ]
+      reason: This capability required to run test
+      capabilities:
+        - method: GET
+          path: /_analyze
+          capabilities: [ wrong_custom_analyzer_returns_400 ]
+
+  - do:
+      catch: bad_request
+      indices.analyze:
+        body:
+          text: the foxes jumping quickly
+          tokenizer:
+            standard
+          filter:
+            type: hunspell
+            locale: en_US
+
+  - match: { status: 400 }
+  - match: { error.type: illegal_argument_exception }
+  - match: { error.reason: "Can not build a custom analyzer" }

modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java

@@ -251,7 +251,7 @@ public class DataStreamGetWriteIndexTests extends ESTestCase {
         MetadataCreateIndexService createIndexService;
         {
             Environment env = mock(Environment.class);
-            when(env.sharedDataFile()).thenReturn(null);
+            when(env.sharedDataDir()).thenReturn(null);
             AllocationService allocationService = mock(AllocationService.class);
             when(allocationService.reroute(any(ClusterState.class), any(String.class), any())).then(i -> i.getArguments()[0]);
             when(allocationService.getShardRoutingRoleStrategy()).thenReturn(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);

modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java

@@ -664,7 +664,7 @@ public class GeoIpDownloaderIT extends AbstractGeoIpIT {
             .map(DiscoveryNode::getId)
             .collect(Collectors.toSet());
         // All nodes share the same geoip base dir in the shared tmp dir:
-        Path geoipBaseTmpDir = internalCluster().getDataNodeInstance(Environment.class).tmpFile().resolve("geoip-databases");
+        Path geoipBaseTmpDir = internalCluster().getDataNodeInstance(Environment.class).tmpDir().resolve("geoip-databases");
         assertThat(Files.exists(geoipBaseTmpDir), is(true));
         final List<Path> geoipTmpDirs;
         try (Stream<Path> files = Files.list(geoipBaseTmpDir)) {
@@ -676,7 +676,7 @@ public class GeoIpDownloaderIT extends AbstractGeoIpIT {
 
     private void setupDatabasesInConfigDirectory() throws Exception {
         StreamSupport.stream(internalCluster().getInstances(Environment.class).spliterator(), false)
-            .map(Environment::configFile)
+            .map(Environment::configDir)
             .map(path -> path.resolve("ingest-geoip"))
             .distinct()
             .forEach(path -> {
@@ -704,7 +704,7 @@ public class GeoIpDownloaderIT extends AbstractGeoIpIT {
 
     private void deleteDatabasesInConfigDirectory() throws Exception {
         StreamSupport.stream(internalCluster().getInstances(Environment.class).spliterator(), false)
-            .map(Environment::configFile)
+            .map(Environment::configDir)
             .map(path -> path.resolve("ingest-geoip"))
             .distinct()
             .forEach(path -> {

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/ConfigDatabases.java

@@ -42,7 +42,7 @@ final class ConfigDatabases implements Closeable {
     private final ConcurrentMap<String, DatabaseReaderLazyLoader> configDatabases;
 
     ConfigDatabases(Environment environment, GeoIpCache cache) {
-        this(environment.configFile().resolve("ingest-geoip"), cache);
+        this(environment.configDir().resolve("ingest-geoip"), cache);
     }
 
     ConfigDatabases(Path geoipConfigDir, GeoIpCache cache) {

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseNodeService.java

@@ -116,7 +116,7 @@ public final class DatabaseNodeService implements IpDatabaseProvider {
         ClusterService clusterService
     ) {
         this(
-            environment.tmpFile(),
+            environment.tmpDir(),
             new OriginSettingClient(client, IngestService.INGEST_ORIGIN),
             cache,
             new ConfigDatabases(environment, cache),

modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java

@@ -41,7 +41,7 @@ public class IngestUserAgentPlugin extends Plugin implements IngestPlugin {
 
     @Override
     public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
-        Path userAgentConfigDirectory = parameters.env.configFile().resolve("ingest-user-agent");
+        Path userAgentConfigDirectory = parameters.env.configDir().resolve("ingest-user-agent");
 
         if (Files.exists(userAgentConfigDirectory) == false && Files.isDirectory(userAgentConfigDirectory)) {
             throw new IllegalStateException(

modules/reindex/src/main/java/org/elasticsearch/reindex/ReindexSslConfig.java

@@ -106,7 +106,7 @@ public class ReindexSslConfig {
                 return settings.getAsList(key);
             }
         };
-        configuration = loader.load(environment.configFile());
+        configuration = loader.load(environment.configDir());
         reload();
 
         final FileChangesListener listener = new FileChangesListener() {

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -369,7 +369,7 @@ class S3Service implements Closeable {
             }
             // Make sure that a readable symlink to the token file exists in the plugin config directory
             // AWS_WEB_IDENTITY_TOKEN_FILE exists but we only use Web Identity Tokens if a corresponding symlink exists and is readable
-            Path webIdentityTokenFileSymlink = environment.configFile().resolve(WEB_IDENTITY_TOKEN_FILE_LOCATION);
+            Path webIdentityTokenFileSymlink = environment.configDir().resolve(WEB_IDENTITY_TOKEN_FILE_LOCATION);
             if (Files.exists(webIdentityTokenFileSymlink) == false) {
                 LOGGER.warn(
                     "Cannot use AWS Web Identity Tokens: AWS_WEB_IDENTITY_TOKEN_FILE is defined but no corresponding symlink exists "

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/CustomWebIdentityTokenCredentialsProviderTests.java

@@ -65,7 +65,7 @@ public class CustomWebIdentityTokenCredentialsProviderTests extends ESTestCase {
         Files.createDirectory(configDirectory.resolve("repository-s3"));
         Files.writeString(configDirectory.resolve("repository-s3/aws-web-identity-token-file"), "YXdzLXdlYi1pZGVudGl0eS10b2tlbi1maWxl");
         Environment environment = Mockito.mock(Environment.class);
-        Mockito.when(environment.configFile()).thenReturn(configDirectory);
+        Mockito.when(environment.configDir()).thenReturn(configDirectory);
         return environment;
     }
 
@@ -212,7 +212,7 @@ public class CustomWebIdentityTokenCredentialsProviderTests extends ESTestCase {
                     latch.countDown();
                 }
             });
-            Files.writeString(environment.configFile().resolve("repository-s3/aws-web-identity-token-file"), newWebIdentityToken);
+            Files.writeString(environment.configDir().resolve("repository-s3/aws-web-identity-token-file"), newWebIdentityToken);
 
             safeAwait(latch);
             assertCredentials(awsCredentialsProvider.getCredentials());

modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java

@@ -158,7 +158,7 @@ public class URLRepository extends BlobStoreRepository {
                 if (normalizedUrl == null) {
                     String logMessage = "The specified url [{}] doesn't start with any repository paths specified by the "
                         + "path.repo setting or by {} setting: [{}] ";
-                    logger.warn(logMessage, urlToCheck, ALLOWED_URLS_SETTING.getKey(), environment.repoFiles());
+                    logger.warn(logMessage, urlToCheck, ALLOWED_URLS_SETTING.getKey(), environment.repoDirs());
                     String exceptionMessage = "file url ["
                         + urlToCheck
                         + "] doesn't match any of the locations specified by path.repo or "

muted-tests.yml

@@ -134,6 +134,12 @@ tests:
 - class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
   method: test {p0=data_stream/120_data_streams_stats/Multiple data stream}
   issue: https://github.com/elastic/elasticsearch/issues/118217
+  # TODO: re-enable after backporting https://github.com/elastic/elasticsearch/pull/119110
+- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
+  method: test {yaml=update/100_synthetic_source/keyword}
+  # TODO: re-enable after backporting https://github.com/elastic/elasticsearch/pull/119110
+- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
+  method: test {yaml=update/100_synthetic_source/stored text}
 - class: org.elasticsearch.xpack.searchablesnapshots.RetrySearchIntegTests
   method: testSearcherId
   issue: https://github.com/elastic/elasticsearch/issues/118374
@@ -385,9 +391,11 @@ tests:
 - class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
   method: testStopQueryLocal
   issue: https://github.com/elastic/elasticsearch/issues/121672
-- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
+- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
-  method: testLookupExplosionBigStringManyMatches
+  issue: https://github.com/elastic/elasticsearch/issues/121411
-  issue: https://github.com/elastic/elasticsearch/issues/121465
+- class: org.elasticsearch.transport.InboundHandlerTests
+  method: testLogsSlowInboundProcessing
+  issue: https://github.com/elastic/elasticsearch/issues/121816
 
 # Examples:
 #

plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuCollationTokenFilterFactory.java

@@ -51,7 +51,7 @@ public class IcuCollationTokenFilterFactory extends AbstractTokenFilterFactory {
         if (rules != null) {
             Exception failureToResolve = null;
             try {
-                rules = Streams.copyToString(Files.newBufferedReader(environment.configFile().resolve(rules), Charset.forName("UTF-8")));
+                rules = Streams.copyToString(Files.newBufferedReader(environment.configDir().resolve(rules), Charset.forName("UTF-8")));
             } catch (IOException | SecurityException | InvalidPathException e) {
                 failureToResolve = e;
             }

plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/IcuTokenizerFactory.java

@@ -99,7 +99,7 @@ public class IcuTokenizerFactory extends AbstractTokenizerFactory {
     // parse a single RBBi rule file
     private static BreakIterator parseRules(String filename, Environment env) throws IOException {
 
-        final Path path = env.configFile().resolve(filename);
+        final Path path = env.configDir().resolve(filename);
         String rules = Files.readAllLines(path).stream().filter((v) -> v.startsWith("#") == false).collect(Collectors.joining("\n"));
 
         return new RuleBasedBreakIterator(rules.toString());

plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java

@@ -81,7 +81,7 @@ class HdfsSecurityContext {
      * Expects keytab file to exist at {@code $CONFIG_DIR$/repository-hdfs/krb5.keytab}
      */
     static Path locateKeytabFile(Environment environment) {
-        Path keytabPath = environment.configFile().resolve("repository-hdfs").resolve("krb5.keytab");
+        Path keytabPath = environment.configDir().resolve("repository-hdfs").resolve("krb5.keytab");
         try {
             if (Files.exists(keytabPath) == false) {
                 throw new RuntimeException("Could not locate keytab at [" + keytabPath + "].");

qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilSecurityTests.java

@@ -103,23 +103,23 @@ public class EvilSecurityTests extends ESTestCase {
         // check that all directories got permissions:
 
         // bin file: ro
-        assertExactPermissions(new FilePermission(environment.binFile().toString(), "read,readlink"), permissions);
+        assertExactPermissions(new FilePermission(environment.binDir().toString(), "read,readlink"), permissions);
         // lib file: ro
-        assertExactPermissions(new FilePermission(environment.libFile().toString(), "read,readlink"), permissions);
+        assertExactPermissions(new FilePermission(environment.libDir().toString(), "read,readlink"), permissions);
         // modules file: ro
-        assertExactPermissions(new FilePermission(environment.modulesFile().toString(), "read,readlink"), permissions);
+        assertExactPermissions(new FilePermission(environment.modulesDir().toString(), "read,readlink"), permissions);
         // config file: ro
-        assertExactPermissions(new FilePermission(environment.configFile().toString(), "read,readlink"), permissions);
+        assertExactPermissions(new FilePermission(environment.configDir().toString(), "read,readlink"), permissions);
         // plugins: ro
-        assertExactPermissions(new FilePermission(environment.pluginsFile().toString(), "read,readlink"), permissions);
+        assertExactPermissions(new FilePermission(environment.pluginsDir().toString(), "read,readlink"), permissions);
 
         // data paths: r/w
-        for (Path dataPath : environment.dataFiles()) {
+        for (Path dataPath : environment.dataDirs()) {
             assertExactPermissions(new FilePermission(dataPath.toString(), "read,readlink,write,delete"), permissions);
         }
-        assertExactPermissions(new FilePermission(environment.sharedDataFile().toString(), "read,readlink,write,delete"), permissions);
+        assertExactPermissions(new FilePermission(environment.sharedDataDir().toString(), "read,readlink,write,delete"), permissions);
         // logs: r/w
-        assertExactPermissions(new FilePermission(environment.logsFile().toString(), "read,readlink,write,delete"), permissions);
+        assertExactPermissions(new FilePermission(environment.logsDir().toString(), "read,readlink,write,delete"), permissions);
         // temp dir: r/w
         assertExactPermissions(new FilePermission(fakeTmpDir.toString(), "read,readlink,write,delete"), permissions);
     }

qa/no-bootstrap-tests/src/test/java/org/elasticsearch/bootstrap/SpawnerNoBootstrapTests.java

@@ -80,8 +80,8 @@ public class SpawnerNoBootstrapTests extends LuceneTestCase {
         Environment environment = TestEnvironment.newEnvironment(settings);
 
         // This plugin will NOT have a controller daemon
-        Path plugin = environment.modulesFile().resolve("a_plugin");
+        Path plugin = environment.modulesDir().resolve("a_plugin");
-        Files.createDirectories(environment.modulesFile());
+        Files.createDirectories(environment.modulesDir());
         Files.createDirectories(plugin);
         PluginTestUtil.writePluginProperties(
             plugin,
@@ -111,8 +111,8 @@ public class SpawnerNoBootstrapTests extends LuceneTestCase {
      * Two plugins - one with a controller daemon and one without.
      */
     public void testControllerSpawn() throws Exception {
-        assertControllerSpawns(Environment::pluginsFile, false);
+        assertControllerSpawns(Environment::pluginsDir, false);
-        assertControllerSpawns(Environment::modulesFile, true);
+        assertControllerSpawns(Environment::modulesDir, true);
     }
 
     private void assertControllerSpawns(final Function<Environment, Path> pluginsDirFinder, boolean expectSpawn) throws Exception {
@@ -131,8 +131,8 @@ public class SpawnerNoBootstrapTests extends LuceneTestCase {
 
         // this plugin will have a controller daemon
         Path plugin = pluginsDirFinder.apply(environment).resolve("test_plugin");
-        Files.createDirectories(environment.modulesFile());
+        Files.createDirectories(environment.modulesDir());
-        Files.createDirectories(environment.pluginsFile());
+        Files.createDirectories(environment.pluginsDir());
         Files.createDirectories(plugin);
         PluginTestUtil.writePluginProperties(
             plugin,
@@ -217,7 +217,7 @@ public class SpawnerNoBootstrapTests extends LuceneTestCase {
 
         Environment environment = TestEnvironment.newEnvironment(settings);
 
-        Path plugin = environment.modulesFile().resolve("test_plugin");
+        Path plugin = environment.modulesDir().resolve("test_plugin");
         Files.createDirectories(plugin);
         PluginTestUtil.writePluginProperties(
             plugin,
@@ -250,10 +250,10 @@ public class SpawnerNoBootstrapTests extends LuceneTestCase {
 
         final Environment environment = TestEnvironment.newEnvironment(settings);
 
-        Files.createDirectories(environment.modulesFile());
+        Files.createDirectories(environment.modulesDir());
-        Files.createDirectories(environment.pluginsFile());
+        Files.createDirectories(environment.pluginsDir());
 
-        final Path desktopServicesStore = environment.modulesFile().resolve(".DS_Store");
+        final Path desktopServicesStore = environment.modulesDir().resolve(".DS_Store");
         Files.createFile(desktopServicesStore);
 
         final Spawner spawner = new Spawner();

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java

@@ -82,8 +82,7 @@ public class DesiredNodesUpgradeIT extends AbstractRollingUpgradeTestCase {
                         Settings.builder().put(NODE_NAME_SETTING.getKey(), nodeName).build(),
                         randomDoubleProcessorCount(),
                         ByteSizeValue.ofGb(randomIntBetween(10, 24)),
-                        ByteSizeValue.ofGb(randomIntBetween(128, 256)),
+                        ByteSizeValue.ofGb(randomIntBetween(128, 256))
-                        null
                     )
                 )
                 .toList();
@@ -94,8 +93,7 @@ public class DesiredNodesUpgradeIT extends AbstractRollingUpgradeTestCase {
                     Settings.builder().put(NODE_NAME_SETTING.getKey(), nodeName).build(),
                     new DesiredNode.ProcessorsRange(minProcessors, minProcessors + randomIntBetween(10, 20)),
                     ByteSizeValue.ofGb(randomIntBetween(10, 24)),
-                    ByteSizeValue.ofGb(randomIntBetween(128, 256)),
+                    ByteSizeValue.ofGb(randomIntBetween(128, 256))
-                    null
                 );
             }).toList();
         }

qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/snapshots/RestGetSnapshotsIT.java

@@ -10,7 +10,6 @@
 package org.elasticsearch.http.snapshots;
 
 import org.apache.http.client.methods.HttpGet;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionFuture;
 import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
@@ -37,7 +36,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -516,10 +514,9 @@ public class RestGetSnapshotsIT extends AbstractSnapshotRestTestCase {
         true,
         (args) -> new GetSnapshotsResponse(
             (List<SnapshotInfo>) args[0],
-            (Map<String, ElasticsearchException>) args[1],
+            (String) args[1],
-            (String) args[2],
+            args[2] == null ? UNKNOWN_COUNT : (int) args[2],
-            args[3] == null ? UNKNOWN_COUNT : (int) args[3],
+            args[3] == null ? UNKNOWN_COUNT : (int) args[3]
-            args[4] == null ? UNKNOWN_COUNT : (int) args[4]
         )
     );
 
@@ -529,11 +526,6 @@ public class RestGetSnapshotsIT extends AbstractSnapshotRestTestCase {
             (p, c) -> SnapshotInfoUtils.snapshotInfoFromXContent(p),
             new ParseField("snapshots")
         );
-        GET_SNAPSHOT_PARSER.declareObject(
-            ConstructingObjectParser.optionalConstructorArg(),
-            (p, c) -> p.map(HashMap::new, ElasticsearchException::fromXContent),
-            new ParseField("failures")
-        );
         GET_SNAPSHOT_PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), new ParseField("next"));
         GET_SNAPSHOT_PARSER.declareIntOrNull(ConstructingObjectParser.optionalConstructorArg(), UNKNOWN_COUNT, new ParseField("total"));
         GET_SNAPSHOT_PARSER.declareIntOrNull(ConstructingObjectParser.optionalConstructorArg(), UNKNOWN_COUNT, new ParseField("remaining"));

qa/smoke-test-multinode/build.gradle

@@ -28,5 +28,7 @@ tasks.named("yamlRestTest").configure {
     'cat.templates/10_basic/No templates',
     'cat.templates/10_basic/Sort templates',
     'cat.templates/10_basic/Multiple template',
+    'update/100_synthetic_source/keyword',
+    'update/100_synthetic_source/stored text'
   ].join(',')
 }

renovate.json

@@ -30,8 +30,8 @@
       "matchDatasources": [
         "docker"
       ],
-      "matchPackagePatterns": [
+      "matchPackageNames": [
-        "^docker.elastic.co/wolfi/chainguard-base$"
+        "/^docker.elastic.co/wolfi/chainguard-base$/"
       ]
     }
   ],

rest-api-spec/build.gradle

@@ -74,4 +74,12 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("index/91_metrics_no_subobjects/Metrics object indexing with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
   task.skipTest("index/91_metrics_no_subobjects/Root without subobjects with synthetic source", "_source.mode mapping attribute is no-op since 9.0.0")
   task.skipTest("indices.create/20_synthetic_source/synthetic_source with copy_to inside nested object", "temporary until backported")
+  task.skipTest(
+    "cluster.desired_nodes/10_basic/Test delete desired nodes with node_version generates a warning",
+    "node_version warning is removed in 9.0"
+  )
+  task.skipTest(
+    "cluster.desired_nodes/10_basic/Test update desired nodes with node_version generates a warning",
+    "node_version warning is removed in 9.0"
+  )
 })

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/10_basic.yml

@@ -59,61 +59,6 @@ teardown:
   - contains: { nodes: { settings: { node: { name: "instance-000187" } }, processors: 8.5, memory: "64gb", storage: "128gb" } }
   - contains: { nodes: { settings: { node: { name: "instance-000188" } }, processors: 16.0, memory: "128gb", storage: "1tb" } }
 ---
-"Test update desired nodes with node_version generates a warning":
-  - skip:
-      reason: "contains is a newly added assertion"
-      features: ["contains", "allowed_warnings"]
-  - do:
-      cluster.state: {}
-
-  # Get master node id
-  - set: { master_node: master }
-
-  - do:
-      nodes.info: {}
-  - set: { nodes.$master.version: es_version }
-
-  - do:
-      _internal.update_desired_nodes:
-        history_id: "test"
-        version: 1
-        body:
-            nodes:
-              - { settings: { "node.name": "instance-000187" }, processors: 8.5, memory: "64gb", storage: "128gb", node_version: $es_version }
-      allowed_warnings:
-        - "[version removal] Specifying node_version in desired nodes requests is deprecated."
-  - match: { replaced_existing_history_id: false }
-
-  - do:
-      _internal.get_desired_nodes: {}
-  - match:
-      $body:
-        history_id: "test"
-        version: 1
-        nodes:
-          - { settings: { node: { name: "instance-000187" } }, processors: 8.5, memory: "64gb", storage: "128gb", node_version: $es_version }
-
-  - do:
-      _internal.update_desired_nodes:
-        history_id: "test"
-        version: 2
-        body:
-          nodes:
-            - { settings: { "node.name": "instance-000187" }, processors: 8.5, memory: "64gb", storage: "128gb", node_version: $es_version }
-            - { settings: { "node.name": "instance-000188" }, processors: 16.0, memory: "128gb", storage: "1tb", node_version: $es_version }
-      allowed_warnings:
-        - "[version removal] Specifying node_version in desired nodes requests is deprecated."
-  - match: { replaced_existing_history_id: false }
-
-  - do:
-      _internal.get_desired_nodes: {}
-
-  - match: { history_id: "test" }
-  - match: { version: 2 }
-  - length: { nodes: 2 }
-  - contains: { nodes: { settings: { node: { name: "instance-000187" } }, processors: 8.5, memory: "64gb", storage: "128gb", node_version: $es_version } }
-  - contains: { nodes: { settings: { node: { name: "instance-000188" } }, processors: 16.0, memory: "128gb", storage: "1tb", node_version: $es_version } }
----
 "Test update move to a new history id":
   - skip:
       reason: "contains is a newly added assertion"
@@ -199,46 +144,6 @@ teardown:
       _internal.get_desired_nodes: {}
   - match: { status: 404 }
 ---
-"Test delete desired nodes with node_version generates a warning":
-  - skip:
-      features: allowed_warnings
-  - do:
-      cluster.state: {}
-
-  - set: { master_node: master }
-
-  - do:
-      nodes.info: {}
-  - set: { nodes.$master.version: es_version }
-
-  - do:
-      _internal.update_desired_nodes:
-        history_id: "test"
-        version: 1
-        body:
-          nodes:
-            - { settings: { "node.external_id": "instance-000187" }, processors: 8.0, memory: "64gb", storage: "128gb", node_version: $es_version }
-      allowed_warnings:
-        - "[version removal] Specifying node_version in desired nodes requests is deprecated."
-  - match: { replaced_existing_history_id: false }
-
-  - do:
-      _internal.get_desired_nodes: {}
-  - match:
-      $body:
-        history_id: "test"
-        version: 1
-        nodes:
-          - { settings: { node: { external_id: "instance-000187" } }, processors: 8.0, memory: "64gb", storage: "128gb", node_version: $es_version }
-
-  - do:
-      _internal.delete_desired_nodes: {}
-
-  - do:
-      catch: missing
-      _internal.get_desired_nodes: {}
-  - match: { status: 404 }
----
 "Test update desired nodes is idempotent":
   - skip:
       reason: "contains is a newly added assertion"

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/update/100_synthetic_source.yml

@@ -6,8 +6,8 @@ setup:
 ---
 keyword:
   - requires:
-      cluster_features: ["gte_v8.4.0"]
+      cluster_features: [ "mapper.synthetic_recovery_source" ]
-      reason: introduced in 8.4.0
+      reason: requires synthetic recovery source
 
   - do:
       indices.create:
@@ -60,13 +60,14 @@ keyword:
         index: test
         run_expensive_tasks: true
   - is_false: test.fields._source
-  - is_true: test.fields._recovery_source
+  # When synthetic source is used there is no _recovery_source field
+  - match: { test.fields._recovery_source: null }
 
 ---
 stored text:
   - requires:
-      cluster_features: ["gte_v8.5.0"]
+      cluster_features: [ "mapper.synthetic_recovery_source" ]
-      reason: introduced in 8.5.0
+      reason: requires synthetic recovery source
 
   - do:
       indices.create:
@@ -121,4 +122,5 @@ stored text:
         index: test
         run_expensive_tasks: true
   - is_false: test.fields._source
-  - is_true: test.fields._recovery_source
+  # When synthetic source is used there is no _recovery_source field
+  - match: { test.fields._recovery_source: null }

server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java

@@ -85,7 +85,7 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
         final Environment environment = internalCluster().getInstance(Environment.class);
         final AtomicReference<AssertionError> reloadSettingsError = new AtomicReference<>();
         // keystore file should be missing for this test case
-        Files.deleteIfExists(KeyStoreWrapper.keystorePath(environment.configFile()));
+        Files.deleteIfExists(KeyStoreWrapper.keystorePath(environment.configDir()));
         final int initialReloadCount = mockReloadablePlugin.getReloadCount();
         final CountDownLatch latch = new CountDownLatch(1);
         executeReloadSecureSettings(Strings.EMPTY_ARRAY, emptyPassword(), new ActionListener<>() {
@@ -129,10 +129,10 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
         final int initialReloadCount = mockReloadablePlugin.getReloadCount();
         // invalid "keystore" file should be present in the config dir
         try (InputStream keystore = ReloadSecureSettingsIT.class.getResourceAsStream("invalid.txt.keystore")) {
-            if (Files.exists(environment.configFile()) == false) {
+            if (Files.exists(environment.configDir()) == false) {
-                Files.createDirectory(environment.configFile());
+                Files.createDirectory(environment.configDir());
             }
-            Files.copy(keystore, KeyStoreWrapper.keystorePath(environment.configFile()), StandardCopyOption.REPLACE_EXISTING);
+            Files.copy(keystore, KeyStoreWrapper.keystorePath(environment.configDir()), StandardCopyOption.REPLACE_EXISTING);
         }
         final CountDownLatch latch = new CountDownLatch(1);
         executeReloadSecureSettings(Strings.EMPTY_ARRAY, emptyPassword(), new ActionListener<>() {
@@ -363,7 +363,7 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
 
         try (KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create()) {
             keyStoreWrapper.setString(VALID_SECURE_SETTING_NAME, new char[0]);
-            keyStoreWrapper.save(environment.configFile(), new char[0], false);
+            keyStoreWrapper.save(environment.configDir(), new char[0], false);
         }
 
         PlainActionFuture<NodesReloadSecureSettingsResponse> actionFuture = new PlainActionFuture<>();
@@ -374,7 +374,7 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
         try (KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create()) {
             assertThat(keyStoreWrapper, notNullValue());
             keyStoreWrapper.setString("some.setting.that.does.not.exist", new char[0]);
-            keyStoreWrapper.save(environment.configFile(), new char[0], false);
+            keyStoreWrapper.save(environment.configDir(), new char[0], false);
         }
 
         actionFuture = new PlainActionFuture<>();
@@ -432,7 +432,7 @@ public class ReloadSecureSettingsIT extends ESIntegTestCase {
 
     private SecureSettings writeEmptyKeystore(Environment environment, char[] password) throws Exception {
         final KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create();
-        keyStoreWrapper.save(environment.configFile(), password, false);
+        keyStoreWrapper.save(environment.configDir(), password, false);
         return keyStoreWrapper;
     }
 

server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java

@@ -220,7 +220,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
 
     public void testIndexDirIsDeletedWhenShardRemoved() throws Exception {
         Environment env = getInstanceFromNode(Environment.class);
-        Path idxPath = env.sharedDataFile().resolve(randomAlphaOfLength(10));
+        Path idxPath = env.sharedDataDir().resolve(randomAlphaOfLength(10));
         logger.info("--> idxPath: [{}]", idxPath);
         Settings idxSettings = Settings.builder().put(IndexMetadata.SETTING_DATA_PATH, idxPath).build();
         createIndex("test", idxSettings);
@@ -254,7 +254,7 @@ public class IndexShardIT extends ESSingleNodeTestCase {
 
     public void testIndexCanChangeCustomDataPath() throws Exception {
         final String index = "test-custom-data-path";
-        final Path sharedDataPath = getInstanceFromNode(Environment.class).sharedDataFile().resolve(randomAsciiLettersOfLength(10));
+        final Path sharedDataPath = getInstanceFromNode(Environment.class).sharedDataDir().resolve(randomAsciiLettersOfLength(10));
         final Path indexDataPath = sharedDataPath.resolve("start-" + randomAsciiLettersOfLength(10));
 
         logger.info("--> creating index [{}] with data_path [{}]", index, indexDataPath);

server/src/internalClusterTest/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandIT.java

@@ -32,7 +32,6 @@ import org.elasticsearch.cli.ProcessInfo;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -531,8 +530,7 @@ public class RemoveCorruptedShardDataCommandIT extends ESIntegTestCase {
             nodeNameToNodeId.put(cursor.getValue().getName(), cursor.getKey());
         }
 
-        final GroupShardsIterator<ShardIterator> shardIterators = state.getRoutingTable()
+        final List<ShardIterator> shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[] { indexName }, false);
-            .activePrimaryShardsGrouped(new String[] { indexName }, false);
         final List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
         final ShardRouting shardRouting = iterators.iterator().next().nextOrNull();
         assertThat(shardRouting, notNullValue());
@@ -562,7 +560,7 @@ public class RemoveCorruptedShardDataCommandIT extends ESIntegTestCase {
             command.findAndProcessShardPath(
                 options,
                 environmentByNodeName.get(nodeName),
-                environmentByNodeName.get(nodeName).dataFiles(),
+                environmentByNodeName.get(nodeName).dataDirs(),
                 state,
                 shardPath -> assertThat(shardPath.resolveIndex(), equalTo(indexPath))
             );
@@ -571,8 +569,7 @@ public class RemoveCorruptedShardDataCommandIT extends ESIntegTestCase {
 
     private Path getPathToShardData(String indexName, String dirSuffix) {
         ClusterState state = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState();
-        GroupShardsIterator<ShardIterator> shardIterators = state.getRoutingTable()
+        List<ShardIterator> shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[] { indexName }, false);
-            .activePrimaryShardsGrouped(new String[] { indexName }, false);
         List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
         ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
         ShardRouting shardRouting = shardIterator.nextOrNull();

server/src/internalClusterTest/java/org/elasticsearch/index/store/CorruptedFileIT.java

@@ -34,7 +34,6 @@ import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -311,8 +310,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
         }
         assertThat(response.getStatus(), is(ClusterHealthStatus.RED));
         ClusterState state = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState();
-        GroupShardsIterator<ShardIterator> shardIterators = state.getRoutingTable()
+        List<ShardIterator> shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[] { "test" }, false);
-            .activePrimaryShardsGrouped(new String[] { "test" }, false);
         for (ShardIterator iterator : shardIterators) {
             ShardRouting routing;
             while ((routing = iterator.nextOrNull()) != null) {
@@ -667,7 +665,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
 
     private int numShards(String... index) {
         ClusterState state = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState();
-        GroupShardsIterator<?> shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(index, false);
+        List<?> shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(index, false);
         return shardIterators.size();
     }
 
@@ -695,8 +693,7 @@ public class CorruptedFileIT extends ESIntegTestCase {
     private ShardRouting corruptRandomPrimaryFile(final boolean includePerCommitFiles) throws IOException {
         ClusterState state = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState();
         Index test = state.metadata().getProject().index("test").getIndex();
-        GroupShardsIterator<ShardIterator> shardIterators = state.getRoutingTable()
+        List<ShardIterator> shardIterators = state.getRoutingTable().activePrimaryShardsGrouped(new String[] { "test" }, false);
-            .activePrimaryShardsGrouped(new String[] { "test" }, false);
         List<ShardIterator> iterators = iterableAsArrayList(shardIterators);
         ShardIterator shardIterator = RandomPicks.randomFrom(random(), iterators);
         ShardRouting shardRouting = shardIterator.nextOrNull();

server/src/internalClusterTest/java/org/elasticsearch/index/suggest/stats/SuggestStatsIT.java

@@ -14,7 +14,6 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.index.search.stats.SearchStats;
@@ -24,6 +23,7 @@ import org.elasticsearch.search.suggest.term.TermSuggestionBuilder;
 import org.elasticsearch.test.ESIntegTestCase;
 
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -146,7 +146,7 @@ public class SuggestStatsIT extends ESIntegTestCase {
 
     private Set<String> nodeIdsWithIndex(String... indices) {
         ClusterState state = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState();
-        GroupShardsIterator<ShardIterator> allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true);
+        List<ShardIterator> allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true);
         Set<String> nodes = new HashSet<>();
         for (ShardIterator shardIterator : allAssignedShardsGrouped) {
             for (ShardRouting routing : shardIterator) {
@@ -161,7 +161,7 @@ public class SuggestStatsIT extends ESIntegTestCase {
 
     protected int numAssignedShards(String... indices) {
         ClusterState state = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState();
-        GroupShardsIterator<?> allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true);
+        List<?> allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true);
         return allAssignedShardsGrouped.size();
     }
 }

server/src/internalClusterTest/java/org/elasticsearch/search/stats/SearchStatsIT.java

@@ -14,7 +14,6 @@ import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.settings.Settings;
@@ -165,7 +164,7 @@ public class SearchStatsIT extends ESIntegTestCase {
 
     private Set<String> nodeIdsWithIndex(String... indices) {
         ClusterState state = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState();
-        GroupShardsIterator<ShardIterator> allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true);
+        List<ShardIterator> allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true);
         Set<String> nodes = new HashSet<>();
         for (ShardIterator shardIterator : allAssignedShardsGrouped) {
             for (ShardRouting routing : shardIterator) {
@@ -248,7 +247,7 @@ public class SearchStatsIT extends ESIntegTestCase {
 
     protected int numAssignedShards(String... indices) {
         ClusterState state = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState();
-        GroupShardsIterator<?> allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true);
+        List<?> allAssignedShardsGrouped = state.routingTable().allAssignedShardsGrouped(indices, true);
         return allAssignedShardsGrouped.size();
     }
 

server/src/internalClusterTest/java/org/elasticsearch/snapshots/MultiClusterRepoAccessIT.java

@@ -195,7 +195,7 @@ public class MultiClusterRepoAccessIT extends AbstractSnapshotIntegTestCase {
         );
 
         assertAcked(clusterAdmin().prepareDeleteRepository(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, repoName));
-        IOUtils.rm(internalCluster().getCurrentMasterNodeInstance(Environment.class).resolveRepoFile(repoPath.toString()));
+        IOUtils.rm(internalCluster().getCurrentMasterNodeInstance(Environment.class).resolveRepoDir(repoPath.toString()));
         createRepository(repoName, "fs", repoPath);
         createFullSnapshot(repoName, "snap-1");
 

server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStatusApisIT.java

@@ -316,7 +316,6 @@ public class SnapshotStatusApisIT extends AbstractSnapshotIntegTestCase {
             .get();
 
         assertTrue(getSnapshotsResponse.getSnapshots().isEmpty());
-        assertTrue(getSnapshotsResponse.getFailures().isEmpty());
     }
 
     public void testGetSnapshotsMultipleRepos() throws Exception {

server/src/main/java/org/elasticsearch/TransportVersion.java

@@ -118,6 +118,14 @@ public record TransportVersion(int id) implements VersionId<TransportVersion> {
         return VersionsHolder.ALL_VERSIONS;
     }
 
+    /**
+     * @return whether this is a known {@link TransportVersion}, i.e. one declared in {@link TransportVersions}. Other versions may exist
+     *         in the wild (they're sent over the wire by numeric ID) but we don't know how to communicate using such versions.
+     */
+    public boolean isKnown() {
+        return VersionsHolder.ALL_VERSIONS_MAP.containsKey(id);
+    }
+
     public static TransportVersion fromString(String str) {
         return TransportVersion.fromId(Integer.parseInt(str));
     }

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -176,6 +176,10 @@ public class TransportVersions {
     public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED_BACKPORT_8_X = def(8_840_0_01);
     public static final TransportVersion ELASTICSEARCH_9_0 = def(9_000_0_00);
     public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00);
+    public static final TransportVersion REMOVE_SNAPSHOT_FAILURES = def(9_002_0_00);
+    public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED = def(9_003_0_00);
+    public static final TransportVersion REMOVE_DESIRED_NODE_VERSION = def(9_004_0_00);
+    public static final TransportVersion ESQL_DRIVER_TASK_DESCRIPTION = def(9_005_0_00);
     /*
      * WARNING: DO NOT MERGE INTO MAIN!
      * This is the transport version used for all multi-project changes.

server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java

@@ -108,7 +108,7 @@ public class TransportNodesReloadSecureSettingsAction extends TransportNodesActi
         Task task
     ) {
         // We default to using an empty string as the keystore password so that we mimic pre 7.3 API behavior
-        try (KeyStoreWrapper keystore = KeyStoreWrapper.load(environment.configFile())) {
+        try (KeyStoreWrapper keystore = KeyStoreWrapper.load(environment.configDir())) {
             // reread keystore from config file
             if (keystore == null) {
                 return new NodesReloadSecureSettingsResponse.NodeResponse(

server/src/main/java/org/elasticsearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java

@@ -22,7 +22,6 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedEx
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.project.ProjectResolver;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -37,6 +36,7 @@ import org.elasticsearch.transport.TransportService;
 
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -117,7 +117,7 @@ public class TransportClusterSearchShardsAction extends TransportMasterNodeReadA
         }
 
         Set<String> nodeIds = new HashSet<>();
-        GroupShardsIterator<ShardIterator> groupShardsIterator = clusterService.operationRouting()
+        List<ShardIterator> groupShardsIterator = clusterService.operationRouting()
             .searchShards(project, concreteIndices, routingMap, request.preference());
         ShardRouting shard;
         ClusterSearchShardsGroup[] groupResponses = new ClusterSearchShardsGroup[groupShardsIterator.size()];

server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java

@@ -9,7 +9,7 @@
 
 package org.elasticsearch.action.admin.cluster.snapshots.get;
 
-import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.Iterators;
@@ -17,12 +17,10 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.xcontent.ChunkedToXContentObject;
 import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.snapshots.SnapshotInfo;
 import org.elasticsearch.xcontent.ToXContent;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -35,9 +33,6 @@ public class GetSnapshotsResponse extends ActionResponse implements ChunkedToXCo
 
     private final List<SnapshotInfo> snapshots;
 
-    @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // always empty, can be dropped
-    private final Map<String, ElasticsearchException> failures;
-
     @Nullable
     private final String next;
 
@@ -45,15 +40,8 @@ public class GetSnapshotsResponse extends ActionResponse implements ChunkedToXCo
 
     private final int remaining;
 
-    public GetSnapshotsResponse(
+    public GetSnapshotsResponse(List<SnapshotInfo> snapshots, @Nullable String next, final int total, final int remaining) {
-        List<SnapshotInfo> snapshots,
-        Map<String, ElasticsearchException> failures,
-        @Nullable String next,
-        final int total,
-        final int remaining
-    ) {
         this.snapshots = List.copyOf(snapshots);
-        this.failures = failures == null ? Map.of() : Map.copyOf(failures);
         this.next = next;
         this.total = total;
         this.remaining = remaining;
@@ -61,7 +49,10 @@ public class GetSnapshotsResponse extends ActionResponse implements ChunkedToXCo
 
     public GetSnapshotsResponse(StreamInput in) throws IOException {
         this.snapshots = in.readCollectionAsImmutableList(SnapshotInfo::readFrom);
-        this.failures = Collections.unmodifiableMap(in.readMap(StreamInput::readException));
+        if (in.getTransportVersion().before(TransportVersions.REMOVE_SNAPSHOT_FAILURES)) {
+            // Deprecated `failures` field
+            in.readMap(StreamInput::readException);
+        }
         this.next = in.readOptionalString();
         this.total = in.readVInt();
         this.remaining = in.readVInt();
@@ -76,25 +67,11 @@ public class GetSnapshotsResponse extends ActionResponse implements ChunkedToXCo
         return snapshots;
     }
 
-    /**
-     * Returns a map of repository name to {@link ElasticsearchException} for each unsuccessful response.
-     */
-    public Map<String, ElasticsearchException> getFailures() {
-        return failures;
-    }
-
     @Nullable
     public String next() {
         return next;
     }
 
-    /**
-     * Returns true if there is at least one failed response.
-     */
-    public boolean isFailed() {
-        return failures.isEmpty() == false;
-    }
-
     public int totalCount() {
         return total;
     }
@@ -106,7 +83,10 @@ public class GetSnapshotsResponse extends ActionResponse implements ChunkedToXCo
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeCollection(snapshots);
-        out.writeMap(failures, StreamOutput::writeException);
+        if (out.getTransportVersion().before(TransportVersions.REMOVE_SNAPSHOT_FAILURES)) {
+            // Deprecated `failures` field
+            out.writeMap(Map.of(), StreamOutput::writeException);
+        }
         out.writeOptionalString(next);
         out.writeVInt(total);
         out.writeVInt(remaining);
@@ -120,18 +100,6 @@ public class GetSnapshotsResponse extends ActionResponse implements ChunkedToXCo
             return b;
         }), Iterators.map(getSnapshots().iterator(), snapshotInfo -> snapshotInfo::toXContentExternal), Iterators.single((b, p) -> {
             b.endArray();
-            if (failures.isEmpty() == false) {
-                b.startObject("failures");
-                for (Map.Entry<String, ElasticsearchException> error : failures.entrySet()) {
-                    b.field(error.getKey(), (bb, pa) -> {
-                        bb.startObject();
-                        error.getValue().toXContent(bb, pa);
-                        bb.endObject();
-                        return bb;
-                    });
-                }
-                b.endObject();
-            }
             if (next != null) {
                 b.field("next", next);
             }
@@ -151,12 +119,12 @@ public class GetSnapshotsResponse extends ActionResponse implements ChunkedToXCo
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         GetSnapshotsResponse that = (GetSnapshotsResponse) o;
-        return Objects.equals(snapshots, that.snapshots) && Objects.equals(failures, that.failures) && Objects.equals(next, that.next);
+        return Objects.equals(snapshots, that.snapshots) && Objects.equals(next, that.next);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(snapshots, failures, next);
+        return Objects.hash(snapshots, next);
     }
 
     @Override

server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java

@@ -543,7 +543,6 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction<GetSn
             }
             return new GetSnapshotsResponse(
                 snapshotInfos,
-                null,
                 remaining > 0 ? sortBy.encodeAfterQueryParam(snapshotInfos.get(snapshotInfos.size() - 1)) : null,
                 totalCount.get(),
                 remaining

server/src/main/java/org/elasticsearch/action/admin/indices/analyze/AnalyzeCapabilities.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.action.admin.indices.analyze;
+
+import java.util.Set;
+
+public final class AnalyzeCapabilities {
+    private AnalyzeCapabilities() {}
+
+    private static final String WRONG_CUSTOM_ANALYZER_RETURNS_400_CAPABILITY = "wrong_custom_analyzer_returns_400";
+
+    public static final Set<String> CAPABILITIES = Set.of(WRONG_CUSTOM_ANALYZER_RETURNS_400_CAPABILITY);
+}

server/src/main/java/org/elasticsearch/action/admin/indices/analyze/TransportAnalyzeAction.java

@@ -147,6 +147,8 @@ public class TransportAnalyzeAction extends TransportSingleShardAction<AnalyzeAc
             if (analyzer != null) {
                 return analyze(request, analyzer, maxTokenCount);
             }
+        } catch (IllegalStateException e) {
+            throw new IllegalArgumentException("Can not build a custom analyzer", e);
         }
 
         // Otherwise we use a built-in analyzer, which should not be closed

server/src/main/java/org/elasticsearch/action/admin/indices/diskusage/TransportAnalyzeIndexDiskUsageAction.java

@@ -23,7 +23,6 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.project.ProjectResolver;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -219,14 +218,10 @@ public class TransportAnalyzeIndexDiskUsageAction extends TransportBroadcastActi
     }
 
     @Override
-    protected GroupShardsIterator<ShardIterator> shards(
+    protected List<ShardIterator> shards(ClusterState clusterState, AnalyzeIndexDiskUsageRequest request, String[] concreteIndices) {
-        ClusterState clusterState,
-        AnalyzeIndexDiskUsageRequest request,
-        String[] concreteIndices
-    ) {
         ProjectState project = projectResolver.getProjectState(clusterState);
-        final GroupShardsIterator<ShardIterator> groups = clusterService.operationRouting()
+        final List<ShardIterator> groups = clusterService.operationRouting().searchShards(project, concreteIndices, null, null);
-            .searchShards(project, concreteIndices, null, null);
+
         for (ShardIterator group : groups) {
             // fails fast if any non-active groups
             if (group.size() == 0) {

server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java

@@ -24,7 +24,6 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression;
 import org.elasticsearch.cluster.project.ProjectResolver;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -158,7 +157,7 @@ public class TransportValidateQueryAction extends TransportBroadcastAction<
     }
 
     @Override
-    protected GroupShardsIterator<ShardIterator> shards(ClusterState clusterState, ValidateQueryRequest request, String[] concreteIndices) {
+    protected List<ShardIterator> shards(ClusterState clusterState, ValidateQueryRequest request, String[] concreteIndices) {
         final String routing;
         if (request.allShards()) {
             routing = null;

server/src/main/java/org/elasticsearch/action/fieldcaps/RequestDispatcher.java

@@ -20,7 +20,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ProjectState;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.project.ProjectResolver;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -99,7 +98,7 @@ final class RequestDispatcher {
         ProjectState project = projectResolver.getProjectState(clusterState);
 
         for (String index : indices) {
-            final GroupShardsIterator<ShardIterator> shardIts;
+            final List<ShardIterator> shardIts;
             try {
                 shardIts = clusterService.operationRouting().searchShards(project, new String[] { index }, null, null);
             } catch (Exception e) {
@@ -256,7 +255,7 @@ final class RequestDispatcher {
         private final Set<ShardId> unmatchedShardIds = new HashSet<>();
         private final Map<ShardId, Exception> failures = new HashMap<>();
 
-        IndexSelector(GroupShardsIterator<ShardIterator> shardIts) {
+        IndexSelector(List<ShardIterator> shardIts) {
             for (ShardIterator shardIt : shardIts) {
                 for (ShardRouting shard : shardIt) {
                     nodeToShards.computeIfAbsent(shard.currentNodeId(), node -> new ArrayList<>()).add(shard);

server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java

@@ -22,7 +22,6 @@ import org.elasticsearch.action.search.TransportSearchAction.SearchTimeProvider;
 import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.action.support.TransportActions;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.util.Maps;
@@ -60,9 +59,9 @@ import java.util.stream.Collectors;
 import static org.elasticsearch.core.Strings.format;
 
 /**
- * This is an abstract base class that encapsulates the logic to fan out to all shards in provided {@link GroupShardsIterator}
+ * This is an abstract base class that encapsulates the logic to fan out to all shards in provided {@link List<SearchShardIterator>}
  * and collect the results. If a shard request returns a failure this class handles the advance to the next replica of the shard until
- * the shards replica iterator is exhausted. Each shard is referenced by position in the {@link GroupShardsIterator} which is later
+ * the shards replica iterator is exhausted. Each shard is referenced by position in the {@link List<SearchShardIterator>} which is later
  * referred to as the {@code shardIndex}.
  * The fan out and collect algorithm is traditionally used as the initial phase which can either be a query execution or collection of
  * distributed frequencies
@@ -90,15 +89,13 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
     private final Object shardFailuresMutex = new Object();
     private final AtomicBoolean hasShardResponse = new AtomicBoolean(false);
     private final AtomicInteger successfulOps = new AtomicInteger();
-    private final AtomicInteger skippedOps = new AtomicInteger();
     private final SearchTimeProvider timeProvider;
     private final SearchResponse.Clusters clusters;
 
-    protected final GroupShardsIterator<SearchShardIterator> toSkipShardsIts;
+    protected final List<SearchShardIterator> toSkipShardsIts;
-    protected final GroupShardsIterator<SearchShardIterator> shardsIts;
+    protected final List<SearchShardIterator> shardsIts;
     private final SearchShardIterator[] shardIterators;
-    private final int expectedTotalOps;
+    private final AtomicInteger outstandingShards;
-    private final AtomicInteger totalOps = new AtomicInteger();
     private final int maxConcurrentRequestsPerNode;
     private final Map<String, PendingExecutions> pendingExecutionsPerNode = new ConcurrentHashMap<>();
     private final boolean throttleConcurrentRequests;
@@ -118,7 +115,7 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
         Executor executor,
         SearchRequest request,
         ActionListener<SearchResponse> listener,
-        GroupShardsIterator<SearchShardIterator> shardsIts,
+        List<SearchShardIterator> shardsIts,
         SearchTimeProvider timeProvider,
         ClusterState clusterState,
         SearchTask task,
@@ -137,20 +134,14 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
                 iterators.add(iterator);
             }
         }
-        this.toSkipShardsIts = new GroupShardsIterator<>(toSkipIterators);
+        this.toSkipShardsIts = toSkipIterators;
-        this.shardsIts = new GroupShardsIterator<>(iterators);
+        this.shardsIts = iterators;
-
+        outstandingShards = new AtomicInteger(shardsIts.size());
         this.shardIterators = iterators.toArray(new SearchShardIterator[0]);
         // we later compute the shard index based on the natural order of the shards
         // that participate in the search request. This means that this number is
         // consistent between two requests that target the same shards.
         Arrays.sort(shardIterators);
-
-        // we need to add 1 for non active partition, since we count it in the total. This means for each shard in the iterator we sum up
-        // it's number of active shards but use 1 as the default if no replica of a shard is active at this point.
-        // on a per shards level we use shardIt.remaining() to increment the totalOps pointer but add 1 for the current shard result
-        // we process hence we add one for the non active partition here.
-        this.expectedTotalOps = shardsIts.totalSizeWith1ForEmpty();
         this.maxConcurrentRequestsPerNode = maxConcurrentRequestsPerNode;
         // in the case were we have less shards than maxConcurrentRequestsPerNode we don't need to throttle
         this.throttleConcurrentRequests = maxConcurrentRequestsPerNode < shardsIts.size();
@@ -179,8 +170,8 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
         SearchSourceBuilder sourceBuilder
     ) {
         progressListener.notifyListShards(
-            SearchProgressListener.buildSearchShards(this.shardsIts),
+            SearchProgressListener.buildSearchShardsFromIter(this.shardsIts),
-            SearchProgressListener.buildSearchShards(toSkipShardsIts),
+            SearchProgressListener.buildSearchShardsFromIter(toSkipShardsIts),
             clusters,
             sourceBuilder == null || sourceBuilder.size() > 0,
             timeProvider
@@ -251,9 +242,8 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
 
     void skipShard(SearchShardIterator iterator) {
         successfulOps.incrementAndGet();
-        skippedOps.incrementAndGet();
         assert iterator.skip();
-        successfulShardExecution(iterator);
+        successfulShardExecution();
     }
 
     private static boolean assertExecuteOnStartThread() {
@@ -380,7 +370,7 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
                             "Partial shards failure (unavailable: {}, successful: {}, skipped: {}, num-shards: {}, phase: {})",
                             discrepancy,
                             successfulOps.get(),
-                            skippedOps.get(),
+                            toSkipShardsIts.size(),
                             getNumShards(),
                             currentPhase
                         );
@@ -449,17 +439,14 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
             }
             onShardGroupFailure(shardIndex, shard, e);
         }
-        final int totalOps = this.totalOps.incrementAndGet();
+        if (lastShard == false) {
-        if (totalOps == expectedTotalOps) {
+            performPhaseOnShard(shardIndex, shardIt, nextShard);
-            onPhaseDone();
-        } else if (totalOps > expectedTotalOps) {
-            throw new AssertionError(
-                "unexpected higher total ops [" + totalOps + "] compared to expected [" + expectedTotalOps + "]",
-                new SearchPhaseExecutionException(getName(), "Shard failures", null, buildShardFailures())
-            );
         } else {
-            if (lastShard == false) {
+            // count down outstanding shards, we're done with this shard as there's no more copies to try
-                performPhaseOnShard(shardIndex, shardIt, nextShard);
+            final int outstanding = outstandingShards.decrementAndGet();
+            assert outstanding >= 0 : "outstanding: " + outstanding;
+            if (outstanding == 0) {
+                onPhaseDone();
             }
         }
     }
@@ -535,10 +522,10 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
         if (logger.isTraceEnabled()) {
             logger.trace("got first-phase result from {}", result != null ? result.getSearchShardTarget() : null);
         }
-        results.consumeResult(result, () -> onShardResultConsumed(result, shardIt));
+        results.consumeResult(result, () -> onShardResultConsumed(result));
     }
 
-    private void onShardResultConsumed(Result result, SearchShardIterator shardIt) {
+    private void onShardResultConsumed(Result result) {
         successfulOps.incrementAndGet();
         // clean a previous error on this shard group (note, this code will be serialized on the same shardIndex value level
         // so its ok concurrency wise to miss potentially the shard failures being created because of another failure
@@ -552,28 +539,14 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
         // cause the successor to read a wrong value from successfulOps if second phase is very fast ie. count etc.
         // increment all the "future" shards to update the total ops since we some may work and some may not...
         // and when that happens, we break on total ops, so we must maintain them
-        successfulShardExecution(shardIt);
+        successfulShardExecution();
     }
 
-    private void successfulShardExecution(SearchShardIterator shardsIt) {
+    private void successfulShardExecution() {
-        final int remainingOpsOnIterator;
+        final int outstanding = outstandingShards.decrementAndGet();
-        if (shardsIt.skip()) {
+        assert outstanding >= 0 : "outstanding: " + outstanding;
-            // It's possible that we're skipping a shard that's unavailable
+        if (outstanding == 0) {
-            // but its range was available in the IndexMetadata, in that
-            // case the shardsIt.remaining() would be 0, expectedTotalOps
-            // accounts for unavailable shards too.
-            remainingOpsOnIterator = Math.max(shardsIt.remaining(), 1);
-        } else {
-            remainingOpsOnIterator = shardsIt.remaining() + 1;
-        }
-        final int xTotalOps = totalOps.addAndGet(remainingOpsOnIterator);
-        if (xTotalOps == expectedTotalOps) {
             onPhaseDone();
-        } else if (xTotalOps > expectedTotalOps) {
-            throw new AssertionError(
-                "unexpected higher total ops [" + xTotalOps + "] compared to expected [" + expectedTotalOps + "]",
-                new SearchPhaseExecutionException(getName(), "Shard failures", null, buildShardFailures())
-            );
         }
     }
 
@@ -640,7 +613,7 @@ abstract class AbstractSearchAsyncAction<Result extends SearchPhaseResult> exten
             scrollId,
             getNumShards(),
             numSuccess,
-            skippedOps.get(),
+            toSkipShardsIts.size(),
             buildTookInMillis(),
             failures,
             clusters,

server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java

@@ -12,7 +12,6 @@ package org.elasticsearch.action.search;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.concurrent.CountDown;
@@ -61,8 +60,8 @@ final class CanMatchPreFilterSearchPhase {
 
     private final Logger logger;
     private final SearchRequest request;
-    private final GroupShardsIterator<SearchShardIterator> shardsIts;
+    private final List<SearchShardIterator> shardsIts;
-    private final ActionListener<GroupShardsIterator<SearchShardIterator>> listener;
+    private final ActionListener<List<SearchShardIterator>> listener;
     private final TransportSearchAction.SearchTimeProvider timeProvider;
     private final BiFunction<String, String, Transport.Connection> nodeIdToConnection;
     private final SearchTransportService searchTransportService;
@@ -86,12 +85,12 @@ final class CanMatchPreFilterSearchPhase {
         Map<String, Float> concreteIndexBoosts,
         Executor executor,
         SearchRequest request,
-        GroupShardsIterator<SearchShardIterator> shardsIts,
+        List<SearchShardIterator> shardsIts,
         TransportSearchAction.SearchTimeProvider timeProvider,
         SearchTask task,
         boolean requireAtLeastOneMatch,
         CoordinatorRewriteContextProvider coordinatorRewriteContextProvider,
-        ActionListener<GroupShardsIterator<SearchShardIterator>> listener
+        ActionListener<List<SearchShardIterator>> listener
     ) {
         this.logger = logger;
         this.searchTransportService = searchTransportService;
@@ -169,10 +168,9 @@ final class CanMatchPreFilterSearchPhase {
         if (matchedShardLevelRequests.isEmpty()) {
             finishPhase();
         } else {
-            GroupShardsIterator<SearchShardIterator> matchingShards = new GroupShardsIterator<>(matchedShardLevelRequests);
             // verify missing shards only for the shards that we hit for the query
-            checkNoMissingShards(matchingShards);
+            checkNoMissingShards(matchedShardLevelRequests);
-            new Round(matchingShards).run();
+            new Round(matchedShardLevelRequests).run();
         }
     }
 
@@ -202,12 +200,12 @@ final class CanMatchPreFilterSearchPhase {
         minAndMaxes[shardIndex] = minAndMax;
     }
 
-    private void checkNoMissingShards(GroupShardsIterator<SearchShardIterator> shards) {
+    private void checkNoMissingShards(List<SearchShardIterator> shards) {
         assert assertSearchCoordinationThread();
         SearchPhase.doCheckNoMissingShards("can_match", request, shards, SearchPhase::makeMissingShardsError);
     }
 
-    private Map<SendingTarget, List<SearchShardIterator>> groupByNode(GroupShardsIterator<SearchShardIterator> shards) {
+    private Map<SendingTarget, List<SearchShardIterator>> groupByNode(List<SearchShardIterator> shards) {
         Map<SendingTarget, List<SearchShardIterator>> requests = new HashMap<>();
         for (int i = 0; i < shards.size(); i++) {
             final SearchShardIterator shardRoutings = shards.get(i);
@@ -230,11 +228,11 @@ final class CanMatchPreFilterSearchPhase {
      * to retry on other available shard copies.
      */
     class Round extends AbstractRunnable {
-        private final GroupShardsIterator<SearchShardIterator> shards;
+        private final List<SearchShardIterator> shards;
         private final CountDown countDown;
         private final AtomicReferenceArray<Exception> failedResponses;
 
-        Round(GroupShardsIterator<SearchShardIterator> shards) {
+        Round(List<SearchShardIterator> shards) {
             this.shards = shards;
             this.countDown = new CountDown(shards.size());
             this.failedResponses = new AtomicReferenceArray<>(shardsIts.size());
@@ -328,7 +326,7 @@ final class CanMatchPreFilterSearchPhase {
                 finishPhase();
             } else {
                 // trigger another round, forcing execution
-                executor.execute(new Round(new GroupShardsIterator<>(remainingShards)) {
+                executor.execute(new Round(remainingShards) {
                     @Override
                     public boolean isForceExecution() {
                         return true;
@@ -419,7 +417,7 @@ final class CanMatchPreFilterSearchPhase {
         listener.onFailure(new SearchPhaseExecutionException("can_match", msg, cause, ShardSearchFailure.EMPTY_ARRAY));
     }
 
-    private synchronized GroupShardsIterator<SearchShardIterator> getIterator(GroupShardsIterator<SearchShardIterator> shardsIts) {
+    private synchronized List<SearchShardIterator> getIterator(List<SearchShardIterator> shardsIts) {
         // TODO: pick the local shard when possible
         if (requireAtLeastOneMatch && numPossibleMatches == 0) {
             // this is a special case where we have no hit but we need to get at least one search response in order
@@ -452,14 +450,10 @@ final class CanMatchPreFilterSearchPhase {
             return shardsIts;
         }
         FieldSortBuilder fieldSort = FieldSortBuilder.getPrimaryFieldSortOrNull(request.source());
-        return new GroupShardsIterator<>(sortShards(shardsIts, minAndMaxes, fieldSort.order()));
+        return sortShards(shardsIts, minAndMaxes, fieldSort.order());
     }
 
-    private static List<SearchShardIterator> sortShards(
+    private static List<SearchShardIterator> sortShards(List<SearchShardIterator> shardsIts, MinAndMax<?>[] minAndMaxes, SortOrder order) {
-        GroupShardsIterator<SearchShardIterator> shardsIts,
-        MinAndMax<?>[] minAndMaxes,
-        SortOrder order
-    ) {
         int bound = shardsIts.size();
         List<Integer> toSort = new ArrayList<>(bound);
         for (int i = 0; i < bound; i++) {

server/src/main/java/org/elasticsearch/action/search/SearchDfsQueryThenFetchAsyncAction.java

@@ -20,7 +20,6 @@ import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
@@ -56,7 +55,7 @@ final class SearchDfsQueryThenFetchAsyncAction extends AbstractSearchAsyncAction
         SearchPhaseResults<SearchPhaseResult> queryPhaseResultConsumer,
         SearchRequest request,
         ActionListener<SearchResponse> listener,
-        GroupShardsIterator<SearchShardIterator> shardsIts,
+        List<SearchShardIterator> shardsIts,
         TransportSearchAction.SearchTimeProvider timeProvider,
         ClusterState clusterState,
         SearchTask task,

server/src/main/java/org/elasticsearch/action/search/SearchPhase.java

@@ -8,11 +8,11 @@
  */
 package org.elasticsearch.action.search;
 
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.transport.Transport;
 
+import java.util.List;
 import java.util.Objects;
 import java.util.function.Function;
 
@@ -45,14 +45,14 @@ abstract class SearchPhase {
             + "]. Consider using `allow_partial_search_results` setting to bypass this error.";
     }
 
-    protected void doCheckNoMissingShards(String phaseName, SearchRequest request, GroupShardsIterator<SearchShardIterator> shardsIts) {
+    protected void doCheckNoMissingShards(String phaseName, SearchRequest request, List<SearchShardIterator> shardsIts) {
         doCheckNoMissingShards(phaseName, request, shardsIts, this::missingShardsErrorMessage);
     }
 
     protected static void doCheckNoMissingShards(
         String phaseName,
         SearchRequest request,
-        GroupShardsIterator<SearchShardIterator> shardsIts,
+        List<SearchShardIterator> shardsIts,
         Function<StringBuilder, String> makeErrorMessage
     ) {
         assert request.allowPartialSearchResults() != null : "SearchRequest missing setting for allowPartialSearchResults";

server/src/main/java/org/elasticsearch/action/search/SearchProgressListener.java

@@ -13,7 +13,6 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.search.TotalHits;
 import org.elasticsearch.action.search.SearchResponse.Clusters;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
 import org.elasticsearch.search.aggregations.InternalAggregations;
@@ -21,7 +20,6 @@ import org.elasticsearch.search.query.QuerySearchResult;
 
 import java.util.List;
 import java.util.Objects;
-import java.util.stream.StreamSupport;
 
 /**
  * A listener that allows to track progress of the {@link TransportSearchAction}.
@@ -225,7 +223,7 @@ public abstract class SearchProgressListener {
             .toList();
     }
 
-    static List<SearchShard> buildSearchShards(GroupShardsIterator<SearchShardIterator> its) {
+    static List<SearchShard> buildSearchShardsFromIter(List<SearchShardIterator> its) {
-        return StreamSupport.stream(its.spliterator(), false).map(e -> new SearchShard(e.getClusterAlias(), e.shardId())).toList();
+        return its.stream().map(e -> new SearchShard(e.getClusterAlias(), e.shardId())).toList();
     }
 }

server/src/main/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncAction.java

@@ -14,7 +14,6 @@ import org.apache.lucene.search.TopFieldDocs;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.search.SearchPhaseResult;
 import org.elasticsearch.search.SearchShardTarget;
@@ -25,6 +24,7 @@ import org.elasticsearch.search.internal.ShardSearchRequest;
 import org.elasticsearch.search.query.QuerySearchResult;
 import org.elasticsearch.transport.Transport;
 
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executor;
 import java.util.function.BiFunction;
@@ -52,7 +52,7 @@ class SearchQueryThenFetchAsyncAction extends AbstractSearchAsyncAction<SearchPh
         SearchPhaseResults<SearchPhaseResult> resultConsumer,
         SearchRequest request,
         ActionListener<SearchResponse> listener,
-        GroupShardsIterator<SearchShardIterator> shardsIts,
+        List<SearchShardIterator> shardsIts,
         TransportSearchAction.SearchTimeProvider timeProvider,
         ClusterState clusterState,
         SearchTask task,

server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java

@@ -23,7 +23,6 @@ import org.elasticsearch.action.support.ChannelActionListener;
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -49,6 +48,7 @@ import org.elasticsearch.transport.TransportResponseHandler;
 import org.elasticsearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executor;
 import java.util.function.BiFunction;
@@ -150,7 +150,7 @@ public class TransportOpenPointInTimeAction extends HandledTransportAction<OpenP
             SearchTask task,
             SearchRequest searchRequest,
             Executor executor,
-            GroupShardsIterator<SearchShardIterator> shardIterators,
+            List<SearchShardIterator> shardIterators,
             TransportSearchAction.SearchTimeProvider timeProvider,
             BiFunction<String, String, Transport.Connection> connectionLookup,
             ClusterState clusterState,
@@ -212,7 +212,7 @@ public class TransportOpenPointInTimeAction extends HandledTransportAction<OpenP
             SearchTask task,
             SearchRequest searchRequest,
             Executor executor,
-            GroupShardsIterator<SearchShardIterator> shardIterators,
+            List<SearchShardIterator> shardIterators,
             TransportSearchAction.SearchTimeProvider timeProvider,
             BiFunction<String, String, Transport.Connection> connectionLookup,
             ClusterState clusterState,

server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java

@@ -11,6 +11,7 @@ package org.elasticsearch.action.search;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionListener;
@@ -44,7 +45,6 @@ import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.project.ProjectResolver;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
@@ -1293,7 +1293,7 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
                 );
             }
         }
-        final GroupShardsIterator<SearchShardIterator> shardIterators = mergeShardsIterators(localShardIterators, remoteShardIterators);
+        final List<SearchShardIterator> shardIterators = mergeShardsIterators(localShardIterators, remoteShardIterators);
 
         failIfOverShardCountLimit(clusterService, shardIterators.size());
 
@@ -1427,7 +1427,7 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
     }
 
     // package private for testing
-    static GroupShardsIterator<SearchShardIterator> mergeShardsIterators(
+    static List<SearchShardIterator> mergeShardsIterators(
         List<SearchShardIterator> localShardIterators,
         List<SearchShardIterator> remoteShardIterators
     ) {
@@ -1437,7 +1437,8 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
         } else {
             shards = CollectionUtils.concatLists(remoteShardIterators, localShardIterators);
         }
-        return GroupShardsIterator.sortAndCreate(shards);
+        CollectionUtil.timSort(shards);
+        return shards;
     }
 
     interface SearchPhaseProvider {
@@ -1445,7 +1446,7 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
             SearchTask task,
             SearchRequest searchRequest,
             Executor executor,
-            GroupShardsIterator<SearchShardIterator> shardIterators,
+            List<SearchShardIterator> shardIterators,
             SearchTimeProvider timeProvider,
             BiFunction<String, String, Transport.Connection> connectionLookup,
             ClusterState clusterState,
@@ -1469,7 +1470,7 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
             SearchTask task,
             SearchRequest searchRequest,
             Executor executor,
-            GroupShardsIterator<SearchShardIterator> shardIterators,
+            List<SearchShardIterator> shardIterators,
             SearchTimeProvider timeProvider,
             BiFunction<String, String, Transport.Connection> connectionLookup,
             ClusterState clusterState,
@@ -1866,7 +1867,7 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
             searchRequest.routing(),
             searchRequest.indices()
         );
-        GroupShardsIterator<ShardIterator> shardRoutings = clusterService.operationRouting()
+        List<ShardIterator> shardRoutings = clusterService.operationRouting()
             .searchShards(
                 projectState,
                 concreteIndices,

server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.action.search;
 
+import org.apache.lucene.util.CollectionUtil;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.RemoteClusterActionType;
@@ -19,7 +20,6 @@ import org.elasticsearch.cluster.ProjectState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.ResolvedExpression;
 import org.elasticsearch.cluster.project.ProjectResolver;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.query.Rewriteable;
@@ -145,15 +145,14 @@ public class TransportSearchShardsAction extends HandledTransportAction<SearchSh
                     concreteIndices
                 );
                 String[] concreteIndexNames = Arrays.stream(concreteIndices).map(Index::getName).toArray(String[]::new);
-                GroupShardsIterator<SearchShardIterator> shardIts = GroupShardsIterator.sortAndCreate(
+                List<SearchShardIterator> shardIts = transportSearchAction.getLocalShardsIterator(
-                    transportSearchAction.getLocalShardsIterator(
+                    project,
-                        project,
+                    searchRequest,
-                        searchRequest,
+                    searchShardsRequest.clusterAlias(),
-                        searchShardsRequest.clusterAlias(),
+                    indicesAndAliases,
-                        indicesAndAliases,
+                    concreteIndexNames
-                        concreteIndexNames
-                    )
                 );
+                CollectionUtil.timSort(shardIts);
                 if (SearchService.canRewriteToMatchNone(searchRequest.source()) == false) {
                     delegate.onResponse(
                         new SearchShardsResponse(toGroups(shardIts), project.cluster().nodes().getAllNodes(), aliasFilters)
@@ -179,7 +178,7 @@ public class TransportSearchShardsAction extends HandledTransportAction<SearchSh
         );
     }
 
-    private static List<SearchShardsGroup> toGroups(GroupShardsIterator<SearchShardIterator> shardIts) {
+    private static List<SearchShardsGroup> toGroups(List<SearchShardIterator> shardIts) {
         List<SearchShardsGroup> groups = new ArrayList<>(shardIts.size());
         for (SearchShardIterator shardIt : shardIts) {
             boolean skip = shardIt.skip();

server/src/main/java/org/elasticsearch/action/support/broadcast/TransportBroadcastAction.java

@@ -22,7 +22,6 @@ import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -36,6 +35,7 @@ import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.transport.Transports;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReferenceArray;
@@ -110,7 +110,7 @@ public abstract class TransportBroadcastAction<
      * on the first shard in it. If the operation fails, it will be retried on the next shard in the iterator.
      */
     @FixForMultiProject // add ProjectMetadata to this method
-    protected abstract GroupShardsIterator<ShardIterator> shards(ClusterState clusterState, Request request, String[] concreteIndices);
+    protected abstract List<ShardIterator> shards(ClusterState clusterState, Request request, String[] concreteIndices);
 
     protected abstract ClusterBlockException checkGlobalBlock(ClusterState state, Request request);
 
@@ -123,7 +123,7 @@ public abstract class TransportBroadcastAction<
         final ActionListener<Response> listener;
         final ClusterState clusterState;
         final DiscoveryNodes nodes;
-        final GroupShardsIterator<ShardIterator> shardsIts;
+        final List<ShardIterator> shardsIts;
         final int expectedOps;
         final AtomicInteger counterOps = new AtomicInteger();
         // ShardResponse or Exception

server/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java

@@ -15,7 +15,6 @@ import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
 import org.elasticsearch.cluster.ProjectState;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.project.ProjectResolver;
-import org.elasticsearch.cluster.routing.GroupShardsIterator;
 import org.elasticsearch.cluster.routing.ShardIterator;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -67,13 +66,8 @@ public class TransportTermVectorsAction extends TransportSingleShardAction<TermV
         final var operationRouting = clusterService.operationRouting();
         if (request.request().doc() != null && request.request().routing() == null) {
             // artificial document without routing specified, ignore its "id" and use either random shard or according to preference
-            GroupShardsIterator<ShardIterator> groupShardsIter = operationRouting.searchShards(
+            return operationRouting.searchShards(project, new String[] { request.concreteIndex() }, null, request.request().preference())
-                project,
+                .getFirst();
-                new String[] { request.concreteIndex() },
-                null,
-                request.request().preference()
-            );
-            return groupShardsIter.iterator().next();
         }
 
         return operationRouting.useOnlyPromotableShardsForStateless(