Add remove index setting command (#109276)

The new subcommand elasticsearch-node remove-index-settings can be used 
to remove index settings from the cluster state in case where it
contains incompatible index settings that prevent the cluster from
forming. This tool can cause data loss and its use should be your last
resort.

Relates #96075

docs/changelog/109276.yaml

@@ -0,0 +1,5 @@
+pr: 109276
+summary: Add remove index setting command
+area: Infra/Settings
+type: enhancement
+issues: []

docs/reference/commands/node-tool.asciidoc

@@ -31,6 +31,10 @@ This tool has a number of modes:
   from the cluster state in case where it contains incompatible settings that
   prevent the cluster from forming.
 
+* `elasticsearch-node remove-index-settings` can be used to remove index settings
+  from the cluster state in case where it contains incompatible index settings that
+  prevent the cluster from forming.
+
 * `elasticsearch-node remove-customs` can be used to remove custom metadata
   from the cluster state in case where it contains broken metadata that
   prevents the cluster state from being loaded.
@@ -107,6 +111,26 @@ The intended use is:
 * Repeat for all other master-eligible nodes
 * Start the nodes
 
+[discrete]
+==== Removing index settings
+
+There may be situations where an index contains index settings
+that prevent the cluster from forming. Since the cluster cannot form,
+it is not possible to remove these settings using the
+<<indices-update-settings>> API.
+
+The `elasticsearch-node remove-index-settings` tool allows you to forcefully remove
+those index settings from the on-disk cluster state. The tool takes a
+list of index settings as parameters that should be removed, and also supports
+wildcard patterns.
+
+The intended use is:
+
+* Stop the node
+* Run `elasticsearch-node remove-index-settings name-of-index-setting-to-remove` on the node
+* Repeat for all nodes
+* Start the nodes
+
 [discrete]
 ==== Removing custom metadata from the cluster state
 
@@ -436,6 +460,37 @@ You can also use wildcards to remove multiple settings, for example using
 node$ ./bin/elasticsearch-node remove-settings xpack.monitoring.*
 ----
 
+[discrete]
+==== Removing index settings
+
+If your indices contain index settings that prevent the cluster
+from forming, you can run the following command to remove one
+or more index settings.
+
+[source,txt]
+----
+node$ ./bin/elasticsearch-node remove-index-settings index.my_plugin.foo
+
+    WARNING: Elasticsearch MUST be stopped before running this tool.
+
+You should only run this tool if you have incompatible index settings in the
+cluster state that prevent the cluster from forming.
+This tool can cause data loss and its use should be your last resort.
+
+Do you want to proceed?
+
+Confirm [y/N] y
+
+Index settings were successfully removed from the cluster state
+----
+
+You can also use wildcards to remove multiple index settings, for example using
+
+[source,txt]
+----
+node$ ./bin/elasticsearch-node remove-index-settings index.my_plugin.*
+----
+
 [discrete]
 ==== Removing custom metadata from the cluster state
 

server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommandIT.java

@@ -0,0 +1,162 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import joptsimple.OptionSet;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cli.MockTerminal;
+import org.elasticsearch.cli.ProcessInfo;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.TestEnvironment;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESIntegTestCase;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.not;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
+public class RemoveIndexSettingsCommandIT extends ESIntegTestCase {
+
+    static final Setting<Integer> FOO = Setting.intSetting("index.foo", 1, Setting.Property.IndexScope, Setting.Property.Dynamic);
+    static final Setting<Integer> BAR = Setting.intSetting("index.bar", 2, Setting.Property.IndexScope, Setting.Property.Final);
+
+    public static class ExtraSettingsPlugin extends Plugin {
+        @Override
+        public List<Setting<?>> getSettings() {
+            return List.of(FOO, BAR);
+        }
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return CollectionUtils.appendToCopy(super.nodePlugins(), ExtraSettingsPlugin.class);
+    }
+
+    public void testRemoveSettingsAbortedByUser() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        var node = internalCluster().startNode();
+        createIndex("test-index", Settings.builder().put(FOO.getKey(), 101).put(BAR.getKey(), 102).build());
+        ensureYellow("test-index");
+        Settings dataPathSettings = internalCluster().dataPathSettings(node);
+        ensureStableCluster(1);
+        internalCluster().stopRandomDataNode();
+
+        Settings nodeSettings = Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build();
+        ElasticsearchException error = expectThrows(
+            ElasticsearchException.class,
+            () -> removeIndexSettings(TestEnvironment.newEnvironment(nodeSettings), true, "index.foo")
+        );
+        assertThat(error.getMessage(), equalTo(ElasticsearchNodeCommand.ABORTED_BY_USER_MSG));
+        internalCluster().startNode(nodeSettings);
+    }
+
+    public void testRemoveSettingsSuccessful() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        var node = internalCluster().startNode();
+        Settings dataPathSettings = internalCluster().dataPathSettings(node);
+
+        int numIndices = randomIntBetween(1, 10);
+        int[] barValues = new int[numIndices];
+        for (int i = 0; i < numIndices; i++) {
+            String index = "test-index-" + i;
+            barValues[i] = between(1, 1000);
+            createIndex(index, Settings.builder().put(FOO.getKey(), between(1, 1000)).put(BAR.getKey(), barValues[i]).build());
+        }
+        int moreIndices = randomIntBetween(1, 10);
+        for (int i = 0; i < moreIndices; i++) {
+            createIndex("more-index-" + i, Settings.EMPTY);
+        }
+        internalCluster().stopNode(node);
+
+        Environment environment = TestEnvironment.newEnvironment(
+            Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build()
+        );
+
+        MockTerminal terminal = removeIndexSettings(environment, false, "index.foo");
+        assertThat(terminal.getOutput(), containsString(RemoveIndexSettingsCommand.SETTINGS_REMOVED_MSG));
+        for (int i = 0; i < numIndices; i++) {
+            assertThat(terminal.getOutput(), containsString("Index setting [index.foo] will be removed from index [[test-index-" + i));
+        }
+        for (int i = 0; i < moreIndices; i++) {
+            assertThat(terminal.getOutput(), not(containsString("Index setting [index.foo] will be removed from index [[more-index-" + i)));
+        }
+        Settings nodeSettings = Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build();
+        internalCluster().startNode(nodeSettings);
+
+        Map<String, Settings> getIndexSettings = client().admin().indices().prepareGetSettings("test-index-*").get().getIndexToSettings();
+        for (int i = 0; i < numIndices; i++) {
+            String index = "test-index-" + i;
+            Settings indexSettings = getIndexSettings.get(index);
+            assertFalse(indexSettings.hasValue("index.foo"));
+            assertThat(indexSettings.get("index.bar"), equalTo(Integer.toString(barValues[i])));
+        }
+        getIndexSettings = client().admin().indices().prepareGetSettings("more-index-*").get().getIndexToSettings();
+        for (int i = 0; i < moreIndices; i++) {
+            assertNotNull(getIndexSettings.get("more-index-" + i));
+        }
+    }
+
+    public void testSettingDoesNotMatch() throws Exception {
+        internalCluster().setBootstrapMasterNodeIndex(0);
+        var node = internalCluster().startNode();
+        createIndex("test-index", Settings.builder().put(FOO.getKey(), 101).put(BAR.getKey(), 102).build());
+        ensureYellow("test-index");
+        Settings dataPathSettings = internalCluster().dataPathSettings(node);
+        ensureStableCluster(1);
+        internalCluster().stopRandomDataNode();
+
+        Settings nodeSettings = Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build();
+        UserException error = expectThrows(
+            UserException.class,
+            () -> removeIndexSettings(TestEnvironment.newEnvironment(nodeSettings), true, "index.not_foo")
+        );
+        assertThat(error.getMessage(), containsString("No index setting matching [index.not_foo] were found on this node"));
+        internalCluster().startNode(nodeSettings);
+    }
+
+    private MockTerminal executeCommand(ElasticsearchNodeCommand command, Environment environment, boolean abort, String... args)
+        throws Exception {
+        final MockTerminal terminal = MockTerminal.create();
+        final OptionSet options = command.getParser().parse(args);
+        final ProcessInfo processInfo = new ProcessInfo(Map.of(), Map.of(), createTempDir());
+        final String input;
+
+        if (abort) {
+            input = randomValueOtherThanMany(c -> c.equalsIgnoreCase("y"), () -> randomAlphaOfLength(1));
+        } else {
+            input = randomBoolean() ? "y" : "Y";
+        }
+
+        terminal.addTextInput(input);
+
+        try {
+            command.execute(terminal, options, environment, processInfo);
+        } finally {
+            assertThat(terminal.getOutput(), containsString(ElasticsearchNodeCommand.STOP_WARNING_MSG));
+        }
+
+        return terminal;
+    }
+
+    private MockTerminal removeIndexSettings(Environment environment, boolean abort, String... args) throws Exception {
+        final MockTerminal terminal = executeCommand(new RemoveIndexSettingsCommand(), environment, abort, args);
+        assertThat(terminal.getOutput(), containsString(RemoveIndexSettingsCommand.CONFIRMATION_MSG));
+        assertThat(terminal.getOutput(), containsString(RemoveIndexSettingsCommand.SETTINGS_REMOVED_MSG));
+        return terminal;
+    }
+}

server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java

@@ -20,6 +20,7 @@ class NodeToolCli extends MultiCommand {
         subcommands.put("detach-cluster", new DetachClusterCommand());
         subcommands.put("override-version", new OverrideNodeVersionCommand());
         subcommands.put("remove-settings", new RemoveSettingsCommand());
+        subcommands.put("remove-index-settings", new RemoveIndexSettingsCommand());
         subcommands.put("remove-customs", new RemoveCustomsCommand());
     }
 }

server/src/main/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommand.java

@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+package org.elasticsearch.cluster.coordination;
+
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.Terminal;
+import org.elasticsearch.cli.UserException;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.Tuple;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.gateway.PersistedClusterStateService;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+
+public class RemoveIndexSettingsCommand extends ElasticsearchNodeCommand {
+
+    static final String SETTINGS_REMOVED_MSG = "Index settings were successfully removed from the cluster state";
+    static final String CONFIRMATION_MSG = DELIMITER
+        + "\n"
+        + "You should only run this tool if you have incompatible index settings in the\n"
+        + "cluster state that prevent the cluster from forming.\n"
+        + "This tool can cause data loss and its use should be your last resort.\n"
+        + "\n"
+        + "Do you want to proceed?\n";
+
+    private final OptionSpec<String> arguments;
+
+    public RemoveIndexSettingsCommand() {
+        super("Removes index settings from the cluster state");
+        arguments = parser.nonOptions("index setting names");
+    }
+
+    @Override
+    protected void processDataPaths(Terminal terminal, Path[] dataPaths, OptionSet options, Environment env) throws IOException,
+        UserException {
+        final List<String> settingsToRemove = arguments.values(options);
+        if (settingsToRemove.isEmpty()) {
+            throw new UserException(ExitCodes.USAGE, "Must supply at least one index setting to remove");
+        }
+
+        final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPaths);
+
+        terminal.println(Terminal.Verbosity.VERBOSE, "Loading cluster state");
+        final Tuple<Long, ClusterState> termAndClusterState = loadTermAndClusterState(persistedClusterStateService, env);
+        final ClusterState oldClusterState = termAndClusterState.v2();
+        final Metadata.Builder newMetadataBuilder = Metadata.builder(oldClusterState.metadata());
+        int changes = 0;
+        for (IndexMetadata indexMetadata : oldClusterState.metadata()) {
+            Settings oldSettings = indexMetadata.getSettings();
+            Settings.Builder newSettings = Settings.builder().put(oldSettings);
+            boolean removed = false;
+            for (String settingToRemove : settingsToRemove) {
+                for (String settingKey : oldSettings.keySet()) {
+                    if (Regex.simpleMatch(settingToRemove, settingKey)) {
+                        terminal.println(
+                            "Index setting [" + settingKey + "] will be removed from index [" + indexMetadata.getIndex() + "]"
+                        );
+                        newSettings.remove(settingKey);
+                        removed = true;
+                    }
+                }
+            }
+            if (removed) {
+                newMetadataBuilder.put(IndexMetadata.builder(indexMetadata).settings(newSettings));
+                changes++;
+            }
+        }
+        if (changes == 0) {
+            throw new UserException(ExitCodes.USAGE, "No index setting matching " + settingsToRemove + " were found on this node");
+        }
+
+        final ClusterState newClusterState = ClusterState.builder(oldClusterState).metadata(newMetadataBuilder).build();
+        terminal.println(
+            Terminal.Verbosity.VERBOSE,
+            "[old cluster state = " + oldClusterState + ", new cluster state = " + newClusterState + "]"
+        );
+
+        confirm(terminal, CONFIRMATION_MSG);
+
+        try (PersistedClusterStateService.Writer writer = persistedClusterStateService.createWriter()) {
+            writer.writeFullStateAndCommit(termAndClusterState.v1(), newClusterState);
+        }
+
+        terminal.println(SETTINGS_REMOVED_MSG);
+    }
+}