Drop alloc filters on mount of searchable snapshot (#70007)

Today when a searchable snapshot is mounted it inherits many of the
settings of the original index, including any index-level allocation
filters that were in place when the snapshot was taken. These filters
typically make no sense on the mounted index, so users must explicitly
override or drop them.

With this commit at mount time we drop all index-level allocation
filters that existed on the original index, although we continue to
respect any allocation filters that were specified as part of the mount
request.

Closes #69759

x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/AllocationFilteringIntegTests.java

@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.searchablesnapshots;
+
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction;
+import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest;
+
+import java.util.List;
+import java.util.Locale;
+
+import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_SETTING;
+import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
+import static org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING;
+import static org.elasticsearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class AllocationFilteringIntegTests extends BaseSearchableSnapshotsIntegTestCase {
+
+    private MountSearchableSnapshotRequest prepareMountRequest(
+        Settings.Builder originalIndexSettings,
+        Settings.Builder mountedIndexSettings
+    ) throws InterruptedException {
+
+        final String fsRepoName = randomAlphaOfLength(10);
+        final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+        final String snapshotName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
+
+        createRepository(fsRepoName, "fs", Settings.builder().put("location", randomRepoPath()));
+        assertAcked(
+            prepareCreate(
+                indexName,
+                Settings.builder()
+                    .put(INDEX_SOFT_DELETES_SETTING.getKey(), true)
+                    .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put(originalIndexSettings.build())
+            )
+        );
+        populateIndex(indexName, 10);
+        ensureGreen(indexName);
+        createFullSnapshot(fsRepoName, snapshotName);
+        assertAcked(client().admin().indices().prepareDelete(indexName));
+
+        final Settings.Builder indexSettingsBuilder = Settings.builder()
+            .put(SearchableSnapshots.SNAPSHOT_CACHE_ENABLED_SETTING.getKey(), true)
+            .put(IndexSettings.INDEX_CHECK_ON_STARTUP.getKey(), Boolean.FALSE.toString())
+            .put(mountedIndexSettings.build());
+
+        return new MountSearchableSnapshotRequest(
+            indexName,
+            fsRepoName,
+            snapshotName,
+            indexName,
+            indexSettingsBuilder.build(),
+            Strings.EMPTY_ARRAY,
+            true,
+            MountSearchableSnapshotRequest.Storage.FULL_COPY
+        );
+    }
+
+    public void testRemovesRequireFilter() throws InterruptedException {
+        runTest(INDEX_ROUTING_REQUIRE_GROUP_SETTING, true, null, true);
+    }
+
+    public void testRemovesIncludeFilter() throws InterruptedException {
+        runTest(INDEX_ROUTING_INCLUDE_GROUP_SETTING, true, null, true);
+    }
+
+    public void testRemovesExcludeFilter() throws InterruptedException {
+        runTest(INDEX_ROUTING_EXCLUDE_GROUP_SETTING, false, null, true);
+    }
+
+    public void testReplacesRequireFilter() throws InterruptedException {
+        runTest(INDEX_ROUTING_REQUIRE_GROUP_SETTING, true, INDEX_ROUTING_REQUIRE_GROUP_SETTING, true);
+    }
+
+    public void testReplacesIncludeFilter() throws InterruptedException {
+        runTest(INDEX_ROUTING_INCLUDE_GROUP_SETTING, true, INDEX_ROUTING_INCLUDE_GROUP_SETTING, true);
+    }
+
+    public void testReplacesExcludeFilter() throws InterruptedException {
+        runTest(INDEX_ROUTING_EXCLUDE_GROUP_SETTING, false, INDEX_ROUTING_EXCLUDE_GROUP_SETTING, false);
+    }
+
+    public void testReplacesIncludeFilterWithExcludeFilter() throws InterruptedException {
+        runTest(INDEX_ROUTING_INCLUDE_GROUP_SETTING, true, INDEX_ROUTING_EXCLUDE_GROUP_SETTING, false);
+    }
+
+    /**
+     * Starts two nodes, allocates the original index on the first node but then excludes that node and verifies that the index can still be
+     * mounted and allocated to the other node.
+     *
+     * @param indexSetting           an allocation filter setting to apply to the original index
+     * @param mountSetting           an (optional) allocation filter setting to apply at mount time
+     * @param indexSettingIsPositive whether {@code indexSetting} is positive (i.e. include/require) or negative (i.e. exclude)
+     * @param mountSettingIsPositive whether {@code mountSetting} is positive (i.e. include/require) or negative (i.e. exclude)
+     */
+    private void runTest(
+        Setting.AffixSetting<String> indexSetting,
+        boolean indexSettingIsPositive,
+        @Nullable Setting.AffixSetting<String> mountSetting,
+        boolean mountSettingIsPositive
+    ) throws InterruptedException {
+        final List<String> nodes = internalCluster().startNodes(2);
+
+        // apply an index setting to restrict the original index to node 0
+        final Settings.Builder indexSettings = Settings.builder()
+            .put(indexSetting.getConcreteSettingForNamespace("_name").getKey(), nodes.get(indexSettingIsPositive ? 0 : 1));
+
+        final Settings.Builder mountSettings = Settings.builder();
+        if (mountSetting != null) {
+            // apply an index setting to restrict the mounted index to node 1
+            mountSettings.put(mountSetting.getConcreteSettingForNamespace("_name").getKey(), nodes.get(mountSettingIsPositive ? 1 : 0));
+        }
+
+        final MountSearchableSnapshotRequest mountRequest = prepareMountRequest(indexSettings, mountSettings);
+
+        // block allocation to node 0 at the cluster level
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(
+                    Settings.builder()
+                        .put(CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey(), nodes.get(0))
+                )
+        );
+
+        // mount snapshot and verify it is allocated as expected
+        final RestoreSnapshotResponse restoreSnapshotResponse = client().execute(MountSearchableSnapshotAction.INSTANCE, mountRequest)
+            .actionGet();
+        assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
+        ensureGreen(mountRequest.mountedIndexName());
+
+        final Settings mountedIndexSettings = client().admin()
+            .indices()
+            .prepareGetSettings(mountRequest.mountedIndexName())
+            .get()
+            .getIndexToSettings()
+            .get(mountRequest.mountedIndexName());
+
+        if (mountSetting != null) {
+            assertTrue(mountedIndexSettings.toString(), mountSetting.getConcreteSettingForNamespace("_name").exists(mountedIndexSettings));
+        }
+
+        if (indexSetting != mountSetting) {
+            assertFalse(mountedIndexSettings.toString(), indexSetting.getConcreteSettingForNamespace("_name").exists(mountedIndexSettings));
+        }
+
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(
+                    Settings.builder().putNull(CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING.getConcreteSettingForNamespace("_name").getKey())
+                )
+        );
+    }
+
+}

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java

@@ -43,10 +43,12 @@ import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots;
 import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants;
 
 import java.util.Arrays;
+import java.util.LinkedHashSet;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 
 import static org.elasticsearch.index.IndexModule.INDEX_RECOVERY_TYPE_SETTING;
 import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
@@ -180,9 +182,6 @@ public class TransportMountSearchableSnapshotAction extends TransportMasterNodeA
             }
             final SnapshotId snapshotId = matchingSnapshotId.get();
 
-            final String[] ignoreIndexSettings = Arrays.copyOf(request.ignoreIndexSettings(), request.ignoreIndexSettings().length + 1);
-            ignoreIndexSettings[ignoreIndexSettings.length - 1] = IndexMetadata.SETTING_DATA_PATH;
-
             final IndexMetadata indexMetadata = repository.getSnapshotIndexMetaData(repoData, snapshotId, indexId);
             if (isSearchableSnapshotStore(indexMetadata.getSettings())) {
                 throw new IllegalArgumentException(
@@ -202,6 +201,16 @@ public class TransportMountSearchableSnapshotAction extends TransportMasterNodeA
                 );
             }
 
+            final Set<String> ignoreIndexSettings = new LinkedHashSet<>(Arrays.asList(request.ignoreIndexSettings()));
+            ignoreIndexSettings.add(IndexMetadata.SETTING_DATA_PATH);
+            for (final String indexSettingKey : indexMetadata.getSettings().keySet()) {
+                if (indexSettingKey.startsWith(IndexMetadata.INDEX_ROUTING_REQUIRE_GROUP_PREFIX)
+                    || indexSettingKey.startsWith(IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_PREFIX)
+                    || indexSettingKey.startsWith(IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_PREFIX)) {
+                    ignoreIndexSettings.add(indexSettingKey);
+                }
+            }
+
             client.admin()
                 .cluster()
                 .restoreSnapshot(
@@ -224,7 +233,7 @@ public class TransportMountSearchableSnapshotAction extends TransportMasterNodeA
                                 .build()
                         )
                         // Pass through ignored index settings
-                        .ignoreIndexSettings(ignoreIndexSettings)
+                        .ignoreIndexSettings(ignoreIndexSettings.toArray(new String[0]))
                         // Don't include global state
                         .includeGlobalState(false)
                         // Don't include aliases