From c62ee88c5f7eb24c56922ecc62c910b25a2b190c Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Fri, 3 Jan 2025 17:44:53 +0100 Subject: [PATCH] Add rolling upgrade BWC tests for N-2 index compatibility (#119468) This change adds rolling upgrade BWC tests for searchable snapshots in N-2 version, so that they are tested in a mixed versions cluster. The tests do not use replicas because it may or may not be assigned depending on which node the primary is assigned on (ie, if the cluster has 1 new node and 2 old node and the primary shard is assigned to the new node, then a replica cannot be assigned until at least another node is upgraded). It also renames the existing full cluster restart BWC tests and mutualizes some common code in AbstractIndexCompatibilityTestCase. Relates ES-10432 --- ...> AbstractIndexCompatibilityTestCase.java} | 117 ++++++++------- ...sterRestartIndexCompatibilityTestCase.java | 65 ++++++++ ...terRestartLuceneIndexCompatibilityIT.java} | 10 +- ...archableSnapshotIndexCompatibilityIT.java} | 16 +- ...lingUpgradeIndexCompatibilityTestCase.java | 104 +++++++++++++ ...earchableSnapshotIndexCompatibilityIT.java | 140 ++++++++++++++++++ 6 files changed, 380 insertions(+), 72 deletions(-) rename qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/{AbstractLuceneIndexCompatibilityTestCase.java => AbstractIndexCompatibilityTestCase.java} (73%) create mode 100644 qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartIndexCompatibilityTestCase.java rename qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/{LuceneCompatibilityIT.java => FullClusterRestartLuceneIndexCompatibilityIT.java} (92%) rename qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/{SearchableSnapshotCompatibilityIT.java => FullClusterRestartSearchableSnapshotIndexCompatibilityIT.java} (91%) create mode 100644 qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeIndexCompatibilityTestCase.java create mode 100644 qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeSearchableSnapshotIndexCompatibilityIT.java diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java similarity index 73% rename from qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java rename to qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java index 1865da06e20c..68375cec5171 100644 --- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java @@ -9,11 +9,6 @@ package org.elasticsearch.lucene; -import com.carrotsearch.randomizedtesting.TestMethodAndParams; -import com.carrotsearch.randomizedtesting.annotations.Name; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; - import org.elasticsearch.client.Request; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Strings; @@ -22,16 +17,18 @@ import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.cluster.util.Version; import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.After; import org.junit.Before; import org.junit.ClassRule; import org.junit.rules.RuleChain; import org.junit.rules.TemporaryFolder; import org.junit.rules.TestRule; -import java.util.Comparator; +import java.io.IOException; +import java.util.HashMap; import java.util.Locale; +import java.util.Map; import java.util.stream.IntStream; -import java.util.stream.Stream; import static org.elasticsearch.test.cluster.util.Version.CURRENT; import static org.elasticsearch.test.cluster.util.Version.fromString; @@ -41,24 +38,21 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -/** - * Test suite for Lucene indices backward compatibility with N-2 versions. The test suite creates a cluster in N-2 version, then upgrades it - * to N-1 version and finally upgrades it to the current version. Test methods are executed after each upgrade. - */ -@TestCaseOrdering(AbstractLuceneIndexCompatibilityTestCase.TestCaseOrdering.class) -public abstract class AbstractLuceneIndexCompatibilityTestCase extends ESRestTestCase { +public abstract class AbstractIndexCompatibilityTestCase extends ESRestTestCase { protected static final Version VERSION_MINUS_2 = fromString(System.getProperty("tests.minimum.index.compatible")); protected static final Version VERSION_MINUS_1 = fromString(System.getProperty("tests.minimum.wire.compatible")); protected static final Version VERSION_CURRENT = CURRENT; - protected static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder(); + protected static final int NODES = 3; + + private static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder(); protected static LocalClusterConfigProvider clusterConfig = c -> {}; private static ElasticsearchCluster cluster = ElasticsearchCluster.local() .distribution(DistributionType.DEFAULT) .version(VERSION_MINUS_2) - .nodes(2) + .nodes(NODES) .setting("path.repo", () -> REPOSITORY_PATH.getRoot().getPath()) .setting("xpack.security.enabled", "false") .setting("xpack.ml.enabled", "false") @@ -71,15 +65,44 @@ public abstract class AbstractLuceneIndexCompatibilityTestCase extends ESRestTes private static boolean upgradeFailed = false; - private final Version clusterVersion; + @Before + public final void maybeUpgradeBeforeTest() throws Exception { + // We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support + // in V10, so we add a check here to ensure we'll revisit this decision once V10 exists. + assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7)); - public AbstractLuceneIndexCompatibilityTestCase(@Name("cluster") Version clusterVersion) { - this.clusterVersion = clusterVersion; + if (upgradeFailed == false) { + try { + maybeUpgrade(); + } catch (Exception e) { + upgradeFailed = true; + throw e; + } + } + + // Skip remaining tests if upgrade failed + assumeFalse("Cluster upgrade failed", upgradeFailed); } - @ParametersFactory - public static Iterable parameters() { - return Stream.of(VERSION_MINUS_2, VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList(); + protected abstract void maybeUpgrade() throws Exception; + + @After + public final void deleteSnapshotBlobCache() throws IOException { + // TODO ES-10475: The .snapshot-blob-cache created in legacy version can block upgrades, we should probably delete it automatically + try { + var request = new Request("DELETE", "/.snapshot-blob-cache"); + request.setOptions( + expectWarnings( + "this request accesses system indices: [.snapshot-blob-cache], but in a future major version, " + + "direct access to system indices will be prevented by default" + ) + ); + adminClient().performRequest(request); + } catch (IOException e) { + if (isNotFoundResponseException(e) == false) { + throw e; + } + } } @Override @@ -92,26 +115,8 @@ public abstract class AbstractLuceneIndexCompatibilityTestCase extends ESRestTes return true; } - @Before - public void maybeUpgrade() throws Exception { - // We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support - // in V10, so we add a check here to ensure we'll revisit this decision once V10 exists. - assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7)); - - var currentVersion = clusterVersion(); - if (currentVersion.before(clusterVersion)) { - try { - cluster.upgradeToVersion(clusterVersion); - closeClients(); - initClient(); - } catch (Exception e) { - upgradeFailed = true; - throw e; - } - } - - // Skip remaining tests if upgrade failed - assumeFalse("Cluster upgrade failed", upgradeFailed); + protected ElasticsearchCluster cluster() { + return cluster; } protected String suffix(String name) { @@ -124,12 +129,18 @@ public abstract class AbstractLuceneIndexCompatibilityTestCase extends ESRestTes .build(); } - protected static Version clusterVersion() throws Exception { - var response = assertOK(client().performRequest(new Request("GET", "/"))); - var responseBody = createFromResponse(response); - var version = Version.fromString(responseBody.evaluate("version.number").toString()); - assertThat("Failed to retrieve cluster version", version, notNullValue()); - return version; + protected static Map nodesVersions() throws Exception { + var nodesInfos = getNodesInfo(adminClient()); + assertThat(nodesInfos.size(), equalTo(NODES)); + var versions = new HashMap(); + for (var nodeInfos : nodesInfos.values()) { + versions.put((String) nodeInfos.get("name"), Version.fromString((String) nodeInfos.get("version"))); + } + return versions; + } + + protected static boolean isFullyUpgradedTo(Version version) throws Exception { + return nodesVersions().values().stream().allMatch(v -> v.equals(version)); } protected static Version indexVersion(String indexName) throws Exception { @@ -181,16 +192,4 @@ public abstract class AbstractLuceneIndexCompatibilityTestCase extends ESRestTes assertThat(responseBody.evaluate("snapshot.shards.total"), equalTo((int) responseBody.evaluate("snapshot.shards.failed"))); assertThat(responseBody.evaluate("snapshot.shards.successful"), equalTo(0)); } - - /** - * Execute the test suite with the parameters provided by the {@link #parameters()} in version order. - */ - public static class TestCaseOrdering implements Comparator { - @Override - public int compare(TestMethodAndParams o1, TestMethodAndParams o2) { - var version1 = (Version) o1.getInstanceArguments().get(0); - var version2 = (Version) o2.getInstanceArguments().get(0); - return version1.compareTo(version2); - } - } } diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartIndexCompatibilityTestCase.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartIndexCompatibilityTestCase.java new file mode 100644 index 000000000000..9ca7132493ae --- /dev/null +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartIndexCompatibilityTestCase.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.lucene; + +import com.carrotsearch.randomizedtesting.TestMethodAndParams; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; + +import org.elasticsearch.test.cluster.util.Version; + +import java.util.Comparator; +import java.util.stream.Stream; + +import static org.elasticsearch.test.cluster.util.Version.CURRENT; +import static org.hamcrest.Matchers.equalTo; + +/** + * Test suite for Lucene indices backward compatibility with N-2 versions after full cluster restart upgrades. The test suite creates a + * cluster in N-2 version, then upgrades it to N-1 version and finally upgrades it to the current version. Test methods are executed after + * each upgrade. + */ +@TestCaseOrdering(FullClusterRestartIndexCompatibilityTestCase.TestCaseOrdering.class) +public abstract class FullClusterRestartIndexCompatibilityTestCase extends AbstractIndexCompatibilityTestCase { + + private final Version clusterVersion; + + public FullClusterRestartIndexCompatibilityTestCase(@Name("cluster") Version clusterVersion) { + this.clusterVersion = clusterVersion; + } + + @ParametersFactory + public static Iterable parameters() { + return Stream.of(VERSION_MINUS_2, VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList(); + } + + @Override + protected void maybeUpgrade() throws Exception { + if (nodesVersions().values().stream().anyMatch(version -> version.before(clusterVersion))) { + cluster().upgradeToVersion(clusterVersion); + closeClients(); + initClient(); + } + assertThat(isFullyUpgradedTo(clusterVersion), equalTo(true)); + } + + /** + * Execute the test suite with the parameters provided by the {@link #parameters()} in version order. + */ + public static class TestCaseOrdering implements Comparator { + @Override + public int compare(TestMethodAndParams o1, TestMethodAndParams o2) { + var version1 = (Version) o1.getInstanceArguments().get(0); + var version2 = (Version) o2.getInstanceArguments().get(0); + return version1.compareTo(version2); + } + } +} diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartLuceneIndexCompatibilityIT.java similarity index 92% rename from qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java rename to qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartLuceneIndexCompatibilityIT.java index 655e30f069f1..15d41cc981ce 100644 --- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartLuceneIndexCompatibilityIT.java @@ -23,13 +23,13 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; -public class LuceneCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase { +public class FullClusterRestartLuceneIndexCompatibilityIT extends FullClusterRestartIndexCompatibilityTestCase { static { clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial"); } - public LuceneCompatibilityIT(Version version) { + public FullClusterRestartLuceneIndexCompatibilityIT(Version version) { super(version); } @@ -42,7 +42,7 @@ public class LuceneCompatibilityIT extends AbstractLuceneIndexCompatibilityTestC final String index = suffix("index"); final int numDocs = 1234; - if (VERSION_MINUS_2.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_MINUS_2)) { logger.debug("--> registering repository [{}]", repository); registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); @@ -65,7 +65,7 @@ public class LuceneCompatibilityIT extends AbstractLuceneIndexCompatibilityTestC return; } - if (VERSION_MINUS_1.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_MINUS_1)) { ensureGreen(index); assertThat(indexVersion(index), equalTo(VERSION_MINUS_2)); @@ -76,7 +76,7 @@ public class LuceneCompatibilityIT extends AbstractLuceneIndexCompatibilityTestC return; } - if (VERSION_CURRENT.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_CURRENT)) { var restoredIndex = suffix("index-restored"); logger.debug("--> restoring index [{}] as [{}]", index, restoredIndex); diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSearchableSnapshotIndexCompatibilityIT.java similarity index 91% rename from qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java rename to qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSearchableSnapshotIndexCompatibilityIT.java index d5db17f257b0..4d59badcba7e 100644 --- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSearchableSnapshotIndexCompatibilityIT.java @@ -17,7 +17,7 @@ import org.elasticsearch.test.cluster.util.Version; import static org.hamcrest.Matchers.equalTo; -public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase { +public class FullClusterRestartSearchableSnapshotIndexCompatibilityIT extends FullClusterRestartIndexCompatibilityTestCase { static { clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial") @@ -25,7 +25,7 @@ public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompat .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB"); } - public SearchableSnapshotCompatibilityIT(Version version) { + public FullClusterRestartSearchableSnapshotIndexCompatibilityIT(Version version) { super(version); } @@ -38,7 +38,7 @@ public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompat final String index = suffix("index"); final int numDocs = 1234; - if (VERSION_MINUS_2.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_MINUS_2)) { logger.debug("--> registering repository [{}]", repository); registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); @@ -61,7 +61,7 @@ public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompat return; } - if (VERSION_MINUS_1.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_MINUS_1)) { ensureGreen(index); assertThat(indexVersion(index), equalTo(VERSION_MINUS_2)); @@ -72,7 +72,7 @@ public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompat return; } - if (VERSION_CURRENT.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_CURRENT)) { var mountedIndex = suffix("index-mounted"); logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex); mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex); @@ -98,7 +98,7 @@ public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompat final String index = suffix("index"); final int numDocs = 4321; - if (VERSION_MINUS_2.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_MINUS_2)) { logger.debug("--> registering repository [{}]", repository); registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); @@ -124,7 +124,7 @@ public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompat return; } - if (VERSION_MINUS_1.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_MINUS_1)) { logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex); mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex); @@ -135,7 +135,7 @@ public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompat return; } - if (VERSION_CURRENT.equals(clusterVersion())) { + if (isFullyUpgradedTo(VERSION_CURRENT)) { ensureGreen(mountedIndex); assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2)); diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeIndexCompatibilityTestCase.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeIndexCompatibilityTestCase.java new file mode 100644 index 000000000000..03b6a9292e35 --- /dev/null +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeIndexCompatibilityTestCase.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.lucene; + +import com.carrotsearch.randomizedtesting.TestMethodAndParams; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; + +import org.elasticsearch.test.cluster.util.Version; + +import java.util.Comparator; +import java.util.List; +import java.util.stream.Stream; + +import static org.elasticsearch.test.cluster.util.Version.CURRENT; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.notNullValue; + +/** + * Test suite for Lucene indices backward compatibility with N-2 versions during rolling upgrades. The test suite creates a cluster in N-2 + * version, then upgrades each node sequentially to N-1 version and finally upgrades each node sequentially to the current version. Test + * methods are executed after each node upgrade. + */ +@TestCaseOrdering(RollingUpgradeIndexCompatibilityTestCase.TestCaseOrdering.class) +public abstract class RollingUpgradeIndexCompatibilityTestCase extends AbstractIndexCompatibilityTestCase { + + private final List nodesVersions; + + public RollingUpgradeIndexCompatibilityTestCase(@Name("cluster") List nodesVersions) { + this.nodesVersions = nodesVersions; + } + + @ParametersFactory + public static Iterable parameters() { + return Stream.of( + // Begin on N-2 + List.of(VERSION_MINUS_2, VERSION_MINUS_2, VERSION_MINUS_2), + // Rolling upgrade to VERSION_MINUS_1 + List.of(VERSION_MINUS_1, VERSION_MINUS_2, VERSION_MINUS_2), + List.of(VERSION_MINUS_1, VERSION_MINUS_1, VERSION_MINUS_2), + List.of(VERSION_MINUS_1, VERSION_MINUS_1, VERSION_MINUS_1), + // Rolling upgrade to CURRENT + List.of(CURRENT, VERSION_MINUS_1, VERSION_MINUS_1), + List.of(CURRENT, CURRENT, VERSION_MINUS_1), + List.of(CURRENT, CURRENT, CURRENT) + ).map(nodesVersion -> new Object[] { nodesVersion }).toList(); + } + + @Override + protected void maybeUpgrade() throws Exception { + assertThat(nodesVersions, hasSize(NODES)); + + for (int i = 0; i < NODES; i++) { + var nodeName = cluster().getName(i); + + var expectedNodeVersion = nodesVersions.get(i); + assertThat(expectedNodeVersion, notNullValue()); + + var currentNodeVersion = nodesVersions().get(nodeName); + assertThat(currentNodeVersion, notNullValue()); + assertThat(currentNodeVersion.onOrBefore(expectedNodeVersion), equalTo(true)); + + if (currentNodeVersion.equals(expectedNodeVersion) == false) { + closeClients(); + cluster().upgradeNodeToVersion(i, expectedNodeVersion); + initClient(); + } + + currentNodeVersion = nodesVersions().get(nodeName); + assertThat(currentNodeVersion, equalTo(expectedNodeVersion)); + } + } + + /** + * Execute the test suite with the parameters provided by the {@link #parameters()} in nodes versions order. + */ + public static class TestCaseOrdering implements Comparator { + @Override + public int compare(TestMethodAndParams o1, TestMethodAndParams o2) { + List nodesVersions1 = asInstanceOf(List.class, o1.getInstanceArguments().get(0)); + assertThat(nodesVersions1, hasSize(NODES)); + List nodesVersions2 = asInstanceOf(List.class, o2.getInstanceArguments().get(0)); + assertThat(nodesVersions2, hasSize(NODES)); + for (int i = 0; i < NODES; i++) { + var nodeVersion1 = asInstanceOf(Version.class, nodesVersions1.get(i)); + var nodeVersion2 = asInstanceOf(Version.class, nodesVersions2.get(i)); + var result = nodeVersion1.compareTo(nodeVersion2); + if (result != 0) { + return result; + } + } + return 0; + } + } +} diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeSearchableSnapshotIndexCompatibilityIT.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeSearchableSnapshotIndexCompatibilityIT.java new file mode 100644 index 000000000000..89c159c15f70 --- /dev/null +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeSearchableSnapshotIndexCompatibilityIT.java @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.lucene; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.repositories.fs.FsRepository; +import org.elasticsearch.test.cluster.util.Version; + +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class RollingUpgradeSearchableSnapshotIndexCompatibilityIT extends RollingUpgradeIndexCompatibilityTestCase { + + static { + clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial") + .setting("xpack.searchable.snapshot.shared_cache.size", "16MB") + .setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB"); + } + + public RollingUpgradeSearchableSnapshotIndexCompatibilityIT(List nodesVersion) { + super(nodesVersion); + } + + /** + * Creates an index and a snapshot on N-2, then mounts the snapshot during rolling upgrades. + */ + public void testMountSearchableSnapshot() throws Exception { + final String repository = suffix("repository"); + final String snapshot = suffix("snapshot"); + final String index = suffix("index-rolling-upgrade"); + final var mountedIndex = suffix("index-rolling-upgrade-mounted"); + final int numDocs = 3145; + + if (isFullyUpgradedTo(VERSION_MINUS_2)) { + logger.debug("--> registering repository [{}]", repository); + registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); + + logger.debug("--> creating index [{}]", index); + createIndex( + client(), + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) + .build() + ); + + logger.debug("--> indexing [{}] docs in [{}]", numDocs, index); + indexDocs(index, numDocs); + + logger.debug("--> creating snapshot [{}]", snapshot); + createSnapshot(client(), repository, snapshot, true); + + logger.debug("--> deleting index [{}]", index); + deleteIndex(index); + return; + } + + boolean success = false; + try { + logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex); + mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex); + + ensureGreen(mountedIndex); + + assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2)); + assertDocCount(client(), mountedIndex, numDocs); + + logger.debug("--> deleting mounted index [{}]", mountedIndex); + deleteIndex(mountedIndex); + + success = true; + } finally { + if (success == false) { + try { + client().performRequest(new Request("DELETE", "/" + mountedIndex)); + } catch (ResponseException e) { + logger.warn("Failed to delete mounted index [" + mountedIndex + ']', e); + } + } + } + } + + /** + * Creates an index and a snapshot on N-2, mounts the snapshot and ensures it remains searchable during rolling upgrades. + */ + public void testSearchableSnapshotUpgrade() throws Exception { + final String mountedIndex = suffix("index-rolling-upgraded-mounted"); + final String repository = suffix("repository"); + final String snapshot = suffix("snapshot"); + final String index = suffix("index-rolling-upgraded"); + final int numDocs = 2143; + + if (isFullyUpgradedTo(VERSION_MINUS_2)) { + logger.debug("--> registering repository [{}]", repository); + registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); + + logger.debug("--> creating index [{}]", index); + createIndex( + client(), + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) + .build() + ); + + logger.debug("--> indexing [{}] docs in [{}]", numDocs, index); + indexDocs(index, numDocs); + + logger.debug("--> creating snapshot [{}]", snapshot); + createSnapshot(client(), repository, snapshot, true); + + logger.debug("--> deleting index [{}]", index); + deleteIndex(index); + + logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex); + mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex); + } + + ensureGreen(mountedIndex); + + assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2)); + assertDocCount(client(), mountedIndex, numDocs); + } +}