From 1b5827da3e1ec2699ece507cbb67fe68f07fc8fe Mon Sep 17 00:00:00 2001 From: William Brafford Date: Tue, 12 Oct 2021 15:44:45 -0400 Subject: [PATCH] Implement GET API for System Feature Upgrades (#78642) (#78860) * Implement GET API for System Feature Upgrades (#78642) * Implement and test get feature upgrade status API * Add integration test for feature upgrade endpoint * Use constant enum for statuses * Add unit tests for transport class methods * Fix bwc tests for 7.x --- .../org/elasticsearch/client/MigrationIT.java | 25 ++-- .../GetFeatureUpgradeStatusResponseTests.java | 17 +-- .../migration/apis/feature_upgrade.asciidoc | 80 ++++++++++-- .../upgrades/FeatureUpgradeIT.java | 115 ++++++++++++++++++ .../system/indices/FeatureUpgradeApiIT.java | 70 +++++++++++ .../GetFeatureUpgradeStatusResponse.java | 77 ++++++++---- ...ransportGetFeatureUpgradeStatusAction.java | 64 ++++++++-- .../GetFeatureUpgradeStatusResponseTests.java | 17 +-- ...ortGetFeatureUpgradeStatusActionTests.java | 99 +++++++++++++++ 9 files changed, 500 insertions(+), 64 deletions(-) create mode 100644 qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java create mode 100644 qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java create mode 100644 server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java index 335263033f75..28334d83de1e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java @@ -9,6 +9,7 @@ package org.elasticsearch.client; import org.elasticsearch.jdk.JavaVersion; +import org.elasticsearch.Version; import org.elasticsearch.client.migration.DeprecationInfoRequest; import org.elasticsearch.client.migration.DeprecationInfoResponse; import org.elasticsearch.client.migration.GetFeatureUpgradeStatusRequest; @@ -20,11 +21,14 @@ import org.elasticsearch.common.settings.Settings; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; public class MigrationIT extends ESRestHighLevelClientTestCase { @@ -51,19 +55,24 @@ public class MigrationIT extends ESRestHighLevelClientTestCase { public void testGetFeatureUpgradeStatus() throws IOException { GetFeatureUpgradeStatusRequest request = new GetFeatureUpgradeStatusRequest(); GetFeatureUpgradeStatusResponse response = highLevelClient().migration().getFeatureUpgradeStatus(request, RequestOptions.DEFAULT); - assertThat(response.getUpgradeStatus(), equalTo("UPGRADE_NEEDED")); - assertThat(response.getFeatureUpgradeStatuses().size(), equalTo(1)); - GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus status = response.getFeatureUpgradeStatuses().get(0); - assertThat(status.getUpgradeStatus(), equalTo("UPGRADE_NEEDED")); - assertThat(status.getMinimumIndexVersion(), equalTo("7.1.1")); - assertThat(status.getFeatureName(), equalTo("security")); - assertThat(status.getIndexVersions().size(), equalTo(1)); + assertThat(response.getUpgradeStatus(), equalTo("NO_UPGRADE_NEEDED")); + assertThat(response.getFeatureUpgradeStatuses().size(), greaterThanOrEqualTo(1)); + Optional optionalTasksStatus = response.getFeatureUpgradeStatuses().stream() + .filter(status -> "tasks".equals(status.getFeatureName())) + .findFirst(); + + assertThat(optionalTasksStatus.isPresent(), is(true)); + + GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus tasksStatus = optionalTasksStatus.get(); + + assertThat(tasksStatus.getUpgradeStatus(), equalTo("NO_UPGRADE_NEEDED")); + assertThat(tasksStatus.getMinimumIndexVersion(), equalTo(Version.CURRENT.toString())); + assertThat(tasksStatus.getFeatureName(), equalTo("tasks")); } public void testPostFeatureUpgradeStatus() throws IOException { PostFeatureUpgradeRequest request = new PostFeatureUpgradeRequest(); PostFeatureUpgradeResponse response = highLevelClient().migration().postFeatureUpgrade(request, RequestOptions.DEFAULT); - // a test like this cannot test actual deprecations assertThat(response.isAccepted(), equalTo(true)); assertThat(response.getFeatures().size(), equalTo(1)); PostFeatureUpgradeResponse.Feature feature = response.getFeatures().get(0); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponseTests.java index 9c9c4ef32b51..e478a2444297 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponseTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.client.migration; +import org.elasticsearch.Version; import org.elasticsearch.client.AbstractResponseTestCase; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; @@ -37,14 +38,14 @@ public class GetFeatureUpgradeStatusResponseTests extends AbstractResponseTestCa randomList(5, () -> new org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus( randomAlphaOfLengthBetween(3, 20), - randomAlphaOfLengthBetween(5, 9), - randomAlphaOfLengthBetween(4, 16), + randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()), + randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()), randomList(4, () -> new org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.IndexVersion( randomAlphaOfLengthBetween(3, 20), - randomAlphaOfLengthBetween(5, 9))) + randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()))) )), - randomAlphaOfLength(5) + randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()) ); } @@ -58,7 +59,7 @@ public class GetFeatureUpgradeStatusResponseTests extends AbstractResponseTestCa org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse serverTestInstance, GetFeatureUpgradeStatusResponse clientInstance) { - assertThat(clientInstance.getUpgradeStatus(), equalTo(serverTestInstance.getUpgradeStatus())); + assertThat(clientInstance.getUpgradeStatus(), equalTo(serverTestInstance.getUpgradeStatus().toString())); assertNotNull(serverTestInstance.getFeatureUpgradeStatuses()); assertNotNull(clientInstance.getFeatureUpgradeStatuses()); @@ -71,8 +72,8 @@ public class GetFeatureUpgradeStatusResponseTests extends AbstractResponseTestCa GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus clientStatus = clientInstance.getFeatureUpgradeStatuses().get(i); assertThat(clientStatus.getFeatureName(), equalTo(serverTestStatus.getFeatureName())); - assertThat(clientStatus.getMinimumIndexVersion(), equalTo(serverTestStatus.getMinimumIndexVersion())); - assertThat(clientStatus.getUpgradeStatus(), equalTo(serverTestStatus.getUpgradeStatus())); + assertThat(clientStatus.getMinimumIndexVersion(), equalTo(serverTestStatus.getMinimumIndexVersion().toString())); + assertThat(clientStatus.getUpgradeStatus(), equalTo(serverTestStatus.getUpgradeStatus().toString())); assertThat(clientStatus.getIndexVersions(), hasSize(serverTestStatus.getIndexVersions().size())); @@ -82,7 +83,7 @@ public class GetFeatureUpgradeStatusResponseTests extends AbstractResponseTestCa GetFeatureUpgradeStatusResponse.IndexVersion clientIndexVersion = clientStatus.getIndexVersions().get(j); assertThat(clientIndexVersion.getIndexName(), equalTo(serverIndexVersion.getIndexName())); - assertThat(clientIndexVersion.getVersion(), equalTo(serverIndexVersion.getVersion())); + assertThat(clientIndexVersion.getVersion(), equalTo(serverIndexVersion.getVersion().toString())); } } } diff --git a/docs/reference/migration/apis/feature_upgrade.asciidoc b/docs/reference/migration/apis/feature_upgrade.asciidoc index 7c50460e8b27..e40176d40fed 100644 --- a/docs/reference/migration/apis/feature_upgrade.asciidoc +++ b/docs/reference/migration/apis/feature_upgrade.asciidoc @@ -43,21 +43,83 @@ Example response: -------------------------------------------------- { "features" : [ + { + "feature_name" : "async_search", + "minimum_index_version" : "7.0.0", + "upgrade_status" : "NO_UPGRADE_NEEDED", + "indices" : [ ] + }, + { + "feature_name" : "enrich", + "minimum_index_version" : "7.0.0", + "upgrade_status" : "NO_UPGRADE_NEEDED", + "indices" : [ ] + }, + { + "feature_name" : "fleet", + "minimum_index_version" : "7.0.0", + "upgrade_status" : "NO_UPGRADE_NEEDED", + "indices" : [ ] + }, + { + "feature_name" : "geoip", + "minimum_index_version" : "7.0.0", + "upgrade_status" : "NO_UPGRADE_NEEDED", + "indices" : [ ] + }, + { + "feature_name" : "kibana", + "minimum_index_version" : "7.0.0", + "upgrade_status" : "NO_UPGRADE_NEEDED", + "indices" : [ ] + }, + { + "feature_name" : "logstash_management", + "minimum_index_version" : "7.0.0", + "upgrade_status" : "NO_UPGRADE_NEEDED", + "indices" : [ ] + }, + { + "feature_name" : "machine_learning", + "minimum_index_version" : "7.0.0", + "upgrade_status" : "NO_UPGRADE_NEEDED", + "indices" : [ ] + }, + { + "feature_name" : "searchable_snapshots", + "minimum_index_version" : "7.0.0", + "upgrade_status" : "NO_UPGRADE_NEEDED", + "indices" : [ ] + }, { "feature_name" : "security", - "minimum_index_version" : "7.1.1", - "upgrade_status" : "UPGRADE_NEEDED", - "indices" : [ - { - "index" : ".security-7", - "version" : "7.1.1" - } - ] + "minimum_index_version" : "7.0.0", + "upgrade_status" : "NO_UPGRADE_NEEDED", + "indices" : [ ] + }, + { + "feature_name" : "tasks", + "minimum_index_version" : "7.0.0", + "upgrade_status" : "NO_UPGRADE_NEEDED", + "indices" : [ ] + }, + { + "feature_name" : "transform", + "minimum_index_version" : "7.0.0", + "upgrade_status" : "NO_UPGRADE_NEEDED", + "indices" : [ ] + }, + { + "feature_name" : "watcher", + "minimum_index_version" : "7.0.0", + "upgrade_status" : "NO_UPGRADE_NEEDED", + "indices" : [ ] } ], - "upgrade_status" : "UPGRADE_NEEDED" + "upgrade_status" : "NO_UPGRADE_NEEDED" } -------------------------------------------------- +// TESTRESPONSE[s/"minimum_index_version" : "7.0.0"/"minimum_index_version" : $body.$_path/] This response tells us that Elasticsearch security needs its internal indices upgraded before we can upgrade the cluster to 8.0. diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java new file mode 100644 index 000000000000..cd48aa606267 --- /dev/null +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.upgrades; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.test.XContentTestUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class FeatureUpgradeIT extends AbstractRollingTestCase { + + @SuppressWarnings("unchecked") + public void testGetFeatureUpgradeStatus() throws Exception { + + final String systemIndexWarning = "this request accesses system indices: [.tasks], but in a future major version, direct " + + "access to system indices will be prevented by default"; + if (CLUSTER_TYPE == ClusterType.OLD) { + // setup - put something in the tasks index + // create index + Request createTestIndex = new Request("PUT", "/feature_test_index_old"); + createTestIndex.setJsonEntity("{\"settings\": {" + + "\"index.number_of_replicas\": 0," + + "\"index.number_of_shards\": 1" + + "}}"); + client().performRequest(createTestIndex); + + Request bulk = new Request("POST", "/_bulk"); + bulk.addParameter("refresh", "true"); + if (UPGRADE_FROM_VERSION.before(Version.V_7_0_0)) { + bulk.setJsonEntity("{\"index\": {\"_index\": \"feature_test_index_old\", \"_type\" : \"_doc\"}}\n" + + "{\"f1\": \"v1\", \"f2\": \"v2\"}\n"); + } else { + bulk.setJsonEntity("{\"index\": {\"_index\": \"feature_test_index_old\"}\n" + + "{\"f1\": \"v1\", \"f2\": \"v2\"}\n"); + } + client().performRequest(bulk); + + // start a async reindex job + Request reindex = new Request("POST", "/_reindex"); + reindex.setJsonEntity( + "{\n" + + " \"source\":{\n" + + " \"index\":\"feature_test_index_old\"\n" + + " },\n" + + " \"dest\":{\n" + + " \"index\":\"feature_test_index_reindex\"\n" + + " }\n" + + "}"); + reindex.addParameter("wait_for_completion", "false"); + Map response = entityAsMap(client().performRequest(reindex)); + String taskId = (String) response.get("task"); + + // wait for task + Request getTask = new Request("GET", "/_tasks/" + taskId); + getTask.addParameter("wait_for_completion", "true"); + client().performRequest(getTask); + + // make sure .tasks index exists + Request getTasksIndex = new Request("GET", "/.tasks"); + getTasksIndex.addParameter("allow_no_indices", "false"); + if (UPGRADE_FROM_VERSION.before(Version.V_7_0_0)) { + getTasksIndex.addParameter("include_type_name", "false"); + } + + getTasksIndex.setOptions(expectVersionSpecificWarnings(v -> { + v.current(systemIndexWarning); + v.compatible(systemIndexWarning); + })); + assertBusy(() -> { + try { + assertThat(client().performRequest(getTasksIndex).getStatusLine().getStatusCode(), is(200)); + } catch (ResponseException e) { + throw new AssertionError(".tasks index does not exist yet"); + } + }); + + } else if (CLUSTER_TYPE == ClusterType.UPGRADED) { + // check results + assertBusy(() -> { + Request clusterStateRequest = new Request("GET", "/_migration/system_features"); + XContentTestUtils.JsonMapView view = new XContentTestUtils.JsonMapView( + entityAsMap(client().performRequest(clusterStateRequest))); + + List> features = view.get("features"); + Map feature = features.stream() + .filter(e -> "tasks".equals(e.get("feature_name"))) + .findFirst() + .orElse(Collections.emptyMap()); + + assertThat(feature.size(), equalTo(4)); + assertThat(feature.get("minimum_index_version"), equalTo(UPGRADE_FROM_VERSION.toString())); + if (UPGRADE_FROM_VERSION.before(TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION)) { + assertThat(feature.get("upgrade_status"), equalTo("UPGRADE_NEEDED")); + } else { + assertThat(feature.get("upgrade_status"), equalTo("NO_UPGRADE_NEEDED")); + } + }); + } + } + +} diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java new file mode 100644 index 000000000000..af9839a772a1 --- /dev/null +++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.system.indices; + +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.XContentTestUtils; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.After; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; + +public class FeatureUpgradeApiIT extends ESRestTestCase { + + static final String BASIC_AUTH_VALUE = basicAuthHeaderValue("rest_user", new SecureString("rest-user-password".toCharArray())); + + @After + public void resetFeatures() throws Exception { + client().performRequest(new Request("POST", "/_features/_reset")); + } + + @Override + protected Settings restClientSettings() { + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", BASIC_AUTH_VALUE).build(); + } + + public void testCreatingSystemIndex() throws Exception { + Response response = client().performRequest(new Request("PUT", "/_net_new_sys_index/_create")); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + } + + @SuppressWarnings("unchecked") + public void testGetFeatureUpgradedStatuses() throws Exception { + client().performRequest(new Request("PUT", "/_net_new_sys_index/_create")); + Response response = client().performRequest(new Request("GET", "/_migration/system_features")); + assertThat(response.getStatusLine().getStatusCode(), is(200)); + XContentTestUtils.JsonMapView view = XContentTestUtils.createJsonMapView(response.getEntity().getContent()); + String upgradeStatus = view.get("upgrade_status"); + assertThat(upgradeStatus, equalTo("NO_UPGRADE_NEEDED")); + List> features = view.get("features"); + Map testFeature = features.stream() + .filter(feature -> "system indices qa".equals(feature.get("feature_name"))) + .findFirst() + .orElse(Collections.emptyMap()); + + assertThat(testFeature.size(), equalTo(4)); + assertThat(testFeature.get("minimum_index_version"), equalTo(Version.CURRENT.toString())); + assertThat(testFeature.get("upgrade_status"), equalTo("NO_UPGRADE_NEEDED")); + assertThat(testFeature.get("indices"), instanceOf(List.class)); + + assertThat((List) testFeature.get("indices"), hasSize(1)); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java index 63e0fd451fad..3e46b4d4b6fa 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.migration; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -27,13 +28,13 @@ import java.util.Objects; public class GetFeatureUpgradeStatusResponse extends ActionResponse implements ToXContentObject { private final List featureUpgradeStatuses; - private final String upgradeStatus; + private final UpgradeStatus upgradeStatus; /** * @param statuses A list of feature statuses * @param upgradeStatus Whether system features need to be upgraded */ - public GetFeatureUpgradeStatusResponse(List statuses, String upgradeStatus) { + public GetFeatureUpgradeStatusResponse(List statuses, UpgradeStatus upgradeStatus) { this.featureUpgradeStatuses = Objects.nonNull(statuses) ? statuses : Collections.emptyList(); this.upgradeStatus = upgradeStatus; } @@ -45,7 +46,7 @@ public class GetFeatureUpgradeStatusResponse extends ActionResponse implements T public GetFeatureUpgradeStatusResponse(StreamInput in) throws IOException { super(in); this.featureUpgradeStatuses = in.readList(FeatureUpgradeStatus::new); - this.upgradeStatus = in.readString(); + this.upgradeStatus = in.readEnum(UpgradeStatus.class); } @Override @@ -64,14 +65,14 @@ public class GetFeatureUpgradeStatusResponse extends ActionResponse implements T @Override public void writeTo(StreamOutput out) throws IOException { out.writeList(this.featureUpgradeStatuses); - out.writeString(upgradeStatus); + out.writeEnum(upgradeStatus); } public List getFeatureUpgradeStatuses() { return featureUpgradeStatuses; } - public String getUpgradeStatus() { + public UpgradeStatus getUpgradeStatus() { return upgradeStatus; } @@ -88,14 +89,28 @@ public class GetFeatureUpgradeStatusResponse extends ActionResponse implements T return Objects.hash(featureUpgradeStatuses, upgradeStatus); } + @Override + public String toString() { + return "GetFeatureUpgradeStatusResponse{" + + "featureUpgradeStatuses=" + featureUpgradeStatuses + + ", upgradeStatus='" + upgradeStatus + '\'' + + '}'; + } + + public enum UpgradeStatus { + UPGRADE_NEEDED, + NO_UPGRADE_NEEDED, + IN_PROGRESS + } + /** * A class for a particular feature, showing whether it needs to be upgraded and the earliest * Elasticsearch version used to create one of this feature's system indices. */ public static class FeatureUpgradeStatus implements Writeable, ToXContentObject { private final String featureName; - private final String minimumIndexVersion; - private final String upgradeStatus; + private final Version minimumIndexVersion; + private final UpgradeStatus upgradeStatus; private final List indexVersions; /** @@ -104,8 +119,8 @@ public class GetFeatureUpgradeStatusResponse extends ActionResponse implements T * @param upgradeStatus Whether the feature needs to be upgraded * @param indexVersions A list of this feature's concrete indices and the Elasticsearch version that created them */ - public FeatureUpgradeStatus(String featureName, String minimumIndexVersion, - String upgradeStatus, List indexVersions) { + public FeatureUpgradeStatus(String featureName, Version minimumIndexVersion, + UpgradeStatus upgradeStatus, List indexVersions) { this.featureName = featureName; this.minimumIndexVersion = minimumIndexVersion; this.upgradeStatus = upgradeStatus; @@ -118,8 +133,8 @@ public class GetFeatureUpgradeStatusResponse extends ActionResponse implements T */ public FeatureUpgradeStatus(StreamInput in) throws IOException { this.featureName = in.readString(); - this.minimumIndexVersion = in.readString(); - this.upgradeStatus = in.readString(); + this.minimumIndexVersion = Version.readVersion(in); + this.upgradeStatus = in.readEnum(UpgradeStatus.class); this.indexVersions = in.readList(IndexVersion::new); } @@ -127,11 +142,11 @@ public class GetFeatureUpgradeStatusResponse extends ActionResponse implements T return this.featureName; } - public String getMinimumIndexVersion() { + public Version getMinimumIndexVersion() { return this.minimumIndexVersion; } - public String getUpgradeStatus() { + public UpgradeStatus getUpgradeStatus() { return this.upgradeStatus; } @@ -142,8 +157,8 @@ public class GetFeatureUpgradeStatusResponse extends ActionResponse implements T @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(this.featureName); - out.writeString(this.minimumIndexVersion); - out.writeString(this.upgradeStatus); + Version.writeVersion(this.minimumIndexVersion, out); + out.writeEnum(this.upgradeStatus); out.writeList(this.indexVersions); } @@ -151,7 +166,7 @@ public class GetFeatureUpgradeStatusResponse extends ActionResponse implements T public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("feature_name", this.featureName); - builder.field("minimum_index_version", this.minimumIndexVersion); + builder.field("minimum_index_version", this.minimumIndexVersion.toString()); builder.field("upgrade_status", this.upgradeStatus); builder.startArray("indices"); for (IndexVersion version : this.indexVersions) { @@ -177,6 +192,16 @@ public class GetFeatureUpgradeStatusResponse extends ActionResponse implements T public int hashCode() { return Objects.hash(featureName, minimumIndexVersion, upgradeStatus, indexVersions); } + + @Override + public String toString() { + return "FeatureUpgradeStatus{" + + "featureName='" + featureName + '\'' + + ", minimumIndexVersion='" + minimumIndexVersion + '\'' + + ", upgradeStatus='" + upgradeStatus + '\'' + + ", indexVersions=" + indexVersions + + '}'; + } } /** @@ -184,13 +209,13 @@ public class GetFeatureUpgradeStatusResponse extends ActionResponse implements T */ public static class IndexVersion implements Writeable, ToXContentObject { private final String indexName; - private final String version; + private final Version version; /** * @param indexName Name of the index * @param version Version of Elasticsearch that created the index */ - public IndexVersion(String indexName, String version) { + public IndexVersion(String indexName, Version version) { this.indexName = indexName; this.version = version; } @@ -201,28 +226,28 @@ public class GetFeatureUpgradeStatusResponse extends ActionResponse implements T */ public IndexVersion(StreamInput in) throws IOException { this.indexName = in.readString(); - this.version = in.readString(); + this.version = Version.readVersion(in); } public String getIndexName() { return this.indexName; } - public String getVersion() { + public Version getVersion() { return this.version; } @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(this.indexName); - out.writeString(this.version); + Version.writeVersion(this.version, out); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("index", this.indexName); - builder.field("version", this.version); + builder.field("version", this.version.toString()); builder.endObject(); return builder; } @@ -239,5 +264,13 @@ public class GetFeatureUpgradeStatusResponse extends ActionResponse implements T public int hashCode() { return Objects.hash(indexName, version); } + + @Override + public String toString() { + return "IndexVersion{" + + "indexName='" + indexName + '\'' + + ", version='" + version + '\'' + + '}'; + } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java index 459c6f393ddb..50c38ca10d28 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.migration; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeAction; @@ -21,8 +22,14 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_UPGRADE_NEEDED; +import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.UPGRADE_NEEDED; /** * Transport class for the get feature upgrade status action @@ -31,6 +38,8 @@ public class TransportGetFeatureUpgradeStatusAction extends TransportMasterNodeA GetFeatureUpgradeStatusRequest, GetFeatureUpgradeStatusResponse> { + public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_7_0_0; + private final SystemIndices systemIndices; @Inject @@ -59,16 +68,51 @@ public class TransportGetFeatureUpgradeStatusAction extends TransportMasterNodeA @Override protected void masterOperation(GetFeatureUpgradeStatusRequest request, ClusterState state, ActionListener listener) throws Exception { - List indexVersions = new ArrayList<>(); - indexVersions.add(new GetFeatureUpgradeStatusResponse.IndexVersion(".security-7", "7.1.1")); - List features = new ArrayList<>(); - features.add(new GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus( - "security", - "7.1.1", - "UPGRADE_NEEDED", + + List features = systemIndices.getFeatures().entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .map(entry -> getFeatureUpgradeStatus(state, entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + + boolean isUpgradeNeeded = features.stream() + .map(GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus::getMinimumIndexVersion) + .min(Version::compareTo) + .orElse(Version.CURRENT) + .before(Version.V_7_0_0); + + listener.onResponse(new GetFeatureUpgradeStatusResponse(features, isUpgradeNeeded ? UPGRADE_NEEDED : NO_UPGRADE_NEEDED)); + } + + // visible for testing + static GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus getFeatureUpgradeStatus( + ClusterState state, String featureName, SystemIndices.Feature feature) { + + List indexVersions = getIndexVersions(state, feature); + + Version minimumVersion = indexVersions.stream() + .map(GetFeatureUpgradeStatusResponse.IndexVersion::getVersion) + .min(Version::compareTo) + .orElse(Version.CURRENT); + + return new GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus( + featureName, + minimumVersion, + minimumVersion.before(Version.V_7_0_0) ? UPGRADE_NEEDED : NO_UPGRADE_NEEDED, indexVersions - )); - listener.onResponse(new GetFeatureUpgradeStatusResponse(features, "UPGRADE_NEEDED")); + ); + } + + // visible for testing + static List getIndexVersions(ClusterState state, SystemIndices.Feature feature) { + return Stream.of(feature.getIndexDescriptors(), feature.getAssociatedIndexDescriptors()) + .flatMap(Collection::stream) + .flatMap(descriptor -> descriptor.getMatchingIndices(state.metadata()).stream()) + .sorted(String::compareTo) + .map(index -> state.metadata().index(index)) + .map(indexMetadata -> new GetFeatureUpgradeStatusResponse.IndexVersion( + indexMetadata.getIndex().getName(), + indexMetadata.getCreationVersion())) + .collect(Collectors.toList()); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java index 58483df600b7..7966c2171ee8 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java @@ -8,12 +8,14 @@ package org.elasticsearch.action.admin.cluster.migration; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.test.AbstractWireSerializingTestCase; import java.io.IOException; import java.util.Collections; +import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.UPGRADE_NEEDED; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -31,7 +33,7 @@ public class GetFeatureUpgradeStatusResponseTests extends AbstractWireSerializin protected GetFeatureUpgradeStatusResponse createTestInstance() { return new GetFeatureUpgradeStatusResponse( randomList(8, GetFeatureUpgradeStatusResponseTests::createFeatureStatus), - randomAlphaOfLengthBetween(4, 16) + randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()) ); } @@ -41,13 +43,14 @@ public class GetFeatureUpgradeStatusResponseTests extends AbstractWireSerializin randomList(8, () -> randomValueOtherThanMany(instance.getFeatureUpgradeStatuses()::contains, GetFeatureUpgradeStatusResponseTests::createFeatureStatus)), - randomValueOtherThan(instance.getUpgradeStatus(), () -> randomAlphaOfLengthBetween(4, 16)) - ); + randomValueOtherThan(instance.getUpgradeStatus(), () -> + randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()))); + } /** If constructor is called with null for a list, we just use an empty list */ public void testConstructorHandlesNullLists() { - GetFeatureUpgradeStatusResponse response = new GetFeatureUpgradeStatusResponse(null, "status"); + GetFeatureUpgradeStatusResponse response = new GetFeatureUpgradeStatusResponse(null, UPGRADE_NEEDED); assertThat(response.getFeatureUpgradeStatuses(), notNullValue()); assertThat(response.getFeatureUpgradeStatuses(), equalTo(Collections.emptyList())); } @@ -55,8 +58,8 @@ public class GetFeatureUpgradeStatusResponseTests extends AbstractWireSerializin private static GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus createFeatureStatus() { return new GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus( randomAlphaOfLengthBetween(3, 20), - randomAlphaOfLengthBetween(5, 9), - randomAlphaOfLengthBetween(4, 16), + randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()), + randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()), randomList(4, GetFeatureUpgradeStatusResponseTests::getIndexVersion) ); } @@ -64,7 +67,7 @@ public class GetFeatureUpgradeStatusResponseTests extends AbstractWireSerializin private static GetFeatureUpgradeStatusResponse.IndexVersion getIndexVersion() { return new GetFeatureUpgradeStatusResponse.IndexVersion( randomAlphaOfLengthBetween(3, 20), - randomAlphaOfLengthBetween(5, 9) + randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()) ); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java new file mode 100644 index 000000000000..c83409e51069 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.admin.cluster.migration; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_UPGRADE_NEEDED; +import static org.hamcrest.Matchers.equalTo; + +public class TransportGetFeatureUpgradeStatusActionTests extends ESTestCase { + + public static String TEST_SYSTEM_INDEX_PATTERN = ".test*"; + private static final ClusterState CLUSTER_STATE = getClusterState(); + private static final SystemIndices.Feature FEATURE = getFeature(); + + public void testGetFeatureStatus() { + GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus status = + TransportGetFeatureUpgradeStatusAction.getFeatureUpgradeStatus( + CLUSTER_STATE, "test-feature", FEATURE); + + assertThat(status.getUpgradeStatus(), equalTo(NO_UPGRADE_NEEDED)); + assertThat(status.getFeatureName(), equalTo("test-feature")); + assertThat(status.getMinimumIndexVersion(), equalTo(Version.V_7_0_0)); + assertThat(status.getIndexVersions().size(), equalTo(2)); // additional testing below + } + + public void testGetIndexVersion() { + List versions = + TransportGetFeatureUpgradeStatusAction.getIndexVersions(CLUSTER_STATE, FEATURE); + + assertThat(versions.size(), equalTo(2)); + + { + GetFeatureUpgradeStatusResponse.IndexVersion version = versions.get(0); + assertThat(version.getVersion(), equalTo(Version.CURRENT)); + assertThat(version.getIndexName(), equalTo(".test-index-1")); + } + { + GetFeatureUpgradeStatusResponse.IndexVersion version = versions.get(1); + assertThat(version.getVersion(), equalTo(Version.V_7_0_0)); + assertThat(version.getIndexName(), equalTo(".test-index-2")); + } + } + + private static SystemIndices.Feature getFeature() { + SystemIndexDescriptor descriptor = new SystemIndexDescriptor(TEST_SYSTEM_INDEX_PATTERN, "descriptor for tests"); + + List descriptors = new ArrayList<>(); + descriptors.add(descriptor); + + // system indices feature object + SystemIndices.Feature feature = new SystemIndices.Feature( + "test-feature", + "feature for tests", + descriptors); + return feature; + } + + private static ClusterState getClusterState() { + IndexMetadata indexMetadata1 = IndexMetadata.builder(".test-index-1") + .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + IndexMetadata indexMetadata2 = IndexMetadata.builder(".test-index-2") + .settings(Settings.builder().put("index.version.created", Version.V_7_0_0).build()) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + + ClusterState clusterState = new ClusterState.Builder(ClusterState.EMPTY_STATE) + .metadata(new Metadata.Builder() + .indices(ImmutableOpenMap.builder() + .fPut(".test-index-1", indexMetadata1) + .fPut(".test-index-2", indexMetadata2) + .build()) + .build()) + .build(); + return clusterState; + } +}