diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/UpgradeTransformsResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/UpgradeTransformsResponseTests.java index 4cb4156c1e98..e541af566794 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/UpgradeTransformsResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/UpgradeTransformsResponseTests.java @@ -37,15 +37,9 @@ public class UpgradeTransformsResponseTests extends ESTestCase { private static void toXContent(UpgradeTransformsResponse response, XContentBuilder builder) throws IOException { builder.startObject(); - if (response.getUpdated() != 0) { - builder.field("updated", response.getUpdated()); - } - if (response.getNoAction() != 0) { - builder.field("no_action", response.getNoAction()); - } - if (response.getNeedsUpdate() != 0) { - builder.field("needs_update", response.getNeedsUpdate()); - } + builder.field("updated", response.getUpdated()); + builder.field("no_action", response.getNoAction()); + builder.field("needs_update", response.getNeedsUpdate()); builder.endObject(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java index f42700c66f3f..ab5f19650e65 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java @@ -8,6 +8,10 @@ package org.elasticsearch.xpack.core.transform; public class TransformDeprecations { + + public static final String UPGRADE_TRANSFORM_URL = "https://ela.st/es-7-upgrade-transforms"; + + // breaking changes base url for the _next_ major release public static final String BREAKING_CHANGES_BASE_URL = "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html"; @@ -15,5 +19,10 @@ public class TransformDeprecations { public static final String AGGS_BREAKING_CHANGES_URL = BREAKING_CHANGES_BASE_URL + "#breaking_80_aggregations_changes"; + public static final String ACTION_UPGRADE_TRANSFORMS_API = "Use the upgrade transforms API to fix your transforms."; + + public static final String ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED = + "[max_page_search_size] is deprecated inside pivot. Use settings instead."; + private TransformDeprecations() {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsAction.java index b32799cd7887..d5449519d262 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsAction.java @@ -125,15 +125,9 @@ public class UpgradeTransformsAction extends ActionType implement new DeprecationIssue( Level.CRITICAL, "Transform [" + id + "] is too old", - TransformDeprecations.BREAKING_CHANGES_BASE_URL, - "The configuration uses an old format, you can use [_update] or [_upgrade] to update", + TransformDeprecations.UPGRADE_TRANSFORM_URL, + TransformDeprecations.ACTION_UPGRADE_TRANSFORMS_API, false, null ) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java index 652ee0577e39..1851b265ec33 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java @@ -13,13 +13,13 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level; import org.elasticsearch.xpack.core.transform.TransformDeprecations; @@ -91,7 +91,7 @@ public class PivotConfig implements Writeable, ToXContentObject { deprecationLogger.critical( DeprecationCategory.API, TransformField.MAX_PAGE_SEARCH_SIZE.getPreferredName(), - "[max_page_search_size] is deprecated inside pivot please use settings instead" + TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED ); } } @@ -187,7 +187,7 @@ public class PivotConfig implements Writeable, ToXContentObject { Level.WARNING, "Transform [" + id + "] uses deprecated max_page_search_size", TransformDeprecations.BREAKING_CHANGES_BASE_URL, - "[max_page_search_size] is deprecated inside pivot please use settings instead", + TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED, false, null ) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index 1febd8d0cf73..b62cf0b2b5cc 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.core.common.validation.SourceDestValidator.Source import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level; import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; +import org.elasticsearch.xpack.core.transform.TransformDeprecations; import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfig; import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfigTests; import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfig; @@ -510,7 +511,7 @@ public class TransformConfigTests extends AbstractSerializingTransformTestCase transformConfigManager.getTransformConfiguration(transformId, listener), transformConfigNew, null, null); // delete old indices + when(clusterService.state()).thenReturn( + createClusterStateWithTransformIndex(oldIndex, TransformInternalIndexConstants.LATEST_INDEX_NAME) + ); + assertAsync(listener -> transformConfigManager.deleteOldIndices(listener), true, null, null); // the config should still be there @@ -724,4 +755,32 @@ public class TransformConfigManagerTests extends TransformSingleNodeTestCase { ); } + private static ClusterState createClusterStateWithTransformIndex(String... indexes) throws IOException { + ImmutableOpenMap.Builder indexMapBuilder = ImmutableOpenMap.builder(); + Metadata.Builder metaBuilder = Metadata.builder(); + ClusterState.Builder csBuilder = ClusterState.builder(ClusterName.DEFAULT); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + + for (String index : indexes) { + IndexMetadata.Builder builder = new IndexMetadata.Builder(index).settings( + Settings.builder() + .put(TransformInternalIndex.settings()) + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT) + .build() + ).numberOfReplicas(0).numberOfShards(1).putMapping(SINGLE_MAPPING_NAME, Strings.toString(TransformInternalIndex.mappings())); + indexMapBuilder.put(index, builder.build()); + + routingTableBuilder.add( + IndexRoutingTable.builder(new Index(index, UUIDs.randomBase64UUID())) + .addShard(TestShardRouting.newShardRouting(index, 0, "node_a", null, true, ShardRoutingState.STARTED)) + .build() + ); + + } + csBuilder.routingTable(routingTableBuilder.build()); + metaBuilder.indices(indexMapBuilder.build()); + csBuilder.metadata(metaBuilder.build()); + + return csBuilder.build(); + } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index 94de9caa4dac..a957d0943a7b 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -318,7 +318,12 @@ public class Transform extends Plugin implements SystemIndexPlugin, PersistentTa return emptyList(); } - TransformConfigManager configManager = new IndexBasedTransformConfigManager(client, xContentRegistry); + TransformConfigManager configManager = new IndexBasedTransformConfigManager( + clusterService, + expressionResolver, + client, + xContentRegistry + ); TransformAuditor auditor = new TransformAuditor(client, clusterService.getNodeName(), clusterService); TransformCheckpointService checkpointService = new TransformCheckpointService( Clock.systemUTC(), diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java index 1fbe2bc1db61..2f514dbe3043 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java @@ -28,9 +28,13 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexNotFoundException; @@ -64,6 +68,7 @@ import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformIn import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.LinkedHashSet; @@ -99,10 +104,19 @@ public class IndexBasedTransformConfigManager implements TransformConfigManager private static final Logger logger = LogManager.getLogger(IndexBasedTransformConfigManager.class); private static final int MAX_RESULTS_WINDOW = 10_000; + private final ClusterService clusterService; + private final IndexNameExpressionResolver indexNameExpressionResolver; private final Client client; private final NamedXContentRegistry xContentRegistry; - public IndexBasedTransformConfigManager(Client client, NamedXContentRegistry xContentRegistry) { + public IndexBasedTransformConfigManager( + ClusterService clusterService, + IndexNameExpressionResolver indexNameExpressionResolver, + Client client, + NamedXContentRegistry xContentRegistry + ) { + this.clusterService = clusterService; + this.indexNameExpressionResolver = indexNameExpressionResolver; this.client = client; this.xContentRegistry = xContentRegistry; } @@ -255,11 +269,42 @@ public class IndexBasedTransformConfigManager implements TransformConfigManager @Override public void deleteOldIndices(ActionListener listener) { - DeleteIndexRequest deleteRequest = new DeleteIndexRequest( - TransformInternalIndexConstants.INDEX_NAME_PATTERN, - TransformInternalIndexConstants.INDEX_NAME_PATTERN_DEPRECATED, - "-" + TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME - ).indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); + ClusterState state = clusterService.state(); + Set indicesToDelete = new HashSet<>(); + + // use the transform context as we access system indexes + try (ThreadContext.StoredContext ctx = client.threadPool().getThreadContext().stashWithOrigin(TRANSFORM_ORIGIN)) { + indicesToDelete.addAll( + Arrays.asList( + indexNameExpressionResolver.concreteIndexNames( + state, + IndicesOptions.lenientExpandHidden(), + TransformInternalIndexConstants.INDEX_NAME_PATTERN + ) + ) + ); + + indicesToDelete.addAll( + Arrays.asList( + indexNameExpressionResolver.concreteIndexNames( + state, + IndicesOptions.lenientExpandHidden(), + TransformInternalIndexConstants.INDEX_NAME_PATTERN_DEPRECATED + ) + ) + ); + + indicesToDelete.remove(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME); + } + + if (indicesToDelete.isEmpty()) { + listener.onResponse(true); + return; + } + + DeleteIndexRequest deleteRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[0])).indicesOptions( + IndicesOptions.LENIENT_EXPAND_OPEN + ); executeAsyncWithOrigin(client, TRANSFORM_ORIGIN, DeleteIndexAction.INSTANCE, deleteRequest, ActionListener.wrap(response -> { if (response.isAcknowledged() == false) { diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java index 582c855a609a..3818a140344b 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java @@ -234,11 +234,20 @@ public class TransformPersistentTasksExecutorTests extends ESTestCase { csBuilder.routingTable(routingTable.build()); csBuilder.metadata(metadata); + ClusterSettings cSettings = new ClusterSettings(Settings.EMPTY, Collections.singleton(Transform.NUM_FAILURE_RETRIES_SETTING)); + ClusterService clusterService = mock(ClusterService.class); + when(clusterService.getClusterSettings()).thenReturn(cSettings); + when(clusterService.state()).thenReturn(TransformInternalIndexTests.randomTransformClusterState()); ClusterState cs = csBuilder.build(); Client client = mock(Client.class); TransformAuditor mockAuditor = mock(TransformAuditor.class); - IndexBasedTransformConfigManager transformsConfigManager = new IndexBasedTransformConfigManager(client, xContentRegistry()); + IndexBasedTransformConfigManager transformsConfigManager = new IndexBasedTransformConfigManager( + clusterService, + TestIndexNameExpressionResolver.newInstance(), + client, + xContentRegistry() + ); TransformCheckpointService transformCheckpointService = new TransformCheckpointService( Clock.systemUTC(), Settings.EMPTY, @@ -253,10 +262,6 @@ public class TransformPersistentTasksExecutorTests extends ESTestCase { mock(SchedulerEngine.class) ); - ClusterSettings cSettings = new ClusterSettings(Settings.EMPTY, Collections.singleton(Transform.NUM_FAILURE_RETRIES_SETTING)); - ClusterService clusterService = mock(ClusterService.class); - when(clusterService.getClusterSettings()).thenReturn(cSettings); - when(clusterService.state()).thenReturn(TransformInternalIndexTests.randomTransformClusterState()); TransformPersistentTasksExecutor executor = new TransformPersistentTasksExecutor( client, transformServices, @@ -514,9 +519,15 @@ public class TransformPersistentTasksExecutorTests extends ESTestCase { } public TransformPersistentTasksExecutor buildTaskExecutor() { + ClusterService clusterService = mock(ClusterService.class); Client client = mock(Client.class); TransformAuditor mockAuditor = mock(TransformAuditor.class); - IndexBasedTransformConfigManager transformsConfigManager = new IndexBasedTransformConfigManager(client, xContentRegistry()); + IndexBasedTransformConfigManager transformsConfigManager = new IndexBasedTransformConfigManager( + clusterService, + TestIndexNameExpressionResolver.newInstance(), + client, + xContentRegistry() + ); TransformCheckpointService transformCheckpointService = new TransformCheckpointService( Clock.systemUTC(), Settings.EMPTY, @@ -532,7 +543,6 @@ public class TransformPersistentTasksExecutorTests extends ESTestCase { ); ClusterSettings cSettings = new ClusterSettings(Settings.EMPTY, Collections.singleton(Transform.NUM_FAILURE_RETRIES_SETTING)); - ClusterService clusterService = mock(ClusterService.class); when(clusterService.getClusterSettings()).thenReturn(cSettings); when(clusterService.state()).thenReturn(TransformInternalIndexTests.randomTransformClusterState()); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java index 887a79945141..55b280c672cc 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java @@ -20,11 +20,6 @@ import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.client.Client; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xcontent.DeprecationHandler; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.search.SearchHit; @@ -32,6 +27,12 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.xcontent.DeprecationHandler; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.transform.TransformDeprecations; import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; import org.elasticsearch.xpack.core.transform.transforms.pivot.AggregationConfig; @@ -136,7 +137,7 @@ public class PivotTests extends ESTestCase { ); assertThat(pivot.getInitialPageSize(), equalTo(Transform.DEFAULT_INITIAL_MAX_PAGE_SEARCH_SIZE)); - assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead"); + assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED); } public void testSearchFailure() throws Exception {