From 47706b505f6510c3314a8a83c8c3f68c3d0eaefc Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Wed, 5 Mar 2025 13:39:58 -0700 Subject: [PATCH] Add index mode to get data stream API (#122486) This commit adds the `index_mode` for both the data stream and each backing index to the output of `GET /_data_stream`. An example looks like: ``` { "data_streams" : [ { "name" : "foo-things", "indices" : [ { "index_name" : ".ds-foo-things-2025.02.13-000001", ... "index_mode" : "standard" } ], ... "index_mode" : "standard" }, { "name" : "logs-foo-bar", "indices" : [ { "index_name" : ".ds-logs-foo-bar-2025.02.13-000001", ... "index_mode" : "logsdb" }, { "index_name" : ".ds-logs-foo-bar-2025.02.13-000002", ... "index_mode" : "logsdb" } ], ... "index_mode" : "logsdb", } ] } ``` --- docs/changelog/122486.yaml | 5 + .../indices/get-data-stream.asciidoc | 376 ++++++++++++++++++ .../action/TransportGetDataStreamsAction.java | 70 +++- .../action/GetDataStreamsResponseTests.java | 30 +- .../TransportGetDataStreamsActionTests.java | 65 +++ .../org/elasticsearch/TransportVersions.java | 1 + .../datastreams/GetDataStreamAction.java | 44 +- .../MetadataIndexTemplateService.java | 7 +- .../datastreams/GetDataStreamActionTests.java | 1 + .../LogsdbIndexModeSettingsProvider.java | 9 +- .../LogsdbIndexModeSettingsProviderTests.java | 3 +- 11 files changed, 585 insertions(+), 26 deletions(-) create mode 100644 docs/changelog/122486.yaml create mode 100644 docs/reference/indices/get-data-stream.asciidoc diff --git a/docs/changelog/122486.yaml b/docs/changelog/122486.yaml new file mode 100644 index 000000000000..027d2a5e63ba --- /dev/null +++ b/docs/changelog/122486.yaml @@ -0,0 +1,5 @@ +pr: 122486 +summary: Add index mode to get data stream API +area: Data streams +type: enhancement +issues: [] diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc new file mode 100644 index 000000000000..a46d247d9a9a --- /dev/null +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -0,0 +1,376 @@ +[role="xpack"] +[[indices-get-data-stream]] +=== Get data stream API +++++ +Get data stream +++++ + +.New API reference +[sidebar] +-- +For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs]. +-- + +Retrieves information about one or more <>. +See <>. + +//// +[source,console] +---- +PUT /_ilm/policy/my-lifecycle-policy +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_primary_shard_size": "25GB" + } + } + }, + "delete": { + "min_age": "30d", + "actions": { + "delete": {} + } + } + } + } +} + +PUT /_index_template/my-index-template +{ + "index_patterns": [ "my-data-stream*" ], + "data_stream": {}, + "template": { + "settings": { + "index.lifecycle.name": "my-lifecycle-policy" + } + }, + "_meta": { + "my-meta-field": "foo" + } +} + +PUT /_data_stream/my-data-stream + +POST /my-data-stream/_rollover + +PUT /_data_stream/my-data-stream-two + +DELETE /_data_stream/my-data-stream*/_lifecycle +---- +// TESTSETUP +//// + +//// +[source,console] +---- +DELETE /_data_stream/* +DELETE /_index_template/* +DELETE /_ilm/policy/my-lifecycle-policy +---- +// TEARDOWN +//// + +[source,console] +---- +GET /_data_stream/my-data-stream +---- + +[[get-data-stream-api-request]] +==== {api-request-title} + +`GET /_data_stream/` + +[[get-data-stream-api-prereqs]] +==== {api-prereq-title} + +* If the {es} {security-features} are enabled, you must have the +`view_index_metadata` or `manage` <> +for the data stream. + +[[get-data-stream-api-path-params]] +==== {api-path-parms-title} + +``:: +(Optional, string) +Comma-separated list of data stream names used to limit the request. Wildcard +(`*`) expressions are supported. If omitted, all data streams will be +returned. + +[role="child_attributes"] +[[get-data-stream-api-query-parms]] +==== {api-query-parms-title} + +include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards] ++ +Defaults to `open`. + +`include_defaults`:: +(Optional, Boolean) Functionality in preview:[]. If `true`, return all default settings in the response. +Defaults to `false`. + +`verbose`:: +(Optional, Boolean). If `true`, Returns the `maximum_timestamp` corresponding to the `@timestamp` field for documents in the data stream. +Defaults to `false`. + +[role="child_attributes"] +[[get-data-stream-api-response-body]] +==== {api-response-body-title} + +`data_streams`:: +(array of objects) +Contains information about retrieved data streams. ++ +.Properties of objects in `data_streams` +[%collapsible%open] +==== +`name`:: +(string) +Name of the data stream. + +`timestamp_field`:: +(object) +Contains information about the data stream's `@timestamp` field. ++ +.Properties of `timestamp_field` +[%collapsible%open] +===== +`name`:: +(string) +Name of the data stream's timestamp field, which must be `@timestamp`. The +`@timestamp` field must be included in every document indexed to the data +stream. +===== + +`indices`:: +(array of objects) +Array of objects containing information about the data stream's backing +indices. ++ +The last item in this array contains information about the stream's current +<>. ++ +.Properties of `indices` objects +[%collapsible%open] +===== +`index_name`:: +(string) +Name of the backing index. For naming conventions, see +<>. + +`index_uuid`:: +(string) +Universally unique identifier (UUID) for the index. + +`prefer_ilm`:: +(boolean) +Functionality in preview:[]. Indicates if this index is configured to prefer {ilm} +when both {ilm-cap} and <> are configured to +manage this index. + +`managed_by`:: +(string) +Functionality in preview:[]. Indicates the system that managed this index. +===== + +`generation`:: +(integer) +Current <> for the data stream. This number +acts as a cumulative count of the stream's rollovers, starting at `1`. + +`_meta`:: +(object) +Custom metadata for the stream, copied from the `_meta` object of the +stream's matching <>. If empty, +the response omits this property. + +`status`:: +(string) +<> of the data stream. ++ +This health status is based on the state of the primary and replica shards of +the stream's backing indices. ++ +.Values for `status` +[%collapsible%open] +===== +`GREEN`::: +All shards are assigned. + +`YELLOW`::: +All primary shards are assigned, but one or more replica shards are +unassigned. + +`RED`::: +One or more primary shards are unassigned, so some data is unavailable. +===== + +`template`:: +(string) +Name of the index template used to create the data stream's backing indices. ++ +The template's index pattern must match the name of this data stream. See +<>. + +`ilm_policy`:: +(string) +Name of the current {ilm-init} lifecycle policy in the stream's matching index +template. This lifecycle policy is set in the `index.lifecycle.name` setting. ++ +If the template does not include a lifecycle policy, this property is not +included in the response. ++ +NOTE: A data stream's backing indices may be assigned different lifecycle +policies. To retrieve the lifecycle policy for individual backing indices, +use the <>. + +`next_generation_managed_by`:: +(string) +Functionality in preview:[]. Indicates the system that will managed the next generation index +(i.e. the next data stream write index). + +`prefer_ilm`:: +(boolean) +Functionality in preview:[]. Indicates if the index template used to create the data +stream's backing indices is configured to prefer {ilm-cap} when both {ilm-cap} and +<> are configured to manage this index. + +`hidden`:: +(Boolean) If `true`, the data stream is <>. + +`system`:: +(Boolean) +If `true`, the data stream is created and managed by an Elastic stack component +and cannot be modified through normal user interaction. + +`allow_custom_routing`:: +(Boolean) +If `true`, the data stream this data stream allows custom routing on write request. + +`replicated`:: +(Boolean) +If `true`, the data stream is created and managed by {ccr} and the local +cluster can not write into this data stream or change its mappings. + +`lifecycle`:: +(object) +Functionality in preview:[]. Contains the configuration for the data stream lifecycle management of this data stream. ++ +.Properties of `lifecycle` +[%collapsible%open] +===== +`data_retention`:: +(string) +If defined, every document added to this data stream will be stored at least for this time frame. Any time after this +duration the document could be deleted. When empty, every document in this data stream will be stored indefinitely. + +`rollover`:: +(object) +The conditions which will trigger the rollover of a backing index as configured by the cluster setting +`cluster.lifecycle.default.rollover`. This property is an implementation detail and it will only be retrieved when the query +param `include_defaults` is set to `true`. The contents of this field are subject to change. +===== + +`rollover_on_write`:: +(Boolean) +If `true`, the next write to this data stream will trigger a rollover first and the document will be +indexed in the new backing index. If the rollover fails the indexing request will fail too. +==== + +[[get-data-stream-api-example]] +==== {api-examples-title} + +[source,console] +---- +GET _data_stream/my-data-stream* +---- + +The API returns the following response: + +[source,console-result] +---- +{ + "data_streams": [ + { + "name": "my-data-stream", + "timestamp_field": { + "name": "@timestamp" + }, + "indices": [ + { + "index_name": ".ds-my-data-stream-2099.03.07-000001", + "index_uuid": "xCEhwsp8Tey0-FLNFYVwSg", + "prefer_ilm": true, + "ilm_policy": "my-lifecycle-policy", + "managed_by": "Index Lifecycle Management", + "index_mode": "standard" + }, + { + "index_name": ".ds-my-data-stream-2099.03.08-000002", + "index_uuid": "PA_JquKGSiKcAKBA8DJ5gw", + "prefer_ilm": true, + "ilm_policy": "my-lifecycle-policy", + "managed_by": "Index Lifecycle Management", + "index_mode": "standard" + } + ], + "generation": 2, + "_meta": { + "my-meta-field": "foo" + }, + "status": "GREEN", + "index_mode": "standard", + "next_generation_managed_by": "Index Lifecycle Management", + "prefer_ilm": true, + "template": "my-index-template", + "ilm_policy": "my-lifecycle-policy", + "hidden": false, + "system": false, + "allow_custom_routing": false, + "replicated": false, + "rollover_on_write": false + }, + { + "name": "my-data-stream-two", + "timestamp_field": { + "name": "@timestamp" + }, + "indices": [ + { + "index_name": ".ds-my-data-stream-two-2099.03.08-000001", + "index_uuid": "3liBu2SYS5axasRt6fUIpA", + "prefer_ilm": true, + "ilm_policy": "my-lifecycle-policy", + "managed_by": "Index Lifecycle Management", + "index_mode": "standard" + } + ], + "generation": 1, + "_meta": { + "my-meta-field": "foo" + }, + "status": "YELLOW", + "index_mode": "standard", + "next_generation_managed_by": "Index Lifecycle Management", + "prefer_ilm": true, + "template": "my-index-template", + "ilm_policy": "my-lifecycle-policy", + "hidden": false, + "system": false, + "allow_custom_routing": false, + "replicated": false, + "rollover_on_write": false + } + ] +} +---- +// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.07-000001"/"index_name": $body.data_streams.0.indices.0.index_name/] +// TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/] +// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.08-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] +// TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] +// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-two-2099.03.08-000001"/"index_name": $body.data_streams.1.indices.0.index_name/] +// TESTRESPONSE[s/"index_uuid": "3liBu2SYS5axasRt6fUIpA"/"index_uuid": $body.data_streams.1.indices.0.index_uuid/] +// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW"/] +// TESTRESPONSE[s/"replicated": false/"replicated": false,"failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java index 5728e5c0a8db..f37158ba3fc6 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.ProjectState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.health.ClusterStateHealth; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings; import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings; @@ -40,6 +41,9 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettingProvider; +import org.elasticsearch.index.IndexSettingProviders; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.indices.SystemDataStreamDescriptor; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; @@ -53,6 +57,7 @@ import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; @@ -68,6 +73,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec private final ClusterSettings clusterSettings; private final DataStreamGlobalRetentionSettings globalRetentionSettings; private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings; + private final IndexSettingProviders indexSettingProviders; private final Client client; @Inject @@ -81,6 +87,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec SystemIndices systemIndices, DataStreamGlobalRetentionSettings globalRetentionSettings, DataStreamFailureStoreSettings dataStreamFailureStoreSettings, + IndexSettingProviders indexSettingProviders, Client client ) { super( @@ -99,6 +106,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec this.globalRetentionSettings = globalRetentionSettings; clusterSettings = clusterService.getClusterSettings(); this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings; + this.indexSettingProviders = indexSettingProviders; this.client = new OriginSettingClient(client, "stack"); } @@ -131,6 +139,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec clusterSettings, globalRetentionSettings, dataStreamFailureStoreSettings, + indexSettingProviders, maxTimestamps ) ); @@ -151,12 +160,43 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec clusterSettings, globalRetentionSettings, dataStreamFailureStoreSettings, + indexSettingProviders, null ) ); } } + /** + * Resolves the index mode ("index.mode" setting) for the given data stream, from the template or additional setting providers + */ + @Nullable + static IndexMode resolveMode( + ProjectState state, + IndexSettingProviders indexSettingProviders, + DataStream dataStream, + Settings settings, + ComposableIndexTemplate indexTemplate + ) { + IndexMode indexMode = state.metadata().retrieveIndexModeFromTemplate(indexTemplate); + for (IndexSettingProvider provider : indexSettingProviders.getIndexSettingProviders()) { + Settings addlSettinsg = provider.getAdditionalIndexSettings( + MetadataIndexTemplateService.VALIDATE_INDEX_NAME, + dataStream.getName(), + indexMode, + state.metadata(), + Instant.now(), + settings, + List.of() + ); + var rawMode = addlSettinsg.get(IndexSettings.MODE.getKey()); + if (rawMode != null) { + indexMode = Enum.valueOf(IndexMode.class, rawMode.toUpperCase(Locale.ROOT)); + } + } + return indexMode; + } + static GetDataStreamAction.Response innerOperation( ProjectState state, GetDataStreamAction.Request request, @@ -165,6 +205,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec ClusterSettings clusterSettings, DataStreamGlobalRetentionSettings globalRetentionSettings, DataStreamFailureStoreSettings dataStreamFailureStoreSettings, + IndexSettingProviders indexSettingProviders, @Nullable Map maxTimestamps ) { List dataStreams = getDataStreams(state.metadata(), indexNameExpressionResolver, request); @@ -177,6 +218,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec final String indexTemplate; boolean indexTemplatePreferIlmValue = true; String ilmPolicyName = null; + IndexMode indexMode = dataStream.getIndexMode(); if (dataStream.isSystem()) { SystemDataStreamDescriptor dataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStream.getName()); indexTemplate = dataStreamDescriptor != null ? dataStreamDescriptor.getDataStreamName() : null; @@ -186,6 +228,15 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec dataStreamDescriptor.getComponentTemplates() ); ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME); + if (indexMode == null) { + indexMode = resolveMode( + state, + indexSettingProviders, + dataStream, + settings, + dataStreamDescriptor.getComposableIndexTemplate() + ); + } indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings); } } else { @@ -193,6 +244,15 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec if (indexTemplate != null) { Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate); ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME); + if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) { + indexMode = resolveMode( + state, + indexSettingProviders, + dataStream, + settings, + state.metadata().templatesV2().get(indexTemplate) + ); + } indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings); } else { LOGGER.warn( @@ -285,7 +345,9 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec timeSeries, backingIndicesSettingsValues, indexTemplatePreferIlmValue, - maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName()) + maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName()), + // Default to standard mode if not specified; should we set this to "unset" or "unspecified" instead? + indexMode == null ? IndexMode.STANDARD.getName() : indexMode.getName() ) ); } @@ -314,7 +376,11 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec } else { managedBy = ManagedBy.UNMANAGED; } - backingIndicesSettingsValues.put(index, new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy)); + String indexMode = IndexSettings.MODE.get(indexMetadata.getSettings()).getName(); + backingIndicesSettingsValues.put( + index, + new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy, indexMode) + ); } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java index 9414943cbb43..d9efa4d458f4 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java @@ -91,13 +91,13 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase String ilmPolicyName = "rollover-30days"; Map indexSettingsValues = Map.of( firstGenerationIndex, - new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM), + new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null), secondGenerationIndex, - new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE), + new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE, null), writeIndex, - new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE), + new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE, null), failureStoreIndex, - new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE) + new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE, null) ); Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo( @@ -109,6 +109,7 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase null, indexSettingsValues, false, + null, null ); Response response = new Response(List.of(dataStreamInfo)); @@ -195,13 +196,13 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase String ilmPolicyName = "rollover-30days"; Map indexSettingsValues = Map.of( firstGenerationIndex, - new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM), + new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null), secondGenerationIndex, - new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM), + new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null), writeIndex, - new Response.IndexProperties(false, null, ManagedBy.UNMANAGED), + new Response.IndexProperties(false, null, ManagedBy.UNMANAGED, null), failureStoreIndex, - new Response.IndexProperties(false, null, ManagedBy.UNMANAGED) + new Response.IndexProperties(false, null, ManagedBy.UNMANAGED, null) ); Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo( @@ -213,6 +214,7 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase null, indexSettingsValues, false, + null, null ); Response response = new Response(List.of(dataStreamInfo)); @@ -309,7 +311,8 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase new Response.IndexProperties( randomBoolean(), randomAlphaOfLengthBetween(50, 100), - randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE + randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE, + null ) ) ); @@ -328,7 +331,8 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase timeSeries, indexSettings, templatePreferIlm, - maximumTimestamp + maximumTimestamp, + null ); } @@ -349,7 +353,8 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase new Response.IndexProperties( randomBoolean(), randomAlphaOfLengthBetween(50, 100), - randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE + randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE, + randomBoolean() ? randomFrom(IndexMode.values()).getName() : null ) ); } @@ -367,7 +372,8 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase timeSeries != null ? new Response.TimeSeries(timeSeries) : null, generateRandomIndexSettingsValues(), randomBoolean(), - usually() ? randomNonNegativeLong() : null + usually() ? randomNonNegativeLong() : null, + usually() ? randomFrom(IndexMode.values()).getName() : null ); } } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index 0963647e2417..15252528e952 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -24,7 +24,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.test.ESTestCase; @@ -32,6 +34,7 @@ import org.elasticsearch.test.ESTestCase; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; +import java.util.Set; import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.getClusterStateWithDataStreams; import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch; @@ -181,6 +184,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat( @@ -213,6 +217,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat( @@ -266,6 +271,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat( @@ -307,6 +313,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); @@ -341,6 +348,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat(response.getGlobalRetention(), nullValue()); @@ -367,6 +375,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase { ClusterSettings.createBuiltInClusterSettings(), withGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat(response.getGlobalRetention(), equalTo(globalRetention)); @@ -394,6 +403,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat(response.getDataStreams(), hasSize(1)); @@ -423,6 +433,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, + new IndexSettingProviders(Set.of()), null ); assertThat(response.getDataStreams(), hasSize(1)); @@ -457,9 +468,63 @@ public class TransportGetDataStreamsActionTests extends ESTestCase { .build() ) ), + new IndexSettingProviders(Set.of()), null ); assertThat(response.getDataStreams(), hasSize(1)); assertThat(response.getDataStreams().getFirst().isFailureStoreEffectivelyEnabled(), is(true)); } + + public void testProvidersAffectMode() { + ClusterState state; + var projectId = randomProjectIdOrDefault(); + { + state = DataStreamTestHelper.getClusterStateWithDataStreams( + projectId, + List.of(Tuple.tuple("data-stream-1", 2)), + List.of(), + System.currentTimeMillis(), + Settings.EMPTY, + 0, + false, + false + ); + } + + var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {}); + var response = TransportGetDataStreamsAction.innerOperation( + state.projectState(projectId), + req, + resolver, + systemIndices, + ClusterSettings.createBuiltInClusterSettings(), + dataStreamGlobalRetentionSettings, + emptyDataStreamFailureStoreSettings, + new IndexSettingProviders( + Set.of( + ( + indexName, + dataStreamName, + templateIndexMode, + metadata, + resolvedAt, + indexTemplateAndCreateRequestSettings, + combinedTemplateMappings) -> Settings.builder().put("index.mode", IndexMode.LOOKUP).build() + ) + ), + null + ); + assertThat(response.getDataStreams().getFirst().getIndexModeName(), equalTo("lookup")); + assertThat( + response.getDataStreams() + .getFirst() + .getIndexSettingsValues() + .values() + .stream() + .findFirst() + .map(GetDataStreamAction.Response.IndexProperties::indexMode) + .orElse("bad"), + equalTo("standard") + ); + } } diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 0dd856ab8c64..ef6a0176b05f 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -183,6 +183,7 @@ public class TransportVersions { public static final TransportVersion STORED_SCRIPT_CONTENT_LENGTH = def(9_019_0_00); public static final TransportVersion JINA_AI_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_020_0_00); public static final TransportVersion RE_REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(9_021_0_00); + public static final TransportVersion INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM = def(9_022_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index 5dd60a1122bf..c0fae1443413 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -232,6 +232,7 @@ public class GetDataStreamAction extends ActionType indexSettingsValues, boolean templatePreferIlmValue, - @Nullable Long maximumTimestamp + @Nullable Long maximumTimestamp, + @Nullable String indexMode ) { this.dataStream = dataStream; this.failureStoreEffectivelyEnabled = failureStoreEffectivelyEnabled; @@ -267,6 +271,7 @@ public class GetDataStreamAction extends ActionType combinedTemplateMappings ) { - if ("validate-index-name".equals(indexName)) { + if (MetadataIndexTemplateService.VALIDATE_INDEX_NAME.equals(indexName)) { // This index name is used when validating component and index templates, we should skip this check in that case. // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) return MappingHints.EMPTY; diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index ca5246cb312c..cf550c3a5ab7 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; @@ -475,7 +476,7 @@ public class LogsdbIndexModeSettingsProviderTests extends ESTestCase { } public void testValidateIndexName() throws IOException { - String indexName = "validate-index-name"; + String indexName = MetadataIndexTemplateService.VALIDATE_INDEX_NAME; String mapping = """ { "_doc": {