github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-06-28 09:28:55 -04:00
Code Issues Projects Releases Packages Wiki Activity

Add index mode to get data stream API (#122486)

Browse source 
This commit adds the `index_mode` for both the data stream and each
backing index to the output of `GET /_data_stream`. An example looks
like:

```
{
  "data_streams" : [
    {
      "name" : "foo-things",
      "indices" : [
        {
          "index_name" : ".ds-foo-things-2025.02.13-000001",
          ...
          "index_mode" : "standard"
        }
      ],
      ...
      "index_mode" : "standard"
    },
    {
      "name" : "logs-foo-bar",
      "indices" : [
        {
          "index_name" : ".ds-logs-foo-bar-2025.02.13-000001",
          ...
          "index_mode" : "logsdb"
        },
        {
          "index_name" : ".ds-logs-foo-bar-2025.02.13-000002",
          ...
          "index_mode" : "logsdb"
        }
      ],
      ...
      "index_mode" : "logsdb",
    }
  ]
}
```
This commit is contained in:
Lee Hinman 2025-03-05 13:39:58 -07:00 • committed by GitHub
parent 2fa6651a68
commit 47706b505f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 585 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

5
docs/changelog/122486.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 122486
summary: Add index mode to get data stream API
area: Data streams
type: enhancement
issues: []

376
docs/reference/indices/get-data-stream.asciidoc Normal file
View file

@ -0,0 +1,376 @@
[role="xpack"]
[[indices-get-data-stream]]
=== Get data stream API
++++
<titleabbrev>Get data stream</titleabbrev>
++++
.New API reference
[sidebar]
--
For the most up-to-date API details, refer to {api-es}/group/endpoint-data-stream[Data stream APIs].
--
Retrieves information about one or more <<data-streams,data streams>>.
See <<get-info-about-data-stream>>.
////
[source,console]
----
PUT /_ilm/policy/my-lifecycle-policy
{
"policy": {
"phases": {
"hot": {
"actions": {
"rollover": {
"max_primary_shard_size": "25GB"
}
}
},
"delete": {
"min_age": "30d",
"actions": {
"delete": {}
}
}
}
}
}
PUT /_index_template/my-index-template
{
"index_patterns": [ "my-data-stream*" ],
"data_stream": {},
"template": {
"settings": {
"index.lifecycle.name": "my-lifecycle-policy"
}
},
"_meta": {
"my-meta-field": "foo"
}
}
PUT /_data_stream/my-data-stream
POST /my-data-stream/_rollover
PUT /_data_stream/my-data-stream-two
DELETE /_data_stream/my-data-stream*/_lifecycle
----
// TESTSETUP
////
////
[source,console]
----
DELETE /_data_stream/*
DELETE /_index_template/*
DELETE /_ilm/policy/my-lifecycle-policy
----
// TEARDOWN
////
[source,console]
----
GET /_data_stream/my-data-stream
----
[[get-data-stream-api-request]]
==== {api-request-title}
`GET /_data_stream/<data-stream>`
[[get-data-stream-api-prereqs]]
==== {api-prereq-title}
* If the {es} {security-features} are enabled, you must have the
`view_index_metadata` or `manage` <<privileges-list-indices,index privilege>>
for the data stream.
[[get-data-stream-api-path-params]]
==== {api-path-parms-title}
`<data-stream>`::
(Optional, string)
Comma-separated list of data stream names used to limit the request. Wildcard
(`*`) expressions are supported. If omitted, all data streams will be
returned.
[role="child_attributes"]
[[get-data-stream-api-query-parms]]
==== {api-query-parms-title}
include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=ds-expand-wildcards]
+
Defaults to `open`.
`include_defaults`::
(Optional, Boolean) Functionality in preview:[]. If `true`, return all default settings in the response.
Defaults to `false`.
`verbose`::
(Optional, Boolean). If `true`, Returns the `maximum_timestamp` corresponding to the `@timestamp` field for documents in the data stream.
Defaults to `false`.
[role="child_attributes"]
[[get-data-stream-api-response-body]]
==== {api-response-body-title}
`data_streams`::
(array of objects)
Contains information about retrieved data streams.
+
.Properties of objects in `data_streams`
[%collapsible%open]
====
`name`::
(string)
Name of the data stream.
`timestamp_field`::
(object)
Contains information about the data stream's `@timestamp` field.
+
.Properties of `timestamp_field`
[%collapsible%open]
=====
`name`::
(string)
Name of the data stream's timestamp field, which must be `@timestamp`. The
`@timestamp` field must be included in every document indexed to the data
stream.
=====
`indices`::
(array of objects)
Array of objects containing information about the data stream's backing
indices.
+
The last item in this array contains information about the stream's current
<<data-stream-write-index,write index>>.
+
.Properties of `indices` objects
[%collapsible%open]
=====
`index_name`::
(string)
Name of the backing index. For naming conventions, see
<<data-streams-generation>>.
`index_uuid`::
(string)
Universally unique identifier (UUID) for the index.
`prefer_ilm`::
(boolean)
Functionality in preview:[]. Indicates if this index is configured to prefer {ilm}
when both {ilm-cap} and <<data-stream-lifecycle, Data stream lifecycle>> are configured to
manage this index.
`managed_by`::
(string)
Functionality in preview:[]. Indicates the system that managed this index.
=====
`generation`::
(integer)
Current <<data-streams-generation,generation>> for the data stream. This number
acts as a cumulative count of the stream's rollovers, starting at `1`.
`_meta`::
(object)
Custom metadata for the stream, copied from the `_meta` object of the
stream's matching <<create-index-template,index template>>. If empty,
the response omits this property.
`status`::
(string)
<<cluster-health,Health status>> of the data stream.
+
This health status is based on the state of the primary and replica shards of
the stream's backing indices.
+
.Values for `status`
[%collapsible%open]
=====
`GREEN`:::
All shards are assigned.
`YELLOW`:::
All primary shards are assigned, but one or more replica shards are
unassigned.
`RED`:::
One or more primary shards are unassigned, so some data is unavailable.
=====
`template`::
(string)
Name of the index template used to create the data stream's backing indices.
+
The template's index pattern must match the name of this data stream. See
<<create-index-template,create an index template>>.
`ilm_policy`::
(string)
Name of the current {ilm-init} lifecycle policy in the stream's matching index
template. This lifecycle policy is set in the `index.lifecycle.name` setting.
+
If the template does not include a lifecycle policy, this property is not
included in the response.
+
NOTE: A data stream's backing indices may be assigned different lifecycle
policies. To retrieve the lifecycle policy for individual backing indices,
use the <<indices-get-settings,get index settings API>>.
`next_generation_managed_by`::
(string)
Functionality in preview:[]. Indicates the system that will managed the next generation index
(i.e. the next data stream write index).
`prefer_ilm`::
(boolean)
Functionality in preview:[]. Indicates if the index template used to create the data
stream's backing indices is configured to prefer {ilm-cap} when both {ilm-cap} and
<<data-stream-lifecycle, Data stream lifecycle>> are configured to manage this index.
`hidden`::
(Boolean) If `true`, the data stream is <<multi-hidden,hidden>>.
`system`::
(Boolean)
If `true`, the data stream is created and managed by an Elastic stack component
and cannot be modified through normal user interaction.
`allow_custom_routing`::
(Boolean)
If `true`, the data stream this data stream allows custom routing on write request.
`replicated`::
(Boolean)
If `true`, the data stream is created and managed by {ccr} and the local
cluster can not write into this data stream or change its mappings.
`lifecycle`::
(object)
Functionality in preview:[]. Contains the configuration for the data stream lifecycle management of this data stream.
+
.Properties of `lifecycle`
[%collapsible%open]
=====
`data_retention`::
(string)
If defined, every document added to this data stream will be stored at least for this time frame. Any time after this
duration the document could be deleted. When empty, every document in this data stream will be stored indefinitely.
`rollover`::
(object)
The conditions which will trigger the rollover of a backing index as configured by the cluster setting
`cluster.lifecycle.default.rollover`. This property is an implementation detail and it will only be retrieved when the query
param `include_defaults` is set to `true`. The contents of this field are subject to change.
=====
`rollover_on_write`::
(Boolean)
If `true`, the next write to this data stream will trigger a rollover first and the document will be
indexed in the new backing index. If the rollover fails the indexing request will fail too.
====
[[get-data-stream-api-example]]
==== {api-examples-title}
[source,console]
----
GET _data_stream/my-data-stream*
----
The API returns the following response:
[source,console-result]
----
{
"data_streams": [
{
"name": "my-data-stream",
"timestamp_field": {
"name": "@timestamp"
},
"indices": [
{
"index_name": ".ds-my-data-stream-2099.03.07-000001",
"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg",
"prefer_ilm": true,
"ilm_policy": "my-lifecycle-policy",
"managed_by": "Index Lifecycle Management",
"index_mode": "standard"
},
{
"index_name": ".ds-my-data-stream-2099.03.08-000002",
"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw",
"prefer_ilm": true,
"ilm_policy": "my-lifecycle-policy",
"managed_by": "Index Lifecycle Management",
"index_mode": "standard"
}
],
"generation": 2,
"_meta": {
"my-meta-field": "foo"
},
"status": "GREEN",
"index_mode": "standard",
"next_generation_managed_by": "Index Lifecycle Management",
"prefer_ilm": true,
"template": "my-index-template",
"ilm_policy": "my-lifecycle-policy",
"hidden": false,
"system": false,
"allow_custom_routing": false,
"replicated": false,
"rollover_on_write": false
},
{
"name": "my-data-stream-two",
"timestamp_field": {
"name": "@timestamp"
},
"indices": [
{
"index_name": ".ds-my-data-stream-two-2099.03.08-000001",
"index_uuid": "3liBu2SYS5axasRt6fUIpA",
"prefer_ilm": true,
"ilm_policy": "my-lifecycle-policy",
"managed_by": "Index Lifecycle Management",
"index_mode": "standard"
}
],
"generation": 1,
"_meta": {
"my-meta-field": "foo"
},
"status": "YELLOW",
"index_mode": "standard",
"next_generation_managed_by": "Index Lifecycle Management",
"prefer_ilm": true,
"template": "my-index-template",
"ilm_policy": "my-lifecycle-policy",
"hidden": false,
"system": false,
"allow_custom_routing": false,
"replicated": false,
"rollover_on_write": false
}
]
}
----
// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.07-000001"/"index_name": $body.data_streams.0.indices.0.index_name/]
// TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/]
// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.08-000002"/"index_name": $body.data_streams.0.indices.1.index_name/]
// TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/]
// TESTRESPONSE[s/"index_name": ".ds-my-data-stream-two-2099.03.08-000001"/"index_name": $body.data_streams.1.indices.0.index_name/]
// TESTRESPONSE[s/"index_uuid": "3liBu2SYS5axasRt6fUIpA"/"index_uuid": $body.data_streams.1.indices.0.index_uuid/]
// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW"/]
// TESTRESPONSE[s/"replicated": false/"replicated": false,"failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/]

70
modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java
View file

@ -24,6 +24,7 @@ import org.elasticsearch.cluster.ProjectState;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.health.ClusterStateHealth;
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
@ -40,6 +41,9 @@ import org.elasticsearch.core.Nullable;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexMode;
import org.elasticsearch.index.IndexSettingProvider;
import org.elasticsearch.index.IndexSettingProviders;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.indices.SystemDataStreamDescriptor;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.injection.guice.Inject;
@ -53,6 +57,7 @@ import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.stream.Collectors;
@ -68,6 +73,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec
private final ClusterSettings clusterSettings;
private final DataStreamGlobalRetentionSettings globalRetentionSettings;
private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings;
private final IndexSettingProviders indexSettingProviders;
private final Client client;
@Inject
@ -81,6 +87,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec
SystemIndices systemIndices,
DataStreamGlobalRetentionSettings globalRetentionSettings,
DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
IndexSettingProviders indexSettingProviders,
Client client
) {
super(
@ -99,6 +106,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec
this.globalRetentionSettings = globalRetentionSettings;
clusterSettings = clusterService.getClusterSettings();
this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings;
this.indexSettingProviders = indexSettingProviders;
this.client = new OriginSettingClient(client, "stack");
}
@ -131,6 +139,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec
clusterSettings,
globalRetentionSettings,
dataStreamFailureStoreSettings,
indexSettingProviders,
maxTimestamps
)
);
@ -151,12 +160,43 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec
clusterSettings,
globalRetentionSettings,
dataStreamFailureStoreSettings,
indexSettingProviders,
null
)
);
}
}
/**
* Resolves the index mode ("index.mode" setting) for the given data stream, from the template or additional setting providers
*/
@Nullable
static IndexMode resolveMode(
ProjectState state,
IndexSettingProviders indexSettingProviders,
DataStream dataStream,
Settings settings,
ComposableIndexTemplate indexTemplate
) {
IndexMode indexMode = state.metadata().retrieveIndexModeFromTemplate(indexTemplate);
for (IndexSettingProvider provider : indexSettingProviders.getIndexSettingProviders()) {
Settings addlSettinsg = provider.getAdditionalIndexSettings(
MetadataIndexTemplateService.VALIDATE_INDEX_NAME,
dataStream.getName(),
indexMode,
state.metadata(),
Instant.now(),
settings,
List.of()
);
var rawMode = addlSettinsg.get(IndexSettings.MODE.getKey());
if (rawMode != null) {
indexMode = Enum.valueOf(IndexMode.class, rawMode.toUpperCase(Locale.ROOT));
}
}
return indexMode;
}
static GetDataStreamAction.Response innerOperation(
ProjectState state,
GetDataStreamAction.Request request,
@ -165,6 +205,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec
ClusterSettings clusterSettings,
DataStreamGlobalRetentionSettings globalRetentionSettings,
DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
IndexSettingProviders indexSettingProviders,
@Nullable Map<String, Long> maxTimestamps
) {
List<DataStream> dataStreams = getDataStreams(state.metadata(), indexNameExpressionResolver, request);
@ -177,6 +218,7 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec
final String indexTemplate;
boolean indexTemplatePreferIlmValue = true;
String ilmPolicyName = null;
IndexMode indexMode = dataStream.getIndexMode();
if (dataStream.isSystem()) {
SystemDataStreamDescriptor dataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStream.getName());
indexTemplate = dataStreamDescriptor != null ? dataStreamDescriptor.getDataStreamName() : null;
@ -186,6 +228,15 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec
dataStreamDescriptor.getComponentTemplates()
);
ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
if (indexMode == null) {
indexMode = resolveMode(
state,
indexSettingProviders,
dataStream,
settings,
dataStreamDescriptor.getComposableIndexTemplate()
);
}
indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);
}
} else {
@ -193,6 +244,15 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec
if (indexTemplate != null) {
Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) {
indexMode = resolveMode(
state,
indexSettingProviders,
dataStream,
settings,
state.metadata().templatesV2().get(indexTemplate)
);
}
indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);
} else {
LOGGER.warn(
@ -285,7 +345,9 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec
timeSeries,
backingIndicesSettingsValues,
indexTemplatePreferIlmValue,
maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName())
maxTimestamps == null ? null : maxTimestamps.get(dataStream.getName()),
// Default to standard mode if not specified; should we set this to "unset" or "unspecified" instead?
indexMode == null ? IndexMode.STANDARD.getName() : indexMode.getName()
)
);
}
@ -314,7 +376,11 @@ public class TransportGetDataStreamsAction extends TransportMasterNodeReadProjec
} else {
managedBy = ManagedBy.UNMANAGED;
}
backingIndicesSettingsValues.put(index, new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy));
String indexMode = IndexSettings.MODE.get(indexMetadata.getSettings()).getName();
backingIndicesSettingsValues.put(
index,
new IndexProperties(preferIlm, indexMetadata.getLifecyclePolicyName(), managedBy, indexMode)
);
}
}

30
modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java
View file

@ -91,13 +91,13 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase
String ilmPolicyName = "rollover-30days";
Map<Index, Response.IndexProperties> indexSettingsValues = Map.of(
firstGenerationIndex,
new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM),
new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null),
secondGenerationIndex,
new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE),
new Response.IndexProperties(false, ilmPolicyName, ManagedBy.LIFECYCLE, null),
writeIndex,
new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE),
new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE, null),
failureStoreIndex,
new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE)
new Response.IndexProperties(false, null, ManagedBy.LIFECYCLE, null)
);
Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo(
@ -109,6 +109,7 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase
null,
indexSettingsValues,
false,
null,
null
);
Response response = new Response(List.of(dataStreamInfo));
@ -195,13 +196,13 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase
String ilmPolicyName = "rollover-30days";
Map<Index, Response.IndexProperties> indexSettingsValues = Map.of(
firstGenerationIndex,
new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM),
new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null),
secondGenerationIndex,
new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM),
new Response.IndexProperties(true, ilmPolicyName, ManagedBy.ILM, null),
writeIndex,
new Response.IndexProperties(false, null, ManagedBy.UNMANAGED),
new Response.IndexProperties(false, null, ManagedBy.UNMANAGED, null),
failureStoreIndex,
new Response.IndexProperties(false, null, ManagedBy.UNMANAGED)
new Response.IndexProperties(false, null, ManagedBy.UNMANAGED, null)
);
Response.DataStreamInfo dataStreamInfo = new Response.DataStreamInfo(
@ -213,6 +214,7 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase
null,
indexSettingsValues,
false,
null,
null
);
Response response = new Response(List.of(dataStreamInfo));
@ -309,7 +311,8 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase
new Response.IndexProperties(
randomBoolean(),
randomAlphaOfLengthBetween(50, 100),
randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE
randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE,
null
)
)
);
@ -328,7 +331,8 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase
timeSeries,
indexSettings,
templatePreferIlm,
maximumTimestamp
maximumTimestamp,
null
);
}
@ -349,7 +353,8 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase
new Response.IndexProperties(
randomBoolean(),
randomAlphaOfLengthBetween(50, 100),
randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE
randomBoolean() ? ManagedBy.ILM : ManagedBy.LIFECYCLE,
randomBoolean() ? randomFrom(IndexMode.values()).getName() : null
)
);
}
@ -367,7 +372,8 @@ public class GetDataStreamsResponseTests extends AbstractWireSerializingTestCase
timeSeries != null ? new Response.TimeSeries(timeSeries) : null,
generateRandomIndexSettingsValues(),
randomBoolean(),
usually() ? randomNonNegativeLong() : null
usually() ? randomNonNegativeLong() : null,
usually() ? randomFrom(IndexMode.values()).getName() : null
);
}
}

65
modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java
View file

@ -24,7 +24,9 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexMode;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.IndexSettingProviders;
import org.elasticsearch.indices.SystemIndices;
import org.elasticsearch.indices.TestIndexNameExpressionResolver;
import org.elasticsearch.test.ESTestCase;
@ -32,6 +34,7 @@ import org.elasticsearch.test.ESTestCase;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Set;
import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.getClusterStateWithDataStreams;
import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch;
@ -181,6 +184,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase {
ClusterSettings.createBuiltInClusterSettings(),
dataStreamGlobalRetentionSettings,
emptyDataStreamFailureStoreSettings,
new IndexSettingProviders(Set.of()),
null
);
assertThat(
@ -213,6 +217,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase {
ClusterSettings.createBuiltInClusterSettings(),
dataStreamGlobalRetentionSettings,
emptyDataStreamFailureStoreSettings,
new IndexSettingProviders(Set.of()),
null
);
assertThat(
@ -266,6 +271,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase {
ClusterSettings.createBuiltInClusterSettings(),
dataStreamGlobalRetentionSettings,
emptyDataStreamFailureStoreSettings,
new IndexSettingProviders(Set.of()),
null
);
assertThat(
@ -307,6 +313,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase {
ClusterSettings.createBuiltInClusterSettings(),
dataStreamGlobalRetentionSettings,
emptyDataStreamFailureStoreSettings,
new IndexSettingProviders(Set.of()),
null
);
@ -341,6 +348,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase {
ClusterSettings.createBuiltInClusterSettings(),
dataStreamGlobalRetentionSettings,
emptyDataStreamFailureStoreSettings,
new IndexSettingProviders(Set.of()),
null
);
assertThat(response.getGlobalRetention(), nullValue());
@ -367,6 +375,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase {
ClusterSettings.createBuiltInClusterSettings(),
withGlobalRetentionSettings,
emptyDataStreamFailureStoreSettings,
new IndexSettingProviders(Set.of()),
null
);
assertThat(response.getGlobalRetention(), equalTo(globalRetention));
@ -394,6 +403,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase {
ClusterSettings.createBuiltInClusterSettings(),
dataStreamGlobalRetentionSettings,
emptyDataStreamFailureStoreSettings,
new IndexSettingProviders(Set.of()),
null
);
assertThat(response.getDataStreams(), hasSize(1));
@ -423,6 +433,7 @@ public class TransportGetDataStreamsActionTests extends ESTestCase {
ClusterSettings.createBuiltInClusterSettings(),
dataStreamGlobalRetentionSettings,
emptyDataStreamFailureStoreSettings,
new IndexSettingProviders(Set.of()),
null
);
assertThat(response.getDataStreams(), hasSize(1));
@ -457,9 +468,63 @@ public class TransportGetDataStreamsActionTests extends ESTestCase {
.build()
)
),
new IndexSettingProviders(Set.of()),
null
);
assertThat(response.getDataStreams(), hasSize(1));
assertThat(response.getDataStreams().getFirst().isFailureStoreEffectivelyEnabled(), is(true));
}
public void testProvidersAffectMode() {
ClusterState state;
var projectId = randomProjectIdOrDefault();
{
state = DataStreamTestHelper.getClusterStateWithDataStreams(
projectId,
List.of(Tuple.tuple("data-stream-1", 2)),
List.of(),
System.currentTimeMillis(),
Settings.EMPTY,
0,
false,
false
);
}
var req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
var response = TransportGetDataStreamsAction.innerOperation(
state.projectState(projectId),
req,
resolver,
systemIndices,
ClusterSettings.createBuiltInClusterSettings(),
dataStreamGlobalRetentionSettings,
emptyDataStreamFailureStoreSettings,
new IndexSettingProviders(
Set.of(
(
indexName,
dataStreamName,
templateIndexMode,
metadata,
resolvedAt,
indexTemplateAndCreateRequestSettings,
combinedTemplateMappings) -> Settings.builder().put("index.mode", IndexMode.LOOKUP).build()
)
),
null
);
assertThat(response.getDataStreams().getFirst().getIndexModeName(), equalTo("lookup"));
assertThat(
response.getDataStreams()
.getFirst()
.getIndexSettingsValues()
.values()
.stream()
.findFirst()
.map(GetDataStreamAction.Response.IndexProperties::indexMode)
.orElse("bad"),
equalTo("standard")
);
}
}

1
server/src/main/java/org/elasticsearch/TransportVersions.java
View file

@ -183,6 +183,7 @@ public class TransportVersions {
public static final TransportVersion STORED_SCRIPT_CONTENT_LENGTH = def(9_019_0_00);
public static final TransportVersion JINA_AI_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_020_0_00);
public static final TransportVersion RE_REMOVE_MIN_COMPATIBLE_SHARD_NODE = def(9_021_0_00);
public static final TransportVersion INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM = def(9_022_0_00);
/*
* STOP! READ THIS FIRST! No, really,

44
server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java
View file

@ -232,6 +232,7 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
);
public static final ParseField FAILURE_STORE_ENABLED = new ParseField("enabled");
public static final ParseField MAXIMUM_TIMESTAMP = new ParseField("maximum_timestamp");
public static final ParseField INDEX_MODE = new ParseField("index_mode");
private final DataStream dataStream;
private final ClusterHealthStatus dataStreamStatus;
@ -246,6 +247,8 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
private final boolean templatePreferIlmValue;
@Nullable
private final Long maximumTimestamp;
@Nullable
private final String indexMode;
public DataStreamInfo(
DataStream dataStream,
@ -256,7 +259,8 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
@Nullable TimeSeries timeSeries,
Map<Index, IndexProperties> indexSettingsValues,
boolean templatePreferIlmValue,
@Nullable Long maximumTimestamp
@Nullable Long maximumTimestamp,
@Nullable String indexMode
) {
this.dataStream = dataStream;
this.failureStoreEffectivelyEnabled = failureStoreEffectivelyEnabled;
@ -267,6 +271,7 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
this.indexSettingsValues = indexSettingsValues;
this.templatePreferIlmValue = templatePreferIlmValue;
this.maximumTimestamp = maximumTimestamp;
this.indexMode = indexMode;
}
@SuppressWarnings("unchecked")
@ -287,6 +292,9 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
: Map.of();
this.templatePreferIlmValue = in.getTransportVersion().onOrAfter(V_8_11_X) ? in.readBoolean() : true;
this.maximumTimestamp = in.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0) ? in.readOptionalVLong() : null;
this.indexMode = in.getTransportVersion().onOrAfter(TransportVersions.INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM)
? in.readOptionalString()
: null;
}
public DataStream getDataStream() {
@ -329,6 +337,11 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
return maximumTimestamp;
}
@Nullable
public String getIndexModeName() {
return indexMode;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
dataStream.writeTo(out);
@ -348,6 +361,9 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_16_0)) {
out.writeOptionalVLong(maximumTimestamp);
}
if (out.getTransportVersion().onOrAfter(TransportVersions.INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM)) {
out.writeOptionalString(indexMode);
}
}
@Override
@ -398,6 +414,9 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
if (this.maximumTimestamp != null) {
builder.field(MAXIMUM_TIMESTAMP.getPreferredName(), this.maximumTimestamp);
}
if (this.indexMode != null) {
builder.field(INDEX_MODE.getPreferredName(), indexMode);
}
addAutoShardingEvent(builder, params, dataStream.getAutoShardingEvent());
if (timeSeries != null) {
builder.startObject(TIME_SERIES.getPreferredName());
@ -441,6 +460,7 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
builder.field(ILM_POLICY_FIELD.getPreferredName(), indexProperties.ilmPolicyName());
}
builder.field(MANAGED_BY.getPreferredName(), indexProperties.managedBy.displayValue);
builder.field(INDEX_MODE.getPreferredName(), indexProperties.indexMode);
}
builder.endObject();
}
@ -500,7 +520,8 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
&& Objects.equals(ilmPolicyName, that.ilmPolicyName)
&& Objects.equals(timeSeries, that.timeSeries)
&& Objects.equals(indexSettingsValues, that.indexSettingsValues)
&& Objects.equals(maximumTimestamp, that.maximumTimestamp);
&& Objects.equals(maximumTimestamp, that.maximumTimestamp)
&& Objects.equals(indexMode, that.indexMode);
}
@Override
@ -514,7 +535,8 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
timeSeries,
indexSettingsValues,
templatePreferIlmValue,
maximumTimestamp
maximumTimestamp,
indexMode
);
}
}
@ -551,9 +573,18 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
* Encapsulates the configured properties we want to display for each backing index.
* They'll usually be settings values, but could also be additional properties derived from settings.
*/
public record IndexProperties(boolean preferIlm, @Nullable String ilmPolicyName, ManagedBy managedBy) implements Writeable {
public record IndexProperties(boolean preferIlm, @Nullable String ilmPolicyName, ManagedBy managedBy, @Nullable String indexMode)
implements
Writeable {
public IndexProperties(StreamInput in) throws IOException {
this(in.readBoolean(), in.readOptionalString(), in.readEnum(ManagedBy.class));
this(
in.readBoolean(),
in.readOptionalString(),
in.readEnum(ManagedBy.class),
in.getTransportVersion().onOrAfter(TransportVersions.INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM)
? in.readOptionalString()
: "unknown"
);
}
@Override
@ -561,6 +592,9 @@ public class GetDataStreamAction extends ActionType<GetDataStreamAction.Response
out.writeBoolean(preferIlm);
out.writeOptionalString(ilmPolicyName);
out.writeEnum(managedBy);
if (out.getTransportVersion().onOrAfter(TransportVersions.INCLUDE_INDEX_MODE_IN_GET_DATA_STREAM)) {
out.writeOptionalString(indexMode);
}
}
}

7
server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java
View file

@ -91,6 +91,9 @@ public class MetadataIndexTemplateService {
public static final String DEFAULT_TIMESTAMP_FIELD = "@timestamp";
public static final CompressedXContent DEFAULT_TIMESTAMP_MAPPING_WITHOUT_ROUTING;
// Names used for validating templates when we do not know the index or data stream name
public static final String VALIDATE_INDEX_NAME = "validate-index-name";
public static final String VALIDATE_DATA_STREAM_NAME = "validate-data-stream-name";
private static final CompressedXContent DEFAULT_TIMESTAMP_MAPPING_WITH_ROUTING;
@ -714,8 +717,8 @@ public class MetadataIndexTemplateService {
var finalSettings = Settings.builder();
for (var provider : indexSettingProviders) {
var newAdditionalSettings = provider.getAdditionalIndexSettings(
"validate-index-name",
indexTemplate.getDataStreamTemplate() != null ? "validate-data-stream-name" : null,
VALIDATE_INDEX_NAME,
indexTemplate.getDataStreamTemplate() != null ? VALIDATE_DATA_STREAM_NAME : null,
projectMetadata.retrieveIndexModeFromTemplate(indexTemplate),
projectMetadata,
now,

1
server/src/test/java/org/elasticsearch/action/datastreams/GetDataStreamActionTests.java
View file

@ -92,6 +92,7 @@ public class GetDataStreamActionTests extends ESTestCase {
null,
Map.of(),
randomBoolean(),
null,
null
);
}

9
x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java
View file

@ -12,6 +12,7 @@ import org.apache.logging.log4j.Logger;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
import org.elasticsearch.cluster.metadata.ProjectMetadata;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.compress.CompressedXContent;
@ -100,7 +101,9 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider {
) {
Settings.Builder settingsBuilder = null;
boolean isLogsDB = templateIndexMode == IndexMode.LOGSDB;
boolean isTemplateValidation = "validate-index-name".equals(indexName);
// This index name is used when validating component and index templates, we should skip this check in that case.
// (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
boolean isTemplateValidation = MetadataIndexTemplateService.VALIDATE_INDEX_NAME.equals(indexName);
// Inject logsdb index mode, based on the logs pattern.
if (isLogsdbEnabled
@ -118,8 +121,6 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider {
if (mappingHints.hasSyntheticSourceUsage
&& supportFallbackToStoredSource.get()
&& minNodeVersion.get().get().onOrAfter(Version.V_8_17_0)) {
// This index name is used when validating component and index templates, we should skip this check in that case.
// (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
boolean legacyLicensedUsageOfSyntheticSourceAllowed = isLegacyLicensedUsageOfSyntheticSourceAllowed(
templateIndexMode,
indexName,
@ -216,7 +217,7 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider {
Settings indexTemplateAndCreateRequestSettings,
List<CompressedXContent> combinedTemplateMappings
) {
if ("validate-index-name".equals(indexName)) {
if (MetadataIndexTemplateService.VALIDATE_INDEX_NAME.equals(indexName)) {
// This index name is used when validating component and index templates, we should skip this check in that case.
// (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
return MappingHints.EMPTY;

3
x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java
View file

@ -14,6 +14,7 @@ import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService;
import org.elasticsearch.cluster.metadata.ProjectMetadata;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.common.compress.CompressedXContent;
@ -475,7 +476,7 @@ public class LogsdbIndexModeSettingsProviderTests extends ESTestCase {
}
public void testValidateIndexName() throws IOException {
String indexName = "validate-index-name";
String indexName = MetadataIndexTemplateService.VALIDATE_INDEX_NAME;
String mapping = """
{
"_doc": {