From 0b58a53a98a324a9053d9bfa145d89f3f09c9c2b Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Tue, 24 Jun 2025 10:30:15 -0500 Subject: [PATCH] Adding the ability to unset data stream settings (#129677) --- ...ansportUpdateDataStreamSettingsAction.java | 11 ++- .../data_stream/240_data_stream_settings.yml | 99 +++++++++++++++++++ .../metadata/MetadataDataStreamsService.java | 9 ++ 3 files changed, 114 insertions(+), 5 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java index af9e848f11ff..4ed25ac8a36c 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamSettingsAction.java @@ -131,7 +131,7 @@ public class TransportUpdateDataStreamSettingsAction extends TransportMasterNode new UpdateDataStreamSettingsAction.DataStreamSettingsResponse( dataStreamName, false, - e.getMessage(), + Strings.hasText(e.getMessage()) ? e.getMessage() : e.toString(), EMPTY, EMPTY, UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult.EMPTY @@ -222,9 +222,12 @@ public class TransportUpdateDataStreamSettingsAction extends TransportMasterNode Map settingsToApply = new HashMap<>(); List appliedToDataStreamOnly = new ArrayList<>(); List appliedToDataStreamAndBackingIndices = new ArrayList<>(); + Settings effectiveSettings = dataStream.getEffectiveSettings( + clusterService.state().projectState(projectResolver.getProjectId()).metadata() + ); for (String settingName : requestSettings.keySet()) { if (APPLY_TO_BACKING_INDICES.contains(settingName)) { - settingsToApply.put(settingName, requestSettings.get(settingName)); + settingsToApply.put(settingName, effectiveSettings.get(settingName)); appliedToDataStreamAndBackingIndices.add(settingName); } else if (APPLY_TO_DATA_STREAM_ONLY.contains(settingName)) { appliedToDataStreamOnly.add(settingName); @@ -242,9 +245,7 @@ public class TransportUpdateDataStreamSettingsAction extends TransportMasterNode true, null, settingsFilter.filter(dataStream.getSettings()), - settingsFilter.filter( - dataStream.getEffectiveSettings(clusterService.state().projectState(projectResolver.getProjectId()).metadata()) - ), + settingsFilter.filter(effectiveSettings), new UpdateDataStreamSettingsAction.DataStreamSettingsResponse.IndicesSettingsResult( appliedToDataStreamOnly, appliedToDataStreamAndBackingIndices, diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml index 8dd15628fa13..ad09a06378c5 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/240_data_stream_settings.yml @@ -299,3 +299,102 @@ setup: - match: { .$idx0name.settings.index.lifecycle.name: "my-policy" } - match: { .$idx1name.settings.index.number_of_shards: "1" } - match: { .$idx1name.settings.index.lifecycle.name: "my-policy" } + +--- +"Test null out settings": + - requires: + cluster_features: [ "logs_stream" ] + reason: requires setting 'logs_stream' to get or set data stream settings + - do: + allowed_warnings: + - "index template [my-template] has index patterns [my-data-stream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-template + body: + index_patterns: [ my-data-stream-* ] + data_stream: { } + template: + settings: + number_of_replicas: 0 + lifecycle.name: my-policy + + - do: + indices.create_data_stream: + name: my-data-stream-1 + + - do: + cluster.health: + index: "my-data-stream-1" + wait_for_status: green + + + + - do: + indices.get_data_stream: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings: {} } + - match: { data_streams.0.effective_settings: null } + + - do: + indices.put_data_stream_settings: + name: my-data-stream-1 + body: + index: + number_of_shards: 2 + lifecycle: + name: my-new-policy + prefer_ilm: true + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.applied_to_data_stream: true } + - match: { data_streams.0.index_settings_results.applied_to_data_stream_only: [index.number_of_shards]} + - length: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: 2 } + - match: { data_streams.0.settings.index.number_of_shards: "2" } + - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" } + - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: "true" } + - match: { data_streams.0.effective_settings.index.number_of_shards: "2" } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" } + - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: "true" } + + - do: + indices.put_data_stream_settings: + name: my-data-stream-1 + body: + index: + number_of_shards: null + lifecycle: + name: null + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.applied_to_data_stream: true } + - match: { data_streams.0.index_settings_results.applied_to_data_stream_only: [index.number_of_shards]} + - length: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: 1 } + - match: { data_streams.0.settings.index.number_of_shards: null } + - match: { data_streams.0.settings.index.lifecycle.name: null } + - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: "true" } + - match: { data_streams.0.effective_settings.index.number_of_shards: null } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-policy" } + - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: "true" } + + - do: + indices.get_data_stream_settings: + name: my-data-stream-1 + - match: { data_streams.0.name: my-data-stream-1 } + - match: { data_streams.0.settings.index.lifecycle.name: null } + - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: "true" } + - match: { data_streams.0.effective_settings.index.number_of_shards: null } + - match: { data_streams.0.effective_settings.index.number_of_replicas: "0" } + - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-policy" } + + - do: + indices.get_data_stream: + name: my-data-stream-1 + - set: { data_streams.0.indices.0.index_name: idx0name } + + - do: + indices.get_settings: + index: my-data-stream-1 + - match: { .$idx0name.settings.index.number_of_shards: "1" } + - match: { .$idx0name.settings.index.lifecycle.name: "my-policy" } + - match: { .$idx0name.settings.index.lifecycle.prefer_ilm: "true" } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java index 75b9f138225c..40a2e23241e8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java @@ -456,6 +456,15 @@ public class MetadataDataStreamsService { Template.Builder templateBuilder = Template.builder(); Settings.Builder mergedSettingsBuilder = Settings.builder().put(existingSettings).put(settingsOverrides); + /* + * A null value for a setting override means that we remove it from the data stream, and let the value from the template (if any) + * be used. + */ + settingsOverrides.keySet().forEach(key -> { + if (mergedSettingsBuilder.get(key) == null) { + mergedSettingsBuilder.remove(key); + } + }); Settings mergedSettings = mergedSettingsBuilder.build(); final ComposableIndexTemplate template = lookupTemplateForDataStream(dataStreamName, projectMetadata);