From ff6465b83b41a65028b05071b4fd62aa2d85de5e Mon Sep 17 00:00:00 2001 From: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Date: Thu, 6 Mar 2025 18:34:57 +0100 Subject: [PATCH] Avoid hoarding cluster state references during rollover (#124107) By keeping a list of all the rollover results in a rollover request batch, we were keeping references to all the intermediate cluster states that we built. We've seen this list take up ~1.4GB with 600 rollover requests in one batch. We only kept the list of results to compute the "reason" for the allocation reroute, so we can easily drop the cluster state reference from the list and only keep what we need. Fixes #123893 --- docs/changelog/124107.yaml | 6 ++++++ .../indices/rollover/LazyRolloverAction.java | 16 ++++------------ .../rollover/TransportRolloverAction.java | 16 ++++------------ 3 files changed, 14 insertions(+), 24 deletions(-) create mode 100644 docs/changelog/124107.yaml diff --git a/docs/changelog/124107.yaml b/docs/changelog/124107.yaml new file mode 100644 index 000000000000..65b3a857639d --- /dev/null +++ b/docs/changelog/124107.yaml @@ -0,0 +1,6 @@ +pr: 124107 +summary: Avoid hoarding cluster state references during rollover +area: Indices APIs +type: bug +issues: + - 123893 diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java index 7917c311f7c3..1fecfc3b7037 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java @@ -36,7 +36,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; @@ -189,7 +188,7 @@ public final class LazyRolloverAction extends ActionType { @Override public ClusterState execute(BatchExecutionContext batchExecutionContext) { final var listener = new AllocationActionMultiListener(threadPool.getThreadContext()); - final var results = new ArrayList(batchExecutionContext.taskContexts().size()); + final var results = new ArrayList(batchExecutionContext.taskContexts().size()); var state = batchExecutionContext.initialState(); Map>>> groupedRequests = new HashMap<>(); for (final var taskContext : batchExecutionContext.taskContexts()) { @@ -214,14 +213,7 @@ public final class LazyRolloverAction extends ActionType { if (state != batchExecutionContext.initialState()) { var reason = new StringBuilder(); - Strings.collectionToDelimitedStringWithLimit( - (Iterable) () -> Iterators.map(results.iterator(), t -> t.sourceIndexName() + "->" + t.rolloverIndexName()), - ",", - "lazy bulk rollover [", - "]", - 1024, - reason - ); + Strings.collectionToDelimitedStringWithLimit(results, ",", "lazy bulk rollover [", "]", 1024, reason); try (var ignored = batchExecutionContext.dropHeadersContext()) { state = allocationService.reroute(state, reason.toString(), listener.reroute()); } @@ -234,7 +226,7 @@ public final class LazyRolloverAction extends ActionType { public ClusterState executeTask( ProjectState currentState, RolloverRequest rolloverRequest, - List results, + ArrayList results, List> rolloverTaskContexts, AllocationActionMultiListener allocationActionMultiListener ) throws Exception { @@ -271,7 +263,7 @@ public final class LazyRolloverAction extends ActionType { null, isFailureStoreRollover ); - results.add(rolloverResult); + results.add(rolloverResult.sourceIndexName() + "->" + rolloverResult.rolloverIndexName()); logger.trace("lazy rollover result [{}]", rolloverResult); final var rolloverIndexName = rolloverResult.rolloverIndexName(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index 451395dfc75b..eeac384188b1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -49,7 +49,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.MasterServiceTaskQueue; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Nullable; @@ -494,7 +493,7 @@ public class TransportRolloverAction extends TransportMasterNodeAction batchExecutionContext) { final var listener = new AllocationActionMultiListener(threadPool.getThreadContext()); - final var results = new ArrayList(batchExecutionContext.taskContexts().size()); + final var results = new ArrayList(batchExecutionContext.taskContexts().size()); var state = batchExecutionContext.initialState(); for (final var taskContext : batchExecutionContext.taskContexts()) { try (var ignored = taskContext.captureResponseHeaders()) { @@ -506,14 +505,7 @@ public class TransportRolloverAction extends TransportMasterNodeAction) () -> Iterators.map(results.iterator(), t -> t.sourceIndexName() + "->" + t.rolloverIndexName()), - ",", - "bulk rollover [", - "]", - 1024, - reason - ); + Strings.collectionToDelimitedStringWithLimit(results, ",", "bulk rollover [", "]", 1024, reason); try (var ignored = batchExecutionContext.dropHeadersContext()) { state = allocationService.reroute(state, reason.toString(), listener.reroute()); } @@ -525,7 +517,7 @@ public class TransportRolloverAction extends TransportMasterNodeAction results, + ArrayList results, TaskContext rolloverTaskContext, AllocationActionMultiListener allocationActionMultiListener ) throws Exception { @@ -596,7 +588,7 @@ public class TransportRolloverAction extends TransportMasterNodeAction" + rolloverResult.rolloverIndexName()); logger.trace("rollover result [{}]", rolloverResult); final var rolloverIndexName = rolloverResult.rolloverIndexName();