Remove non-test usages of Metadata.Builder#putCustom (#128801)

This removes all non-test usages of
```
Metadata.Builder.putCustom(String type, ProjectCustom custom)
```
And replaces it with appropriate calls to the equivalent method on
`ProjectMetadata.Builder`.

In most cases this _does not_ make the code project aware, but does
reduce the number of deprecated methods in use.

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportDeleteDatabaseConfigurationAction.java

@@ -21,7 +21,7 @@ import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.SimpleBatchedExecutor;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
@@ -103,17 +103,17 @@ public class TransportDeleteDatabaseConfigurationAction extends TransportMasterN
             ClusterStateTaskListener {
 
         ClusterState execute(ClusterState currentState) throws Exception {
-            final IngestGeoIpMetadata geoIpMeta = currentState.metadata()
+            final var project = currentState.metadata().getProject();
-                .getProject()
+            final IngestGeoIpMetadata geoIpMeta = project.custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY);
-                .custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY);
 
             logger.debug("deleting database configuration [{}]", databaseId);
             Map<String, DatabaseConfigurationMetadata> databases = new HashMap<>(geoIpMeta.getDatabases());
             databases.remove(databaseId);
 
-            Metadata currentMeta = currentState.metadata();
             return ClusterState.builder(currentState)
-                .metadata(Metadata.builder(currentMeta).putCustom(IngestGeoIpMetadata.TYPE, new IngestGeoIpMetadata(databases)))
+                .putProjectMetadata(
+                    ProjectMetadata.builder(project).putCustom(IngestGeoIpMetadata.TYPE, new IngestGeoIpMetadata(databases))
+                )
                 .build();
         }
 

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java

@@ -20,7 +20,7 @@ import org.elasticsearch.cluster.ClusterStateTaskListener;
 import org.elasticsearch.cluster.SimpleBatchedExecutor;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
@@ -130,9 +130,8 @@ public class TransportPutDatabaseConfigurationAction extends TransportMasterNode
             ClusterStateTaskListener {
 
         ClusterState execute(ClusterState currentState) throws Exception {
-            IngestGeoIpMetadata geoIpMeta = currentState.metadata()
+            final var project = currentState.metadata().getProject();
-                .getProject()
+            IngestGeoIpMetadata geoIpMeta = project.custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY);
-                .custom(IngestGeoIpMetadata.TYPE, IngestGeoIpMetadata.EMPTY);
 
             String id = database.id();
             final DatabaseConfigurationMetadata existingDatabase = geoIpMeta.getDatabases().get(id);
@@ -160,9 +159,8 @@ public class TransportPutDatabaseConfigurationAction extends TransportMasterNode
                 logger.debug("updating existing database configuration [{}]", id);
             }
 
-            Metadata currentMeta = currentState.metadata();
             return ClusterState.builder(currentState)
-                .metadata(Metadata.builder(currentMeta).putCustom(IngestGeoIpMetadata.TYPE, geoIpMeta))
+                .putProjectMetadata(ProjectMetadata.builder(project).putCustom(IngestGeoIpMetadata.TYPE, geoIpMeta))
                 .build();
         }
 

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -1712,6 +1712,7 @@ public class Metadata implements Diffable<Metadata>, ChunkedToXContent {
             return this;
         }
 
+        @Deprecated(forRemoval = true)
         public Builder putCustom(String type, ProjectCustom custom) {
             return putProjectCustom(type, custom);
         }

server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java

@@ -2593,7 +2593,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
             new ClusterStateUpdateTask() {
                 @Override
                 public ClusterState execute(ClusterState currentState) {
-                    final RepositoriesMetadata state = RepositoriesMetadata.get(currentState);
+                    final var project = currentState.metadata().getDefaultProject();
+                    final RepositoriesMetadata state = RepositoriesMetadata.get(project);
                     final RepositoryMetadata repoState = state.repository(metadata.name());
                     if (repoState.generation() != corruptedGeneration) {
                         throw new IllegalStateException(
@@ -2605,8 +2606,8 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                         );
                     }
                     return ClusterState.builder(currentState)
-                        .metadata(
+                        .putProjectMetadata(
-                            Metadata.builder(currentState.metadata())
+                            ProjectMetadata.builder(project)
                                 .putCustom(
                                     RepositoriesMetadata.TYPE,
                                     state.withUpdatedGeneration(
@@ -2615,7 +2616,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                                         repoState.pendingGeneration()
                                     )
                                 )
-                                .build()
                         )
                         .build();
                 }
@@ -2787,12 +2787,13 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
                         + "] must be larger than latest known generation ["
                         + latestKnownRepoGen.get()
                         + "]";
+                final var project = currentState.metadata().getDefaultProject();
                 return ClusterState.builder(currentState)
-                    .metadata(
+                    .putProjectMetadata(
-                        Metadata.builder(currentState.getMetadata())
+                        ProjectMetadata.builder(project)
                             .putCustom(
                                 RepositoriesMetadata.TYPE,
-                                RepositoriesMetadata.get(currentState).withUpdatedGeneration(repoName, safeGeneration, newGen)
+                                RepositoriesMetadata.get(project).withUpdatedGeneration(repoName, safeGeneration, newGen)
                             )
                             .build()
                     )

server/src/main/java/org/elasticsearch/script/ScriptService.java

@@ -22,7 +22,6 @@ import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationCategory;
@@ -733,11 +732,11 @@ public class ScriptService implements Closeable, ClusterStateApplier, ScriptComp
         submitUnbatchedTask(clusterService, "put-script-" + request.id(), new AckedClusterStateUpdateTask(request, listener) {
             @Override
             public ClusterState execute(ClusterState currentState) {
-                ScriptMetadata smd = currentState.metadata().getProject().custom(ScriptMetadata.TYPE);
+                final var project = currentState.metadata().getProject();
-                smd = ScriptMetadata.putStoredScript(smd, request.id(), source);
+                final ScriptMetadata originalSmd = project.custom(ScriptMetadata.TYPE);
-                Metadata.Builder mdb = Metadata.builder(currentState.getMetadata()).putCustom(ScriptMetadata.TYPE, smd);
+                final ScriptMetadata updatedSmd = ScriptMetadata.putStoredScript(originalSmd, request.id(), source);
 
-                return ClusterState.builder(currentState).metadata(mdb).build();
+                return currentState.copyAndUpdateProject(project.id(), builder -> builder.putCustom(ScriptMetadata.TYPE, updatedSmd));
             }
         });
     }
@@ -750,11 +749,11 @@ public class ScriptService implements Closeable, ClusterStateApplier, ScriptComp
         submitUnbatchedTask(clusterService, "delete-script-" + request.id(), new AckedClusterStateUpdateTask(request, listener) {
             @Override
             public ClusterState execute(ClusterState currentState) {
-                ScriptMetadata smd = currentState.metadata().getProject().custom(ScriptMetadata.TYPE);
+                final var project = currentState.metadata().getProject();
-                smd = ScriptMetadata.deleteStoredScript(smd, request.id());
+                final ScriptMetadata originalSmd = project.custom(ScriptMetadata.TYPE);
-                Metadata.Builder mdb = Metadata.builder(currentState.getMetadata()).putCustom(ScriptMetadata.TYPE, smd);
+                final ScriptMetadata updatedSmd = ScriptMetadata.deleteStoredScript(originalSmd, request.id());
 
-                return ClusterState.builder(currentState).metadata(mdb).build();
+                return currentState.copyAndUpdateProject(project.id(), builder -> builder.putCustom(ScriptMetadata.TYPE, updatedSmd));
             }
         });
     }

server/src/main/java/org/elasticsearch/snapshots/RestoreService.java

@@ -1583,6 +1583,8 @@ public final class RestoreService implements ClusterStateApplier {
         }
 
         private void applyGlobalStateRestore(ClusterState currentState, Metadata.Builder mdBuilder) {
+            @FixForMultiProject
+            final var projectBuilder = mdBuilder.getProject(ProjectId.DEFAULT);
             if (metadata.persistentSettings() != null) {
                 Settings settings = metadata.persistentSettings();
                 if (request.skipOperatorOnlyState()) {
@@ -1607,13 +1609,13 @@ public final class RestoreService implements ClusterStateApplier {
             if (metadata.getProject().templates() != null) {
                 // TODO: Should all existing templates be deleted first?
                 for (IndexTemplateMetadata cursor : metadata.getProject().templates().values()) {
-                    mdBuilder.put(cursor);
+                    projectBuilder.put(cursor);
                 }
             }
 
             // override existing restorable customs (as there might be nothing in snapshot to override them)
             mdBuilder.removeCustomIf((key, value) -> value.isRestorable());
-            mdBuilder.removeProjectCustomIf((key, value) -> value.isRestorable());
+            projectBuilder.removeCustomIf((key, value) -> value.isRestorable());
 
             // restore customs from the snapshot
             if (metadata.customs() != null) {
@@ -1630,7 +1632,7 @@ public final class RestoreService implements ClusterStateApplier {
                 for (var entry : metadata.getProject().customs().entrySet()) {
                     if (entry.getValue().isRestorable()) {
                         // Also, don't restore data streams here, we already added them to the metadata builder above
-                        mdBuilder.putCustom(entry.getKey(), entry.getValue());
+                        projectBuilder.putCustom(entry.getKey(), entry.getValue());
                     }
                 }
             }

server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java

@@ -4090,10 +4090,11 @@ public final class SnapshotsService extends AbstractLifecycleComponent implement
             // Handle the tasks to apply the shard snapshot updates (ShardSnapshotUpdate tasks).
             SnapshotsInProgress snapshotsInProgress = shardsUpdateContext.computeUpdatedState();
 
-            final RegisteredPolicySnapshots.Builder registeredPolicySnapshots = state.metadata()
+            final var project = state.metadata().getProject();
-                .getProject()
+            final RegisteredPolicySnapshots.Builder registeredPolicySnapshots = project.custom(
-                .custom(RegisteredPolicySnapshots.TYPE, RegisteredPolicySnapshots.EMPTY)
+                RegisteredPolicySnapshots.TYPE,
-                .builder();
+                RegisteredPolicySnapshots.EMPTY
+            ).builder();
             // Handle the tasks to create new snapshots (CreateSnapshotTask tasks).
             for (final var taskContext : batchExecutionContext.taskContexts()) {
                 if (taskContext.getTask() instanceof CreateSnapshotTask task) {
@@ -4135,7 +4136,9 @@ public final class SnapshotsService extends AbstractLifecycleComponent implement
 
             return ClusterState.builder(state)
                 .putCustom(SnapshotsInProgress.TYPE, snapshotsInProgress)
-                .metadata(Metadata.builder(state.metadata()).putCustom(RegisteredPolicySnapshots.TYPE, registeredPolicySnapshots.build()))
+                .putProjectMetadata(
+                    ProjectMetadata.builder(project).putCustom(RegisteredPolicySnapshots.TYPE, registeredPolicySnapshots.build())
+                )
                 .build();
         }
 

test/framework/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreTestUtil.java

@@ -18,6 +18,8 @@ import org.elasticsearch.cluster.ClusterStateApplier;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.metadata.RepositoriesMetadata;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
@@ -391,18 +393,25 @@ public final class BlobStoreTestUtil {
     /**
      * Creates a mocked {@link ClusterService} for use in {@link BlobStoreRepository} related tests that mocks out all the necessary
      * functionality to make {@link BlobStoreRepository} work. Initializes the cluster state with a {@link RepositoriesMetadata} instance
-     * that contains the given {@code metadata}.
+     * that contains the given {@code repositoryMetadata}.
      *
-     * @param metadata RepositoryMetadata to initialize the cluster state with
+     * @param repositoryMetadata RepositoryMetadata to initialize the cluster state with
      * @return Mock ClusterService
      */
-    public static ClusterService mockClusterService(RepositoryMetadata metadata) {
+    public static ClusterService mockClusterService(RepositoryMetadata repositoryMetadata) {
         return mockClusterService(
             ClusterState.builder(ClusterState.EMPTY_STATE)
                 .metadata(
                     Metadata.builder()
                         .clusterUUID(UUIDs.randomBase64UUID(random()))
-                        .putCustom(RepositoriesMetadata.TYPE, new RepositoriesMetadata(Collections.singletonList(metadata)))
+                        .put(
+                            ProjectMetadata.builder(ProjectId.DEFAULT)
+                                .putCustom(
+                                    RepositoriesMetadata.TYPE,
+                                    new RepositoriesMetadata(Collections.singletonList(repositoryMetadata))
+                                )
+                                .build()
+                        )
                         .build()
                 )
                 .build()

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java

@@ -23,6 +23,7 @@ import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.routing.IndexRoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
@@ -879,7 +880,8 @@ public class AutoFollowCoordinator extends AbstractLifecycleComponent implements
 
         static Function<ClusterState, ClusterState> recordLeaderIndexAsFollowFunction(String name, Index indexToFollow) {
             return currentState -> {
-                AutoFollowMetadata currentAutoFollowMetadata = currentState.metadata().getProject().custom(AutoFollowMetadata.TYPE);
+                final var project = currentState.metadata().getProject();
+                AutoFollowMetadata currentAutoFollowMetadata = project.custom(AutoFollowMetadata.TYPE);
                 Map<String, List<String>> newFollowedIndexUUIDS = new HashMap<>(currentAutoFollowMetadata.getFollowedLeaderIndexUUIDs());
                 if (newFollowedIndexUUIDS.containsKey(name) == false) {
                     // A delete auto follow pattern request can have removed the auto follow pattern while we want to update
@@ -900,9 +902,7 @@ public class AutoFollowCoordinator extends AbstractLifecycleComponent implements
                     currentAutoFollowMetadata.getHeaders()
                 );
                 return ClusterState.builder(currentState)
-                    .metadata(
+                    .putProjectMetadata(ProjectMetadata.builder(project).putCustom(AutoFollowMetadata.TYPE, newAutoFollowMetadata).build())
-                        Metadata.builder(currentState.getMetadata()).putCustom(AutoFollowMetadata.TYPE, newAutoFollowMetadata).build()
-                    )
                     .build();
             };
         }
@@ -920,7 +920,8 @@ public class AutoFollowCoordinator extends AbstractLifecycleComponent implements
             final List<String> autoFollowPatternNames
         ) {
             return currentState -> {
-                AutoFollowMetadata currentAutoFollowMetadata = currentState.metadata().getProject().custom(AutoFollowMetadata.TYPE);
+                final var currentProject = currentState.metadata().getProject();
+                AutoFollowMetadata currentAutoFollowMetadata = currentProject.custom(AutoFollowMetadata.TYPE);
                 Map<String, List<String>> autoFollowPatternNameToFollowedIndexUUIDs = new HashMap<>(
                     currentAutoFollowMetadata.getFollowedLeaderIndexUUIDs()
                 );
@@ -958,8 +959,8 @@ public class AutoFollowCoordinator extends AbstractLifecycleComponent implements
                         currentAutoFollowMetadata.getHeaders()
                     );
                     return ClusterState.builder(currentState)
-                        .metadata(
+                        .putProjectMetadata(
-                            Metadata.builder(currentState.getMetadata()).putCustom(AutoFollowMetadata.TYPE, newAutoFollowMetadata).build()
+                            ProjectMetadata.builder(currentProject).putCustom(AutoFollowMetadata.TYPE, newAutoFollowMetadata).build()
                         )
                         .build();
                 } else {

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportActivateAutoFollowPatternAction.java

@@ -76,7 +76,8 @@ public class TransportActivateAutoFollowPatternAction extends AcknowledgedTransp
     }
 
     static ClusterState innerActivate(final Request request, ClusterState currentState) {
-        final AutoFollowMetadata autoFollowMetadata = currentState.metadata().getProject().custom(AutoFollowMetadata.TYPE);
+        final var project = currentState.metadata().getProject();
+        final AutoFollowMetadata autoFollowMetadata = project.custom(AutoFollowMetadata.TYPE);
         if (autoFollowMetadata == null) {
             throw new ResourceNotFoundException("auto-follow pattern [{}] is missing", request.getName());
         }
@@ -114,8 +115,9 @@ public class TransportActivateAutoFollowPatternAction extends AcknowledgedTransp
             )
         );
 
-        return currentState.copyAndUpdateMetadata(
+        return currentState.copyAndUpdateProject(
-            metadata -> metadata.putCustom(
+            project.id(),
+            builder -> builder.putCustom(
                 AutoFollowMetadata.TYPE,
                 new AutoFollowMetadata(newPatterns, autoFollowMetadata.getFollowedLeaderIndexUUIDs(), autoFollowMetadata.getHeaders())
             )

x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java

@@ -151,7 +151,8 @@ public class TransportPutAutoFollowPatternAction extends AcknowledgedTransportMa
         // auto patterns are always overwritten
         // only already followed index uuids are updated
 
-        AutoFollowMetadata currentAutoFollowMetadata = localState.metadata().getProject().custom(AutoFollowMetadata.TYPE);
+        final var localProject = localState.metadata().getProject();
+        AutoFollowMetadata currentAutoFollowMetadata = localProject.custom(AutoFollowMetadata.TYPE);
         Map<String, List<String>> followedLeaderIndices;
         Map<String, AutoFollowPattern> patterns;
         Map<String, Map<String, String>> headers;
@@ -215,8 +216,9 @@ public class TransportPutAutoFollowPatternAction extends AcknowledgedTransportMa
         );
         patterns.put(request.getName(), autoFollowPattern);
 
-        return localState.copyAndUpdateMetadata(
+        return localState.copyAndUpdateProject(
-            metadata -> metadata.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, followedLeaderIndices, headers))
+            localProject.id(),
+            builder -> builder.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, followedLeaderIndices, headers))
         );
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OperationModeUpdateTask.java

@@ -14,7 +14,6 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.Priority;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Strings;
@@ -91,7 +90,8 @@ public class OperationModeUpdateTask extends ClusterStateUpdateTask {
             return currentState;
         }
 
-        final OperationMode currentMode = currentILMMode(currentState.metadata().getProject());
+        final var project = currentState.metadata().getProject();
+        final OperationMode currentMode = currentILMMode(project);
         if (currentMode.equals(ilmMode)) {
             // No need for a new state
             return currentState;
@@ -106,12 +106,8 @@ public class OperationModeUpdateTask extends ClusterStateUpdateTask {
         }
 
         logger.info("updating ILM operation mode to {}", newMode);
-        return ClusterState.builder(currentState)
+        final var updatedMetadata = new LifecycleOperationMetadata(newMode, currentSLMMode(currentState));
-            .metadata(
+        return currentState.copyAndUpdateProject(project.id(), b -> b.putCustom(LifecycleOperationMetadata.TYPE, updatedMetadata));
-                Metadata.builder(currentState.metadata())
-                    .putCustom(LifecycleOperationMetadata.TYPE, new LifecycleOperationMetadata(newMode, currentSLMMode(currentState)))
-            )
-            .build();
     }
 
     private ClusterState updateSLMState(final ClusterState currentState) {
@@ -119,6 +115,7 @@ public class OperationModeUpdateTask extends ClusterStateUpdateTask {
             return currentState;
         }
 
+        final var project = currentState.metadata().getProject();
         final OperationMode currentMode = currentSLMMode(currentState);
         if (currentMode.equals(slmMode)) {
             // No need for a new state
@@ -134,15 +131,8 @@ public class OperationModeUpdateTask extends ClusterStateUpdateTask {
         }
 
         logger.info("updating SLM operation mode to {}", newMode);
-        return ClusterState.builder(currentState)
+        final var updatedMetadata = new LifecycleOperationMetadata(currentILMMode(project), newMode);
-            .metadata(
+        return currentState.copyAndUpdateProject(project.id(), b -> b.putCustom(LifecycleOperationMetadata.TYPE, updatedMetadata));
-                Metadata.builder(currentState.metadata())
-                    .putCustom(
-                        LifecycleOperationMetadata.TYPE,
-                        new LifecycleOperationMetadata(currentILMMode(currentState.metadata().getProject()), newMode)
-                    )
-            )
-            .build();
     }
 
     @Override

x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/MigrationResultsUpdateTask.java

@@ -12,7 +12,6 @@ import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.core.SuppressForbidden;
 
@@ -70,14 +69,13 @@ public class MigrationResultsUpdateTask extends ClusterStateUpdateTask {
 
     @Override
     public ClusterState execute(ClusterState currentState) throws Exception {
-        FeatureMigrationResults currentResults = currentState.metadata().getProject().custom(FeatureMigrationResults.TYPE);
+        final var project = currentState.metadata().getProject();
+        FeatureMigrationResults currentResults = project.custom(FeatureMigrationResults.TYPE);
         if (currentResults == null) {
             currentResults = new FeatureMigrationResults(new HashMap<>());
         }
         FeatureMigrationResults newResults = currentResults.withResult(featureName, status);
-        final Metadata newMetadata = Metadata.builder(currentState.metadata()).putCustom(FeatureMigrationResults.TYPE, newResults).build();
+        return currentState.copyAndUpdateProject(project.id(), builder -> builder.putCustom(FeatureMigrationResults.TYPE, newResults));
-        final ClusterState newState = ClusterState.builder(currentState).metadata(newMetadata).build();
-        return newState;
     }
 
     @Override

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAction.java

@@ -22,7 +22,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
@@ -228,7 +227,6 @@ public class TransportDeleteTrainedModelAction extends AcknowledgedTransportMast
         submitUnbatchedTask("delete-trained-model-alias", new AckedClusterStateUpdateTask(request, nameDeletionListener) {
             @Override
             public ClusterState execute(final ClusterState currentState) {
-                final ClusterState.Builder builder = ClusterState.builder(currentState);
                 final ModelAliasMetadata currentMetadata = ModelAliasMetadata.fromState(currentState);
                 if (currentMetadata.modelAliases().isEmpty()) {
                     return currentState;
@@ -237,10 +235,8 @@ public class TransportDeleteTrainedModelAction extends AcknowledgedTransportMast
                 logger.info("[{}] delete model model_aliases {}", request.getId(), modelAliases);
                 modelAliases.forEach(newMetadata::remove);
                 final ModelAliasMetadata modelAliasMetadata = new ModelAliasMetadata(newMetadata);
-                builder.metadata(
+                final var project = currentState.metadata().getProject();
-                    Metadata.builder(currentState.getMetadata()).putCustom(ModelAliasMetadata.NAME, modelAliasMetadata).build()
+                return currentState.copyAndUpdateProject(project.id(), b -> b.putCustom(ModelAliasMetadata.NAME, modelAliasMetadata));
-                );
-                return builder.build();
             }
         });
     }

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportDeleteTrainedModelAliasAction.java

@@ -19,7 +19,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
@@ -108,7 +107,8 @@ public class TransportDeleteTrainedModelAliasAction extends AcknowledgedTranspor
                 request.getModelId()
             );
         }
-        IngestMetadata currentIngestMetadata = currentState.metadata().getProject().custom(IngestMetadata.TYPE);
+        final var project = currentState.metadata().getProject();
+        IngestMetadata currentIngestMetadata = project.custom(IngestMetadata.TYPE);
         Set<String> referencedModels = InferenceProcessorInfoExtractor.getModelIdsFromInferenceProcessors(currentIngestMetadata);
         if (referencedModels.contains(request.getModelAlias())) {
             throw new ElasticsearchStatusException(
@@ -117,15 +117,13 @@ public class TransportDeleteTrainedModelAliasAction extends AcknowledgedTranspor
                 request.getModelAlias()
             );
         }
-        final ClusterState.Builder builder = ClusterState.builder(currentState);
         final Map<String, ModelAliasMetadata.ModelAliasEntry> newMetadata = new HashMap<>(currentMetadata.modelAliases());
         logger.info("deleting model_alias [{}] that refers to model [{}]", request.getModelAlias(), request.getModelId());
         inferenceAuditor.info(referencedModel, String.format(Locale.ROOT, "deleting model_alias [%s]", request.getModelAlias()));
 
         newMetadata.remove(request.getModelAlias());
         final ModelAliasMetadata modelAliasMetadata = new ModelAliasMetadata(newMetadata);
-        builder.metadata(Metadata.builder(currentState.getMetadata()).putCustom(ModelAliasMetadata.NAME, modelAliasMetadata).build());
+        return currentState.copyAndUpdateProject(project.id(), builder -> builder.putCustom(ModelAliasMetadata.NAME, modelAliasMetadata));
-        return builder.build();
     }
 
     @Override

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutTrainedModelAliasAction.java

@@ -18,7 +18,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -257,7 +256,6 @@ public class TransportPutTrainedModelAliasAction extends AcknowledgedTransportMa
     }
 
     static ClusterState updateModelAlias(final ClusterState currentState, final PutTrainedModelAliasAction.Request request) {
-        final ClusterState.Builder builder = ClusterState.builder(currentState);
         final ModelAliasMetadata currentMetadata = ModelAliasMetadata.fromState(currentState);
         String currentModelId = currentMetadata.getModelId(request.getModelAlias());
         final Map<String, ModelAliasMetadata.ModelAliasEntry> newMetadata = new HashMap<>(currentMetadata.modelAliases());
@@ -273,8 +271,8 @@ public class TransportPutTrainedModelAliasAction extends AcknowledgedTransportMa
         }
         newMetadata.put(request.getModelAlias(), new ModelAliasMetadata.ModelAliasEntry(request.getModelId()));
         final ModelAliasMetadata modelAliasMetadata = new ModelAliasMetadata(newMetadata);
-        builder.metadata(Metadata.builder(currentState.getMetadata()).putCustom(ModelAliasMetadata.NAME, modelAliasMetadata).build());
+        final var project = currentState.metadata().getProject();
-        return builder.build();
+        return currentState.copyAndUpdateProject(project.id(), builder -> builder.putCustom(ModelAliasMetadata.NAME, modelAliasMetadata));
     }
 
     @Override

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportSetUpgradeModeAction.java

@@ -87,11 +87,10 @@ public class TransportSetUpgradeModeAction extends AbstractTransportSetUpgradeMo
     @Override
     protected ClusterState createUpdatedState(SetUpgradeModeActionRequest request, ClusterState currentState) {
         logger.trace("Executing cluster state update");
-        MlMetadata.Builder builder = new MlMetadata.Builder(currentState.metadata().getProject().custom(MlMetadata.TYPE));
+        final var project = currentState.metadata().getProject();
+        MlMetadata.Builder builder = new MlMetadata.Builder(project.custom(MlMetadata.TYPE));
         builder.isUpgradeMode(request.enabled());
-        ClusterState.Builder newState = ClusterState.builder(currentState);
+        return currentState.copyAndUpdateProject(project.id(), b -> b.putCustom(MlMetadata.TYPE, builder.build()));
-        newState.metadata(Metadata.builder(currentState.getMetadata()).putCustom(MlMetadata.TYPE, builder.build()).build());
-        return newState.build();
     }
 
     protected void upgradeModeSuccessfullyChanged(

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/assignment/TrainedModelAssignmentClusterService.java

@@ -21,8 +21,8 @@ import org.elasticsearch.cluster.ClusterChangedEvent;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.NodesShutdownMetadata;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.cluster.service.ClusterService;
@@ -523,14 +523,14 @@ public class TrainedModelAssignmentClusterService implements ClusterStateListene
 
     private static ClusterState forceUpdate(ClusterState currentState, TrainedModelAssignmentMetadata.Builder modelAssignments) {
         logger.debug(() -> format("updated assignments: %s", modelAssignments.build()));
-        Metadata.Builder metadata = Metadata.builder(currentState.metadata());
+        ProjectMetadata.Builder builder = ProjectMetadata.builder(currentState.metadata().getProject());
         if (currentState.getMinTransportVersion().onOrAfter(RENAME_ALLOCATION_TO_ASSIGNMENT_TRANSPORT_VERSION)) {
-            metadata.putCustom(TrainedModelAssignmentMetadata.NAME, modelAssignments.build())
+            builder.putCustom(TrainedModelAssignmentMetadata.NAME, modelAssignments.build())
-                .removeProjectCustom(TrainedModelAssignmentMetadata.DEPRECATED_NAME);
+                .removeCustom(TrainedModelAssignmentMetadata.DEPRECATED_NAME);
         } else {
-            metadata.putCustom(TrainedModelAssignmentMetadata.DEPRECATED_NAME, modelAssignments.buildOld());
+            builder.putCustom(TrainedModelAssignmentMetadata.DEPRECATED_NAME, modelAssignments.buildOld());
         }
-        return ClusterState.builder(currentState).metadata(metadata).build();
+        return ClusterState.builder(currentState).putProjectMetadata(builder).build();
     }
 
     ClusterState createModelAssignment(ClusterState currentState, CreateTrainedModelAssignmentAction.Request request) throws Exception {

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/TrainedModelCacheMetadataService.java

@@ -17,7 +17,7 @@ import org.elasticsearch.cluster.ClusterStateListener;
 import org.elasticsearch.cluster.ClusterStateTaskExecutor;
 import org.elasticsearch.cluster.ClusterStateTaskExecutor.TaskContext;
 import org.elasticsearch.cluster.ClusterStateTaskListener;
-import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.cluster.service.MasterServiceTaskQueue;
 import org.elasticsearch.common.Priority;
@@ -124,7 +124,10 @@ public class TrainedModelCacheMetadataService implements ClusterStateListener {
             }
 
             return ClusterState.builder(initialState)
-                .metadata(Metadata.builder(initialState.metadata()).putCustom(TrainedModelCacheMetadata.NAME, currentCacheMetadata))
+                .putProjectMetadata(
+                    ProjectMetadata.builder(initialState.metadata().getProject())
+                        .putCustom(TrainedModelCacheMetadata.NAME, currentCacheMetadata)
+                )
                 .build();
         }
     }

x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/SnapshotLifecycleTask.java

@@ -17,7 +17,6 @@ import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.SnapshotsInProgress;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.scheduler.SchedulerEngine;
@@ -271,12 +270,9 @@ public class SnapshotLifecycleTask implements SchedulerEngine.Listener {
 
         @Override
         public ClusterState execute(ClusterState currentState) throws Exception {
-            SnapshotLifecycleMetadata snapMeta = currentState.metadata()
+            final var project = currentState.metadata().getProject();
-                .getProject()
+            SnapshotLifecycleMetadata snapMeta = project.custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY);
-                .custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY);
+            RegisteredPolicySnapshots registeredSnapshots = project.custom(RegisteredPolicySnapshots.TYPE, RegisteredPolicySnapshots.EMPTY);
-            RegisteredPolicySnapshots registeredSnapshots = currentState.metadata()
-                .getProject()
-                .custom(RegisteredPolicySnapshots.TYPE, RegisteredPolicySnapshots.EMPTY);
 
             Map<String, SnapshotLifecyclePolicyMetadata> snapLifecycles = new HashMap<>(snapMeta.getSnapshotConfigurations());
             SnapshotLifecyclePolicyMetadata policyMetadata = snapLifecycles.get(policyName);
@@ -349,11 +345,11 @@ public class SnapshotLifecycleTask implements SchedulerEngine.Listener {
                 currentSLMMode(currentState),
                 newStats
             );
-            Metadata newMeta = Metadata.builder(currentState.metadata())
+            return currentState.copyAndUpdateProject(
-                .putCustom(SnapshotLifecycleMetadata.TYPE, lifecycleMetadata)
+                project.id(),
+                builder -> builder.putCustom(SnapshotLifecycleMetadata.TYPE, lifecycleMetadata)
                     .putCustom(RegisteredPolicySnapshots.TYPE, new RegisteredPolicySnapshots(newRegistered))
-                .build();
+            );
-            return ClusterState.builder(currentState).metadata(newMeta).build();
         }
 
         @Override

x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/UpdateSnapshotLifecycleStatsTask.java

@@ -37,7 +37,8 @@ public class UpdateSnapshotLifecycleStatsTask extends ClusterStateUpdateTask {
     @Override
     public ClusterState execute(ClusterState currentState) {
         final Metadata currentMeta = currentState.metadata();
-        final SnapshotLifecycleMetadata currentSlmMeta = currentMeta.getProject().custom(SnapshotLifecycleMetadata.TYPE);
+        final var project = currentMeta.getProject();
+        final SnapshotLifecycleMetadata currentSlmMeta = project.custom(SnapshotLifecycleMetadata.TYPE);
 
         if (currentSlmMeta == null) {
             return currentState;
@@ -50,9 +51,7 @@ public class UpdateSnapshotLifecycleStatsTask extends ClusterStateUpdateTask {
             newMetrics
         );
 
-        return ClusterState.builder(currentState)
+        return currentState.copyAndUpdateProject(project.id(), builder -> builder.putCustom(SnapshotLifecycleMetadata.TYPE, newSlmMeta));
-            .metadata(Metadata.builder(currentMeta).putCustom(SnapshotLifecycleMetadata.TYPE, newSlmMeta))
-            .build();
     }
 
     @Override

x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportDeleteSnapshotLifecycleAction.java

@@ -17,7 +17,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.SuppressForbidden;
@@ -82,7 +81,8 @@ public class TransportDeleteSnapshotLifecycleAction extends TransportMasterNodeA
 
         @Override
         public ClusterState execute(ClusterState currentState) {
-            SnapshotLifecycleMetadata snapMeta = currentState.metadata().getProject().custom(SnapshotLifecycleMetadata.TYPE);
+            final var project = currentState.metadata().getProject();
+            SnapshotLifecycleMetadata snapMeta = project.custom(SnapshotLifecycleMetadata.TYPE);
             if (snapMeta == null) {
                 throw new ResourceNotFoundException("snapshot lifecycle policy not found: {}", request.getLifecycleId());
             }
@@ -101,20 +101,13 @@ public class TransportDeleteSnapshotLifecycleAction extends TransportMasterNodeA
                 .filter(e -> e.getKey().equals(request.getLifecycleId()) == false)
                 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
 
-            Metadata metadata = currentState.metadata();
+            return currentState.copyAndUpdateProject(
-            return ClusterState.builder(currentState)
+                project.id(),
-                .metadata(
+                builder -> builder.putCustom(
-                    Metadata.builder(metadata)
-                        .putCustom(
                     SnapshotLifecycleMetadata.TYPE,
-                            new SnapshotLifecycleMetadata(
+                    new SnapshotLifecycleMetadata(newConfigs, currentMode, snapMeta.getStats().removePolicy(request.getLifecycleId()))
-                                newConfigs,
-                                currentMode,
-                                snapMeta.getStats().removePolicy(request.getLifecycleId())
                 )
-                        )
+            );
-                )
-                .build();
         }
     }
 

x-pack/plugin/slm/src/main/java/org/elasticsearch/xpack/slm/action/TransportPutSnapshotLifecycleAction.java

@@ -18,7 +18,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.core.Nullable;
@@ -121,9 +120,8 @@ public class TransportPutSnapshotLifecycleAction extends TransportMasterNodeActi
 
         @Override
         public ClusterState execute(ClusterState currentState) {
-            SnapshotLifecycleMetadata snapMeta = currentState.metadata()
+            final var project = currentState.metadata().getProject();
-                .getProject()
+            SnapshotLifecycleMetadata snapMeta = project.custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY);
-                .custom(SnapshotLifecycleMetadata.TYPE, SnapshotLifecycleMetadata.EMPTY);
             var currentMode = LifecycleOperationMetadata.currentSLMMode(currentState);
             final SnapshotLifecyclePolicyMetadata existingPolicyMetadata = snapMeta.getSnapshotConfigurations()
                 .get(request.getLifecycleId());
@@ -149,15 +147,8 @@ public class TransportPutSnapshotLifecycleAction extends TransportMasterNodeActi
                 logger.info("updating existing snapshot lifecycle [{}]", newLifecycle.getId());
             }
 
-            return ClusterState.builder(currentState)
+            final var updatedMetadata = new SnapshotLifecycleMetadata(snapLifecycles, currentMode, snapMeta.getStats());
-                .metadata(
+            return currentState.copyAndUpdateProject(project.id(), b -> b.putCustom(SnapshotLifecycleMetadata.TYPE, updatedMetadata));
-                    Metadata.builder(currentState.metadata())
-                        .putCustom(
-                            SnapshotLifecycleMetadata.TYPE,
-                            new SnapshotLifecycleMetadata(snapLifecycles, currentMode, snapMeta.getStats())
-                        )
-                )
-                .build();
         }
     }
 

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportSetTransformUpgradeModeAction.java

@@ -77,8 +77,10 @@ public class TransportSetTransformUpgradeModeAction extends AbstractTransportSet
     @Override
     protected ClusterState createUpdatedState(SetUpgradeModeActionRequest request, ClusterState state) {
         var updatedTransformMetadata = TransformMetadata.getTransformMetadata(state).builder().upgradeMode(request.enabled()).build();
-        var updatedClusterMetadata = state.metadata().copyAndUpdate(b -> b.putCustom(TransformMetadata.TYPE, updatedTransformMetadata));
+        return state.copyAndUpdateProject(
-        return state.copyAndUpdate(b -> b.metadata(updatedClusterMetadata));
+            state.metadata().getProject().id(),
+            b -> b.putCustom(TransformMetadata.TYPE, updatedTransformMetadata)
+        );
     }
 
     @Override

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/transport/actions/TransportWatcherServiceAction.java

@@ -18,7 +18,6 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateUpdateTask;
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.injection.guice.Inject;
@@ -74,15 +73,14 @@ public class TransportWatcherServiceAction extends AcknowledgedTransportMasterNo
                     XPackPlugin.checkReadyForXPackCustomMetadata(clusterState);
 
                     WatcherMetadata newWatcherMetadata = new WatcherMetadata(manuallyStopped);
-                    WatcherMetadata currentMetadata = clusterState.metadata().getProject().custom(WatcherMetadata.TYPE);
+                    final var project = clusterState.metadata().getProject();
+                    WatcherMetadata currentMetadata = project.custom(WatcherMetadata.TYPE);
 
                     // adhere to the contract of returning the original state if nothing has changed
                     if (newWatcherMetadata.equals(currentMetadata)) {
                         return clusterState;
                     } else {
-                        ClusterState.Builder builder = new ClusterState.Builder(clusterState);
+                        return clusterState.copyAndUpdateProject(project.id(), b -> b.putCustom(WatcherMetadata.TYPE, newWatcherMetadata));
-                        builder.metadata(Metadata.builder(clusterState.getMetadata()).putCustom(WatcherMetadata.TYPE, newWatcherMetadata));
-                        return builder.build();
                     }
                 }