Make enrich project-aware (#124099)

Makes the execution and use of enrich policies project-aware.
Note: this does not make the enrich cache project-aware. That is to be
handled in a follow-up PR.

libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineBridge.java

@@ -8,6 +8,7 @@
  */
 package org.elasticsearch.logstashbridge.ingest;
 
+import org.elasticsearch.core.FixForMultiProject;
 import org.elasticsearch.ingest.Pipeline;
 import org.elasticsearch.logstashbridge.StableBridgeAPI;
 import org.elasticsearch.logstashbridge.script.ScriptServiceBridge;
@@ -20,6 +21,7 @@ public class PipelineBridge extends StableBridgeAPI.Proxy<Pipeline> {
         return new PipelineBridge(pipeline);
     }
 
+    @FixForMultiProject(description = "should we pass a non-null project ID here?")
     public static PipelineBridge create(
         String id,
         Map<String, Object> config,
@@ -27,7 +29,13 @@ public class PipelineBridge extends StableBridgeAPI.Proxy<Pipeline> {
         ScriptServiceBridge scriptServiceBridge
     ) throws Exception {
         return wrap(
-            Pipeline.create(id, config, StableBridgeAPI.unwrap(processorFactories), StableBridgeAPI.unwrapNullable(scriptServiceBridge))
+            Pipeline.create(
+                id,
+                config,
+                StableBridgeAPI.unwrap(processorFactories),
+                StableBridgeAPI.unwrapNullable(scriptServiceBridge),
+                null
+            )
         );
     }
 

libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ProcessorBridge.java

@@ -8,6 +8,7 @@
  */
 package org.elasticsearch.logstashbridge.ingest;
 
+import org.elasticsearch.core.FixForMultiProject;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.ingest.Processor;
@@ -118,7 +119,7 @@ public interface ProcessorBridge extends StableBridgeAPI<Processor> {
         @Override
         default Processor.Factory unwrap() {
             final Factory stableAPIFactory = this;
-            return (registry, tag, description, config) -> stableAPIFactory.create(
+            return (registry, tag, description, config, projectId) -> stableAPIFactory.create(
                 StableBridgeAPI.wrap(registry, Factory::wrap),
                 tag,
                 description,
@@ -131,6 +132,7 @@ public interface ProcessorBridge extends StableBridgeAPI<Processor> {
                 super(delegate);
             }
 
+            @FixForMultiProject(description = "should we pass a non-null project ID here?")
             @Override
             public ProcessorBridge create(
                 final Map<String, Factory> registry,
@@ -138,7 +140,9 @@ public interface ProcessorBridge extends StableBridgeAPI<Processor> {
                 final String description,
                 final Map<String, Object> config
             ) throws Exception {
-                return ProcessorBridge.wrap(this.delegate.create(StableBridgeAPI.unwrap(registry), processorTag, description, config));
+                return ProcessorBridge.wrap(
+                    this.delegate.create(StableBridgeAPI.unwrap(registry), processorTag, description, config, null)
+                );
             }
 
             @Override

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java

@@ -403,9 +403,9 @@ public class IngestFailureStoreMetricsIT extends ESIntegTestCase {
             Map<String, Processor.Factory> processors = new HashMap<>();
             processors.put(
                 "drop",
-                (factories, tag, description, config) -> new TestProcessor(tag, "drop", description, ingestDocument -> null)
+                (factories, tag, description, config, projectId) -> new TestProcessor(tag, "drop", description, ingestDocument -> null)
             );
-            processors.put("reroute", (factories, tag, description, config) -> {
+            processors.put("reroute", (factories, tag, description, config, projectId) -> {
                 String destination = (String) config.remove("destination");
                 return new TestProcessor(
                     tag,
@@ -416,7 +416,12 @@ public class IngestFailureStoreMetricsIT extends ESIntegTestCase {
             });
             processors.put(
                 "fail",
-                (processorFactories, tag, description, config) -> new TestProcessor(tag, "fail", description, new RuntimeException())
+                (processorFactories, tag, description, config, projectId) -> new TestProcessor(
+                    tag,
+                    "fail",
+                    description,
+                    new RuntimeException()
+                )
             );
             return processors;
         }

modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java

@@ -15,6 +15,7 @@ import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.Office;
 import org.apache.tika.metadata.TikaCoreProperties;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
@@ -232,7 +233,8 @@ public final class AttachmentProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) {
             String field = readStringProperty(TYPE, processorTag, config, "field");
             String resourceName = readOptionalStringProperty(TYPE, processorTag, config, "resource_name");

modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java

@@ -36,7 +36,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
 
         String processorTag = randomAlphaOfLength(10);
 
-        AttachmentProcessor processor = factory.create(null, processorTag, null, config);
+        AttachmentProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("attachment"));
@@ -57,7 +57,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         config.put("indexed_chars", indexedChars);
 
         String processorTag = randomAlphaOfLength(10);
-        AttachmentProcessor processor = factory.create(null, processorTag, null, config);
+        AttachmentProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getIndexedChars(), is(indexedChars));
         assertFalse(processor.isIgnoreMissing());
@@ -73,7 +73,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("target_field", "_field");
-        AttachmentProcessor processor = factory.create(null, null, null, config);
+        AttachmentProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("_field"));
         assertFalse(processor.isIgnoreMissing());
@@ -97,7 +97,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("properties", fieldNames);
-        AttachmentProcessor processor = factory.create(null, null, null, config);
+        AttachmentProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
         assertFalse(processor.isIgnoreMissing());
@@ -114,7 +114,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", Collections.singletonList("invalid"));
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[properties] illegal field option [invalid]"));
@@ -128,7 +128,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", "invalid");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("exception expected");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
@@ -148,7 +148,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
 
         String processorTag = randomAlphaOfLength(10);
 
-        AttachmentProcessor processor = factory.create(null, processorTag, null, config);
+        AttachmentProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("attachment"));
@@ -169,7 +169,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
 
         String processorTag = randomAlphaOfLength(10);
 
-        AttachmentProcessor processor = factory.create(null, processorTag, null, config);
+        AttachmentProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("attachment"));

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -108,7 +109,8 @@ abstract class AbstractStringProcessor<T> extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String tag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(processorType, tag, config, "field");
             boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(processorType, tag, config, "ignore_missing", false);

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -73,7 +74,8 @@ public final class AppendProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value");

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
@@ -297,7 +298,8 @@ public final class CommunityIdProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String sourceIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_ip", DEFAULT_SOURCE_IP);
             String sourcePortField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_port", DEFAULT_SOURCE_PORT);

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
@@ -216,7 +217,8 @@ public final class ConvertProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -93,7 +94,8 @@ public final class CsvProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String quote = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "quote", "\"");

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
@@ -150,7 +151,8 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String tag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "locale");
             String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "timezone");

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.ingest.common;
 
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.util.LocaleUtils;
@@ -183,7 +184,8 @@ public final class DateProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET_FIELD);

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.dissect.DissectParser;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
@@ -60,7 +61,8 @@ public final class DissectProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String pattern = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "pattern");

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -120,7 +121,8 @@ public final class DotExpanderProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> processorFactories,
             String tag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
             if (field.contains(".") == false && field.equals("*") == false) {

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -60,7 +61,8 @@ public final class FailProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String message = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "message");
             TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FingerprintProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.hash.Murmur3Hasher;
 import org.elasticsearch.common.util.ByteUtils;
@@ -227,7 +228,8 @@ public final class FingerprintProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             List<String> fields = ConfigurationUtils.readList(TYPE, processorTag, config, "fields");
             if (fields.size() < 1) {

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.ingest.AbstractProcessor;
@@ -235,8 +236,13 @@ public final class ForEachProcessor extends AbstractProcessor implements Wrappin
         }
 
         @Override
-        public ForEachProcessor create(Map<String, Processor.Factory> factories, String tag, String description, Map<String, Object> config)
+        public ForEachProcessor create(
-            throws Exception {
+            Map<String, Processor.Factory> factories,
+            String tag,
+            String description,
+            Map<String, Object> config,
+            ProjectId projectId
+        ) throws Exception {
             String field = readStringProperty(TYPE, tag, config, "field");
             boolean ignoreMissing = readBooleanProperty(TYPE, tag, config, "ignore_missing", false);
             Map<String, Map<String, Object>> processorConfig = readMap(TYPE, tag, config, "processor");
@@ -245,7 +251,7 @@ public final class ForEachProcessor extends AbstractProcessor implements Wrappin
                 throw newConfigurationException(TYPE, tag, "processor", "Must specify exactly one processor type");
             }
             Map.Entry<String, Map<String, Object>> entry = entries.iterator().next();
-            Processor processor = ConfigurationUtils.readProcessor(factories, scriptService, entry.getKey(), entry.getValue());
+            Processor processor = ConfigurationUtils.readProcessor(factories, scriptService, entry.getKey(), entry.getValue(), projectId);
             return new ForEachProcessor(tag, description, field, processor, ignoreMissing);
         }
     }

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java

@@ -11,6 +11,7 @@ package org.elasticsearch.ingest.common;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.grok.Grok;
 import org.elasticsearch.grok.GrokBuiltinPatterns;
 import org.elasticsearch.grok.MatcherWatchdog;
@@ -150,7 +151,8 @@ public final class GrokProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             List<String> matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns");

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -71,7 +72,8 @@ public final class JoinProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String separator = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator");

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
@@ -217,7 +218,8 @@ public final class JsonProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String targetField = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "target_field");

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.core.Predicates;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
@@ -256,7 +257,8 @@ public final class KeyValueProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             TemplateScript.Factory fieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", field, scriptService);

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.network.CIDRUtils;
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.ingest.AbstractProcessor;
@@ -251,7 +252,8 @@ public class NetworkDirectionProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             final String sourceIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_ip", DEFAULT_SOURCE_IP);
             final String destIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "destination_ip", DEFAULT_DEST_IP);

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java

@@ -11,6 +11,7 @@ package org.elasticsearch.ingest.common;
 
 import org.apache.http.conn.util.PublicSuffixMatcher;
 import org.apache.http.conn.util.PublicSuffixMatcherLoader;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.ingest.AbstractProcessor;
@@ -140,7 +141,8 @@ public class RegisteredDomainProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String tag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, tag, config, "target_field", DEFAULT_TARGET_FIELD);

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -109,7 +110,8 @@ public final class RemoveProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             final List<TemplateScript.Factory> compiledTemplatesToRemove = getTemplates(processorTag, config, "field");
             final List<TemplateScript.Factory> compiledTemplatesToKeep = getTemplates(processorTag, config, "keep");

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -110,7 +111,8 @@ public final class RenameProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             TemplateScript.Factory fieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", field, scriptService);

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
@@ -175,7 +176,8 @@ public final class RerouteProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> processorFactories,
             String tag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             List<DataStreamValueSource> type;
             try {

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -105,7 +106,8 @@ public final class ScriptProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             try (
                 XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent).map(config);

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -109,7 +110,8 @@ public final class SetProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String copyFrom = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "copy_from");

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -115,7 +116,8 @@ public final class SortProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, FIELD);
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestDocument;
@@ -99,7 +100,8 @@ public final class SplitProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TerminateProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
@@ -45,7 +46,8 @@ public class TerminateProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> processorFactories,
             String tag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) {
             return new TerminateProcessor(tag, description);
         }

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.common;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.ConfigurationUtils;
@@ -184,7 +185,8 @@ public class UriPartsProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", "url");

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java

@@ -37,7 +37,7 @@ public abstract class AbstractStringProcessorFactoryTestCase extends ESTestCase
         Map<String, Object> config = new HashMap<>();
         config.put("field", fieldName);
 
-        AbstractStringProcessor<?> processor = factory.create(null, processorTag, null, modifyConfig(config));
+        AbstractStringProcessor<?> processor = factory.create(null, processorTag, null, modifyConfig(config), null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo(fieldName));
         assertThat(processor.isIgnoreMissing(), is(false));
@@ -54,7 +54,7 @@ public abstract class AbstractStringProcessorFactoryTestCase extends ESTestCase
         config.put("field", fieldName);
         config.put("ignore_missing", true);
 
-        AbstractStringProcessor<?> processor = factory.create(null, processorTag, null, modifyConfig(config));
+        AbstractStringProcessor<?> processor = factory.create(null, processorTag, null, modifyConfig(config), null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo(fieldName));
         assertThat(processor.isIgnoreMissing(), is(true));
@@ -72,7 +72,7 @@ public abstract class AbstractStringProcessorFactoryTestCase extends ESTestCase
         config.put("field", fieldName);
         config.put("target_field", targetFieldName);
 
-        AbstractStringProcessor<?> processor = factory.create(null, processorTag, null, modifyConfig(config));
+        AbstractStringProcessor<?> processor = factory.create(null, processorTag, null, modifyConfig(config), null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo(fieldName));
         assertThat(processor.isIgnoreMissing(), is(false));
@@ -84,7 +84,7 @@ public abstract class AbstractStringProcessorFactoryTestCase extends ESTestCase
         AbstractStringProcessor.Factory factory = newFactory();
         Map<String, Object> config = new HashMap<>();
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java

@@ -43,7 +43,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         }
         config.put("value", value);
         String processorTag = randomAlphaOfLength(10);
-        AppendProcessor appendProcessor = factory.create(null, processorTag, null, config);
+        AppendProcessor appendProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(appendProcessor.getTag(), equalTo(processorTag));
         assertThat(appendProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
         assertThat(appendProcessor.getValue().copyAndResolve(Map.of()), equalTo(value));
@@ -53,7 +53,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("value", "value1");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -64,7 +64,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[value] required property is missing"));
@@ -76,7 +76,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("value", null);
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[value] required property is missing"));
@@ -91,7 +91,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(
             ElasticsearchException.class,
-            () -> factory.create(null, processorTag, null, config)
+            () -> factory.create(null, processorTag, null, config, null)
         );
         assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
         assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));
@@ -105,7 +105,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         config.put("value", "value1");
         config.put("media_type", expectedMediaType);
         String processorTag = randomAlphaOfLength(10);
-        AppendProcessor appendProcessor = factory.create(null, processorTag, null, config);
+        AppendProcessor appendProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(appendProcessor.getTag(), equalTo(processorTag));
 
         // invalid media type
@@ -117,7 +117,10 @@ public class AppendProcessorFactoryTests extends ESTestCase {
         config2.put("field", "field1");
         config2.put("value", "value1");
         config2.put("media_type", expectedMediaType);
-        ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config2));
+        ElasticsearchException e = expectThrows(
+            ElasticsearchException.class,
+            () -> factory.create(null, processorTag, null, config2, null)
+        );
         assertThat(e.getMessage(), containsString("property does not contain a supported media type [" + expectedMediaType + "]"));
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java

@@ -65,7 +65,7 @@ public class CommunityIdProcessorFactoryTests extends ESTestCase {
         config.put("ignore_missing", ignoreMissing);
 
         String processorTag = randomAlphaOfLength(10);
-        CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config);
+        CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(communityIdProcessor.getTag(), equalTo(processorTag));
         assertThat(communityIdProcessor.getSourceIpField(), equalTo(sourceIpField));
         assertThat(communityIdProcessor.getSourcePortField(), equalTo(sourcePortField));
@@ -87,27 +87,27 @@ public class CommunityIdProcessorFactoryTests extends ESTestCase {
         // negative seeds are rejected
         int tooSmallSeed = randomIntBetween(Integer.MIN_VALUE, -1);
         config.put("seed", Integer.toString(tooSmallSeed));
-        ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config));
+        ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null));
         assertThat(e.getMessage(), containsString("must be a value between 0 and 65535"));
 
         // seeds >= 2^16 are rejected
         int tooBigSeed = randomIntBetween(65536, Integer.MAX_VALUE);
         config.put("seed", Integer.toString(tooBigSeed));
-        e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config));
+        e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null));
         assertThat(e.getMessage(), containsString("must be a value between 0 and 65535"));
 
         // seeds between 0 and 2^16-1 are accepted
         int justRightSeed = randomIntBetween(0, 65535);
         byte[] expectedSeed = new byte[] { (byte) (justRightSeed >> 8), (byte) justRightSeed };
         config.put("seed", Integer.toString(justRightSeed));
-        CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config);
+        CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(communityIdProcessor.getSeed(), equalTo(expectedSeed));
     }
 
     public void testRequiredFields() throws Exception {
         HashMap<String, Object> config = new HashMap<>();
         String processorTag = randomAlphaOfLength(10);
-        CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config);
+        CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(communityIdProcessor.getTag(), equalTo(processorTag));
         assertThat(communityIdProcessor.getSourceIpField(), equalTo(DEFAULT_SOURCE_IP));
         assertThat(communityIdProcessor.getSourcePortField(), equalTo(DEFAULT_SOURCE_PORT));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java

@@ -28,7 +28,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("type", type.toString());
         String processorTag = randomAlphaOfLength(10);
-        ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config);
+        ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(convertProcessor.getTag(), equalTo(processorTag));
         assertThat(convertProcessor.getField(), equalTo("field1"));
         assertThat(convertProcessor.getTargetField(), equalTo("field1"));
@@ -43,7 +43,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("type", type);
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[type] type [" + type + "] not supported, cannot convert field."));
@@ -59,7 +59,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         String type = "type-" + randomAlphaOfLengthBetween(1, 10);
         config.put("type", type);
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -71,7 +71,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[type] required property is missing"));
@@ -86,7 +86,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         config.put("target_field", "field2");
         config.put("type", type.toString());
         String processorTag = randomAlphaOfLength(10);
-        ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config);
+        ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(convertProcessor.getTag(), equalTo(processorTag));
         assertThat(convertProcessor.getField(), equalTo("field1"));
         assertThat(convertProcessor.getTargetField(), equalTo("field2"));
@@ -102,7 +102,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
         config.put("type", type.toString());
         config.put("ignore_missing", true);
         String processorTag = randomAlphaOfLength(10);
-        ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config);
+        ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(convertProcessor.getTag(), equalTo(processorTag));
         assertThat(convertProcessor.getField(), equalTo("field1"));
         assertThat(convertProcessor.getTargetField(), equalTo("field1"));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java

@@ -31,7 +31,7 @@ public class CsvProcessorFactoryTests extends ESTestCase {
         properties.put("empty_value", "empty");
         properties.put("trim", true);
         properties.put("ignore_missing", true);
-        CsvProcessor csv = factory.create(null, "csv", null, properties);
+        CsvProcessor csv = factory.create(null, "csv", null, properties, null);
         assertThat(csv, notNullValue());
         assertThat(csv.field, equalTo("field"));
         assertThat(csv.headers, equalTo(new String[] { "target" }));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java

@@ -28,7 +28,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("date_rounding", "y");
 
-        DateIndexNameProcessor processor = factory.create(null, null, null, config);
+        DateIndexNameProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getDateFormats().size(), equalTo(1));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getIndexNamePrefixTemplate().newInstance(Map.of()).execute(), equalTo(""));
@@ -45,7 +45,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("date_rounding", "y");
         config.put("date_formats", List.of("UNIX", "UNIX_MS"));
 
-        DateIndexNameProcessor processor = factory.create(null, null, null, config);
+        DateIndexNameProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getDateFormats().size(), equalTo(2));
 
         config = new HashMap<>();
@@ -54,7 +54,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("date_rounding", "y");
         config.put("index_name_format", "yyyyMMdd");
 
-        processor = factory.create(null, null, null, config);
+        processor = factory.create(null, null, null, config, null);
         assertThat(processor.getIndexNameFormatTemplate().newInstance(Map.of()).execute(), equalTo("yyyyMMdd"));
 
         config = new HashMap<>();
@@ -63,7 +63,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("date_rounding", "y");
         config.put("timezone", "+02:00");
 
-        processor = factory.create(null, null, null, config);
+        processor = factory.create(null, null, null, config, null);
         assertThat(processor.getTimezone(), equalTo(ZoneOffset.ofHours(2)));
 
         config = new HashMap<>();
@@ -71,7 +71,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         config.put("index_name_prefix", "_prefix");
         config.put("date_rounding", "y");
 
-        processor = factory.create(null, null, null, config);
+        processor = factory.create(null, null, null, config, null);
         assertThat(processor.getIndexNamePrefixTemplate().newInstance(Map.of()).execute(), equalTo("_prefix"));
     }
 
@@ -79,12 +79,15 @@ public class DateIndexNameFactoryTests extends ESTestCase {
         DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance());
         Map<String, Object> config = new HashMap<>();
         config.put("date_rounding", "y");
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> factory.create(null, null, null, config, null)
+        );
         assertThat(e.getMessage(), equalTo("[field] required property is missing"));
 
         config.clear();
         config.put("field", "_field");
-        e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
+        e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config, null));
         assertThat(e.getMessage(), equalTo("[date_rounding] required property is missing"));
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java

@@ -38,7 +38,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("field", sourceField);
         config.put("formats", List.of("dd/MM/yyyyy"));
         String processorTag = randomAlphaOfLength(10);
-        DateProcessor processor = factory.create(null, processorTag, null, config);
+        DateProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo(sourceField));
         assertThat(processor.getTargetField(), equalTo(DateProcessor.DEFAULT_TARGET_FIELD));
@@ -54,7 +54,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("formats", List.of("dd/MM/yyyyy"));
 
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("processor creation should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[field] required property is missing"));
@@ -69,7 +69,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("target_field", targetField);
 
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("processor creation should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[formats] required property is missing"));
@@ -84,7 +84,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         Locale locale = randomFrom(Locale.GERMANY, Locale.FRENCH, Locale.ROOT);
         config.put("locale", locale.toLanguageTag());
 
-        DateProcessor processor = factory.create(null, null, null, config);
+        DateProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getLocale().newInstance(Map.of()).execute(), equalTo(locale.toLanguageTag()));
     }
 
@@ -96,7 +96,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
 
         ZoneId timezone = randomZone();
         config.put("timezone", timezone.getId());
-        DateProcessor processor = factory.create(null, null, null, config);
+        DateProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getTimezone().newInstance(Map.of()).execute(), equalTo(timezone.getId()));
     }
 
@@ -106,7 +106,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("field", sourceField);
         config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
 
-        DateProcessor processor = factory.create(null, null, null, config);
+        DateProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getFormats(), equalTo(List.of("dd/MM/yyyy", "dd-MM-yyyy")));
     }
 
@@ -117,7 +117,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("formats", "dd/MM/yyyy");
 
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("processor creation should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[formats] property isn't a list, but of type [java.lang.String]"));
@@ -132,7 +132,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("target_field", targetField);
         config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
 
-        DateProcessor processor = factory.create(null, null, null, config);
+        DateProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getTargetField(), equalTo(targetField));
     }
 
@@ -145,7 +145,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("target_field", targetField);
         config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
         config.put("output_format", outputFormat);
-        DateProcessor processor = factory.create(null, null, null, config);
+        DateProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getOutputFormat(), equalTo(outputFormat));
     }
 
@@ -156,7 +156,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("field", sourceField);
         config.put("target_field", targetField);
         config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
-        DateProcessor processor = factory.create(null, null, null, config);
+        DateProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getOutputFormat(), equalTo(DateProcessor.DEFAULT_OUTPUT_FORMAT));
     }
 
@@ -170,7 +170,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
         config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
         config.put("output_format", outputFormat);
 
-        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, null, null, config));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, null, null, config, null));
         assertThat(e.getMessage(), containsString("invalid output format [" + outputFormat + "]"));
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java

@@ -36,7 +36,7 @@ public class DissectProcessorFactoryTests extends ESTestCase {
         config.put("append_separator", appendSeparator);
         config.put("ignore_missing", true);
 
-        DissectProcessor processor = factory.create(null, processorTag, null, config);
+        DissectProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.field, equalTo(fieldName));
         assertThat(processor.pattern, equalTo(pattern));
@@ -49,7 +49,7 @@ public class DissectProcessorFactoryTests extends ESTestCase {
         DissectProcessor.Factory factory = new DissectProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         config.put("pattern", "%{a},%{b},%{c}");
-        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config));
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null));
         assertThat(e.getMessage(), equalTo("[field] required property is missing"));
     }
 
@@ -57,7 +57,7 @@ public class DissectProcessorFactoryTests extends ESTestCase {
         DissectProcessor.Factory factory = new DissectProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         config.put("field", randomAlphaOfLength(10));
-        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config));
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null));
         assertThat(e.getMessage(), equalTo("[pattern] required property is missing"));
     }
 
@@ -66,7 +66,7 @@ public class DissectProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("pattern", "%{a},%{b},%{c}");
         config.put("field", randomAlphaOfLength(10));
-        DissectProcessor processor = factory.create(null, "_tag", null, config);
+        DissectProcessor processor = factory.create(null, "_tag", null, config, null);
         assertThat(processor.appendSeparator, equalTo(""));
         assertThat(processor.ignoreMissing, is(false));
     }
@@ -76,6 +76,6 @@ public class DissectProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("pattern", "no keys defined");
         config.put("field", randomAlphaOfLength(10));
-        expectThrows(DissectException.class, () -> factory.create(null, "_tag", null, config));
+        expectThrows(DissectException.class, () -> factory.create(null, "_tag", null, config, null));
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorFactoryTests.java

@@ -26,13 +26,13 @@ public class DotExpanderProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field.field");
         config.put("path", "_path");
-        DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config);
+        DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config, null);
         assertThat(processor.getField(), equalTo("_field.field"));
         assertThat(processor.getPath(), equalTo("_path"));
 
         config = new HashMap<>();
         config.put("field", "_field.field");
-        processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config);
+        processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config, null);
         assertThat(processor.getField(), equalTo("_field.field"));
         assertThat(processor.getPath(), nullValue());
     }
@@ -45,7 +45,7 @@ public class DotExpanderProcessorFactoryTests extends ESTestCase {
             Map<String, Object> config = new HashMap<>();
             config.put("field", field);
             config.put("path", "_path");
-            DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config);
+            DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config, null);
             assertThat(processor.getField(), equalTo(field));
             assertThat(processor.getPath(), equalTo("_path"));
         }
@@ -56,7 +56,7 @@ public class DotExpanderProcessorFactoryTests extends ESTestCase {
 
         Map<String, Object> config = new HashMap<>();
         config.put("path", "_path");
-        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config));
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null));
         assertThat(e.getMessage(), equalTo("[field] required property is missing"));
     }
 
@@ -66,7 +66,7 @@ public class DotExpanderProcessorFactoryTests extends ESTestCase {
         for (String field : fields) {
             Map<String, Object> config = new HashMap<>();
             config.put("field", field);
-            Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config));
+            Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null));
             assertThat(e.getMessage(), equalTo("[field] field does not contain a dot and is not a wildcard"));
         }
 
@@ -74,7 +74,7 @@ public class DotExpanderProcessorFactoryTests extends ESTestCase {
         for (String field : fields) {
             Map<String, Object> config = new HashMap<>();
             config.put("field", field);
-            Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config));
+            Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null));
             assertThat(e.getMessage(), equalTo("[field] Field can't start or end with a dot"));
         }
 
@@ -82,7 +82,7 @@ public class DotExpanderProcessorFactoryTests extends ESTestCase {
         for (String field : fields) {
             Map<String, Object> config = new HashMap<>();
             config.put("field", field);
-            Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config));
+            Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null));
             assertThat(e.getMessage(), equalTo("[field] No space between dots"));
         }
     }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java

@@ -33,7 +33,7 @@ public class FailProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("message", "error");
         String processorTag = randomAlphaOfLength(10);
-        FailProcessor failProcessor = factory.create(null, processorTag, null, config);
+        FailProcessor failProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(failProcessor.getTag(), equalTo(processorTag));
         assertThat(failProcessor.getMessage().newInstance(Map.of()).execute(), equalTo("error"));
     }
@@ -41,7 +41,7 @@ public class FailProcessorFactoryTests extends ESTestCase {
     public void testCreateMissingMessageField() throws Exception {
         Map<String, Object> config = new HashMap<>();
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[message] required property is missing"));
@@ -55,7 +55,7 @@ public class FailProcessorFactoryTests extends ESTestCase {
         String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(
             ElasticsearchException.class,
-            () -> factory.create(null, processorTag, null, config)
+            () -> factory.create(null, processorTag, null, config, null)
         );
         assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
         assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java

@@ -48,7 +48,7 @@ public class FingerprintProcessorFactoryTests extends ESTestCase {
         config.put("ignore_missing", ignoreMissing);
 
         String processorTag = randomAlphaOfLength(10);
-        FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config);
+        FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(fingerprintProcessor.getTag(), equalTo(processorTag));
         assertThat(fingerprintProcessor.getFields(), equalTo(sortedFieldList));
         assertThat(fingerprintProcessor.getTargetField(), equalTo(targetField));
@@ -68,7 +68,7 @@ public class FingerprintProcessorFactoryTests extends ESTestCase {
         config.put("method", method);
 
         String processorTag = randomAlphaOfLength(10);
-        FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config);
+        FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(fingerprintProcessor.getTag(), equalTo(processorTag));
         assertThat(fingerprintProcessor.getFields(), equalTo(sortedFieldList));
         assertThat(fingerprintProcessor.getThreadLocalHasher().get().getAlgorithm(), equalTo(method));
@@ -80,7 +80,7 @@ public class FingerprintProcessorFactoryTests extends ESTestCase {
         );
         config.put("fields", fieldList);
         config.put("method", invalidMethod);
-        ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config));
+        ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null));
         assertThat(e.getMessage(), containsString("[" + invalidMethod + "] must be one of the supported hash methods ["));
     }
 
@@ -93,17 +93,17 @@ public class FingerprintProcessorFactoryTests extends ESTestCase {
         config.put("fields", fieldList);
 
         String processorTag = randomAlphaOfLength(10);
-        FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config);
+        FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(fingerprintProcessor.getTag(), equalTo(processorTag));
         assertThat(fingerprintProcessor.getFields(), equalTo(sortedFieldList));
 
         // fields is a list of length zero
         config.put("fields", List.of());
-        ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config));
+        ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null));
         assertThat(e.getMessage(), containsString("must specify at least one field"));
 
         // fields is missing
-        e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config));
+        e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null));
         assertThat(e.getMessage(), containsString("[fields] required property is missing"));
     }
 
@@ -115,7 +115,7 @@ public class FingerprintProcessorFactoryTests extends ESTestCase {
         HashMap<String, Object> config = new HashMap<>();
         config.put("fields", fieldList);
 
-        FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config);
+        FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(fingerprintProcessor.getTag(), equalTo(processorTag));
         assertThat(fingerprintProcessor.getFields(), equalTo(sortedFieldList));
         assertThat(fingerprintProcessor.getTargetField(), equalTo(FingerprintProcessor.Factory.DEFAULT_TARGET));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorTests.java

@@ -215,7 +215,7 @@ public class FingerprintProcessorTests extends ESTestCase {
         if (salt != null) {
             config.put("salt", salt);
         }
-        FingerprintProcessor fp = factory.create(null, randomAlphaOfLength(10), null, config);
+        FingerprintProcessor fp = factory.create(null, randomAlphaOfLength(10), null, config, null);
 
         byte[] expectedBytes = new byte[0];
         if (salt != null) {
@@ -257,7 +257,7 @@ public class FingerprintProcessorTests extends ESTestCase {
             config.put("fields", List.of("foo", "bar"));
             config.put("method", FingerprintProcessor.Factory.SUPPORTED_DIGESTS[k]);
 
-            FingerprintProcessor fp = factory.create(null, randomAlphaOfLength(10), null, config);
+            FingerprintProcessor fp = factory.create(null, randomAlphaOfLength(10), null, config, null);
             var input = TestIngestDocument.withDefaultVersion(inputMap);
             var output = fp.execute(input);
             assertTrue(output.hasField("fingerprint"));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java

@@ -32,13 +32,13 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
     public void testCreate() throws Exception {
         Processor processor = new TestProcessor(ingestDocument -> {});
         Map<String, Processor.Factory> registry = new HashMap<>();
-        registry.put("_name", (r, t, description, c) -> processor);
+        registry.put("_name", (r, t, description, c, projectId) -> processor);
         ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("processor", Map.of("_name", Collections.emptyMap()));
-        ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config);
+        ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config, null);
         assertThat(forEachProcessor, notNullValue());
         assertThat(forEachProcessor.getField(), equalTo("_field"));
         assertThat(forEachProcessor.getInnerProcessor(), sameInstance(processor));
@@ -48,14 +48,14 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
     public void testSetIgnoreMissing() throws Exception {
         Processor processor = new TestProcessor(ingestDocument -> {});
         Map<String, Processor.Factory> registry = new HashMap<>();
-        registry.put("_name", (r, t, description, c) -> processor);
+        registry.put("_name", (r, t, description, c, projectId) -> processor);
         ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("processor", Map.of("_name", Collections.emptyMap()));
         config.put("ignore_missing", true);
-        ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config);
+        ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config, null);
         assertThat(forEachProcessor, notNullValue());
         assertThat(forEachProcessor.getField(), equalTo("_field"));
         assertThat(forEachProcessor.getInnerProcessor(), sameInstance(processor));
@@ -65,8 +65,8 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
     public void testCreateWithTooManyProcessorTypes() throws Exception {
         Processor processor = new TestProcessor(ingestDocument -> {});
         Map<String, Processor.Factory> registry = new HashMap<>();
-        registry.put("_first", (r, t, description, c) -> processor);
+        registry.put("_first", (r, t, description, c, projectId) -> processor);
-        registry.put("_second", (r, t, description, c) -> processor);
+        registry.put("_second", (r, t, description, c, projectId) -> processor);
         ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
 
         Map<String, Object> config = new HashMap<>();
@@ -75,7 +75,10 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
         processorTypes.put("_first", Map.of());
         processorTypes.put("_second", Map.of());
         config.put("processor", processorTypes);
-        Exception exception = expectThrows(ElasticsearchParseException.class, () -> forEachFactory.create(registry, null, null, config));
+        Exception exception = expectThrows(
+            ElasticsearchParseException.class,
+            () -> forEachFactory.create(registry, null, null, config, null)
+        );
         assertThat(exception.getMessage(), equalTo("[processor] Must specify exactly one processor type"));
     }
 
@@ -86,7 +89,7 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
         config.put("processor", Map.of("_name", Collections.emptyMap()));
         Exception expectedException = expectThrows(
             ElasticsearchParseException.class,
-            () -> forEachFactory.create(Map.of(), null, null, config)
+            () -> forEachFactory.create(Map.of(), null, null, config, null)
         );
         assertThat(expectedException.getMessage(), equalTo("No processor type exists with name [_name]"));
     }
@@ -94,11 +97,11 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
     public void testCreateWithMissingField() throws Exception {
         Processor processor = new TestProcessor(ingestDocument -> {});
         Map<String, Processor.Factory> registry = new HashMap<>();
-        registry.put("_name", (r, t, description, c) -> processor);
+        registry.put("_name", (r, t, description, c, projectId) -> processor);
         ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
         Map<String, Object> config = new HashMap<>();
         config.put("processor", List.of(Map.of("_name", Map.of())));
-        Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(registry, null, null, config));
+        Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(registry, null, null, config, null));
         assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
     }
 
@@ -106,7 +109,7 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
         ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(Map.of(), null, null, config));
+        Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(Map.of(), null, null, config, null));
         assertThat(exception.getMessage(), equalTo("[processor] required property is missing"));
     }
 

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java

@@ -30,7 +30,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("patterns", List.of("(?<foo>\\w+)"));
         String processorTag = randomAlphaOfLength(10);
-        GrokProcessor processor = factory.create(null, processorTag, null, config);
+        GrokProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getMatchField(), equalTo("_field"));
         assertThat(processor.getGrok(), notNullValue());
@@ -45,7 +45,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         config.put("patterns", List.of("(?<foo>\\w+)"));
         config.put("ignore_missing", true);
         String processorTag = randomAlphaOfLength(10);
-        GrokProcessor processor = factory.create(null, processorTag, null, config);
+        GrokProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getMatchField(), equalTo("_field"));
         assertThat(processor.getGrok(), notNullValue());
@@ -56,7 +56,10 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
         Map<String, Object> config = new HashMap<>();
         config.put("patterns", List.of("(?<foo>\\w+)"));
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> factory.create(null, null, null, config, null)
+        );
         assertThat(e.getMessage(), equalTo("[field] required property is missing"));
     }
 
@@ -64,7 +67,10 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
         Map<String, Object> config = new HashMap<>();
         config.put("field", "foo");
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> factory.create(null, null, null, config, null)
+        );
         assertThat(e.getMessage(), equalTo("[patterns] required property is missing"));
     }
 
@@ -73,7 +79,10 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "foo");
         config.put("patterns", List.of());
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> factory.create(null, null, null, config, null)
+        );
         assertThat(e.getMessage(), equalTo("[patterns] List of patterns must not be empty"));
     }
 
@@ -84,7 +93,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("patterns", List.of("%{MY_PATTERN:name}!"));
         config.put("pattern_definitions", Map.of("MY_PATTERN", "foo"));
-        GrokProcessor processor = factory.create(null, null, null, config);
+        GrokProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getMatchField(), equalTo("_field"));
         assertThat(processor.getGrok(), notNullValue());
         assertThat(processor.getGrok().match("foo!"), equalTo(true));
@@ -95,7 +104,10 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("patterns", List.of("["));
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> factory.create(null, null, null, config, null)
+        );
         assertThat(e.getMessage(), equalTo("[patterns] Invalid regex pattern found in: [[]. premature end of char-class"));
     }
 
@@ -105,7 +117,10 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("patterns", List.of("%{MY_PATTERN:name}!"));
         config.put("pattern_definitions", Map.of("MY_PATTERN", "["));
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> factory.create(null, null, null, config, null)
+        );
         assertThat(
             e.getMessage(),
             equalTo("[patterns] Invalid regex pattern found in: [%{MY_PATTERN:name}!]. premature end of char-class")
@@ -119,7 +134,10 @@ public class GrokProcessorFactoryTests extends ESTestCase {
         config.put("patterns", List.of("(?<foo>\\w+)"));
         String invalidEcsMode = randomAlphaOfLength(3);
         config.put("ecs_compatibility", invalidEcsMode);
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> factory.create(null, null, null, config, null)
+        );
         assertThat(e.getMessage(), equalTo("[ecs_compatibility] unsupported mode '" + invalidEcsMode + "'"));
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java

@@ -44,7 +44,7 @@ public class GsubProcessorFactoryTests extends AbstractStringProcessorFactoryTes
         config.put("field", "field1");
         config.put("replacement", "-");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[pattern] required property is missing"));
@@ -57,7 +57,7 @@ public class GsubProcessorFactoryTests extends AbstractStringProcessorFactoryTes
         config.put("field", "field1");
         config.put("pattern", "\\.");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[replacement] required property is missing"));
@@ -71,7 +71,7 @@ public class GsubProcessorFactoryTests extends AbstractStringProcessorFactoryTes
         config.put("pattern", "[");
         config.put("replacement", "-");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), containsString("[pattern] Invalid regex pattern. Unclosed character class"));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java

@@ -25,7 +25,7 @@ public class JoinProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("separator", "-");
         String processorTag = randomAlphaOfLength(10);
-        JoinProcessor joinProcessor = factory.create(null, processorTag, null, config);
+        JoinProcessor joinProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(joinProcessor.getTag(), equalTo(processorTag));
         assertThat(joinProcessor.getField(), equalTo("field1"));
         assertThat(joinProcessor.getSeparator(), equalTo("-"));
@@ -37,7 +37,7 @@ public class JoinProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("separator", "-");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -49,7 +49,7 @@ public class JoinProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[separator] required property is missing"));
@@ -63,7 +63,7 @@ public class JoinProcessorFactoryTests extends ESTestCase {
         config.put("separator", "-");
         config.put("target_field", "target");
         String processorTag = randomAlphaOfLength(10);
-        JoinProcessor joinProcessor = factory.create(null, processorTag, null, config);
+        JoinProcessor joinProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(joinProcessor.getTag(), equalTo(processorTag));
         assertThat(joinProcessor.getField(), equalTo("field1"));
         assertThat(joinProcessor.getSeparator(), equalTo("-"));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java

@@ -31,7 +31,7 @@ public class JsonProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", randomField);
         config.put("target_field", randomTargetField);
-        JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config);
+        JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config, null);
         assertThat(jsonProcessor.getTag(), equalTo(processorTag));
         assertThat(jsonProcessor.getField(), equalTo(randomField));
         assertThat(jsonProcessor.getTargetField(), equalTo(randomTargetField));
@@ -43,7 +43,7 @@ public class JsonProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", randomField);
         config.put("add_to_root", true);
-        JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config);
+        JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config, null);
         assertThat(jsonProcessor.getTag(), equalTo(processorTag));
         assertThat(jsonProcessor.getField(), equalTo(randomField));
         assertThat(jsonProcessor.getTargetField(), equalTo(randomField));
@@ -55,7 +55,7 @@ public class JsonProcessorFactoryTests extends ESTestCase {
         String randomField = randomAlphaOfLength(10);
         Map<String, Object> config = new HashMap<>();
         config.put("field", randomField);
-        JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config);
+        JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config, null);
         assertThat(jsonProcessor.getTag(), equalTo(processorTag));
         assertThat(jsonProcessor.getField(), equalTo(randomField));
         assertThat(jsonProcessor.getTargetField(), equalTo(randomField));
@@ -66,7 +66,7 @@ public class JsonProcessorFactoryTests extends ESTestCase {
         String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(
             ElasticsearchParseException.class,
-            () -> FACTORY.create(null, processorTag, null, config)
+            () -> FACTORY.create(null, processorTag, null, config, null)
         );
         assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
     }
@@ -79,7 +79,7 @@ public class JsonProcessorFactoryTests extends ESTestCase {
         {
             Map<String, Object> strictConfig = new HashMap<>();
             strictConfig.put("field", fieldName);
-            JsonProcessor strictProcessor = FACTORY.create(null, processorTag, null, strictConfig);
+            JsonProcessor strictProcessor = FACTORY.create(null, processorTag, null, strictConfig, null);
             IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> strictProcessor.execute(document));
             assertThat(exception.getMessage(), containsString("is not valid JSON and the strict_json_parsing parameter is true"));
         }
@@ -88,7 +88,7 @@ public class JsonProcessorFactoryTests extends ESTestCase {
             Map<String, Object> lenientConfig = new HashMap<>();
             lenientConfig.put("field", fieldName);
             lenientConfig.put("strict_json_parsing", false);
-            JsonProcessor lenientProcessor = FACTORY.create(null, processorTag, null, lenientConfig);
+            JsonProcessor lenientProcessor = FACTORY.create(null, processorTag, null, lenientConfig, null);
             IngestDocument result = lenientProcessor.execute(document);
             assertThat(result.getSource().get(fieldName), equalTo(123));
         }
@@ -103,7 +103,7 @@ public class JsonProcessorFactoryTests extends ESTestCase {
         config.put("add_to_root", true);
         ElasticsearchException exception = expectThrows(
             ElasticsearchParseException.class,
-            () -> FACTORY.create(null, randomAlphaOfLength(10), null, config)
+            () -> FACTORY.create(null, randomAlphaOfLength(10), null, config, null)
         );
         assertThat(exception.getMessage(), equalTo("[target_field] Cannot set a target field while also setting `add_to_root` to true"));
     }
@@ -151,6 +151,6 @@ public class JsonProcessorFactoryTests extends ESTestCase {
         if (mergeStrategy != null) {
             config.put("add_to_root_conflict_strategy", mergeStrategy);
         }
-        return FACTORY.create(null, randomAlphaOfLength(10), null, config);
+        return FACTORY.create(null, randomAlphaOfLength(10), null, config, null);
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java

@@ -40,7 +40,7 @@ public class KeyValueProcessorFactoryTests extends ESTestCase {
         config.put("field_split", "&");
         config.put("value_split", "=");
         String processorTag = randomAlphaOfLength(10);
-        KeyValueProcessor processor = factory.create(null, processorTag, null, config);
+        KeyValueProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
         assertThat(processor.getFieldSplit(), equalTo("&"));
@@ -60,7 +60,7 @@ public class KeyValueProcessorFactoryTests extends ESTestCase {
         config.put("exclude_keys", List.of());
         config.put("ignore_missing", true);
         String processorTag = randomAlphaOfLength(10);
-        KeyValueProcessor processor = factory.create(null, processorTag, null, config);
+        KeyValueProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
         assertThat(processor.getFieldSplit(), equalTo("&"));
@@ -76,7 +76,7 @@ public class KeyValueProcessorFactoryTests extends ESTestCase {
         String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(
             ElasticsearchParseException.class,
-            () -> factory.create(null, processorTag, null, config)
+            () -> factory.create(null, processorTag, null, config, null)
         );
         assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
     }
@@ -87,7 +87,7 @@ public class KeyValueProcessorFactoryTests extends ESTestCase {
         String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(
             ElasticsearchParseException.class,
-            () -> factory.create(null, processorTag, null, config)
+            () -> factory.create(null, processorTag, null, config, null)
         );
         assertThat(exception.getMessage(), equalTo("[field_split] required property is missing"));
     }
@@ -99,7 +99,7 @@ public class KeyValueProcessorFactoryTests extends ESTestCase {
         String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(
             ElasticsearchParseException.class,
-            () -> factory.create(null, processorTag, null, config)
+            () -> factory.create(null, processorTag, null, config, null)
         );
         assertThat(exception.getMessage(), equalTo("[value_split] required property is missing"));
     }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java

@@ -251,6 +251,6 @@ public class KeyValueProcessorTests extends ESTestCase {
         if (prefix != null) {
             config.put("prefix", prefix);
         }
-        return FACTORY.create(null, randomAlphaOfLength(10), null, config);
+        return FACTORY.create(null, randomAlphaOfLength(10), null, config, null);
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java

@@ -50,7 +50,7 @@ public class NetworkDirectionProcessorFactoryTests extends ESTestCase {
         config.put("ignore_missing", ignoreMissing);
 
         String processorTag = randomAlphaOfLength(10);
-        NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config);
+        NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(networkProcessor.getTag(), equalTo(processorTag));
         assertThat(networkProcessor.getSourceIpField(), equalTo(sourceIpField));
         assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField));
@@ -75,7 +75,7 @@ public class NetworkDirectionProcessorFactoryTests extends ESTestCase {
         config.put("ignore_missing", ignoreMissing);
 
         String processorTag = randomAlphaOfLength(10);
-        NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config);
+        NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(networkProcessor.getTag(), equalTo(processorTag));
         assertThat(networkProcessor.getSourceIpField(), equalTo(sourceIpField));
         assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField));
@@ -88,7 +88,7 @@ public class NetworkDirectionProcessorFactoryTests extends ESTestCase {
         HashMap<String, Object> config = new HashMap<>();
         String processorTag = randomAlphaOfLength(10);
         try {
-            factory.create(null, processorTag, null, config);
+            factory.create(null, processorTag, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[internal_networks] or [internal_networks_field] must be specified"));
@@ -102,7 +102,7 @@ public class NetworkDirectionProcessorFactoryTests extends ESTestCase {
         internalNetworks.add("10.0.0.0/8");
         config.put("internal_networks", internalNetworks);
 
-        NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config);
+        NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(networkProcessor.getTag(), equalTo(processorTag));
         assertThat(networkProcessor.getSourceIpField(), equalTo(DEFAULT_SOURCE_IP));
         assertThat(networkProcessor.getDestinationIpField(), equalTo(DEFAULT_DEST_IP));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java

@@ -139,7 +139,8 @@ public class NetworkDirectionProcessorTests extends ESTestCase {
             null,
             processorTag,
             null,
-            config
+            config,
+            null
         );
         IngestDocument input = TestIngestDocument.withDefaultVersion(source);
         IngestDocument output = processor.execute(input);
@@ -158,7 +159,7 @@ public class NetworkDirectionProcessorTests extends ESTestCase {
         config.put("internal_networks", networks);
         ElasticsearchParseException e = expectThrows(
             ElasticsearchParseException.class,
-            () -> new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config)
+            () -> new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config, null)
         );
         assertThat(
             e.getMessage(),
@@ -184,7 +185,8 @@ public class NetworkDirectionProcessorTests extends ESTestCase {
             null,
             processorTag,
             null,
-            config
+            config,
+            null
         );
 
         IngestDocument input = TestIngestDocument.withDefaultVersion(source);

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java

@@ -38,7 +38,7 @@ public class RegisteredDomainProcessorFactoryTests extends ESTestCase {
         config.put("ignore_missing", ignoreMissing);
 
         String processorTag = randomAlphaOfLength(10);
-        RegisteredDomainProcessor publicSuffixProcessor = factory.create(null, processorTag, null, config);
+        RegisteredDomainProcessor publicSuffixProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(publicSuffixProcessor.getTag(), equalTo(processorTag));
         assertThat(publicSuffixProcessor.getTargetField(), equalTo(targetField));
         assertThat(publicSuffixProcessor.getIgnoreMissing(), equalTo(ignoreMissing));
@@ -51,7 +51,7 @@ public class RegisteredDomainProcessorFactoryTests extends ESTestCase {
         config.put("field", field);
 
         String processorTag = randomAlphaOfLength(10);
-        RegisteredDomainProcessor publicSuffixProcessor = factory.create(null, processorTag, null, config);
+        RegisteredDomainProcessor publicSuffixProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(publicSuffixProcessor.getTargetField(), equalTo(RegisteredDomainProcessor.Factory.DEFAULT_TARGET_FIELD));
     }
 
@@ -59,7 +59,7 @@ public class RegisteredDomainProcessorFactoryTests extends ESTestCase {
         HashMap<String, Object> config = new HashMap<>();
         String processorTag = randomAlphaOfLength(10);
         try {
-            factory.create(null, processorTag, null, config);
+            factory.create(null, processorTag, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java

@@ -34,7 +34,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         String processorTag = randomAlphaOfLength(10);
-        RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config);
+        RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(removeProcessor.getTag(), equalTo(processorTag));
         assertThat(removeProcessor.getFieldsToRemove().get(0).newInstance(Map.of()).execute(), equalTo("field1"));
     }
@@ -43,7 +43,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("keep", List.of("field1", "field2"));
         String processorTag = randomAlphaOfLength(10);
-        RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config);
+        RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(removeProcessor.getTag(), equalTo(processorTag));
         assertThat(removeProcessor.getFieldsToKeep().get(0).newInstance(Map.of()).execute(), equalTo("field1"));
         assertThat(removeProcessor.getFieldsToKeep().get(1).newInstance(Map.of()).execute(), equalTo("field2"));
@@ -53,7 +53,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", List.of("field1", "field2"));
         String processorTag = randomAlphaOfLength(10);
-        RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config);
+        RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(removeProcessor.getTag(), equalTo(processorTag));
         assertThat(
             removeProcessor.getFieldsToRemove().stream().map(template -> template.newInstance(Map.of()).execute()).toList(),
@@ -64,7 +64,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
     public void testCreateMissingField() throws Exception {
         Map<String, Object> config = new HashMap<>();
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[keep] or [field] must be specified"));
@@ -76,7 +76,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("keep", "field2");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[keep] and [field] cannot both be used in the same processor"));
@@ -90,7 +90,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
         String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(
             ElasticsearchException.class,
-            () -> factory.create(null, processorTag, null, config)
+            () -> factory.create(null, processorTag, null, config, null)
         );
         assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
         assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java

@@ -46,7 +46,7 @@ public class RemoveProcessorTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", fieldName);
         String processorTag = randomAlphaOfLength(10);
-        Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config);
+        Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config, null);
         try {
             processor.execute(ingestDocument);
             fail("remove field should have failed");
@@ -62,7 +62,7 @@ public class RemoveProcessorTests extends ESTestCase {
         config.put("field", fieldName);
         config.put("ignore_missing", true);
         String processorTag = randomAlphaOfLength(10);
-        Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config);
+        Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config, null);
         processor.execute(ingestDocument);
     }
 

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java

@@ -33,7 +33,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
         config.put("field", "old_field");
         config.put("target_field", "new_field");
         String processorTag = randomAlphaOfLength(10);
-        RenameProcessor renameProcessor = factory.create(null, processorTag, null, config);
+        RenameProcessor renameProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(renameProcessor.getTag(), equalTo(processorTag));
         assertThat(renameProcessor.getField().newInstance(Map.of()).execute(), equalTo("old_field"));
         assertThat(renameProcessor.getTargetField().newInstance(Map.of()).execute(), equalTo("new_field"));
@@ -47,7 +47,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
         config.put("target_field", "new_field");
         config.put("ignore_missing", true);
         String processorTag = randomAlphaOfLength(10);
-        RenameProcessor renameProcessor = factory.create(null, processorTag, null, config);
+        RenameProcessor renameProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(renameProcessor.getTag(), equalTo(processorTag));
         assertThat(renameProcessor.getField().newInstance(Map.of()).execute(), equalTo("old_field"));
         assertThat(renameProcessor.getTargetField().newInstance(Map.of()).execute(), equalTo("new_field"));
@@ -60,7 +60,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
         config.put("target_field", "new_field");
         config.put("override", true);
         String processorTag = randomAlphaOfLength(10);
-        RenameProcessor renameProcessor = factory.create(null, processorTag, null, config);
+        RenameProcessor renameProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(renameProcessor.getTag(), equalTo(processorTag));
         assertThat(renameProcessor.getField().newInstance(Map.of()).execute(), equalTo("old_field"));
         assertThat(renameProcessor.getTargetField().newInstance(Map.of()).execute(), equalTo("new_field"));
@@ -71,7 +71,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("target_field", "new_field");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -82,7 +82,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "old_field");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[target_field] required property is missing"));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorFactoryTests.java

@@ -74,6 +74,6 @@ public class RerouteProcessorFactoryTests extends ESTestCase {
     }
 
     private static RerouteProcessor create(Map<String, Object> config) throws Exception {
-        return new RerouteProcessor.Factory().create(null, null, null, new HashMap<>(config));
+        return new RerouteProcessor.Factory().create(null, null, null, new HashMap<>(config), null);
     }
 }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java

@@ -53,7 +53,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
         Map<String, Object> configMap = new HashMap<>();
         String randomType = randomFrom("id", "source");
         configMap.put(randomType, "foo");
-        ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap);
+        ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap, null);
         assertThat(processor.getScript().getLang(), equalTo(randomType.equals("id") ? null : Script.DEFAULT_SCRIPT_LANG));
         assertThat(processor.getScript().getType().toString(), equalTo(INGEST_SCRIPT_PARAM_TO_TYPE.get(randomType)));
         assertThat(processor.getScript().getParams(), equalTo(Map.of()));
@@ -69,7 +69,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
         Map<String, Object> randomParams = Map.of(randomAlphaOfLength(10), randomAlphaOfLength(10));
         configMap.put(randomType, "foo");
         configMap.put("params", randomParams);
-        ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap);
+        ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap, null);
         assertThat(processor.getScript().getLang(), equalTo(randomType.equals("id") ? null : Script.DEFAULT_SCRIPT_LANG));
         assertThat(processor.getScript().getType().toString(), equalTo(INGEST_SCRIPT_PARAM_TO_TYPE.get(randomType)));
         assertThat(processor.getScript().getParams(), equalTo(randomParams));
@@ -83,7 +83,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
 
         XContentParseException exception = expectThrows(
             XContentParseException.class,
-            () -> factory.create(null, randomAlphaOfLength(10), null, configMap)
+            () -> factory.create(null, randomAlphaOfLength(10), null, configMap, null)
         );
         assertThat(exception.getMessage(), containsString("[script] failed to parse field [source]"));
     }
@@ -94,7 +94,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
 
         IllegalArgumentException exception = expectThrows(
             IllegalArgumentException.class,
-            () -> factory.create(null, randomAlphaOfLength(10), null, configMap)
+            () -> factory.create(null, randomAlphaOfLength(10), null, configMap, null)
         );
 
         assertThat(exception.getMessage(), is("must specify either [source] for an inline script or [id] for a stored script"));
@@ -108,7 +108,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
         Map<String, Object> configMap = new HashMap<>();
         configMap.put("inline", "code");
 
-        factory.create(null, randomAlphaOfLength(10), null, configMap);
+        factory.create(null, randomAlphaOfLength(10), null, configMap, null);
         assertWarnings("Deprecated field [inline] used, expected [source] instead");
     }
 
@@ -130,7 +130,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
 
         ElasticsearchException exception = expectThrows(
             ElasticsearchException.class,
-            () -> factory.create(null, randomAlphaOfLength(10), null, configMap)
+            () -> factory.create(null, randomAlphaOfLength(10), null, configMap, null)
         );
 
         assertThat(exception.getMessage(), is("compile-time exception"));
@@ -151,7 +151,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
 
         Map<String, Object> configMap = new HashMap<>();
         configMap.put("source", scriptName);
-        ScriptProcessor processor = factory.create(null, null, randomAlphaOfLength(10), configMap);
+        ScriptProcessor processor = factory.create(null, null, randomAlphaOfLength(10), configMap, null);
         assertThat(processor.getScript().getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG));
         assertThat(processor.getScript().getType(), equalTo(ScriptType.INLINE));
         assertThat(processor.getScript().getParams(), equalTo(Map.of()));
@@ -167,7 +167,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
         factory = new ScriptProcessor.Factory(mockedScriptService);
         Map<String, Object> configMap = new HashMap<>();
         configMap.put("id", "script_name");
-        ScriptProcessor processor = factory.create(null, null, randomAlphaOfLength(10), configMap);
+        ScriptProcessor processor = factory.create(null, null, randomAlphaOfLength(10), configMap, null);
         assertNull(processor.getScript().getLang());
         assertThat(processor.getScript().getType(), equalTo(ScriptType.STORED));
         assertThat(processor.getScript().getParams(), equalTo(Map.of()));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java

@@ -37,7 +37,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("value", "value1");
         String processorTag = randomAlphaOfLength(10);
-        SetProcessor setProcessor = factory.create(null, processorTag, null, config);
+        SetProcessor setProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
         assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
         assertThat(setProcessor.getValue().copyAndResolve(Map.of()), equalTo("value1"));
@@ -52,7 +52,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         config.put("value", "value1");
         config.put("override", overrideEnabled);
         String processorTag = randomAlphaOfLength(10);
-        SetProcessor setProcessor = factory.create(null, processorTag, null, config);
+        SetProcessor setProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
         assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
         assertThat(setProcessor.getValue().copyAndResolve(Map.of()), equalTo("value1"));
@@ -66,7 +66,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         config.put("value", "value1");
         config.put("ignore_empty_value", ignoreEmptyValueEnabled);
         String processorTag = randomAlphaOfLength(10);
-        SetProcessor setProcessor = factory.create(null, processorTag, null, config);
+        SetProcessor setProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
         assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
         assertThat(setProcessor.getValue().copyAndResolve(Map.of()), equalTo("value1"));
@@ -77,7 +77,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("value", "value1");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -88,7 +88,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[value] required property is missing"));
@@ -100,7 +100,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("value", null);
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[value] required property is missing"));
@@ -115,7 +115,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(
             ElasticsearchException.class,
-            () -> factory.create(null, processorTag, null, config)
+            () -> factory.create(null, processorTag, null, config, null)
         );
         assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
         assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));
@@ -126,7 +126,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("copy_from", "field2");
         String processorTag = randomAlphaOfLength(10);
-        SetProcessor setProcessor = factory.create(null, processorTag, null, config);
+        SetProcessor setProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
         assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
         assertThat(setProcessor.getCopyFrom(), equalTo("field2"));
@@ -140,7 +140,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         String processorTag = randomAlphaOfLength(10);
         ElasticsearchException exception = expectThrows(
             ElasticsearchException.class,
-            () -> factory.create(null, processorTag, null, config)
+            () -> factory.create(null, processorTag, null, config, null)
         );
         assertThat(exception.getMessage(), equalTo("[copy_from] cannot set both `copy_from` and `value` in the same processor"));
     }
@@ -153,7 +153,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         config.put("value", "value1");
         config.put("media_type", expectedMediaType);
         String processorTag = randomAlphaOfLength(10);
-        SetProcessor setProcessor = factory.create(null, processorTag, null, config);
+        SetProcessor setProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
 
         // invalid media type
@@ -165,7 +165,10 @@ public class SetProcessorFactoryTests extends ESTestCase {
         config2.put("field", "field1");
         config2.put("value", "value1");
         config2.put("media_type", expectedMediaType);
-        ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config2));
+        ElasticsearchException e = expectThrows(
+            ElasticsearchException.class,
+            () -> factory.create(null, processorTag, null, config2, null)
+        );
         assertThat(e.getMessage(), containsString("property does not contain a supported media type [" + expectedMediaType + "]"));
     }
 
@@ -176,7 +179,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
         config.put("field", "");
         config.put("value", "value1");
         String processorTag = randomAlphaOfLength(10);
-        SetProcessor setProcessor = factory.create(null, processorTag, null, config);
+        SetProcessor setProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(setProcessor.getTag(), equalTo(processorTag));
         assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo(""));
         assertThat(setProcessor.getValue().copyAndResolve(Map.of()), equalTo("value1"));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java

@@ -28,7 +28,7 @@ public class SortProcessorFactoryTests extends ESTestCase {
         config.put("field", fieldName);
 
         SortProcessor.Factory factory = new SortProcessor.Factory();
-        SortProcessor processor = factory.create(null, processorTag, null, config);
+        SortProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo(fieldName));
         assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.ASCENDING));
@@ -44,7 +44,7 @@ public class SortProcessorFactoryTests extends ESTestCase {
         config.put("order", "desc");
 
         SortProcessor.Factory factory = new SortProcessor.Factory();
-        SortProcessor processor = factory.create(null, processorTag, null, config);
+        SortProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo(fieldName));
         assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.DESCENDING));
@@ -61,7 +61,7 @@ public class SortProcessorFactoryTests extends ESTestCase {
         config.put("target_field", targetFieldName);
 
         SortProcessor.Factory factory = new SortProcessor.Factory();
-        SortProcessor processor = factory.create(null, processorTag, null, config);
+        SortProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo(fieldName));
         assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.ASCENDING));
@@ -78,7 +78,7 @@ public class SortProcessorFactoryTests extends ESTestCase {
 
         SortProcessor.Factory factory = new SortProcessor.Factory();
         try {
-            factory.create(null, processorTag, null, config);
+            factory.create(null, processorTag, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[order] Sort direction [invalid] not recognized. Valid values are: [asc, desc]"));
@@ -89,7 +89,7 @@ public class SortProcessorFactoryTests extends ESTestCase {
         SortProcessor.Factory factory = new SortProcessor.Factory();
         Map<String, Object> config = new HashMap<>();
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java

@@ -25,7 +25,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
         config.put("field", "field1");
         config.put("separator", "\\.");
         String processorTag = randomAlphaOfLength(10);
-        SplitProcessor splitProcessor = factory.create(null, processorTag, null, config);
+        SplitProcessor splitProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(splitProcessor.getTag(), equalTo(processorTag));
         assertThat(splitProcessor.getField(), equalTo("field1"));
         assertThat(splitProcessor.getSeparator(), equalTo("\\."));
@@ -38,7 +38,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("separator", "\\.");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -50,7 +50,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "field1");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[separator] required property is missing"));
@@ -64,7 +64,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
         config.put("separator", "\\.");
         config.put("target_field", "target");
         String processorTag = randomAlphaOfLength(10);
-        SplitProcessor splitProcessor = factory.create(null, processorTag, null, config);
+        SplitProcessor splitProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(splitProcessor.getTag(), equalTo(processorTag));
         assertThat(splitProcessor.getField(), equalTo("field1"));
         assertThat(splitProcessor.getSeparator(), equalTo("\\."));
@@ -81,7 +81,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
         config.put("target_field", "target");
         config.put("preserve_trailing", true);
         String processorTag = randomAlphaOfLength(10);
-        SplitProcessor splitProcessor = factory.create(null, processorTag, null, config);
+        SplitProcessor splitProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(splitProcessor.getTag(), equalTo(processorTag));
         assertThat(splitProcessor.getField(), equalTo("field1"));
         assertThat(splitProcessor.getSeparator(), equalTo("\\."));

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java

@@ -97,7 +97,7 @@ public class SplitProcessorTests extends ESTestCase {
         Map<String, Object> splitConfig = new HashMap<>();
         splitConfig.put("field", "flags");
         splitConfig.put("separator", "\\|");
-        Processor splitProcessor = (new SplitProcessor.Factory()).create(null, null, null, splitConfig);
+        Processor splitProcessor = (new SplitProcessor.Factory()).create(null, null, null, splitConfig, null);
         Map<String, Object> source = new HashMap<>();
         source.put("flags", "new|hot|super|fun|interesting");
         IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(source);

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java

@@ -45,7 +45,7 @@ public class UriPartsProcessorFactoryTests extends ESTestCase {
         config.put("ignore_missing", ignoreMissing);
 
         String processorTag = randomAlphaOfLength(10);
-        UriPartsProcessor uriPartsProcessor = factory.create(null, processorTag, null, config);
+        UriPartsProcessor uriPartsProcessor = factory.create(null, processorTag, null, config, null);
         assertThat(uriPartsProcessor.getTag(), equalTo(processorTag));
         assertThat(uriPartsProcessor.getField(), equalTo(field));
         assertThat(uriPartsProcessor.getTargetField(), equalTo(targetField));
@@ -58,7 +58,7 @@ public class UriPartsProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("value", "value1");
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@@ -69,7 +69,7 @@ public class UriPartsProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", null);
         try {
-            factory.create(null, null, null, config);
+            factory.create(null, null, null, config, null);
             fail("factory create should have failed");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[field] required property is missing"));

modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java

@@ -811,7 +811,7 @@ public class GeoIpDownloaderIT extends AbstractGeoIpIT {
         @Override
         public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
             Map<String, Processor.Factory> procMap = new HashMap<>();
-            procMap.put(NON_GEO_PROCESSOR_TYPE, (factories, tag, description, config) -> {
+            procMap.put(NON_GEO_PROCESSOR_TYPE, (factories, tag, description, config, projectId) -> {
                 readStringProperty(NON_GEO_PROCESSOR_TYPE, tag, config, "randomField");
                 return new AbstractProcessor(tag, description) {
                     @Override

modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java

@@ -74,12 +74,19 @@ public class ReloadingDatabasesWhilePerformingGeoLookupsIT extends ESTestCase {
         databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
         lazyLoadReaders(databaseNodeService);
 
-        final GeoIpProcessor processor1 = (GeoIpProcessor) factory.create(null, "_tag", null, new HashMap<>(Map.of("field", "_field")));
+        final GeoIpProcessor processor1 = (GeoIpProcessor) factory.create(
+            null,
+            "_tag",
+            null,
+            new HashMap<>(Map.of("field", "_field")),
+            null
+        );
         final GeoIpProcessor processor2 = (GeoIpProcessor) factory.create(
             null,
             "_tag",
             null,
-            new HashMap<>(Map.of("field", "_field", "database_file", "GeoLite2-City-Test.mmdb"))
+            new HashMap<>(Map.of("field", "_field", "database_file", "GeoLite2-City-Test.mmdb")),
+            null
         );
 
         final AtomicBoolean completed = new AtomicBoolean(false);

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.geoip;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
@@ -227,7 +228,8 @@ public final class GeoIpProcessor extends AbstractProcessor {
             final Map<String, Processor.Factory> registry,
             final String processorTag,
             final String description,
-            final Map<String, Object> config
+            final Map<String, Object> config,
+            final ProjectId projectId
         ) throws IOException {
             String ipField = readStringProperty(type, processorTag, config, "field");
             String targetField = readStringProperty(type, processorTag, config, "target_field", type);

modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java

@@ -95,7 +95,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         String processorTag = randomAlphaOfLength(10);
 
-        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config);
+        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
@@ -112,7 +112,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config.put("ignore_missing", true);
         String processorTag = randomAlphaOfLength(10);
 
-        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config);
+        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
@@ -129,7 +129,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config.put("database_file", "GeoLite2-Country.mmdb");
         String processorTag = randomAlphaOfLength(10);
 
-        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config);
+        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null);
 
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
@@ -147,7 +147,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config.put("database_file", "GeoLite2-ASN.mmdb");
         String processorTag = randomAlphaOfLength(10);
 
-        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config);
+        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null);
 
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
@@ -162,7 +162,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("target_field", "_field");
-        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config);
+        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("_field"));
         assertFalse(processor.isIgnoreMissing());
@@ -173,7 +173,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", "GeoLite2-Country.mmdb");
-        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config);
+        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("geoip"));
         assertThat(processor.getDatabaseType(), equalTo("GeoLite2-Country"));
@@ -190,7 +190,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         asnOnlyProperties.remove(Property.IP);
         String asnProperty = RandomPicks.randomFrom(Randomness.get(), asnOnlyProperties).toString();
         config.put("properties", List.of(asnProperty));
-        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config, null));
         assertThat(
             e.getMessage(),
             equalTo(
@@ -211,7 +211,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         cityOnlyProperties.remove(Property.IP);
         String cityProperty = RandomPicks.randomFrom(Randomness.get(), cityOnlyProperties).toString();
         config.put("properties", List.of(cityProperty));
-        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config, null));
         assertThat(
             e.getMessage(),
             equalTo("[properties] illegal property value [" + cityProperty + "]. valid values are [IP, ASN, ORGANIZATION_NAME, NETWORK]")
@@ -226,7 +226,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", "does-not-exist.mmdb");
-        Processor processor = factory.create(null, null, null, config);
+        Processor processor = factory.create(null, null, null, config, null);
         assertThat(processor, instanceOf(GeoIpProcessor.DatabaseUnavailableProcessor.class));
     }
 
@@ -237,7 +237,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", randomFrom(DEFAULT_DATABASES));
-        Processor processor = factory.create(null, null, null, config);
+        Processor processor = factory.create(null, null, null, config, null);
         assertThat(processor, instanceOf(GeoIpProcessor.DatabaseUnavailableProcessor.class));
     }
 
@@ -259,7 +259,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("properties", fieldNames);
-        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config);
+        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
         assertFalse(processor.isIgnoreMissing());
@@ -271,7 +271,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config1 = new HashMap<>();
         config1.put("field", "_field");
         config1.put("properties", List.of("invalid"));
-        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1));
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1, null));
         assertThat(
             e.getMessage(),
             equalTo(
@@ -285,7 +285,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config2 = new HashMap<>();
         config2.put("field", "_field");
         config2.put("properties", "invalid");
-        e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config2));
+        e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config2, null));
         assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
     }
 
@@ -301,7 +301,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config1 = new HashMap<>();
         config1.put("field", "_field");
         config1.put("properties", List.of("ip"));
-        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1));
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1, null));
         assertThat(
             e.getMessage(),
             equalTo("[database_file] Unsupported database type [some-unsupported-database] for file [GeoLite2-City.mmdb]")
@@ -320,7 +320,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config1 = new HashMap<>();
         config1.put("field", "_field");
         config1.put("properties", List.of("ip"));
-        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1));
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1, null));
         assertThat(e.getMessage(), equalTo("[database_file] Unsupported database type [null] for file [GeoLite2-City.mmdb]"));
     }
 
@@ -336,7 +336,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config1.put("database_file", "some-ipinfo-database.mmdb");
         config1.put("field", "_field");
         config1.put("properties", List.of("ip"));
-        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1));
+        Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1, null));
         assertThat(
             e.getMessage(),
             equalTo(
@@ -358,7 +358,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config1.put("database_file", "some-custom-database.mmdb");
         config1.put("field", "_field");
         config1.put("properties", List.of("ip"));
-        factory.create(null, null, null, config1);
+        factory.create(null, null, null, config1, null);
         assertWarnings(GeoIpProcessor.UNSUPPORTED_DATABASE_DEPRECATION_MESSAGE.replaceAll("\\{}", "some_custom_database.mmdb-City"));
     }
 
@@ -393,7 +393,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", "GeoLite2-City.mmdb");
-        final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config);
+        final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config, null);
 
         // these are lazy loaded until first use, so we expect null here
         assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-City.mmdb").databaseReader.get());
@@ -404,7 +404,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", "GeoLite2-Country.mmdb");
-        final GeoIpProcessor country = (GeoIpProcessor) factory.create(null, "_tag", null, config);
+        final GeoIpProcessor country = (GeoIpProcessor) factory.create(null, "_tag", null, config, null);
 
         // these are lazy loaded until first use, so we expect null here
         assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-Country.mmdb").databaseReader.get());
@@ -415,7 +415,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", "GeoLite2-ASN.mmdb");
-        final GeoIpProcessor asn = (GeoIpProcessor) factory.create(null, "_tag", null, config);
+        final GeoIpProcessor asn = (GeoIpProcessor) factory.create(null, "_tag", null, config, null);
 
         // these are lazy loaded until first use, so we expect null here
         assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-ASN.mmdb").databaseReader.get());
@@ -462,7 +462,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
         config.put("database_file", "GeoIP2-City.mmdb");
-        final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config);
+        final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config, null);
 
         // these are lazy loaded until first use, so we expect null here
         assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoIP2-City.mmdb").databaseReader.get());
@@ -478,7 +478,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("field", randomIdentifier());
         config.put("download_database_on_pipeline_creation", randomBoolean());
-        factory.create(null, null, null, config);
+        factory.create(null, null, null, config, null);
         // Check all the config params were consumed.
         assertThat(config, anEmptyMap());
     }
@@ -498,7 +498,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         String processorTag = randomAlphaOfLength(10);
 
-        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config);
+        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null);
 
         processor.execute(RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>(Map.of("_field", "89.160.20.128"))));
     }
@@ -507,7 +507,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
         GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService);
         Map<String, Object> config = new HashMap<>();
         config.put("field", "source_field");
-        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config);
+        GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null);
         Map<String, Object> document = Map.of("source_field", "89.160.20.128");
         {
             IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>(document));
@@ -558,7 +558,8 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
                 null,
                 null,
                 null,
-                config
+                config,
+                null
             );
             processor.execute(ingestDocument);
             assertThat(ingestDocument.getSourceAndMetadata().get("geoip"), nullValue());
@@ -580,7 +581,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
             document.put("source_field", "89.160.20.128");
             IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
 
-            GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config);
+            GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null);
             processor.execute(ingestDocument);
             assertThat(ingestDocument.getSourceAndMetadata().get("tags"), nullValue());
             Map<?, ?> geoData = (Map<?, ?>) ingestDocument.getSourceAndMetadata().get("geoip");

modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest.useragent;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.core.UpdateForV10;
@@ -191,7 +192,8 @@ public class UserAgentProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> factories,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) {
             String field = readStringProperty(TYPE, processorTag, config, "field");
             String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "user_agent");

modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java

@@ -74,7 +74,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
 
         String processorTag = randomAlphaOfLength(10);
 
-        UserAgentProcessor processor = factory.create(null, processorTag, null, config);
+        UserAgentProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("user_agent"));
@@ -95,7 +95,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
 
         String processorTag = randomAlphaOfLength(10);
 
-        UserAgentProcessor processor = factory.create(null, processorTag, null, config);
+        UserAgentProcessor processor = factory.create(null, processorTag, null, config, null);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("user_agent"));
@@ -113,7 +113,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("target_field", "_target_field");
 
-        UserAgentProcessor processor = factory.create(null, null, null, config);
+        UserAgentProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getTargetField(), equalTo("_target_field"));
     }
@@ -125,7 +125,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("regex_file", regexWithoutDevicesFilename);
 
-        UserAgentProcessor processor = factory.create(null, null, null, config);
+        UserAgentProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0));
         assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
@@ -140,7 +140,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("extract_device_type", extractDeviceType);
 
-        UserAgentProcessor processor = factory.create(null, null, null, config);
+        UserAgentProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.isExtractDeviceType(), equalTo(extractDeviceType));
     }
@@ -152,7 +152,10 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("regex_file", "does-not-exist.yml");
 
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> factory.create(null, null, null, config, null)
+        );
         assertThat(e.getMessage(), equalTo("[regex_file] regex file [does-not-exist.yml] doesn't exist (has to exist at node startup)"));
     }
 
@@ -172,7 +175,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", fieldNames);
 
-        UserAgentProcessor processor = factory.create(null, null, null, config);
+        UserAgentProcessor processor = factory.create(null, null, null, config, null);
         assertThat(processor.getField(), equalTo("_field"));
         assertThat(processor.getProperties(), equalTo(properties));
     }
@@ -184,7 +187,10 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", Collections.singletonList("invalid"));
 
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> factory.create(null, null, null, config, null)
+        );
         assertThat(
             e.getMessage(),
             equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " + "ORIGINAL, VERSION]")
@@ -198,7 +204,10 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
         config.put("field", "_field");
         config.put("properties", "invalid");
 
-        ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
+        ElasticsearchParseException e = expectThrows(
+            ElasticsearchParseException.class,
+            () -> factory.create(null, null, null, config, null)
+        );
         assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
     }
 }

server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java

@@ -358,7 +358,7 @@ public class FinalPipelineIT extends ESIntegTestCase {
                 "final",
                 getFinal(parameters, randomBoolean()),
                 "request",
-                (processorFactories, tag, description, config) -> new AbstractProcessor(tag, description) {
+                (processorFactories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) {
                     @Override
                     public IngestDocument execute(final IngestDocument ingestDocument) throws Exception {
                         ingestDocument.setFieldValue("request", true);
@@ -371,7 +371,7 @@ public class FinalPipelineIT extends ESIntegTestCase {
                     }
                 },
                 "changing_dest",
-                (processorFactories, tag, description, config) -> new AbstractProcessor(tag, description) {
+                (processorFactories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) {
                     @Override
                     public IngestDocument execute(final IngestDocument ingestDocument) throws Exception {
                         ingestDocument.setFieldValue(IngestDocument.Metadata.INDEX.getFieldName(), "target");
@@ -385,7 +385,7 @@ public class FinalPipelineIT extends ESIntegTestCase {
 
                 },
                 "reroute",
-                (processorFactories, tag, description, config) -> {
+                (processorFactories, tag, description, config, projectId) -> {
                     final String dest = Objects.requireNonNullElse(
                         ConfigurationUtils.readOptionalStringProperty(description, tag, config, "dest"),
                         "target"
@@ -408,7 +408,7 @@ public class FinalPipelineIT extends ESIntegTestCase {
         }
 
         private static Processor.Factory getDefault(Processor.Parameters parameters, boolean async) {
-            return (factories, tag, description, config) -> new AbstractProcessor(tag, description) {
+            return (factories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) {
 
                 @Override
                 public void execute(IngestDocument ingestDocument, BiConsumer<IngestDocument, Exception> handler) {
@@ -446,7 +446,7 @@ public class FinalPipelineIT extends ESIntegTestCase {
         }
 
         private static Processor.Factory getFinal(Processor.Parameters parameters, boolean async) {
-            return (processorFactories, tag, description, config) -> {
+            return (processorFactories, tag, description, config, projectId) -> {
                 final String exists = (String) config.remove("exists");
                 return new AbstractProcessor(tag, description) {
 

server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java

@@ -96,7 +96,7 @@ public class IngestAsyncProcessorIT extends ESSingleNodeTestCase {
 
         @Override
         public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
-            return Map.of("test-async", (factories, tag, description, config) -> new AbstractProcessor(tag, description) {
+            return Map.of("test-async", (factories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) {
 
                 @Override
                 public void execute(IngestDocument ingestDocument, BiConsumer<IngestDocument, Exception> handler) {
@@ -129,7 +129,7 @@ public class IngestAsyncProcessorIT extends ESSingleNodeTestCase {
                     return true;
                 }
 
-            }, "test", (processorFactories, tag, description, config) -> new AbstractProcessor(tag, description) {
+            }, "test", (processorFactories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) {
                 @Override
                 public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
                     String id = (String) ingestDocument.getSourceAndMetadata().get("_id");

server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java

@@ -386,11 +386,16 @@ public class IngestClientIT extends ESIntegTestCase {
             factories.put(PipelineProcessor.TYPE, new PipelineProcessor.Factory(parameters.ingestService));
             factories.put(
                 "fail",
-                (processorFactories, tag, description, config) -> new TestProcessor(tag, "fail", description, new RuntimeException())
+                (processorFactories, tag, description, config, projectId) -> new TestProcessor(
+                    tag,
+                    "fail",
+                    description,
+                    new RuntimeException()
+                )
             );
             factories.put(
                 "onfailure_processor",
-                (processorFactories, tag, description, config) -> new TestProcessor(tag, "fail", description, document -> {
+                (processorFactories, tag, description, config, projectId) -> new TestProcessor(tag, "fail", description, document -> {
                     String onFailurePipeline = document.getFieldValue("_ingest.on_failure_pipeline", String.class);
                     document.setFieldValue("readme", "pipeline with id [" + onFailurePipeline + "] is a bad pipeline");
                 })

server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java

@@ -262,7 +262,7 @@ public class IngestFileSettingsIT extends ESIntegTestCase {
         @Override
         public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
             Map<String, Processor.Factory> processors = new HashMap<>();
-            processors.put("test", (factories, tag, description, config) -> {
+            processors.put("test", (factories, tag, description, config, projectId) -> {
                 String field = (String) config.remove("field");
                 String value = (String) config.remove("value");
                 return new FakeProcessor("test", tag, description, (ingestDocument) -> ingestDocument.setFieldValue(field, value));

server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java

@@ -166,7 +166,7 @@ public class IngestStatsNamesAndTypesIT extends ESIntegTestCase {
         @Override
         public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
             Map<String, Processor.Factory> processors = new HashMap<>();
-            processors.put("set", (factories, tag, description, config) -> {
+            processors.put("set", (factories, tag, description, config, projectId) -> {
                 String field = (String) config.remove("field");
                 String value = (String) config.remove("value");
                 return new FakeProcessor("set", tag, description, (ingestDocument) -> ingestDocument.setFieldValue(field, value));

server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java

@@ -149,14 +149,20 @@ public class SimulatePipelineRequest extends ActionRequest implements ToXContent
         return new Parsed(pipeline, ingestDocumentList, verbose);
     }
 
-    static Parsed parse(Map<String, Object> config, boolean verbose, IngestService ingestService, RestApiVersion restApiVersion)
+    static Parsed parse(
-        throws Exception {
+        ProjectId projectId,
+        Map<String, Object> config,
+        boolean verbose,
+        IngestService ingestService,
+        RestApiVersion restApiVersion
+    ) throws Exception {
         Map<String, Object> pipelineConfig = ConfigurationUtils.readMap(null, null, config, Fields.PIPELINE);
         Pipeline pipeline = Pipeline.create(
             SIMULATED_PIPELINE_ID,
             pipelineConfig,
             ingestService.getProcessorFactories(),
-            ingestService.getScriptService()
+            ingestService.getScriptService(),
+            projectId
         );
         List<IngestDocument> ingestDocumentList = parseDocs(config, restApiVersion);
         return new Parsed(pipeline, ingestDocumentList, verbose);

server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java

@@ -113,6 +113,7 @@ public class SimulatePipelineTransportAction extends HandledTransportAction<Simu
                     );
                 } else {
                     simulateRequest = SimulatePipelineRequest.parse(
+                        projectId,
                         source,
                         request.isVerbose(),
                         ingestService,

server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java

@@ -12,6 +12,7 @@ package org.elasticsearch.ingest;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.util.Maps;
@@ -487,7 +488,8 @@ public final class ConfigurationUtils {
     public static List<Processor> readProcessorConfigs(
         List<Map<String, Object>> processorConfigs,
         ScriptService scriptService,
-        Map<String, Processor.Factory> processorFactories
+        Map<String, Processor.Factory> processorFactories,
+        ProjectId projectId
     ) throws Exception {
         Exception exception = null;
         List<Processor> processors = new ArrayList<>();
@@ -498,7 +500,7 @@ public final class ConfigurationUtils {
                         if (entry.getValue() == null) {
                             throw newConfigurationException(entry.getKey(), null, null, "processor config cannot be [null]");
                         } else {
-                            processors.add(readProcessor(processorFactories, scriptService, entry.getKey(), entry.getValue()));
+                            processors.add(readProcessor(processorFactories, scriptService, entry.getKey(), entry.getValue(), projectId));
                         }
                     } catch (Exception e) {
                         exception = ExceptionsHelper.useOrSuppress(exception, e);
@@ -575,14 +577,15 @@ public final class ConfigurationUtils {
         Map<String, Processor.Factory> processorFactories,
         ScriptService scriptService,
         String type,
-        Object config
+        Object config,
+        ProjectId projectId
     ) throws Exception {
         if (config instanceof Map) {
-            return readProcessor(processorFactories, scriptService, type, (Map<String, Object>) config);
+            return readProcessor(processorFactories, scriptService, type, (Map<String, Object>) config, projectId);
         } else if (config instanceof String && "script".equals(type)) {
             Map<String, Object> normalizedScript = Maps.newMapWithExpectedSize(1);
             normalizedScript.put(ScriptType.INLINE.getParseField().getPreferredName(), config);
-            return readProcessor(processorFactories, scriptService, type, normalizedScript);
+            return readProcessor(processorFactories, scriptService, type, normalizedScript, projectId);
         } else {
             throw newConfigurationException(type, null, null, "property isn't a map, but of type [" + config.getClass().getName() + "]");
         }
@@ -592,7 +595,8 @@ public final class ConfigurationUtils {
         Map<String, Processor.Factory> processorFactories,
         ScriptService scriptService,
         String type,
-        Map<String, Object> config
+        Map<String, Object> config,
+        ProjectId projectId
     ) throws Exception {
         String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY);
         String description = ConfigurationUtils.readOptionalStringProperty(null, tag, config, DESCRIPTION_KEY);
@@ -607,14 +611,19 @@ public final class ConfigurationUtils {
                 Pipeline.ON_FAILURE_KEY
             );
 
-            List<Processor> onFailureProcessors = readProcessorConfigs(onFailureProcessorConfigs, scriptService, processorFactories);
+            List<Processor> onFailureProcessors = readProcessorConfigs(
+                onFailureProcessorConfigs,
+                scriptService,
+                processorFactories,
+                projectId
+            );
 
             if (onFailureProcessorConfigs != null && onFailureProcessors.isEmpty()) {
                 throw newConfigurationException(type, tag, Pipeline.ON_FAILURE_KEY, "processors list cannot be empty");
             }
 
             try {
-                Processor processor = factory.create(processorFactories, tag, description, config);
+                Processor processor = factory.create(processorFactories, tag, description, config, projectId);
                 if (config.isEmpty() == false) {
                     throw new ElasticsearchParseException(
                         "processor [{}] doesn't support one or more provided configuration parameters {}",

server/src/main/java/org/elasticsearch/ingest/DropProcessor.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
+
 import java.util.Map;
 
 /**
@@ -40,7 +42,8 @@ public final class DropProcessor extends AbstractProcessor {
             final Map<String, Processor.Factory> processorFactories,
             final String tag,
             final String description,
-            final Map<String, Object> config
+            final Map<String, Object> config,
+            final ProjectId projectId
         ) {
             return new DropProcessor(tag, description);
         }

server/src/main/java/org/elasticsearch/ingest/IngestService.java

@@ -372,6 +372,10 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
         return scriptService;
     }
 
+    public ProjectResolver getProjectResolver() {
+        return projectResolver;
+    }
+
     /**
      * Deletes the pipeline specified by id in the request.
      */
@@ -522,7 +526,7 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
         }
 
         nodeInfoListener.accept(listener.delegateFailureAndWrap((l, nodeInfos) -> {
-            validatePipelineRequest(request, nodeInfos);
+            validatePipelineRequest(projectId, request, nodeInfos);
 
             taskQueue.submitTask(
                 "put-pipeline-" + request.getId(),
@@ -532,14 +536,14 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
         }));
     }
 
-    public void validatePipelineRequest(PutPipelineRequest request, NodesInfoResponse nodeInfos) throws Exception {
+    public void validatePipelineRequest(ProjectId projectId, PutPipelineRequest request, NodesInfoResponse nodeInfos) throws Exception {
         final Map<String, Object> config = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2();
         Map<DiscoveryNode, IngestInfo> ingestInfos = new HashMap<>();
         for (NodeInfo nodeInfo : nodeInfos.getNodes()) {
             ingestInfos.put(nodeInfo.getNode(), nodeInfo.getInfo(IngestInfo.class));
         }
 
-        validatePipeline(ingestInfos, request.getId(), config);
+        validatePipeline(ingestInfos, projectId, request.getId(), config);
     }
 
     public static boolean isNoOpPipelineUpdate(ProjectMetadata metadata, PutPipelineRequest request) {
@@ -729,8 +733,12 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
     }
 
     @UpdateForV10(owner = DATA_MANAGEMENT) // Change deprecation log for special characters in name to a failure
-    void validatePipeline(Map<DiscoveryNode, IngestInfo> ingestInfos, String pipelineId, Map<String, Object> pipelineConfig)
+    void validatePipeline(
-        throws Exception {
+        Map<DiscoveryNode, IngestInfo> ingestInfos,
+        ProjectId projectId,
+        String pipelineId,
+        Map<String, Object> pipelineConfig
+    ) throws Exception {
         if (ingestInfos.isEmpty()) {
             throw new IllegalStateException("Ingest info is empty");
         }
@@ -746,7 +754,7 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
             deprecationLogger.critical(DeprecationCategory.API, "pipeline_name_special_chars", e.getMessage());
         }
 
-        Pipeline pipeline = Pipeline.create(pipelineId, pipelineConfig, processorFactories, scriptService);
+        Pipeline pipeline = Pipeline.create(pipelineId, pipelineConfig, processorFactories, scriptService, projectId);
         List<Exception> exceptions = new ArrayList<>();
         for (Processor processor : pipeline.flattenAllProcessors()) {
 
@@ -1394,7 +1402,8 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
                     newConfiguration.getId(),
                     newConfiguration.getConfig(false),
                     processorFactories,
-                    scriptService
+                    scriptService,
+                    projectId
                 );
                 newPipelines.put(newConfiguration.getId(), new PipelineHolder(newConfiguration, newPipeline));
 
@@ -1523,7 +1532,7 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
     public synchronized void reloadPipeline(ProjectId projectId, String id) throws Exception {
         var originalPipelines = this.pipelines.getOrDefault(projectId, ImmutableOpenMap.of());
         PipelineHolder holder = originalPipelines.get(id);
-        Pipeline updatedPipeline = Pipeline.create(id, holder.configuration.getConfig(false), processorFactories, scriptService);
+        Pipeline updatedPipeline = Pipeline.create(id, holder.configuration.getConfig(false), processorFactories, scriptService, projectId);
         ImmutableOpenMap<String, PipelineHolder> updatedPipelines = ImmutableOpenMap.builder(originalPipelines)
             .fPut(id, new PipelineHolder(holder.configuration, updatedPipeline))
             .build();

server/src/main/java/org/elasticsearch/ingest/Pipeline.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.script.ScriptService;
 
@@ -89,19 +90,26 @@ public final class Pipeline {
         String id,
         Map<String, Object> config,
         Map<String, Processor.Factory> processorFactories,
-        ScriptService scriptService
+        ScriptService scriptService,
+        ProjectId projectId
     ) throws Exception {
         String description = ConfigurationUtils.readOptionalStringProperty(null, null, config, DESCRIPTION_KEY);
         Integer version = ConfigurationUtils.readIntProperty(null, null, config, VERSION_KEY, null);
         Map<String, Object> metadata = ConfigurationUtils.readOptionalMap(null, null, config, META_KEY);
         Boolean deprecated = ConfigurationUtils.readOptionalBooleanProperty(null, null, config, DEPRECATED_KEY);
         List<Map<String, Object>> processorConfigs = ConfigurationUtils.readList(null, null, config, PROCESSORS_KEY);
-        List<Processor> processors = ConfigurationUtils.readProcessorConfigs(processorConfigs, scriptService, processorFactories);
+        List<Processor> processors = ConfigurationUtils.readProcessorConfigs(
+            processorConfigs,
+            scriptService,
+            processorFactories,
+            projectId
+        );
         List<Map<String, Object>> onFailureProcessorConfigs = ConfigurationUtils.readOptionalList(null, null, config, ON_FAILURE_KEY);
         List<Processor> onFailureProcessors = ConfigurationUtils.readProcessorConfigs(
             onFailureProcessorConfigs,
             scriptService,
-            processorFactories
+            processorFactories,
+            projectId
         );
         if (config.isEmpty() == false) {
             throw new ElasticsearchParseException(

server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java

@@ -9,6 +9,7 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.script.TemplateScript;
 
 import java.util.Map;
@@ -93,7 +94,8 @@ public class PipelineProcessor extends AbstractProcessor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             TemplateScript.Factory pipelineTemplate = ConfigurationUtils.readTemplateProperty(
                 TYPE,

server/src/main/java/org/elasticsearch/ingest/Processor.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.ingest;
 
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.grok.MatcherWatchdog;
@@ -84,12 +85,12 @@ public interface Processor {
      *
      * Override this method to perform additional post-construction validation that should be performed at the rest/transport level.
      * If there's an issue with the processor, then indicate that by throwing an exception. See
-     * {@link IngestService#validatePipeline(Map, String, Map)}} for the call site where there is invoked in a try/catch.
+     * {@link IngestService#validatePipeline(Map, ProjectId, String, Map)}} for the call site where there is invoked in a try/catch.
      *
      * An example of where this would be needed is a processor that interacts with external state like the license state -- it may
      * be okay to create that processor on day 1 with license state A, but later illegal to create a similar processor on day 2 with
      * state B. We want to reject put requests on day 2 (at the rest/transport level), but still allow for restarting nodes in the
-     * cluster (so we can't throw exceptions from {@link Factory#create(Map, String, String, Map)}).
+     * cluster (so we can't throw exceptions from {@link Factory#create(Map, String, String, Map, ProjectId)}).
      */
     default void extraValidation() throws Exception {}
 
@@ -100,15 +101,22 @@ public interface Processor {
 
         /**
          * Creates a processor based on the specified map of maps config.
-         *  @param processorFactories Other processors which may be created inside this processor
+         *
+         * @param processorFactories Other processors which may be created inside this processor
          * @param tag The tag for the processor
          * @param description A short description of what this processor does
          * @param config The configuration for the processor
+         * @param projectId The project for which the processor is created
          *
          * <b>Note:</b> Implementations are responsible for removing the used configuration keys, so that after
          */
-        Processor create(Map<String, Factory> processorFactories, String tag, String description, Map<String, Object> config)
+        Processor create(
-            throws Exception;
+            Map<String, Factory> processorFactories,
+            String tag,
+            String description,
+            Map<String, Object> config,
+            ProjectId projectId
+        ) throws Exception;
     }
 
     /**

server/src/main/java/org/elasticsearch/ingest/SimulateIngestService.java

@@ -57,7 +57,8 @@ public class SimulateIngestService extends IngestService {
                     pipelineId,
                     entry.getValue(),
                     ingestService.getProcessorFactories(),
-                    ingestService.getScriptService()
+                    ingestService.getScriptService(),
+                    ingestService.getProjectResolver().getProjectId()
                 );
                 parsedPipelineSubstitutions.put(pipelineId, pipeline);
             }

server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java

@@ -261,7 +261,7 @@ public class SimulateExecutionServiceTests extends ESTestCase {
 
     public void testDropDocument() throws Exception {
         TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field", "value"));
-        Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of());
+        Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of(), null);
         Pipeline pipeline = new Pipeline("_id", "_description", version, null, new CompoundProcessor(processor1, processor2));
 
         CountDownLatch latch = new CountDownLatch(1);
@@ -281,7 +281,7 @@ public class SimulateExecutionServiceTests extends ESTestCase {
 
     public void testDropDocumentVerbose() throws Exception {
         TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field", "value"));
-        Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of());
+        Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of(), null);
         Pipeline pipeline = new Pipeline("_id", "_description", version, null, new CompoundProcessor(processor1, processor2));
 
         CountDownLatch latch = new CountDownLatch(1);
@@ -304,7 +304,7 @@ public class SimulateExecutionServiceTests extends ESTestCase {
 
     public void testDropDocumentVerboseExtraProcessor() throws Exception {
         TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field1", "value"));
-        Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of());
+        Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of(), null);
         TestProcessor processor3 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field2", "value"));
         Pipeline pipeline = new Pipeline("_id", "_description", version, null, new CompoundProcessor(processor1, processor2, processor3));
 

server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java

@@ -60,7 +60,7 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
         Pipeline pipeline = new Pipeline(SIMULATED_PIPELINE_ID, null, null, null, pipelineCompoundProcessor);
         Map<String, Processor.Factory> registry = Collections.singletonMap(
             "mock_processor",
-            (factories, tag, description, config) -> processor
+            (factories, tag, description, config, projectId) -> processor
         );
         ingestService = mock(IngestService.class);
         when(ingestService.getPipeline(any(), eq(SIMULATED_PIPELINE_ID))).thenReturn(pipeline);
@@ -189,7 +189,9 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
 
         requestContent.put(Fields.PIPELINE, pipelineConfig);
 
+        var projectId = randomProjectIdOrDefault();
         SimulatePipelineRequest.Parsed actualRequest = SimulatePipelineRequest.parse(
+            projectId,
             requestContent,
             false,
             ingestService,
@@ -256,6 +258,7 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
     }
 
     public void testNotValidDocs() {
+        var projectId = randomProjectIdOrDefault();
         Map<String, Object> requestContent = new HashMap<>();
         List<Map<String, Object>> docs = new ArrayList<>();
         Map<String, Object> pipelineConfig = new HashMap<>();
@@ -265,7 +268,7 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
         requestContent.put(Fields.PIPELINE, pipelineConfig);
         Exception e1 = expectThrows(
             IllegalArgumentException.class,
-            () -> SimulatePipelineRequest.parse(requestContent, false, ingestService, RestApiVersion.current())
+            () -> SimulatePipelineRequest.parse(projectId, requestContent, false, ingestService, RestApiVersion.current())
         );
         assertThat(e1.getMessage(), equalTo("must specify at least one document in [docs]"));
 
@@ -276,7 +279,7 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
         requestContent.put(Fields.PIPELINE, pipelineConfig);
         Exception e2 = expectThrows(
             IllegalArgumentException.class,
-            () -> SimulatePipelineRequest.parse(requestContent, false, ingestService, RestApiVersion.current())
+            () -> SimulatePipelineRequest.parse(projectId, requestContent, false, ingestService, RestApiVersion.current())
         );
         assertThat(e2.getMessage(), equalTo("malformed [docs] section, should include an inner object"));
 
@@ -285,7 +288,7 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
         requestContent.put(Fields.PIPELINE, pipelineConfig);
         Exception e3 = expectThrows(
             ElasticsearchParseException.class,
-            () -> SimulatePipelineRequest.parse(requestContent, false, ingestService, RestApiVersion.current())
+            () -> SimulatePipelineRequest.parse(projectId, requestContent, false, ingestService, RestApiVersion.current())
         );
         assertThat(e3.getMessage(), containsString("required property is missing"));
     }
@@ -358,7 +361,9 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
             pipelineConfig.put("on_failure", onFailureProcessors);
         }
         requestContent.put(Fields.PIPELINE, pipelineConfig);
+        var projectId = randomProjectIdOrDefault();
         SimulatePipelineRequest.Parsed actualRequest = SimulatePipelineRequest.parse(
+            projectId,
             requestContent,
             false,
             ingestService,

server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java

@@ -11,6 +11,7 @@ package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.script.TemplateScript;
 import org.elasticsearch.test.ESTestCase;
@@ -40,6 +41,7 @@ public class ConfigurationUtilsTests extends ESTestCase {
 
     private final ScriptService scriptService = mock(ScriptService.class);
 
+    private final ProjectId projectId = randomProjectIdOrDefault();
     private Map<String, Object> config;
 
     @Before
@@ -148,7 +150,7 @@ public class ConfigurationUtilsTests extends ESTestCase {
         Processor processor = mock(Processor.class);
         Map<String, Processor.Factory> registry = Collections.singletonMap(
             "test_processor",
-            (factories, tag, description, config) -> processor
+            (factories, tag, description, config, projectId) -> processor
         );
 
         List<Map<String, Object>> config = new ArrayList<>();
@@ -156,7 +158,7 @@ public class ConfigurationUtilsTests extends ESTestCase {
         config.add(Collections.singletonMap("test_processor", emptyConfig));
         config.add(Collections.singletonMap("test_processor", emptyConfig));
 
-        List<Processor> result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry);
+        List<Processor> result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry, projectId);
         assertThat(result.size(), equalTo(2));
         assertThat(result.get(0), sameInstance(processor));
         assertThat(result.get(1), sameInstance(processor));
@@ -169,7 +171,7 @@ public class ConfigurationUtilsTests extends ESTestCase {
         config.add(Collections.singletonMap("unknown_processor", unknownTaggedConfig));
         ElasticsearchParseException e = expectThrows(
             ElasticsearchParseException.class,
-            () -> ConfigurationUtils.readProcessorConfigs(config, scriptService, registry)
+            () -> ConfigurationUtils.readProcessorConfigs(config, scriptService, registry, projectId)
         );
         assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]"));
         assertThat(e.getMetadata("es.processor_tag"), equalTo(Collections.singletonList("my_unknown")));
@@ -186,7 +188,7 @@ public class ConfigurationUtilsTests extends ESTestCase {
         config2.add(Collections.singletonMap("second_unknown_processor", secondUnknownTaggedConfig));
         e = expectThrows(
             ElasticsearchParseException.class,
-            () -> ConfigurationUtils.readProcessorConfigs(config2, scriptService, registry)
+            () -> ConfigurationUtils.readProcessorConfigs(config2, scriptService, registry, projectId)
         );
         assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]"));
         assertThat(e.getMetadata("es.processor_tag"), equalTo(Collections.singletonList("my_unknown")));
@@ -205,7 +207,7 @@ public class ConfigurationUtilsTests extends ESTestCase {
         config3.add(Collections.singletonMap("test_processor", null));
         ElasticsearchParseException e3 = expectThrows(
             ElasticsearchParseException.class,
-            () -> ConfigurationUtils.readProcessorConfigs(config3, scriptService, registry)
+            () -> ConfigurationUtils.readProcessorConfigs(config3, scriptService, registry, projectId)
         );
         assertThat(e3.getMetadata("es.processor_type"), equalTo(Collections.singletonList("test_processor")));
         assertThat(e3.getMessage(), equalTo("processor config cannot be [null]"));
@@ -213,15 +215,18 @@ public class ConfigurationUtilsTests extends ESTestCase {
 
     public void testReadProcessorNullDescription() throws Exception {
         Processor processor = new TestProcessor("tag", "type", null, (ingestDocument) -> {});
-        Map<String, Processor.Factory> registry = Collections.singletonMap("test_processor", (factories, tag, description, config) -> {
+        Map<String, Processor.Factory> registry = Collections.singletonMap(
-            assertNull(description);
+            "test_processor",
-            return processor;
+            (factories, tag, description, config, projectId) -> {
-        });
+                assertNull(description);
+                return processor;
+            }
+        );
 
         List<Map<String, Object>> config = new ArrayList<>();
         Map<String, Object> emptyConfig = Collections.emptyMap();
         config.add(Collections.singletonMap("test_processor", emptyConfig));
-        List<Processor> result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry);
+        List<Processor> result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry, projectId);
         assertThat(result.size(), equalTo(1));
         assertThat(result.get(0), sameInstance(processor));
     }
@@ -229,40 +234,46 @@ public class ConfigurationUtilsTests extends ESTestCase {
     public void testReadProcessorDescription() throws Exception {
         String testDescription = randomAlphaOfLengthBetween(10, 20);
         Processor processor = new TestProcessor("tag", "type", testDescription, (ingestDocument) -> {});
-        Map<String, Processor.Factory> registry = Collections.singletonMap("test_processor", (factories, tag, description, config) -> {
+        Map<String, Processor.Factory> registry = Collections.singletonMap(
-            assertThat(description, equalTo(processor.getDescription()));
+            "test_processor",
-            return processor;
+            (factories, tag, description, config, projectId) -> {
-        });
+                assertThat(description, equalTo(processor.getDescription()));
+                return processor;
+            }
+        );
 
         List<Map<String, Object>> config = new ArrayList<>();
         Map<String, Object> processorConfig = new HashMap<>();
         processorConfig.put(ConfigurationUtils.DESCRIPTION_KEY, testDescription);
         config.add(Collections.singletonMap("test_processor", processorConfig));
-        List<Processor> result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry);
+        List<Processor> result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry, projectId);
         assertThat(result.size(), equalTo(1));
         assertThat(result.get(0), sameInstance(processor));
     }
 
     public void testReadProcessorFromObjectOrMap() throws Exception {
         Processor processor = mock(Processor.class);
-        Map<String, Processor.Factory> registry = Collections.singletonMap("script", (processorFactories, tag, description, config) -> {
+        Map<String, Processor.Factory> registry = Collections.singletonMap(
-            config.clear();
+            "script",
-            return processor;
+            (processorFactories, tag, description, config, projectId) -> {
-        });
+                config.clear();
+                return processor;
+            }
+        );
 
         Object emptyConfig = Collections.emptyMap();
-        Processor processor1 = ConfigurationUtils.readProcessor(registry, scriptService, "script", emptyConfig);
+        Processor processor1 = ConfigurationUtils.readProcessor(registry, scriptService, "script", emptyConfig, projectId);
         assertThat(processor1, sameInstance(processor));
 
         Object inlineScript = "test_script";
-        Processor processor2 = ConfigurationUtils.readProcessor(registry, scriptService, "script", inlineScript);
+        Processor processor2 = ConfigurationUtils.readProcessor(registry, scriptService, "script", inlineScript, projectId);
         assertThat(processor2, sameInstance(processor));
 
         Object invalidConfig = 12L;
 
         ElasticsearchParseException ex = expectThrows(
             ElasticsearchParseException.class,
-            () -> ConfigurationUtils.readProcessor(registry, scriptService, "unknown_processor", invalidConfig)
+            () -> ConfigurationUtils.readProcessor(registry, scriptService, "unknown_processor", invalidConfig, projectId)
         );
         assertThat(ex.getMessage(), equalTo("property isn't a map, but of type [" + invalidConfig.getClass().getName() + "]"));
     }

server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java

@@ -135,7 +135,7 @@ public class IngestServiceTests extends ESTestCase {
     private static final IngestPlugin DUMMY_PLUGIN = new IngestPlugin() {
         @Override
         public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
-            return Map.of("foo", (factories, tag, description, config) -> null);
+            return Map.of("foo", (factories, tag, description, config, projectId) -> null);
         }
     };
 
@@ -347,11 +347,11 @@ public class IngestServiceTests extends ESTestCase {
 
     public void testInnerUpdatePipelinesValidation() {
         Map<String, Processor.Factory> processors = new HashMap<>();
-        processors.put("fail_validation", (factories, tag, description, config) -> {
+        processors.put("fail_validation", (factories, tag, description, config, projectId) -> {
             // ordinary validation issues happen at processor construction time
             throw newConfigurationException("fail_validation", tag, "no_property_name", "validation failure reason");
         });
-        processors.put("fail_extra_validation", (factories, tag, description, config) -> {
+        processors.put("fail_extra_validation", (factories, tag, description, config, projectId) -> {
             // 'extra validation' issues happen post- processor construction time
             return new FakeProcessor("fail_extra_validation", tag, description, ingestDocument -> {}) {
                 @Override
@@ -444,16 +444,17 @@ public class IngestServiceTests extends ESTestCase {
         PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """
             {"processors": [{"set" : {"field": "_field", "value": "_value"}}]}""");
 
+        var projectId = randomProjectIdOrDefault();
         var pipelineConfig = XContentHelper.convertToMap(putRequest.getSource(), false, putRequest.getXContentType()).v2();
         Exception e = expectThrows(
             IllegalStateException.class,
-            () -> ingestService.validatePipeline(Map.of(), putRequest.getId(), pipelineConfig)
+            () -> ingestService.validatePipeline(Map.of(), projectId, putRequest.getId(), pipelineConfig)
         );
         assertEquals("Ingest info is empty", e.getMessage());
 
         DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("_node_id", buildNewFakeTransportAddress(), Map.of(), Set.of());
         IngestInfo ingestInfo = new IngestInfo(List.of(new ProcessorInfo("set")));
-        ingestService.validatePipeline(Map.of(discoveryNode, ingestInfo), putRequest.getId(), pipelineConfig);
+        ingestService.validatePipeline(Map.of(discoveryNode, ingestInfo), projectId, putRequest.getId(), pipelineConfig);
     }
 
     public void testValidateNotInUse() {
@@ -618,7 +619,7 @@ public class IngestServiceTests extends ESTestCase {
         boolean[] externalProperty = new boolean[] { false };
 
         Map<String, Processor.Factory> processorFactories = new HashMap<>();
-        processorFactories.put("set", (factories, tag, description, config) -> {
+        processorFactories.put("set", (factories, tag, description, config, projectId) -> {
             String field = (String) config.remove("field");
             String value = (String) config.remove("value");
             if (externalProperty[0]) {
@@ -685,7 +686,7 @@ public class IngestServiceTests extends ESTestCase {
         );
 
         Map<String, Processor.Factory> processors = new HashMap<>();
-        processors.put("complexSet", (factories, tag, description, config) -> {
+        processors.put("complexSet", (factories, tag, description, config, projectId) -> {
             String field = (String) config.remove("field");
             String value = (String) config.remove("value");
 
@@ -1017,6 +1018,7 @@ public class IngestServiceTests extends ESTestCase {
 
     public void testValidateProcessorTypeOnAllNodes() throws Exception {
         IngestService ingestService = createWithProcessors();
+        var projectId = randomProjectIdOrDefault();
         PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """
             {
               "processors": [
@@ -1045,7 +1047,7 @@ public class IngestServiceTests extends ESTestCase {
 
         ElasticsearchParseException e = expectThrows(
             ElasticsearchParseException.class,
-            () -> ingestService.validatePipeline(ingestInfos, putRequest.getId(), pipelineConfig)
+            () -> ingestService.validatePipeline(ingestInfos, projectId, putRequest.getId(), pipelineConfig)
         );
         assertEquals("Processor type [remove] is not installed on node [" + node2 + "]", e.getMessage());
         assertEquals("remove", e.getMetadata("es.processor_type").get(0));
@@ -1053,14 +1055,15 @@ public class IngestServiceTests extends ESTestCase {
 
         var pipelineConfig2 = XContentHelper.convertToMap(putRequest.getSource(), false, putRequest.getXContentType()).v2();
         ingestInfos.put(node2, new IngestInfo(List.of(new ProcessorInfo("set"), new ProcessorInfo("remove"))));
-        ingestService.validatePipeline(ingestInfos, putRequest.getId(), pipelineConfig2);
+        ingestService.validatePipeline(ingestInfos, projectId, putRequest.getId(), pipelineConfig2);
     }
 
     public void testValidateConfigurationExceptions() {
-        IngestService ingestService = createWithProcessors(Map.of("fail_validation", (factories, tag, description, config) -> {
+        IngestService ingestService = createWithProcessors(Map.of("fail_validation", (factories, tag, description, config, projectId) -> {
             // ordinary validation issues happen at processor construction time
             throw newConfigurationException("fail_validation", tag, "no_property_name", "validation failure reason");
         }));
+        var projectId = randomProjectIdOrDefault();
         PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """
             {
               "processors": [
@@ -1079,22 +1082,30 @@ public class IngestServiceTests extends ESTestCase {
 
         ElasticsearchParseException e = expectThrows(
             ElasticsearchParseException.class,
-            () -> ingestService.validatePipeline(ingestInfos, putRequest.getId(), pipelineConfig)
+            () -> ingestService.validatePipeline(ingestInfos, projectId, putRequest.getId(), pipelineConfig)
         );
         assertEquals("[no_property_name] validation failure reason", e.getMessage());
         assertEquals("fail_validation", e.getMetadata("es.processor_type").get(0));
     }
 
     public void testValidateExtraValidationConfigurationExceptions() {
-        IngestService ingestService = createWithProcessors(Map.of("fail_extra_validation", (factories, tag, description, config) -> {
+        IngestService ingestService = createWithProcessors(
-            // 'extra validation' issues happen post- processor construction time
+            Map.of("fail_extra_validation", (factories, tag, description, config, projectId) -> {
-            return new FakeProcessor("fail_extra_validation", tag, description, ingestDocument -> {}) {
+                // 'extra validation' issues happen post- processor construction time
-                @Override
+                return new FakeProcessor("fail_extra_validation", tag, description, ingestDocument -> {}) {
-                public void extraValidation() throws Exception {
+                    @Override
-                    throw newConfigurationException("fail_extra_validation", tag, "no_property_name", "extra validation failure reason");
+                    public void extraValidation() throws Exception {
-                }
+                        throw newConfigurationException(
-            };
+                            "fail_extra_validation",
-        }));
+                            tag,
+                            "no_property_name",
+                            "extra validation failure reason"
+                        );
+                    }
+                };
+            })
+        );
+        var projectId = randomProjectIdOrDefault();
         PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """
             {
               "processors": [
@@ -1113,13 +1124,14 @@ public class IngestServiceTests extends ESTestCase {
 
         ElasticsearchParseException e = expectThrows(
             ElasticsearchParseException.class,
-            () -> ingestService.validatePipeline(ingestInfos, putRequest.getId(), pipelineConfig)
+            () -> ingestService.validatePipeline(ingestInfos, projectId, putRequest.getId(), pipelineConfig)
         );
         assertEquals("[no_property_name] extra validation failure reason", e.getMessage());
         assertEquals("fail_extra_validation", e.getMetadata("es.processor_type").get(0));
     }
 
     public void testValidatePipelineName() throws Exception {
+        var projectId = randomProjectIdOrDefault();
         IngestService ingestService = createWithProcessors();
         for (Character badChar : List.of('\\', '/', '*', '?', '"', '<', '>', '|', ' ', ',')) {
             PutPipelineRequest putRequest = new PutPipelineRequest(
@@ -1135,7 +1147,7 @@ public class IngestServiceTests extends ESTestCase {
             Map<DiscoveryNode, IngestInfo> ingestInfos = new HashMap<>();
             ingestInfos.put(node1, new IngestInfo(List.of(new ProcessorInfo("set"))));
             final String name = randomAlphaOfLength(5) + badChar + randomAlphaOfLength(5);
-            ingestService.validatePipeline(ingestInfos, name, pipelineConfig);
+            ingestService.validatePipeline(ingestInfos, projectId, name, pipelineConfig);
             assertCriticalWarnings(
                 "Pipeline name ["
                     + name
@@ -1147,7 +1159,7 @@ public class IngestServiceTests extends ESTestCase {
 
     public void testExecuteIndexPipelineExistsButFailedParsing() {
         IngestService ingestService = createWithProcessors(
-            Map.of("mock", (factories, tag, description, config) -> new AbstractProcessor("mock", "description") {
+            Map.of("mock", (factories, tag, description, config, projectId) -> new AbstractProcessor("mock", "description") {
                 @Override
                 public IngestDocument execute(IngestDocument ingestDocument) {
                     throw new IllegalStateException("error");
@@ -1213,7 +1225,7 @@ public class IngestServiceTests extends ESTestCase {
 
     public void testExecuteBulkPipelineDoesNotExist() {
         IngestService ingestService = createWithProcessors(
-            Map.of("mock", (factories, tag, description, config) -> mockCompoundProcessor())
+            Map.of("mock", (factories, tag, description, config, projectId) -> mockCompoundProcessor())
         );
 
         PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
@@ -1267,7 +1279,7 @@ public class IngestServiceTests extends ESTestCase {
 
     public void testExecuteSuccess() {
         IngestService ingestService = createWithProcessors(
-            Map.of("mock", (factories, tag, description, config) -> mockCompoundProcessor())
+            Map.of("mock", (factories, tag, description, config, projectId) -> mockCompoundProcessor())
         );
         PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
         var projectId = randomProjectIdOrDefault();
@@ -1304,7 +1316,7 @@ public class IngestServiceTests extends ESTestCase {
         IngestService ingestService = createWithProcessors(
             Map.of(
                 "set",
-                (factories, tag, description, config) -> new FakeProcessor(
+                (factories, tag, description, config, projectId) -> new FakeProcessor(
                     "set",
                     "",
                     "",
@@ -1380,7 +1392,7 @@ public class IngestServiceTests extends ESTestCase {
 
     public void testExecutePropagateAllMetadataUpdates() throws Exception {
         final CompoundProcessor processor = mockCompoundProcessor();
-        IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> processor));
+        IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config, projectId) -> processor));
         PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
         var projectId = randomProjectIdOrDefault();
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
@@ -1454,9 +1466,9 @@ public class IngestServiceTests extends ESTestCase {
         IngestService ingestService = createWithProcessors(
             Map.of(
                 "mock",
-                (factories, tag, description, config) -> processor,
+                (factories, tag, description, config, projectId) -> processor,
                 "set",
-                (factories, tag, description, config) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail())
+                (factories, tag, description, config, projectId) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail())
             )
         );
         PutPipelineRequest putRequest1 = putJsonPipelineRequest("_id1", "{\"processors\": [{\"mock\" : {}}]}");
@@ -1526,7 +1538,9 @@ public class IngestServiceTests extends ESTestCase {
             List.of(processor),
             List.of(new CompoundProcessor(onFailureProcessor))
         );
-        IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> compoundProcessor));
+        IngestService ingestService = createWithProcessors(
+            Map.of("mock", (factories, tag, description, config, projectId) -> compoundProcessor)
+        );
         PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
         var projectId = randomProjectIdOrDefault();
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
@@ -1572,7 +1586,9 @@ public class IngestServiceTests extends ESTestCase {
             List.of(processor),
             List.of(new CompoundProcessor(false, processors, onFailureProcessors))
         );
-        IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> compoundProcessor));
+        IngestService ingestService = createWithProcessors(
+            Map.of("mock", (factories, tag, description, config, projectId) -> compoundProcessor)
+        );
         PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
         var projectId = randomProjectIdOrDefault();
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
@@ -1650,7 +1666,7 @@ public class IngestServiceTests extends ESTestCase {
             handler.accept(null, error);
             return null;
         }).when(processor).execute(any(), any());
-        IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> processor));
+        IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config, projectId) -> processor));
         PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
         var projectId = randomProjectIdOrDefault();
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
@@ -1691,9 +1707,9 @@ public class IngestServiceTests extends ESTestCase {
         IngestService ingestService = createWithProcessors(
             Map.of(
                 "mock",
-                (factories, tag, description, config) -> processor,
+                (factories, tag, description, config, projectId) -> processor,
                 "set",
-                (factories, tag, description, config) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail())
+                (factories, tag, description, config, projectId) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail())
             )
         );
         PutPipelineRequest putRequest1 = putJsonPipelineRequest("_id1", "{\"processors\": [{\"mock\" : {}}]}");
@@ -1742,9 +1758,9 @@ public class IngestServiceTests extends ESTestCase {
         IngestService ingestService = createWithProcessors(
             Map.of(
                 "mock",
-                (factories, tag, description, config) -> processor,
+                (factories, tag, description, config, projectId) -> processor,
                 "set",
-                (factories, tag, description, config) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail())
+                (factories, tag, description, config, projectId) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail())
             )
         );
         PutPipelineRequest putRequest1 = putJsonPipelineRequest("_id1", "{\"processors\": [{\"mock\" : {}}]}");
@@ -1799,7 +1815,9 @@ public class IngestServiceTests extends ESTestCase {
             List.of(processor),
             List.of(new CompoundProcessor(false, processors, onFailureProcessors))
         );
-        IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> compoundProcessor));
+        IngestService ingestService = createWithProcessors(
+            Map.of("mock", (factories, tag, description, config, projectId) -> compoundProcessor)
+        );
         PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
         var projectId = randomProjectIdOrDefault();
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
@@ -1875,7 +1893,7 @@ public class IngestServiceTests extends ESTestCase {
             handler.accept(null, error);
             return null;
         }).when(processor).execute(any(), any());
-        IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> processor));
+        IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config, projectId) -> processor));
         PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
         var projectId = randomProjectIdOrDefault();
         ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
@@ -1939,7 +1957,7 @@ public class IngestServiceTests extends ESTestCase {
             return null;
         }).when(processor).execute(any(), any());
         Map<String, Processor.Factory> map = Maps.newMapWithExpectedSize(2);
-        map.put("mock", (factories, tag, description, config) -> processor);
+        map.put("mock", (factories, tag, description, config, projectId) -> processor);
 
         IngestService ingestService = createWithProcessors(map);
         PutPipelineRequest putRequest = putJsonPipelineRequest("_id", """
@@ -2007,14 +2025,20 @@ public class IngestServiceTests extends ESTestCase {
         IngestService ingestService = createWithProcessors(
             Map.of(
                 "pipeline",
-                (factories, tag, description, config) -> new PipelineProcessor(tag, description, (params) -> new TemplateScript(params) {
+                (factories, tag, description, config, projectId) -> new PipelineProcessor(
-                    @Override
+                    tag,
-                    public String execute() {
+                    description,
-                        return "_id3";
+                    (params) -> new TemplateScript(params) {
-                    } // this pipeline processor will always execute the '_id3' processor
+                        @Override
-                }, false, pipelineIngestService),
+                        public String execute() {
+                            return "_id3";
+                        } // this pipeline processor will always execute the '_id3' processor
+                    },
+                    false,
+                    pipelineIngestService
+                ),
                 "mock",
-                (factories, tag, description, config) -> processor
+                (factories, tag, description, config, projectId) -> processor
             )
         );
 
@@ -2116,8 +2140,8 @@ public class IngestServiceTests extends ESTestCase {
             return null;
         }).when(processorFailure).execute(any(IngestDocument.class), any());
         Map<String, Processor.Factory> map = Maps.newMapWithExpectedSize(2);
-        map.put("mock", (factories, tag, description, config) -> processor);
+        map.put("mock", (factories, tag, description, config, projectId) -> processor);
-        map.put("failure-mock", (factories, tag, description, config) -> processorFailure);
+        map.put("failure-mock", (factories, tag, description, config, projectId) -> processorFailure);
         map.put("drop", new DropProcessor.Factory());
         IngestService ingestService = createWithProcessors(map);
 
@@ -2324,7 +2348,7 @@ public class IngestServiceTests extends ESTestCase {
     public void testExecuteWithDrop() {
         Map<String, Processor.Factory> factories = new HashMap<>();
         factories.put("drop", new DropProcessor.Factory());
-        factories.put("mock", (processorFactories, tag, description, config) -> new Processor() {
+        factories.put("mock", (processorFactories, tag, description, config, projectId) -> new Processor() {
             @Override
             public IngestDocument execute(final IngestDocument ingestDocument) {
                 throw new AssertionError("Document should have been dropped but reached this processor");
@@ -2397,7 +2421,7 @@ public class IngestServiceTests extends ESTestCase {
         IngestPlugin testPlugin = new IngestPlugin() {
             @Override
             public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
-                return Map.of("test", (factories, tag, description, config) -> {
+                return Map.of("test", (factories, tag, description, config, projectId) -> {
                     assertThat(counter.compareAndSet(1, 2), is(true));
                     return new FakeProcessor("test", tag, description, ingestDocument -> {});
                 });
@@ -2438,7 +2462,7 @@ public class IngestServiceTests extends ESTestCase {
         AtomicReference<Object> reference = new AtomicReference<>();
         Consumer<IngestDocument> executor = doc -> reference.set(doc.getFieldValueAsBytes("data"));
         final IngestService ingestService = createWithProcessors(
-            Map.of("foo", (factories, tag, description, config) -> new FakeProcessor("foo", tag, description, executor))
+            Map.of("foo", (factories, tag, description, config, projectId) -> new FakeProcessor("foo", tag, description, executor))
         );
 
         var projectId = randomProjectIdOrDefault();
@@ -2481,9 +2505,9 @@ public class IngestServiceTests extends ESTestCase {
         IngestService ingestService = createWithProcessors(
             Map.of(
                 "mock",
-                (factories, tag, description, config) -> mockCompoundProcessor(),
+                (factories, tag, description, config, projectId) -> mockCompoundProcessor(),
                 "set",
-                (factories, tag, description, config) -> new FakeProcessor(
+                (factories, tag, description, config, projectId) -> new FakeProcessor(
                     "set",
                     tag,
                     description,
@@ -3206,12 +3230,12 @@ public class IngestServiceTests extends ESTestCase {
 
     private static IngestService createWithProcessors() {
         Map<String, Processor.Factory> processors = new HashMap<>();
-        processors.put("set", (factories, tag, description, config) -> {
+        processors.put("set", (factories, tag, description, config, projectId) -> {
             String field = (String) config.remove("field");
             String value = (String) config.remove("value");
             return new FakeProcessor("set", tag, description, (ingestDocument) -> ingestDocument.setFieldValue(field, value));
         });
-        processors.put("remove", (factories, tag, description, config) -> {
+        processors.put("remove", (factories, tag, description, config, projectId) -> {
             String field = (String) config.remove("field");
             return new WrappingProcessorImpl("remove", tag, description, (ingestDocument -> ingestDocument.removeField(field))) {
             };

server/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.ingest;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.test.ESTestCase;
@@ -26,6 +27,7 @@ import static org.mockito.Mockito.mock;
 
 public class PipelineFactoryTests extends ESTestCase {
 
+    private final ProjectId projectId = randomProjectIdOrDefault();
     private final Integer version = randomBoolean() ? randomInt() : null;
     private final String versionString = version != null ? Integer.toString(version) : null;
     private final ScriptService scriptService = mock(ScriptService.class);
@@ -45,7 +47,7 @@ public class PipelineFactoryTests extends ESTestCase {
         pipelineConfig.put(Pipeline.DEPRECATED_KEY, deprecated);
         pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig0), Map.of("test", processorConfig1)));
         Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
-        Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService);
+        Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null);
         assertThat(pipeline.getId(), equalTo("_id"));
         assertThat(pipeline.getDescription(), equalTo("_description"));
         assertThat(pipeline.getVersion(), equalTo(version));
@@ -65,7 +67,7 @@ public class PipelineFactoryTests extends ESTestCase {
             pipelineConfig.put(Pipeline.META_KEY, metadata);
         }
         try {
-            Pipeline.create("_id", pipelineConfig, Map.of(), scriptService);
+            Pipeline.create("_id", pipelineConfig, Map.of(), scriptService, null);
             fail("should fail, missing required [processors] field");
         } catch (ElasticsearchParseException e) {
             assertThat(e.getMessage(), equalTo("[processors] required property is missing"));
@@ -80,7 +82,7 @@ public class PipelineFactoryTests extends ESTestCase {
             pipelineConfig.put(Pipeline.META_KEY, metadata);
         }
         pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of());
-        Pipeline pipeline = Pipeline.create("_id", pipelineConfig, null, scriptService);
+        Pipeline pipeline = Pipeline.create("_id", pipelineConfig, null, scriptService, null);
         assertThat(pipeline.getId(), equalTo("_id"));
         assertThat(pipeline.getDescription(), equalTo("_description"));
         assertThat(pipeline.getVersion(), equalTo(version));
@@ -98,7 +100,7 @@ public class PipelineFactoryTests extends ESTestCase {
         pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig)));
         pipelineConfig.put(Pipeline.ON_FAILURE_KEY, List.of(Map.of("test", processorConfig)));
         Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
-        Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService);
+        Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null);
         assertThat(pipeline.getId(), equalTo("_id"));
         assertThat(pipeline.getDescription(), equalTo("_description"));
         assertThat(pipeline.getVersion(), equalTo(version));
@@ -121,7 +123,7 @@ public class PipelineFactoryTests extends ESTestCase {
         Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
         Exception e = expectThrows(
             ElasticsearchParseException.class,
-            () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService)
+            () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null)
         );
         assertThat(e.getMessage(), equalTo("pipeline [_id] cannot have an empty on_failure option defined"));
     }
@@ -139,7 +141,7 @@ public class PipelineFactoryTests extends ESTestCase {
         Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
         Exception e = expectThrows(
             ElasticsearchParseException.class,
-            () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService)
+            () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null)
         );
         assertThat(e.getMessage(), equalTo("[on_failure] processors list cannot be empty"));
     }
@@ -157,7 +159,7 @@ public class PipelineFactoryTests extends ESTestCase {
         }
         pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig)));
 
-        Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService);
+        Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null);
         assertThat(pipeline.getId(), equalTo("_id"));
         assertThat(pipeline.getDescription(), equalTo("_description"));
         assertThat(pipeline.getVersion(), equalTo(version));
@@ -182,7 +184,7 @@ public class PipelineFactoryTests extends ESTestCase {
         Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
         Exception e = expectThrows(
             ElasticsearchParseException.class,
-            () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService)
+            () -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null)
         );
         assertThat(e.getMessage(), equalTo("processor [test] doesn't support one or more provided configuration parameters [unused]"));
     }
@@ -199,7 +201,7 @@ public class PipelineFactoryTests extends ESTestCase {
         }
         pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig)));
         Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
-        Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService);
+        Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null);
         assertThat(pipeline.getId(), equalTo("_id"));
         assertThat(pipeline.getDescription(), equalTo("_description"));
         assertThat(pipeline.getVersion(), equalTo(version));

server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java

@@ -61,7 +61,7 @@ public class PipelineProcessorTests extends ESTestCase {
         PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService);
         Map<String, Object> config = new HashMap<>();
         config.put("name", pipelineId);
-        factory.create(Map.of(), null, null, config).execute(testIngestDocument, (result, e) -> {});
+        factory.create(Map.of(), null, null, config, null).execute(testIngestDocument, (result, e) -> {});
         assertIngestDocument(testIngestDocument, safeGet(invoked));
     }
 
@@ -72,7 +72,7 @@ public class PipelineProcessorTests extends ESTestCase {
         Map<String, Object> config = new HashMap<>();
         config.put("name", "missingPipelineId");
         IllegalStateException[] e = new IllegalStateException[1];
-        factory.create(Map.of(), null, null, config).execute(testIngestDocument, (result, e1) -> e[0] = (IllegalStateException) e1);
+        factory.create(Map.of(), null, null, config, null).execute(testIngestDocument, (result, e1) -> e[0] = (IllegalStateException) e1);
         assertEquals("Pipeline processor configured for non-existent pipeline [missingPipelineId]", e[0].getMessage());
     }
 
@@ -86,7 +86,7 @@ public class PipelineProcessorTests extends ESTestCase {
 
         var r = new IngestDocument[1];
         var e = new Exception[1];
-        var processor = factory.create(Map.of(), null, null, config);
+        var processor = factory.create(Map.of(), null, null, config, null);
         processor.execute(testIngestDocument, (result, e1) -> {
             r[0] = result;
             e[0] = e1;
@@ -108,7 +108,7 @@ public class PipelineProcessorTests extends ESTestCase {
             null,
             null,
             null,
-            new CompoundProcessor(factory.create(Map.of(), null, null, outerConfig))
+            new CompoundProcessor(factory.create(Map.of(), null, null, outerConfig, null))
         );
         Map<String, Object> innerConfig = new HashMap<>();
         innerConfig.put("name", outerPipelineId);
@@ -117,13 +117,14 @@ public class PipelineProcessorTests extends ESTestCase {
             null,
             null,
             null,
-            new CompoundProcessor(factory.create(Map.of(), null, null, innerConfig))
+            new CompoundProcessor(factory.create(Map.of(), null, null, innerConfig, null))
         );
         when(ingestService.getPipeline(outerPipelineId)).thenReturn(outer);
         when(ingestService.getPipeline(innerPipelineId)).thenReturn(inner);
         outerConfig.put("name", innerPipelineId);
         ElasticsearchException[] e = new ElasticsearchException[1];
-        factory.create(Map.of(), null, null, outerConfig).execute(testIngestDocument, (result, e1) -> e[0] = (ElasticsearchException) e1);
+        factory.create(Map.of(), null, null, outerConfig, null)
+            .execute(testIngestDocument, (result, e1) -> e[0] = (ElasticsearchException) e1);
         assertEquals("Cycle detected for pipeline: inner", e[0].getRootCause().getMessage());
     }
 
@@ -136,7 +137,7 @@ public class PipelineProcessorTests extends ESTestCase {
         PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService);
         Pipeline inner = new Pipeline(innerPipelineId, null, null, null, new CompoundProcessor());
         when(ingestService.getPipeline(innerPipelineId)).thenReturn(inner);
-        Processor outerProc = factory.create(Map.of(), null, null, outerConfig);
+        Processor outerProc = factory.create(Map.of(), null, null, outerConfig, null);
         outerProc.execute(testIngestDocument, (result, e) -> {});
         outerProc.execute(testIngestDocument, (result, e) -> {});
     }
@@ -150,11 +151,11 @@ public class PipelineProcessorTests extends ESTestCase {
 
         Map<String, Object> pipeline1ProcessorConfig = new HashMap<>();
         pipeline1ProcessorConfig.put("name", pipeline2Id);
-        PipelineProcessor pipeline1Processor = factory.create(Map.of(), null, null, pipeline1ProcessorConfig);
+        PipelineProcessor pipeline1Processor = factory.create(Map.of(), null, null, pipeline1ProcessorConfig, null);
 
         Map<String, Object> pipeline2ProcessorConfig = new HashMap<>();
         pipeline2ProcessorConfig.put("name", pipeline3Id);
-        PipelineProcessor pipeline2Processor = factory.create(Map.of(), null, null, pipeline2ProcessorConfig);
+        PipelineProcessor pipeline2Processor = factory.create(Map.of(), null, null, pipeline2ProcessorConfig, null);
 
         LongSupplier relativeTimeProvider = mock(LongSupplier.class);
         when(relativeTimeProvider.getAsLong()).thenReturn(0L);

server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java

@@ -12,6 +12,7 @@ package org.elasticsearch.ingest;
 import org.elasticsearch.action.bulk.FailureStoreMetrics;
 import org.elasticsearch.action.bulk.SimulateBulkRequest;
 import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.project.TestProjectResolvers;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.bytes.BytesArray;
@@ -48,21 +49,21 @@ public class SimulateIngestServiceTests extends ESTestCase {
         Map<String, Processor.Factory> processors = new HashMap<>();
         processors.put(
             "processor1",
-            (factories, tag, description, config) -> new FakeProcessor("processor1", tag, description, ingestDocument -> {}) {
+            (factories, tag, description, config, projectId) -> new FakeProcessor("processor1", tag, description, ingestDocument -> {}) {
             }
         );
         processors.put(
             "processor2",
-            (factories, tag, description, config) -> new FakeProcessor("processor2", tag, description, ingestDocument -> {}) {
+            (factories, tag, description, config, projectId) -> new FakeProcessor("processor2", tag, description, ingestDocument -> {}) {
             }
         );
         processors.put(
             "processor3",
-            (factories, tag, description, config) -> new FakeProcessor("processor3", tag, description, ingestDocument -> {}) {
+            (factories, tag, description, config, projectId) -> new FakeProcessor("processor3", tag, description, ingestDocument -> {}) {
             }
         );
-        IngestService ingestService = createWithProcessors(processors);
         final var projectId = randomProjectIdOrDefault();
+        IngestService ingestService = createWithProcessors(projectId, processors);
         ingestService.innerUpdatePipelines(projectId, ingestMetadata);
         {
             // First we make sure that if there are no substitutions that we get our original pipeline back:
@@ -113,7 +114,7 @@ public class SimulateIngestServiceTests extends ESTestCase {
         }
     }
 
-    private static IngestService createWithProcessors(Map<String, Processor.Factory> processors) {
+    private static IngestService createWithProcessors(ProjectId projectId, Map<String, Processor.Factory> processors) {
         Client client = mock(Client.class);
         ThreadPool threadPool = mock(ThreadPool.class);
         when(threadPool.generic()).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE);
@@ -134,7 +135,7 @@ public class SimulateIngestServiceTests extends ESTestCase {
             client,
             null,
             FailureStoreMetrics.NOOP,
-            TestProjectResolvers.singleProjectOnly()
+            TestProjectResolvers.singleProject(projectId)
         );
     }
 }

server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java

@@ -306,7 +306,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
         })));
         when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
 
-        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig, null);
         CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
 
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@@ -375,7 +375,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
                 null,
                 new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()),
                 scriptService,
-                factory.create(Map.of(), "pipeline1", null, pipelineConfig2)
+                factory.create(Map.of(), "pipeline1", null, pipelineConfig2, null)
             ),
             new TestProcessor(ingestDocument -> {
                 ingestDocument.setFieldValue(key3, randomInt());
@@ -389,7 +389,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
         when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1);
         when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2);
 
-        PipelineProcessor pipelineProcessor = factory.create(Map.of(), "pipeline0", null, pipelineConfig0);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), "pipeline0", null, pipelineConfig0, null);
         CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
 
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@@ -468,7 +468,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
                 null,
                 new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()),
                 scriptService,
-                factory.create(Map.of(), null, null, pipelineConfig2)
+                factory.create(Map.of(), null, null, pipelineConfig2, null)
             ),
             new TestProcessor(ingestDocument -> {
                 ingestDocument.setFieldValue(key3, randomInt());
@@ -482,7 +482,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
         when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1);
         when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2);
 
-        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig0);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig0, null);
         CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
 
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@@ -545,7 +545,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
         ));
         when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
 
-        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig, null);
         CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
 
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@@ -608,7 +608,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
         );
         when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
 
-        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig, null);
         CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
 
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@@ -650,7 +650,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
             null,
             null,
             null,
-            new CompoundProcessor(factory.create(Map.of(), null, null, pipelineConfig2))
+            new CompoundProcessor(factory.create(Map.of(), null, null, pipelineConfig2, null))
         );
 
         Pipeline pipeline2 = new Pipeline(
@@ -658,13 +658,13 @@ public class TrackingResultProcessorTests extends ESTestCase {
             null,
             null,
             null,
-            new CompoundProcessor(factory.create(Map.of(), null, null, pipelineConfig1))
+            new CompoundProcessor(factory.create(Map.of(), null, null, pipelineConfig1, null))
         );
 
         when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1);
         when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2);
 
-        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig0);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig0, null);
         CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
 
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@@ -694,7 +694,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
                 null,
                 null,
                 null,
-                new CompoundProcessor(factory.create(Map.of(), null, null, nextPipelineConfig))
+                new CompoundProcessor(factory.create(Map.of(), null, null, nextPipelineConfig, null))
             );
             when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
         }
@@ -708,7 +708,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
         String firstPipelineId = "pipeline0";
         Map<String, Object> firstPipelineConfig = new HashMap<>();
         firstPipelineConfig.put("name", firstPipelineId);
-        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, firstPipelineConfig);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, firstPipelineConfig, null);
         CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
 
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@@ -747,7 +747,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
                 null,
                 null,
                 null,
-                new CompoundProcessor(factory.create(Map.of(), null, null, nextPipelineConfig))
+                new CompoundProcessor(factory.create(Map.of(), null, null, nextPipelineConfig, null))
             );
             when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
         }
@@ -761,7 +761,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
         String firstPipelineId = "pipeline0";
         Map<String, Object> firstPipelineConfig = new HashMap<>();
         firstPipelineConfig.put("name", firstPipelineId);
-        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, firstPipelineConfig);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, firstPipelineConfig, null);
         CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
 
         CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@@ -789,7 +789,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
         PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService);
 
         String key1 = randomAlphaOfLength(10);
-        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig);
+        PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig, null);
         Pipeline pipeline = new Pipeline(pipelineId, null, null, null, new CompoundProcessor(new TestProcessor(ingestDocument -> {
             ingestDocument.setFieldValue(key1, randomInt());
         })));

test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java

@@ -20,7 +20,7 @@ import java.util.Map;
 public class IngestTestPlugin extends Plugin implements IngestPlugin {
     @Override
     public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
-        return Map.of("test", (factories, tag, description, config) -> new TestProcessor("id", "test", "description", doc -> {
+        return Map.of("test", (factories, tag, description, config, projectId) -> new TestProcessor("id", "test", "description", doc -> {
             doc.setFieldValue("processed", true);
             if (doc.hasField("fail") && doc.getFieldValue("fail", Boolean.class)) {
                 throw new IllegalArgumentException("test processor failed");

test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.ingest;
 
+import org.elasticsearch.cluster.metadata.ProjectId;
+
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiConsumer;
@@ -102,7 +104,8 @@ public class TestProcessor implements Processor {
             Map<String, Processor.Factory> registry,
             String processorTag,
             String description,
-            Map<String, Object> config
+            Map<String, Object> config,
+            ProjectId projectId
         ) throws Exception {
             return new TestProcessor(processorTag, "test-processor", description, ingestDocument -> {});
         }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidatorTests.java

@@ -599,7 +599,8 @@ public class SourceDestValidatorTests extends ESTestCase {
             Arrays.asList(Collections.singletonMap("test", processorConfig0), Collections.singletonMap("test", processorConfig1))
         );
         Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
-        Pipeline pipeline = Pipeline.create("missing-pipeline", pipelineConfig, processorRegistry, null);
+        var projectId = randomProjectIdOrDefault();
+        Pipeline pipeline = Pipeline.create("missing-pipeline", pipelineConfig, processorRegistry, null, projectId);
         when(ingestService.getPipeline("missing-pipeline")).thenReturn(pipeline);
 
         assertValidation(

x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java

@@ -13,6 +13,7 @@ import org.elasticsearch.common.cache.Cache;
 import org.elasticsearch.common.cache.CacheBuilder;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.core.FixForMultiProject;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.xpack.core.enrich.action.EnrichStatsAction;
@@ -89,6 +90,7 @@ public final class EnrichCache {
      * @param searchResponseFetcher The function used to compute the value to be put in the cache, if there is no value in the cache already
      * @param listener A listener to be notified of the value in the cache
      */
+    @FixForMultiProject(description = "The enrich cache will currently leak data between projects. We need to either disable or fix it.")
     public void computeIfAbsent(
         String enrichIndex,
         Object lookupValue,

x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutor.java

@@ -18,6 +18,7 @@ import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest;
 import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
@@ -115,19 +116,20 @@ public class EnrichPolicyExecutor {
     }
 
     public void runPolicyLocally(
+        ProjectId projectId,
         ExecuteEnrichPolicyTask task,
         String policyName,
         String enrichIndexName,
         ActionListener<ExecuteEnrichPolicyStatus> listener
     ) {
         try {
-            EnrichPolicy policy = EnrichStore.getPolicy(policyName, clusterService.state().metadata().getProject());
+            EnrichPolicy policy = EnrichStore.getPolicy(policyName, clusterService.state().metadata().getProject(projectId));
             if (policy == null) {
                 throw new ResourceNotFoundException("policy [{}] does not exist", policyName);
             }
 
             task.setStatus(new ExecuteEnrichPolicyStatus(ExecuteEnrichPolicyStatus.PolicyPhases.SCHEDULED));
-            var policyRunner = createPolicyRunner(policyName, policy, enrichIndexName, task);
+            var policyRunner = createPolicyRunner(projectId, policyName, policy, enrichIndexName, task);
             threadPool.executor(ThreadPool.Names.GENERIC)
                 .execute(ActionRunnable.wrap(ActionListener.assertOnce(listener), policyRunner::run));
         } catch (Exception e) {
@@ -209,12 +211,14 @@ public class EnrichPolicyExecutor {
     }
 
     private EnrichPolicyRunner createPolicyRunner(
+        ProjectId projectId,
         String policyName,
         EnrichPolicy policy,
         String enrichIndexName,
         ExecuteEnrichPolicyTask task
     ) {
         return new EnrichPolicyRunner(
+            projectId,
             policyName,
             policy,
             task,

x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyReindexPipeline.java

@@ -12,7 +12,7 @@ import org.elasticsearch.action.ingest.PutPipelineRequest;
 import org.elasticsearch.action.ingest.PutPipelineTransportAction;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
-import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.ingest.IngestMetadata;
 import org.elasticsearch.ingest.PipelineConfiguration;
@@ -47,11 +47,11 @@ public class EnrichPolicyReindexPipeline {
 
     /**
      * Checks if the current version of the pipeline definition is installed in the cluster
-     * @param clusterState The cluster state to check
+     * @param project The project metadata to check
      * @return true if a pipeline exists that is compatible with this version of Enrich, false otherwise
      */
-    static boolean exists(ClusterState clusterState) {
+    static boolean exists(ProjectMetadata project) {
-        final IngestMetadata ingestMetadata = clusterState.getMetadata().getProject().custom(IngestMetadata.TYPE);
+        final IngestMetadata ingestMetadata = project.custom(IngestMetadata.TYPE);
         // we ensure that we both have the pipeline and its version represents the current (or later) version
         if (ingestMetadata != null) {
             final PipelineConfiguration pipeline = ingestMetadata.getPipelines().get(pipelineName());

x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java

@@ -40,10 +40,11 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.FilterClient;
 import org.elasticsearch.client.internal.OriginSettingClient;
-import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
@@ -102,6 +103,7 @@ public class EnrichPolicyRunner {
      */
     static final TimeValue ENRICH_MASTER_REQUEST_TIMEOUT = TimeValue.THIRTY_SECONDS;
 
+    private final ProjectId projectId;
     private final String policyName;
     private final EnrichPolicy policy;
     private final ExecuteEnrichPolicyTask task;
@@ -114,6 +116,7 @@ public class EnrichPolicyRunner {
     private final int maxForceMergeAttempts;
 
     EnrichPolicyRunner(
+        ProjectId projectId,
         String policyName,
         EnrichPolicy policy,
         ExecuteEnrichPolicyTask task,
@@ -125,6 +128,7 @@ public class EnrichPolicyRunner {
         int fetchSize,
         int maxForceMergeAttempts
     ) {
+        this.projectId = projectId;
         this.policyName = Objects.requireNonNull(policyName);
         this.policy = Objects.requireNonNull(policy);
         this.task = Objects.requireNonNull(task);
@@ -480,7 +484,7 @@ public class EnrichPolicyRunner {
 
     private void prepareReindexOperation(ActionListener<AcknowledgedResponse> listener) {
         // Check to make sure that the enrich pipeline exists, and create it if it is missing.
-        if (EnrichPolicyReindexPipeline.exists(clusterService.state())) {
+        if (EnrichPolicyReindexPipeline.exists(clusterService.state().getMetadata().getProject(projectId))) {
             listener.onResponse(null);
         } else {
             EnrichPolicyReindexPipeline.create(enrichOriginClient(), listener);
@@ -700,8 +704,8 @@ public class EnrichPolicyRunner {
      * and recreated with invalid mappings/data. We validate that the mapping exists and that it contains the expected meta fields on it to
      * guard against accidental removal and recreation during policy execution.
      */
-    private void validateIndexBeforePromotion(String destinationIndexName, ClusterState clusterState) {
+    private void validateIndexBeforePromotion(String destinationIndexName, ProjectMetadata project) {
-        IndexMetadata destinationIndex = clusterState.metadata().getProject().index(destinationIndexName);
+        IndexMetadata destinationIndex = project.index(destinationIndexName);
         if (destinationIndex == null) {
             throw new IndexNotFoundException(
                 "was not able to promote it as part of executing enrich policy [" + policyName + "]",
@@ -749,12 +753,12 @@ public class EnrichPolicyRunner {
         String enrichIndexBase = EnrichPolicy.getBaseName(policyName);
         logger.debug("Policy [{}]: Promoting new enrich index [{}] to alias [{}]", policyName, enrichIndexName, enrichIndexBase);
         GetAliasesRequest aliasRequest = new GetAliasesRequest(ENRICH_MASTER_REQUEST_TIMEOUT, enrichIndexBase);
-        ClusterState clusterState = clusterService.state();
+        final var project = clusterService.state().metadata().getProject(projectId);
-        validateIndexBeforePromotion(enrichIndexName, clusterState);
+        validateIndexBeforePromotion(enrichIndexName, project);
-        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(clusterState, aliasRequest);
+        String[] concreteIndices = indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(project, aliasRequest);
         String[] aliases = aliasRequest.aliases();
         IndicesAliasesRequest aliasToggleRequest = new IndicesAliasesRequest(ENRICH_MASTER_REQUEST_TIMEOUT, ENRICH_MASTER_REQUEST_TIMEOUT);
-        String[] indices = clusterState.metadata().getProject().findAliases(aliases, concreteIndices).keySet().toArray(new String[0]);
+        String[] indices = project.findAliases(aliases, concreteIndices).keySet().toArray(new String[0]);
         if (indices.length > 0) {
             aliasToggleRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove().indices(indices).alias(enrichIndexBase));
         }

x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java

@@ -14,6 +14,8 @@ import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.common.geo.Orientation;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
@@ -50,20 +52,26 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer<Cluste
     }
 
     @Override
-    public Processor create(Map<String, Processor.Factory> processorFactories, String tag, String description, Map<String, Object> config)
+    public Processor create(
-        throws Exception {
+        Map<String, Processor.Factory> processorFactories,
+        String tag,
+        String description,
+        Map<String, Object> config,
+        ProjectId projectId
+    ) throws Exception {
         final String policyName = ConfigurationUtils.readStringProperty(TYPE, tag, config, "policy_name");
         final String indexAlias = EnrichPolicy.getBaseName(policyName);
         if (metadata == null) {
             throw new IllegalStateException("enrich processor factory has not yet been initialized with cluster state");
         }
-        IndexAbstraction indexAbstraction = metadata.getProject().getIndicesLookup().get(indexAlias);
+        final var project = metadata.getProject(projectId);
+        IndexAbstraction indexAbstraction = project.getIndicesLookup().get(indexAlias);
         if (indexAbstraction == null) {
             throw new IllegalArgumentException("no enrich index exists for policy with name [" + policyName + "]");
         }
         assert indexAbstraction.getType() == IndexAbstraction.Type.ALIAS;
         assert indexAbstraction.getIndices().size() == 1;
-        IndexMetadata imd = metadata.getProject().index(indexAbstraction.getIndices().get(0));
+        IndexMetadata imd = project.index(indexAbstraction.getIndices().get(0));
 
         Map<String, Object> mappingAsMap = imd.mapping().sourceAsMap();
         String policyType = (String) XContentMapValues.extractValue(
@@ -80,7 +88,7 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer<Cluste
         if (maxMatches < 1 || maxMatches > 128) {
             throw ConfigurationUtils.newConfigurationException(TYPE, tag, "max_matches", "should be between 1 and 128");
         }
-        var searchRunner = createSearchRunner(indexAlias, client, enrichCache);
+        var searchRunner = createSearchRunner(project, indexAlias);
         switch (policyType) {
             case EnrichPolicy.MATCH_TYPE:
             case EnrichPolicy.RANGE_TYPE:
@@ -125,12 +133,12 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer<Cluste
         metadata = state.getMetadata();
     }
 
-    private SearchRunner createSearchRunner(String indexAlias, Client client, EnrichCache enrichCache) {
+    private SearchRunner createSearchRunner(ProjectMetadata project, String indexAlias) {
         Client originClient = new OriginSettingClient(client, ENRICH_ORIGIN);
         return (value, maxMatches, reqSupplier, handler) -> {
             // intentionally non-locking for simplicity...it's OK if we re-put the same key/value in the cache during a race condition.
             enrichCache.computeIfAbsent(
-                getEnrichIndexKey(indexAlias),
+                getEnrichIndexKey(project, indexAlias),
                 value,
                 maxMatches,
                 (searchResponseActionListener) -> originClient.execute(
@@ -143,8 +151,8 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer<Cluste
         };
     }
 
-    private String getEnrichIndexKey(String indexAlias) {
+    private String getEnrichIndexKey(ProjectMetadata project, String indexAlias) {
-        IndexAbstraction ia = metadata.getProject().getIndicesLookup().get(indexAlias);
+        IndexAbstraction ia = project.getIndicesLookup().get(indexAlias);
         if (ia == null) {
             throw new IndexNotFoundException("no generated enrich index [" + indexAlias + "]");
         }