github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-06-27 17:10:22 -04:00
Code Issues Projects Releases Packages Wiki Activity

Make enrich project-aware (#124099)

Browse source 
Makes the execution and use of enrich policies project-aware.
Note: this does not make the enrich cache project-aware. That is to be
handled in a follow-up PR.
This commit is contained in:
Niels Bauman 2025-03-06 19:20:46 +01:00 • committed by GitHub
parent ff6465b83b
commit 20e186a252
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
119 changed files with 863 additions and 541 deletions
Download patch file Download diff file Expand all files Collapse all files

10
libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineBridge.java
View file

@ -8,6 +8,7 @@
*/
package org.elasticsearch.logstashbridge.ingest;
import org.elasticsearch.core.FixForMultiProject;
import org.elasticsearch.ingest.Pipeline;
import org.elasticsearch.logstashbridge.StableBridgeAPI;
import org.elasticsearch.logstashbridge.script.ScriptServiceBridge;
@ -20,6 +21,7 @@ public class PipelineBridge extends StableBridgeAPI.Proxy<Pipeline> {
return new PipelineBridge(pipeline);
}
@FixForMultiProject(description = "should we pass a non-null project ID here?")
public static PipelineBridge create(
String id,
Map<String, Object> config,
@ -27,7 +29,13 @@ public class PipelineBridge extends StableBridgeAPI.Proxy<Pipeline> {
ScriptServiceBridge scriptServiceBridge
) throws Exception {
return wrap(
Pipeline.create(id, config, StableBridgeAPI.unwrap(processorFactories), StableBridgeAPI.unwrapNullable(scriptServiceBridge))
Pipeline.create(
id,
config,
StableBridgeAPI.unwrap(processorFactories),
StableBridgeAPI.unwrapNullable(scriptServiceBridge),
null
)
);
}

8
libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ProcessorBridge.java
View file

@ -8,6 +8,7 @@
*/
package org.elasticsearch.logstashbridge.ingest;
import org.elasticsearch.core.FixForMultiProject;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.ingest.IngestService;
import org.elasticsearch.ingest.Processor;
@ -118,7 +119,7 @@ public interface ProcessorBridge extends StableBridgeAPI<Processor> {
@Override
default Processor.Factory unwrap() {
final Factory stableAPIFactory = this;
return (registry, tag, description, config) -> stableAPIFactory.create(
return (registry, tag, description, config, projectId) -> stableAPIFactory.create(
StableBridgeAPI.wrap(registry, Factory::wrap),
tag,
description,
@ -131,6 +132,7 @@ public interface ProcessorBridge extends StableBridgeAPI<Processor> {
super(delegate);
}
@FixForMultiProject(description = "should we pass a non-null project ID here?")
@Override
public ProcessorBridge create(
final Map<String, Factory> registry,
@ -138,7 +140,9 @@ public interface ProcessorBridge extends StableBridgeAPI<Processor> {
final String description,
final Map<String, Object> config
) throws Exception {
return ProcessorBridge.wrap(this.delegate.create(StableBridgeAPI.unwrap(registry), processorTag, description, config));
return ProcessorBridge.wrap(
this.delegate.create(StableBridgeAPI.unwrap(registry), processorTag, description, config, null)
);
}
@Override

11
modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java
View file

@ -403,9 +403,9 @@ public class IngestFailureStoreMetricsIT extends ESIntegTestCase {
Map<String, Processor.Factory> processors = new HashMap<>();
processors.put(
"drop",
(factories, tag, description, config) -> new TestProcessor(tag, "drop", description, ingestDocument -> null)
(factories, tag, description, config, projectId) -> new TestProcessor(tag, "drop", description, ingestDocument -> null)
);
processors.put("reroute", (factories, tag, description, config) -> {
processors.put("reroute", (factories, tag, description, config, projectId) -> {
String destination = (String) config.remove("destination");
return new TestProcessor(
tag,
@ -416,7 +416,12 @@ public class IngestFailureStoreMetricsIT extends ESIntegTestCase {
});
processors.put(
"fail",
(processorFactories, tag, description, config) -> new TestProcessor(tag, "fail", description, new RuntimeException())
(processorFactories, tag, description, config, projectId) -> new TestProcessor(
tag,
"fail",
description,
new RuntimeException()
)
);
return processors;
}

4
modules/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java
View file

@ -15,6 +15,7 @@ import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.Office;
import org.apache.tika.metadata.TikaCoreProperties;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.logging.DeprecationCategory;
import org.elasticsearch.common.logging.DeprecationLogger;
@ -232,7 +233,8 @@ public final class AttachmentProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) {
String field = readStringProperty(TYPE, processorTag, config, "field");
String resourceName = readOptionalStringProperty(TYPE, processorTag, config, "resource_name");

16
modules/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorFactoryTests.java
View file

@ -36,7 +36,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
AttachmentProcessor processor = factory.create(null, processorTag, null, config);
AttachmentProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("attachment"));
@ -57,7 +57,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
config.put("indexed_chars", indexedChars);
String processorTag = randomAlphaOfLength(10);
AttachmentProcessor processor = factory.create(null, processorTag, null, config);
AttachmentProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getIndexedChars(), is(indexedChars));
assertFalse(processor.isIgnoreMissing());
@ -73,7 +73,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("target_field", "_field");
AttachmentProcessor processor = factory.create(null, null, null, config);
AttachmentProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("_field"));
assertFalse(processor.isIgnoreMissing());
@ -97,7 +97,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("properties", fieldNames);
AttachmentProcessor processor = factory.create(null, null, null, config);
AttachmentProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getProperties(), equalTo(properties));
assertFalse(processor.isIgnoreMissing());
@ -114,7 +114,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field");
config.put("properties", Collections.singletonList("invalid"));
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("exception expected");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), containsString("[properties] illegal field option [invalid]"));
@ -128,7 +128,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field");
config.put("properties", "invalid");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("exception expected");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
@ -148,7 +148,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
AttachmentProcessor processor = factory.create(null, processorTag, null, config);
AttachmentProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("attachment"));
@ -169,7 +169,7 @@ public class AttachmentProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
AttachmentProcessor processor = factory.create(null, processorTag, null, config);
AttachmentProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("attachment"));

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
@ -108,7 +109,8 @@ abstract class AbstractStringProcessor<T> extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String tag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = ConfigurationUtils.readStringProperty(processorType, tag, config, "field");
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(processorType, tag, config, "ignore_missing", false);

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
@ -73,7 +74,8 @@ public final class AppendProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value");

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CommunityIdProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.common.network.InetAddresses;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
@ -297,7 +298,8 @@ public final class CommunityIdProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String sourceIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_ip", DEFAULT_SOURCE_IP);
String sourcePortField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_port", DEFAULT_SOURCE_PORT);

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.common.network.InetAddresses;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
@ -216,7 +217,8 @@ public final class ConvertProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String typeProperty = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type");

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/CsvProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
@ -93,7 +94,8 @@ public final class CsvProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String quote = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "quote", "\"");

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java
View file

@ -10,6 +10,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
@ -150,7 +151,8 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String tag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String localeString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "locale");
String timezoneString = ConfigurationUtils.readOptionalStringProperty(TYPE, tag, config, "timezone");

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java
View file

@ -10,6 +10,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.util.LocaleUtils;
@ -183,7 +184,8 @@ public final class DateProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET_FIELD);

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.dissect.DissectParser;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
@ -60,7 +61,8 @@ public final class DissectProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String pattern = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "pattern");

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
@ -120,7 +121,8 @@ public final class DotExpanderProcessor extends AbstractProcessor {
Map<String, Processor.Factory> processorFactories,
String tag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
if (field.contains(".") == false && field.equals("*") == false) {

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
@ -60,7 +61,8 @@ public final class FailProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String message = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "message");
TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FingerprintProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.hash.Murmur3Hasher;
import org.elasticsearch.common.util.ByteUtils;
@ -227,7 +228,8 @@ public final class FingerprintProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
List<String> fields = ConfigurationUtils.readList(TYPE, processorTag, config, "fields");
if (fields.size() < 1) {

12
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.util.Maps;
import org.elasticsearch.ingest.AbstractProcessor;
@ -235,8 +236,13 @@ public final class ForEachProcessor extends AbstractProcessor implements Wrappin
}
@Override
public ForEachProcessor create(Map<String, Processor.Factory> factories, String tag, String description, Map<String, Object> config)
throws Exception {
public ForEachProcessor create(
Map<String, Processor.Factory> factories,
String tag,
String description,
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = readStringProperty(TYPE, tag, config, "field");
boolean ignoreMissing = readBooleanProperty(TYPE, tag, config, "ignore_missing", false);
Map<String, Map<String, Object>> processorConfig = readMap(TYPE, tag, config, "processor");
@ -245,7 +251,7 @@ public final class ForEachProcessor extends AbstractProcessor implements Wrappin
throw newConfigurationException(TYPE, tag, "processor", "Must specify exactly one processor type");
}
Map.Entry<String, Map<String, Object>> entry = entries.iterator().next();
Processor processor = ConfigurationUtils.readProcessor(factories, scriptService, entry.getKey(), entry.getValue());
Processor processor = ConfigurationUtils.readProcessor(factories, scriptService, entry.getKey(), entry.getValue(), projectId);
return new ForEachProcessor(tag, description, field, processor, ignoreMissing);
}
}

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java
View file

@ -11,6 +11,7 @@ package org.elasticsearch.ingest.common;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.grok.Grok;
import org.elasticsearch.grok.GrokBuiltinPatterns;
import org.elasticsearch.grok.MatcherWatchdog;
@ -150,7 +151,8 @@ public final class GrokProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String matchField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
List<String> matchPatterns = ConfigurationUtils.readList(TYPE, processorTag, config, "patterns");

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
@ -71,7 +72,8 @@ public final class JoinProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String separator = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "separator");

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.core.Strings;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
@ -217,7 +218,8 @@ public final class JsonProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String targetField = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "target_field");

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.core.Predicates;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
@ -256,7 +257,8 @@ public final class KeyValueProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
TemplateScript.Factory fieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", field, scriptService);

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.common.network.CIDRUtils;
import org.elasticsearch.common.network.InetAddresses;
import org.elasticsearch.ingest.AbstractProcessor;
@ -251,7 +252,8 @@ public class NetworkDirectionProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
final String sourceIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_ip", DEFAULT_SOURCE_IP);
final String destIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "destination_ip", DEFAULT_DEST_IP);

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RegisteredDomainProcessor.java
View file

@ -11,6 +11,7 @@ package org.elasticsearch.ingest.common;
import org.apache.http.conn.util.PublicSuffixMatcher;
import org.apache.http.conn.util.PublicSuffixMatcherLoader;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.common.Strings;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.ingest.AbstractProcessor;
@ -140,7 +141,8 @@ public class RegisteredDomainProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String tag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field");
String targetField = ConfigurationUtils.readStringProperty(TYPE, tag, config, "target_field", DEFAULT_TARGET_FIELD);

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
@ -109,7 +110,8 @@ public final class RemoveProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
final List<TemplateScript.Factory> compiledTemplatesToRemove = getTemplates(processorTag, config, "field");
final List<TemplateScript.Factory> compiledTemplatesToKeep = getTemplates(processorTag, config, "keep");

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
@ -110,7 +111,8 @@ public final class RenameProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
TemplateScript.Factory fieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", field, scriptService);

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RerouteProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
@ -175,7 +176,8 @@ public final class RerouteProcessor extends AbstractProcessor {
Map<String, Processor.Factory> processorFactories,
String tag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
List<DataStreamValueSource> type;
try {

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.XContentHelper;
@ -105,7 +106,8 @@ public final class ScriptProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
try (
XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent).map(config);

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
@ -109,7 +110,8 @@ public final class SetProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String copyFrom = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "copy_from");

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
@ -115,7 +116,8 @@ public final class SortProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, FIELD);
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", field);

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
import org.elasticsearch.ingest.IngestDocument;
@ -99,7 +100,8 @@ public final class SplitProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/TerminateProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.IngestDocument;
import org.elasticsearch.ingest.Processor;
@ -45,7 +46,8 @@ public class TerminateProcessor extends AbstractProcessor {
Map<String, Processor.Factory> processorFactories,
String tag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) {
return new TerminateProcessor(tag, description);
}

4
modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/UriPartsProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.common;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.core.SuppressForbidden;
import org.elasticsearch.ingest.AbstractProcessor;
import org.elasticsearch.ingest.ConfigurationUtils;
@ -184,7 +185,8 @@ public class UriPartsProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", "url");

8
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AbstractStringProcessorFactoryTestCase.java
View file

@ -37,7 +37,7 @@ public abstract class AbstractStringProcessorFactoryTestCase extends ESTestCase
Map<String, Object> config = new HashMap<>();
config.put("field", fieldName);
AbstractStringProcessor<?> processor = factory.create(null, processorTag, null, modifyConfig(config));
AbstractStringProcessor<?> processor = factory.create(null, processorTag, null, modifyConfig(config), null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo(fieldName));
assertThat(processor.isIgnoreMissing(), is(false));
@ -54,7 +54,7 @@ public abstract class AbstractStringProcessorFactoryTestCase extends ESTestCase
config.put("field", fieldName);
config.put("ignore_missing", true);
AbstractStringProcessor<?> processor = factory.create(null, processorTag, null, modifyConfig(config));
AbstractStringProcessor<?> processor = factory.create(null, processorTag, null, modifyConfig(config), null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo(fieldName));
assertThat(processor.isIgnoreMissing(), is(true));
@ -72,7 +72,7 @@ public abstract class AbstractStringProcessorFactoryTestCase extends ESTestCase
config.put("field", fieldName);
config.put("target_field", targetFieldName);
AbstractStringProcessor<?> processor = factory.create(null, processorTag, null, modifyConfig(config));
AbstractStringProcessor<?> processor = factory.create(null, processorTag, null, modifyConfig(config), null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo(fieldName));
assertThat(processor.isIgnoreMissing(), is(false));
@ -84,7 +84,7 @@ public abstract class AbstractStringProcessorFactoryTestCase extends ESTestCase
AbstractStringProcessor.Factory factory = newFactory();
Map<String, Object> config = new HashMap<>();
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[field] required property is missing"));

17
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorFactoryTests.java
View file

@ -43,7 +43,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
}
config.put("value", value);
String processorTag = randomAlphaOfLength(10);
AppendProcessor appendProcessor = factory.create(null, processorTag, null, config);
AppendProcessor appendProcessor = factory.create(null, processorTag, null, config, null);
assertThat(appendProcessor.getTag(), equalTo(processorTag));
assertThat(appendProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
assertThat(appendProcessor.getValue().copyAndResolve(Map.of()), equalTo(value));
@ -53,7 +53,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("value", "value1");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@ -64,7 +64,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "field1");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[value] required property is missing"));
@ -76,7 +76,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
config.put("field", "field1");
config.put("value", null);
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[value] required property is missing"));
@ -91,7 +91,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
ElasticsearchException exception = expectThrows(
ElasticsearchException.class,
() -> factory.create(null, processorTag, null, config)
() -> factory.create(null, processorTag, null, config, null)
);
assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));
@ -105,7 +105,7 @@ public class AppendProcessorFactoryTests extends ESTestCase {
config.put("value", "value1");
config.put("media_type", expectedMediaType);
String processorTag = randomAlphaOfLength(10);
AppendProcessor appendProcessor = factory.create(null, processorTag, null, config);
AppendProcessor appendProcessor = factory.create(null, processorTag, null, config, null);
assertThat(appendProcessor.getTag(), equalTo(processorTag));
// invalid media type
@ -117,7 +117,10 @@ public class AppendProcessorFactoryTests extends ESTestCase {
config2.put("field", "field1");
config2.put("value", "value1");
config2.put("media_type", expectedMediaType);
ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config2));
ElasticsearchException e = expectThrows(
ElasticsearchException.class,
() -> factory.create(null, processorTag, null, config2, null)
);
assertThat(e.getMessage(), containsString("property does not contain a supported media type [" + expectedMediaType + "]"));
}
}

10
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorFactoryTests.java
View file

@ -65,7 +65,7 @@ public class CommunityIdProcessorFactoryTests extends ESTestCase {
config.put("ignore_missing", ignoreMissing);
String processorTag = randomAlphaOfLength(10);
CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config);
CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config, null);
assertThat(communityIdProcessor.getTag(), equalTo(processorTag));
assertThat(communityIdProcessor.getSourceIpField(), equalTo(sourceIpField));
assertThat(communityIdProcessor.getSourcePortField(), equalTo(sourcePortField));
@ -87,27 +87,27 @@ public class CommunityIdProcessorFactoryTests extends ESTestCase {
// negative seeds are rejected
int tooSmallSeed = randomIntBetween(Integer.MIN_VALUE, -1);
config.put("seed", Integer.toString(tooSmallSeed));
ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config));
ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null));
assertThat(e.getMessage(), containsString("must be a value between 0 and 65535"));
// seeds >= 2^16 are rejected
int tooBigSeed = randomIntBetween(65536, Integer.MAX_VALUE);
config.put("seed", Integer.toString(tooBigSeed));
e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config));
e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null));
assertThat(e.getMessage(), containsString("must be a value between 0 and 65535"));
// seeds between 0 and 2^16-1 are accepted
int justRightSeed = randomIntBetween(0, 65535);
byte[] expectedSeed = new byte[] { (byte) (justRightSeed >> 8), (byte) justRightSeed };
config.put("seed", Integer.toString(justRightSeed));
CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config);
CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config, null);
assertThat(communityIdProcessor.getSeed(), equalTo(expectedSeed));
}
public void testRequiredFields() throws Exception {
HashMap<String, Object> config = new HashMap<>();
String processorTag = randomAlphaOfLength(10);
CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config);
CommunityIdProcessor communityIdProcessor = factory.create(null, processorTag, null, config, null);
assertThat(communityIdProcessor.getTag(), equalTo(processorTag));
assertThat(communityIdProcessor.getSourceIpField(), equalTo(DEFAULT_SOURCE_IP));
assertThat(communityIdProcessor.getSourcePortField(), equalTo(DEFAULT_SOURCE_PORT));

12
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorFactoryTests.java
View file

@ -28,7 +28,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
config.put("field", "field1");
config.put("type", type.toString());
String processorTag = randomAlphaOfLength(10);
ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config);
ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config, null);
assertThat(convertProcessor.getTag(), equalTo(processorTag));
assertThat(convertProcessor.getField(), equalTo("field1"));
assertThat(convertProcessor.getTargetField(), equalTo("field1"));
@ -43,7 +43,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
config.put("field", "field1");
config.put("type", type);
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[type] type [" + type + "] not supported, cannot convert field."));
@ -59,7 +59,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
String type = "type-" + randomAlphaOfLengthBetween(1, 10);
config.put("type", type);
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@ -71,7 +71,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "field1");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[type] required property is missing"));
@ -86,7 +86,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
config.put("target_field", "field2");
config.put("type", type.toString());
String processorTag = randomAlphaOfLength(10);
ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config);
ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config, null);
assertThat(convertProcessor.getTag(), equalTo(processorTag));
assertThat(convertProcessor.getField(), equalTo("field1"));
assertThat(convertProcessor.getTargetField(), equalTo("field2"));
@ -102,7 +102,7 @@ public class ConvertProcessorFactoryTests extends ESTestCase {
config.put("type", type.toString());
config.put("ignore_missing", true);
String processorTag = randomAlphaOfLength(10);
ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config);
ConvertProcessor convertProcessor = factory.create(null, processorTag, null, config, null);
assertThat(convertProcessor.getTag(), equalTo(processorTag));
assertThat(convertProcessor.getField(), equalTo("field1"));
assertThat(convertProcessor.getTargetField(), equalTo("field1"));

2
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CsvProcessorFactoryTests.java
View file

@ -31,7 +31,7 @@ public class CsvProcessorFactoryTests extends ESTestCase {
properties.put("empty_value", "empty");
properties.put("trim", true);
properties.put("ignore_missing", true);
CsvProcessor csv = factory.create(null, "csv", null, properties);
CsvProcessor csv = factory.create(null, "csv", null, properties, null);
assertThat(csv, notNullValue());
assertThat(csv.field, equalTo("field"));
assertThat(csv.headers, equalTo(new String[] { "target" }));

17
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java
View file

@ -28,7 +28,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
config.put("field", "_field");
config.put("date_rounding", "y");
DateIndexNameProcessor processor = factory.create(null, null, null, config);
DateIndexNameProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getDateFormats().size(), equalTo(1));
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getIndexNamePrefixTemplate().newInstance(Map.of()).execute(), equalTo(""));
@ -45,7 +45,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
config.put("date_rounding", "y");
config.put("date_formats", List.of("UNIX", "UNIX_MS"));
DateIndexNameProcessor processor = factory.create(null, null, null, config);
DateIndexNameProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getDateFormats().size(), equalTo(2));
config = new HashMap<>();
@ -54,7 +54,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
config.put("date_rounding", "y");
config.put("index_name_format", "yyyyMMdd");
processor = factory.create(null, null, null, config);
processor = factory.create(null, null, null, config, null);
assertThat(processor.getIndexNameFormatTemplate().newInstance(Map.of()).execute(), equalTo("yyyyMMdd"));
config = new HashMap<>();
@ -63,7 +63,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
config.put("date_rounding", "y");
config.put("timezone", "+02:00");
processor = factory.create(null, null, null, config);
processor = factory.create(null, null, null, config, null);
assertThat(processor.getTimezone(), equalTo(ZoneOffset.ofHours(2)));
config = new HashMap<>();
@ -71,7 +71,7 @@ public class DateIndexNameFactoryTests extends ESTestCase {
config.put("index_name_prefix", "_prefix");
config.put("date_rounding", "y");
processor = factory.create(null, null, null, config);
processor = factory.create(null, null, null, config, null);
assertThat(processor.getIndexNamePrefixTemplate().newInstance(Map.of()).execute(), equalTo("_prefix"));
}
@ -79,12 +79,15 @@ public class DateIndexNameFactoryTests extends ESTestCase {
DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance());
Map<String, Object> config = new HashMap<>();
config.put("date_rounding", "y");
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> factory.create(null, null, null, config, null)
);
assertThat(e.getMessage(), equalTo("[field] required property is missing"));
config.clear();
config.put("field", "_field");
e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config, null));
assertThat(e.getMessage(), equalTo("[date_rounding] required property is missing"));
}
}

22
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateProcessorFactoryTests.java
View file

@ -38,7 +38,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
config.put("field", sourceField);
config.put("formats", List.of("dd/MM/yyyyy"));
String processorTag = randomAlphaOfLength(10);
DateProcessor processor = factory.create(null, processorTag, null, config);
DateProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo(sourceField));
assertThat(processor.getTargetField(), equalTo(DateProcessor.DEFAULT_TARGET_FIELD));
@ -54,7 +54,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
config.put("formats", List.of("dd/MM/yyyyy"));
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("processor creation should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), containsString("[field] required property is missing"));
@ -69,7 +69,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
config.put("target_field", targetField);
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("processor creation should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), containsString("[formats] required property is missing"));
@ -84,7 +84,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
Locale locale = randomFrom(Locale.GERMANY, Locale.FRENCH, Locale.ROOT);
config.put("locale", locale.toLanguageTag());
DateProcessor processor = factory.create(null, null, null, config);
DateProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getLocale().newInstance(Map.of()).execute(), equalTo(locale.toLanguageTag()));
}
@ -96,7 +96,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
ZoneId timezone = randomZone();
config.put("timezone", timezone.getId());
DateProcessor processor = factory.create(null, null, null, config);
DateProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getTimezone().newInstance(Map.of()).execute(), equalTo(timezone.getId()));
}
@ -106,7 +106,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
config.put("field", sourceField);
config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
DateProcessor processor = factory.create(null, null, null, config);
DateProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getFormats(), equalTo(List.of("dd/MM/yyyy", "dd-MM-yyyy")));
}
@ -117,7 +117,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
config.put("formats", "dd/MM/yyyy");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("processor creation should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), containsString("[formats] property isn't a list, but of type [java.lang.String]"));
@ -132,7 +132,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
config.put("target_field", targetField);
config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
DateProcessor processor = factory.create(null, null, null, config);
DateProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getTargetField(), equalTo(targetField));
}
@ -145,7 +145,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
config.put("target_field", targetField);
config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
config.put("output_format", outputFormat);
DateProcessor processor = factory.create(null, null, null, config);
DateProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getOutputFormat(), equalTo(outputFormat));
}
@ -156,7 +156,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
config.put("field", sourceField);
config.put("target_field", targetField);
config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
DateProcessor processor = factory.create(null, null, null, config);
DateProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getOutputFormat(), equalTo(DateProcessor.DEFAULT_OUTPUT_FORMAT));
}
@ -170,7 +170,7 @@ public class DateProcessorFactoryTests extends ESTestCase {
config.put("formats", List.of("dd/MM/yyyy", "dd-MM-yyyy"));
config.put("output_format", outputFormat);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, null, null, config));
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> factory.create(null, null, null, config, null));
assertThat(e.getMessage(), containsString("invalid output format [" + outputFormat + "]"));
}
}

10
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorFactoryTests.java
View file

@ -36,7 +36,7 @@ public class DissectProcessorFactoryTests extends ESTestCase {
config.put("append_separator", appendSeparator);
config.put("ignore_missing", true);
DissectProcessor processor = factory.create(null, processorTag, null, config);
DissectProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.field, equalTo(fieldName));
assertThat(processor.pattern, equalTo(pattern));
@ -49,7 +49,7 @@ public class DissectProcessorFactoryTests extends ESTestCase {
DissectProcessor.Factory factory = new DissectProcessor.Factory();
Map<String, Object> config = new HashMap<>();
config.put("pattern", "%{a},%{b},%{c}");
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null));
assertThat(e.getMessage(), equalTo("[field] required property is missing"));
}
@ -57,7 +57,7 @@ public class DissectProcessorFactoryTests extends ESTestCase {
DissectProcessor.Factory factory = new DissectProcessor.Factory();
Map<String, Object> config = new HashMap<>();
config.put("field", randomAlphaOfLength(10));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null));
assertThat(e.getMessage(), equalTo("[pattern] required property is missing"));
}
@ -66,7 +66,7 @@ public class DissectProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("pattern", "%{a},%{b},%{c}");
config.put("field", randomAlphaOfLength(10));
DissectProcessor processor = factory.create(null, "_tag", null, config);
DissectProcessor processor = factory.create(null, "_tag", null, config, null);
assertThat(processor.appendSeparator, equalTo(""));
assertThat(processor.ignoreMissing, is(false));
}
@ -76,6 +76,6 @@ public class DissectProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("pattern", "no keys defined");
config.put("field", randomAlphaOfLength(10));
expectThrows(DissectException.class, () -> factory.create(null, "_tag", null, config));
expectThrows(DissectException.class, () -> factory.create(null, "_tag", null, config, null));
}
}

14
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorFactoryTests.java
View file

@ -26,13 +26,13 @@ public class DotExpanderProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "_field.field");
config.put("path", "_path");
DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config);
DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config, null);
assertThat(processor.getField(), equalTo("_field.field"));
assertThat(processor.getPath(), equalTo("_path"));
config = new HashMap<>();
config.put("field", "_field.field");
processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config);
processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config, null);
assertThat(processor.getField(), equalTo("_field.field"));
assertThat(processor.getPath(), nullValue());
}
@ -45,7 +45,7 @@ public class DotExpanderProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", field);
config.put("path", "_path");
DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config);
DotExpanderProcessor processor = (DotExpanderProcessor) factory.create(null, "_tag", null, config, null);
assertThat(processor.getField(), equalTo(field));
assertThat(processor.getPath(), equalTo("_path"));
}
@ -56,7 +56,7 @@ public class DotExpanderProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("path", "_path");
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null));
assertThat(e.getMessage(), equalTo("[field] required property is missing"));
}
@ -66,7 +66,7 @@ public class DotExpanderProcessorFactoryTests extends ESTestCase {
for (String field : fields) {
Map<String, Object> config = new HashMap<>();
config.put("field", field);
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null));
assertThat(e.getMessage(), equalTo("[field] field does not contain a dot and is not a wildcard"));
}
@ -74,7 +74,7 @@ public class DotExpanderProcessorFactoryTests extends ESTestCase {
for (String field : fields) {
Map<String, Object> config = new HashMap<>();
config.put("field", field);
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null));
assertThat(e.getMessage(), equalTo("[field] Field can't start or end with a dot"));
}
@ -82,7 +82,7 @@ public class DotExpanderProcessorFactoryTests extends ESTestCase {
for (String field : fields) {
Map<String, Object> config = new HashMap<>();
config.put("field", field);
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, "_tag", null, config, null));
assertThat(e.getMessage(), equalTo("[field] No space between dots"));
}
}

6
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FailProcessorFactoryTests.java
View file

@ -33,7 +33,7 @@ public class FailProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("message", "error");
String processorTag = randomAlphaOfLength(10);
FailProcessor failProcessor = factory.create(null, processorTag, null, config);
FailProcessor failProcessor = factory.create(null, processorTag, null, config, null);
assertThat(failProcessor.getTag(), equalTo(processorTag));
assertThat(failProcessor.getMessage().newInstance(Map.of()).execute(), equalTo("error"));
}
@ -41,7 +41,7 @@ public class FailProcessorFactoryTests extends ESTestCase {
public void testCreateMissingMessageField() throws Exception {
Map<String, Object> config = new HashMap<>();
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[message] required property is missing"));
@ -55,7 +55,7 @@ public class FailProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
ElasticsearchException exception = expectThrows(
ElasticsearchException.class,
() -> factory.create(null, processorTag, null, config)
() -> factory.create(null, processorTag, null, config, null)
);
assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));

14
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorFactoryTests.java
View file

@ -48,7 +48,7 @@ public class FingerprintProcessorFactoryTests extends ESTestCase {
config.put("ignore_missing", ignoreMissing);
String processorTag = randomAlphaOfLength(10);
FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config);
FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config, null);
assertThat(fingerprintProcessor.getTag(), equalTo(processorTag));
assertThat(fingerprintProcessor.getFields(), equalTo(sortedFieldList));
assertThat(fingerprintProcessor.getTargetField(), equalTo(targetField));
@ -68,7 +68,7 @@ public class FingerprintProcessorFactoryTests extends ESTestCase {
config.put("method", method);
String processorTag = randomAlphaOfLength(10);
FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config);
FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config, null);
assertThat(fingerprintProcessor.getTag(), equalTo(processorTag));
assertThat(fingerprintProcessor.getFields(), equalTo(sortedFieldList));
assertThat(fingerprintProcessor.getThreadLocalHasher().get().getAlgorithm(), equalTo(method));
@ -80,7 +80,7 @@ public class FingerprintProcessorFactoryTests extends ESTestCase {
);
config.put("fields", fieldList);
config.put("method", invalidMethod);
ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config));
ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null));
assertThat(e.getMessage(), containsString("[" + invalidMethod + "] must be one of the supported hash methods ["));
}
@ -93,17 +93,17 @@ public class FingerprintProcessorFactoryTests extends ESTestCase {
config.put("fields", fieldList);
String processorTag = randomAlphaOfLength(10);
FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config);
FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config, null);
assertThat(fingerprintProcessor.getTag(), equalTo(processorTag));
assertThat(fingerprintProcessor.getFields(), equalTo(sortedFieldList));
// fields is a list of length zero
config.put("fields", List.of());
ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config));
ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null));
assertThat(e.getMessage(), containsString("must specify at least one field"));
// fields is missing
e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config));
e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config, null));
assertThat(e.getMessage(), containsString("[fields] required property is missing"));
}
@ -115,7 +115,7 @@ public class FingerprintProcessorFactoryTests extends ESTestCase {
HashMap<String, Object> config = new HashMap<>();
config.put("fields", fieldList);
FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config);
FingerprintProcessor fingerprintProcessor = factory.create(null, processorTag, null, config, null);
assertThat(fingerprintProcessor.getTag(), equalTo(processorTag));
assertThat(fingerprintProcessor.getFields(), equalTo(sortedFieldList));
assertThat(fingerprintProcessor.getTargetField(), equalTo(FingerprintProcessor.Factory.DEFAULT_TARGET));

4
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorTests.java
View file

@ -215,7 +215,7 @@ public class FingerprintProcessorTests extends ESTestCase {
if (salt != null) {
config.put("salt", salt);
}
FingerprintProcessor fp = factory.create(null, randomAlphaOfLength(10), null, config);
FingerprintProcessor fp = factory.create(null, randomAlphaOfLength(10), null, config, null);
byte[] expectedBytes = new byte[0];
if (salt != null) {
@ -257,7 +257,7 @@ public class FingerprintProcessorTests extends ESTestCase {
config.put("fields", List.of("foo", "bar"));
config.put("method", FingerprintProcessor.Factory.SUPPORTED_DIGESTS[k]);
FingerprintProcessor fp = factory.create(null, randomAlphaOfLength(10), null, config);
FingerprintProcessor fp = factory.create(null, randomAlphaOfLength(10), null, config, null);
var input = TestIngestDocument.withDefaultVersion(inputMap);
var output = fp.execute(input);
assertTrue(output.hasField("fingerprint"));

25
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorFactoryTests.java
View file

@ -32,13 +32,13 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
public void testCreate() throws Exception {
Processor processor = new TestProcessor(ingestDocument -> {});
Map<String, Processor.Factory> registry = new HashMap<>();
registry.put("_name", (r, t, description, c) -> processor);
registry.put("_name", (r, t, description, c, projectId) -> processor);
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("processor", Map.of("_name", Collections.emptyMap()));
ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config);
ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config, null);
assertThat(forEachProcessor, notNullValue());
assertThat(forEachProcessor.getField(), equalTo("_field"));
assertThat(forEachProcessor.getInnerProcessor(), sameInstance(processor));
@ -48,14 +48,14 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
public void testSetIgnoreMissing() throws Exception {
Processor processor = new TestProcessor(ingestDocument -> {});
Map<String, Processor.Factory> registry = new HashMap<>();
registry.put("_name", (r, t, description, c) -> processor);
registry.put("_name", (r, t, description, c, projectId) -> processor);
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("processor", Map.of("_name", Collections.emptyMap()));
config.put("ignore_missing", true);
ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config);
ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config, null);
assertThat(forEachProcessor, notNullValue());
assertThat(forEachProcessor.getField(), equalTo("_field"));
assertThat(forEachProcessor.getInnerProcessor(), sameInstance(processor));
@ -65,8 +65,8 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
public void testCreateWithTooManyProcessorTypes() throws Exception {
Processor processor = new TestProcessor(ingestDocument -> {});
Map<String, Processor.Factory> registry = new HashMap<>();
registry.put("_first", (r, t, description, c) -> processor);
registry.put("_second", (r, t, description, c) -> processor);
registry.put("_first", (r, t, description, c, projectId) -> processor);
registry.put("_second", (r, t, description, c, projectId) -> processor);
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
Map<String, Object> config = new HashMap<>();
@ -75,7 +75,10 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
processorTypes.put("_first", Map.of());
processorTypes.put("_second", Map.of());
config.put("processor", processorTypes);
Exception exception = expectThrows(ElasticsearchParseException.class, () -> forEachFactory.create(registry, null, null, config));
Exception exception = expectThrows(
ElasticsearchParseException.class,
() -> forEachFactory.create(registry, null, null, config, null)
);
assertThat(exception.getMessage(), equalTo("[processor] Must specify exactly one processor type"));
}
@ -86,7 +89,7 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
config.put("processor", Map.of("_name", Collections.emptyMap()));
Exception expectedException = expectThrows(
ElasticsearchParseException.class,
() -> forEachFactory.create(Map.of(), null, null, config)
() -> forEachFactory.create(Map.of(), null, null, config, null)
);
assertThat(expectedException.getMessage(), equalTo("No processor type exists with name [_name]"));
}
@ -94,11 +97,11 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
public void testCreateWithMissingField() throws Exception {
Processor processor = new TestProcessor(ingestDocument -> {});
Map<String, Processor.Factory> registry = new HashMap<>();
registry.put("_name", (r, t, description, c) -> processor);
registry.put("_name", (r, t, description, c, projectId) -> processor);
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
Map<String, Object> config = new HashMap<>();
config.put("processor", List.of(Map.of("_name", Map.of())));
Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(registry, null, null, config));
Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(registry, null, null, config, null));
assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
}
@ -106,7 +109,7 @@ public class ForEachProcessorFactoryTests extends ESTestCase {
ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(Map.of(), null, null, config));
Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(Map.of(), null, null, config, null));
assertThat(exception.getMessage(), equalTo("[processor] required property is missing"));
}

36
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorFactoryTests.java
View file

@ -30,7 +30,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
config.put("field", "_field");
config.put("patterns", List.of("(?<foo>\\w+)"));
String processorTag = randomAlphaOfLength(10);
GrokProcessor processor = factory.create(null, processorTag, null, config);
GrokProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getMatchField(), equalTo("_field"));
assertThat(processor.getGrok(), notNullValue());
@ -45,7 +45,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
config.put("patterns", List.of("(?<foo>\\w+)"));
config.put("ignore_missing", true);
String processorTag = randomAlphaOfLength(10);
GrokProcessor processor = factory.create(null, processorTag, null, config);
GrokProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getMatchField(), equalTo("_field"));
assertThat(processor.getGrok(), notNullValue());
@ -56,7 +56,10 @@ public class GrokProcessorFactoryTests extends ESTestCase {
GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
Map<String, Object> config = new HashMap<>();
config.put("patterns", List.of("(?<foo>\\w+)"));
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> factory.create(null, null, null, config, null)
);
assertThat(e.getMessage(), equalTo("[field] required property is missing"));
}
@ -64,7 +67,10 @@ public class GrokProcessorFactoryTests extends ESTestCase {
GrokProcessor.Factory factory = new GrokProcessor.Factory(MatcherWatchdog.noop());
Map<String, Object> config = new HashMap<>();
config.put("field", "foo");
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> factory.create(null, null, null, config, null)
);
assertThat(e.getMessage(), equalTo("[patterns] required property is missing"));
}
@ -73,7 +79,10 @@ public class GrokProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "foo");
config.put("patterns", List.of());
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> factory.create(null, null, null, config, null)
);
assertThat(e.getMessage(), equalTo("[patterns] List of patterns must not be empty"));
}
@ -84,7 +93,7 @@ public class GrokProcessorFactoryTests extends ESTestCase {
config.put("field", "_field");
config.put("patterns", List.of("%{MY_PATTERN:name}!"));
config.put("pattern_definitions", Map.of("MY_PATTERN", "foo"));
GrokProcessor processor = factory.create(null, null, null, config);
GrokProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getMatchField(), equalTo("_field"));
assertThat(processor.getGrok(), notNullValue());
assertThat(processor.getGrok().match("foo!"), equalTo(true));
@ -95,7 +104,10 @@ public class GrokProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("patterns", List.of("["));
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> factory.create(null, null, null, config, null)
);
assertThat(e.getMessage(), equalTo("[patterns] Invalid regex pattern found in: [[]. premature end of char-class"));
}
@ -105,7 +117,10 @@ public class GrokProcessorFactoryTests extends ESTestCase {
config.put("field", "_field");
config.put("patterns", List.of("%{MY_PATTERN:name}!"));
config.put("pattern_definitions", Map.of("MY_PATTERN", "["));
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> factory.create(null, null, null, config, null)
);
assertThat(
e.getMessage(),
equalTo("[patterns] Invalid regex pattern found in: [%{MY_PATTERN:name}!]. premature end of char-class")
@ -119,7 +134,10 @@ public class GrokProcessorFactoryTests extends ESTestCase {
config.put("patterns", List.of("(?<foo>\\w+)"));
String invalidEcsMode = randomAlphaOfLength(3);
config.put("ecs_compatibility", invalidEcsMode);
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> factory.create(null, null, null, config, null)
);
assertThat(e.getMessage(), equalTo("[ecs_compatibility] unsupported mode '" + invalidEcsMode + "'"));
}
}

6
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GsubProcessorFactoryTests.java
View file

@ -44,7 +44,7 @@ public class GsubProcessorFactoryTests extends AbstractStringProcessorFactoryTes
config.put("field", "field1");
config.put("replacement", "-");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[pattern] required property is missing"));
@ -57,7 +57,7 @@ public class GsubProcessorFactoryTests extends AbstractStringProcessorFactoryTes
config.put("field", "field1");
config.put("pattern", "\\.");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[replacement] required property is missing"));
@ -71,7 +71,7 @@ public class GsubProcessorFactoryTests extends AbstractStringProcessorFactoryTes
config.put("pattern", "[");
config.put("replacement", "-");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), containsString("[pattern] Invalid regex pattern. Unclosed character class"));

8
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JoinProcessorFactoryTests.java
View file

@ -25,7 +25,7 @@ public class JoinProcessorFactoryTests extends ESTestCase {
config.put("field", "field1");
config.put("separator", "-");
String processorTag = randomAlphaOfLength(10);
JoinProcessor joinProcessor = factory.create(null, processorTag, null, config);
JoinProcessor joinProcessor = factory.create(null, processorTag, null, config, null);
assertThat(joinProcessor.getTag(), equalTo(processorTag));
assertThat(joinProcessor.getField(), equalTo("field1"));
assertThat(joinProcessor.getSeparator(), equalTo("-"));
@ -37,7 +37,7 @@ public class JoinProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("separator", "-");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@ -49,7 +49,7 @@ public class JoinProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "field1");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[separator] required property is missing"));
@ -63,7 +63,7 @@ public class JoinProcessorFactoryTests extends ESTestCase {
config.put("separator", "-");
config.put("target_field", "target");
String processorTag = randomAlphaOfLength(10);
JoinProcessor joinProcessor = factory.create(null, processorTag, null, config);
JoinProcessor joinProcessor = factory.create(null, processorTag, null, config, null);
assertThat(joinProcessor.getTag(), equalTo(processorTag));
assertThat(joinProcessor.getField(), equalTo("field1"));
assertThat(joinProcessor.getSeparator(), equalTo("-"));

16
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorFactoryTests.java
View file

@ -31,7 +31,7 @@ public class JsonProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", randomField);
config.put("target_field", randomTargetField);
JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config);
JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config, null);
assertThat(jsonProcessor.getTag(), equalTo(processorTag));
assertThat(jsonProcessor.getField(), equalTo(randomField));
assertThat(jsonProcessor.getTargetField(), equalTo(randomTargetField));
@ -43,7 +43,7 @@ public class JsonProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", randomField);
config.put("add_to_root", true);
JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config);
JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config, null);
assertThat(jsonProcessor.getTag(), equalTo(processorTag));
assertThat(jsonProcessor.getField(), equalTo(randomField));
assertThat(jsonProcessor.getTargetField(), equalTo(randomField));
@ -55,7 +55,7 @@ public class JsonProcessorFactoryTests extends ESTestCase {
String randomField = randomAlphaOfLength(10);
Map<String, Object> config = new HashMap<>();
config.put("field", randomField);
JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config);
JsonProcessor jsonProcessor = FACTORY.create(null, processorTag, null, config, null);
assertThat(jsonProcessor.getTag(), equalTo(processorTag));
assertThat(jsonProcessor.getField(), equalTo(randomField));
assertThat(jsonProcessor.getTargetField(), equalTo(randomField));
@ -66,7 +66,7 @@ public class JsonProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
ElasticsearchException exception = expectThrows(
ElasticsearchParseException.class,
() -> FACTORY.create(null, processorTag, null, config)
() -> FACTORY.create(null, processorTag, null, config, null)
);
assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
}
@ -79,7 +79,7 @@ public class JsonProcessorFactoryTests extends ESTestCase {
{
Map<String, Object> strictConfig = new HashMap<>();
strictConfig.put("field", fieldName);
JsonProcessor strictProcessor = FACTORY.create(null, processorTag, null, strictConfig);
JsonProcessor strictProcessor = FACTORY.create(null, processorTag, null, strictConfig, null);
IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> strictProcessor.execute(document));
assertThat(exception.getMessage(), containsString("is not valid JSON and the strict_json_parsing parameter is true"));
}
@ -88,7 +88,7 @@ public class JsonProcessorFactoryTests extends ESTestCase {
Map<String, Object> lenientConfig = new HashMap<>();
lenientConfig.put("field", fieldName);
lenientConfig.put("strict_json_parsing", false);
JsonProcessor lenientProcessor = FACTORY.create(null, processorTag, null, lenientConfig);
JsonProcessor lenientProcessor = FACTORY.create(null, processorTag, null, lenientConfig, null);
IngestDocument result = lenientProcessor.execute(document);
assertThat(result.getSource().get(fieldName), equalTo(123));
}
@ -103,7 +103,7 @@ public class JsonProcessorFactoryTests extends ESTestCase {
config.put("add_to_root", true);
ElasticsearchException exception = expectThrows(
ElasticsearchParseException.class,
() -> FACTORY.create(null, randomAlphaOfLength(10), null, config)
() -> FACTORY.create(null, randomAlphaOfLength(10), null, config, null)
);
assertThat(exception.getMessage(), equalTo("[target_field] Cannot set a target field while also setting `add_to_root` to true"));
}
@ -151,6 +151,6 @@ public class JsonProcessorFactoryTests extends ESTestCase {
if (mergeStrategy != null) {
config.put("add_to_root_conflict_strategy", mergeStrategy);
}
return FACTORY.create(null, randomAlphaOfLength(10), null, config);
return FACTORY.create(null, randomAlphaOfLength(10), null, config, null);
}
}

10
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorFactoryTests.java
View file

@ -40,7 +40,7 @@ public class KeyValueProcessorFactoryTests extends ESTestCase {
config.put("field_split", "&");
config.put("value_split", "=");
String processorTag = randomAlphaOfLength(10);
KeyValueProcessor processor = factory.create(null, processorTag, null, config);
KeyValueProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
assertThat(processor.getFieldSplit(), equalTo("&"));
@ -60,7 +60,7 @@ public class KeyValueProcessorFactoryTests extends ESTestCase {
config.put("exclude_keys", List.of());
config.put("ignore_missing", true);
String processorTag = randomAlphaOfLength(10);
KeyValueProcessor processor = factory.create(null, processorTag, null, config);
KeyValueProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
assertThat(processor.getFieldSplit(), equalTo("&"));
@ -76,7 +76,7 @@ public class KeyValueProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
ElasticsearchException exception = expectThrows(
ElasticsearchParseException.class,
() -> factory.create(null, processorTag, null, config)
() -> factory.create(null, processorTag, null, config, null)
);
assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
}
@ -87,7 +87,7 @@ public class KeyValueProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
ElasticsearchException exception = expectThrows(
ElasticsearchParseException.class,
() -> factory.create(null, processorTag, null, config)
() -> factory.create(null, processorTag, null, config, null)
);
assertThat(exception.getMessage(), equalTo("[field_split] required property is missing"));
}
@ -99,7 +99,7 @@ public class KeyValueProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
ElasticsearchException exception = expectThrows(
ElasticsearchParseException.class,
() -> factory.create(null, processorTag, null, config)
() -> factory.create(null, processorTag, null, config, null)
);
assertThat(exception.getMessage(), equalTo("[value_split] required property is missing"));
}

2
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/KeyValueProcessorTests.java
View file

@ -251,6 +251,6 @@ public class KeyValueProcessorTests extends ESTestCase {
if (prefix != null) {
config.put("prefix", prefix);
}
return FACTORY.create(null, randomAlphaOfLength(10), null, config);
return FACTORY.create(null, randomAlphaOfLength(10), null, config, null);
}
}

8
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java
View file

@ -50,7 +50,7 @@ public class NetworkDirectionProcessorFactoryTests extends ESTestCase {
config.put("ignore_missing", ignoreMissing);
String processorTag = randomAlphaOfLength(10);
NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config);
NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config, null);
assertThat(networkProcessor.getTag(), equalTo(processorTag));
assertThat(networkProcessor.getSourceIpField(), equalTo(sourceIpField));
assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField));
@ -75,7 +75,7 @@ public class NetworkDirectionProcessorFactoryTests extends ESTestCase {
config.put("ignore_missing", ignoreMissing);
String processorTag = randomAlphaOfLength(10);
NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config);
NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config, null);
assertThat(networkProcessor.getTag(), equalTo(processorTag));
assertThat(networkProcessor.getSourceIpField(), equalTo(sourceIpField));
assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField));
@ -88,7 +88,7 @@ public class NetworkDirectionProcessorFactoryTests extends ESTestCase {
HashMap<String, Object> config = new HashMap<>();
String processorTag = randomAlphaOfLength(10);
try {
factory.create(null, processorTag, null, config);
factory.create(null, processorTag, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[internal_networks] or [internal_networks_field] must be specified"));
@ -102,7 +102,7 @@ public class NetworkDirectionProcessorFactoryTests extends ESTestCase {
internalNetworks.add("10.0.0.0/8");
config.put("internal_networks", internalNetworks);
NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config);
NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config, null);
assertThat(networkProcessor.getTag(), equalTo(processorTag));
assertThat(networkProcessor.getSourceIpField(), equalTo(DEFAULT_SOURCE_IP));
assertThat(networkProcessor.getDestinationIpField(), equalTo(DEFAULT_DEST_IP));

8
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java
View file

@ -139,7 +139,8 @@ public class NetworkDirectionProcessorTests extends ESTestCase {
null,
processorTag,
null,
config
config,
null
);
IngestDocument input = TestIngestDocument.withDefaultVersion(source);
IngestDocument output = processor.execute(input);
@ -158,7 +159,7 @@ public class NetworkDirectionProcessorTests extends ESTestCase {
config.put("internal_networks", networks);
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config)
() -> new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config, null)
);
assertThat(
e.getMessage(),
@ -184,7 +185,8 @@ public class NetworkDirectionProcessorTests extends ESTestCase {
null,
processorTag,
null,
config
config,
null
);
IngestDocument input = TestIngestDocument.withDefaultVersion(source);

6
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorFactoryTests.java
View file

@ -38,7 +38,7 @@ public class RegisteredDomainProcessorFactoryTests extends ESTestCase {
config.put("ignore_missing", ignoreMissing);
String processorTag = randomAlphaOfLength(10);
RegisteredDomainProcessor publicSuffixProcessor = factory.create(null, processorTag, null, config);
RegisteredDomainProcessor publicSuffixProcessor = factory.create(null, processorTag, null, config, null);
assertThat(publicSuffixProcessor.getTag(), equalTo(processorTag));
assertThat(publicSuffixProcessor.getTargetField(), equalTo(targetField));
assertThat(publicSuffixProcessor.getIgnoreMissing(), equalTo(ignoreMissing));
@ -51,7 +51,7 @@ public class RegisteredDomainProcessorFactoryTests extends ESTestCase {
config.put("field", field);
String processorTag = randomAlphaOfLength(10);
RegisteredDomainProcessor publicSuffixProcessor = factory.create(null, processorTag, null, config);
RegisteredDomainProcessor publicSuffixProcessor = factory.create(null, processorTag, null, config, null);
assertThat(publicSuffixProcessor.getTargetField(), equalTo(RegisteredDomainProcessor.Factory.DEFAULT_TARGET_FIELD));
}
@ -59,7 +59,7 @@ public class RegisteredDomainProcessorFactoryTests extends ESTestCase {
HashMap<String, Object> config = new HashMap<>();
String processorTag = randomAlphaOfLength(10);
try {
factory.create(null, processorTag, null, config);
factory.create(null, processorTag, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[field] required property is missing"));

12
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorFactoryTests.java
View file

@ -34,7 +34,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "field1");
String processorTag = randomAlphaOfLength(10);
RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config);
RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config, null);
assertThat(removeProcessor.getTag(), equalTo(processorTag));
assertThat(removeProcessor.getFieldsToRemove().get(0).newInstance(Map.of()).execute(), equalTo("field1"));
}
@ -43,7 +43,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("keep", List.of("field1", "field2"));
String processorTag = randomAlphaOfLength(10);
RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config);
RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config, null);
assertThat(removeProcessor.getTag(), equalTo(processorTag));
assertThat(removeProcessor.getFieldsToKeep().get(0).newInstance(Map.of()).execute(), equalTo("field1"));
assertThat(removeProcessor.getFieldsToKeep().get(1).newInstance(Map.of()).execute(), equalTo("field2"));
@ -53,7 +53,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", List.of("field1", "field2"));
String processorTag = randomAlphaOfLength(10);
RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config);
RemoveProcessor removeProcessor = factory.create(null, processorTag, null, config, null);
assertThat(removeProcessor.getTag(), equalTo(processorTag));
assertThat(
removeProcessor.getFieldsToRemove().stream().map(template -> template.newInstance(Map.of()).execute()).toList(),
@ -64,7 +64,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
public void testCreateMissingField() throws Exception {
Map<String, Object> config = new HashMap<>();
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[keep] or [field] must be specified"));
@ -76,7 +76,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
config.put("field", "field1");
config.put("keep", "field2");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[keep] and [field] cannot both be used in the same processor"));
@ -90,7 +90,7 @@ public class RemoveProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
ElasticsearchException exception = expectThrows(
ElasticsearchException.class,
() -> factory.create(null, processorTag, null, config)
() -> factory.create(null, processorTag, null, config, null)
);
assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));

4
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java
View file

@ -46,7 +46,7 @@ public class RemoveProcessorTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", fieldName);
String processorTag = randomAlphaOfLength(10);
Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config);
Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config, null);
try {
processor.execute(ingestDocument);
fail("remove field should have failed");
@ -62,7 +62,7 @@ public class RemoveProcessorTests extends ESTestCase {
config.put("field", fieldName);
config.put("ignore_missing", true);
String processorTag = randomAlphaOfLength(10);
Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config);
Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, null, config, null);
processor.execute(ingestDocument);
}

10
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java
View file

@ -33,7 +33,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
config.put("field", "old_field");
config.put("target_field", "new_field");
String processorTag = randomAlphaOfLength(10);
RenameProcessor renameProcessor = factory.create(null, processorTag, null, config);
RenameProcessor renameProcessor = factory.create(null, processorTag, null, config, null);
assertThat(renameProcessor.getTag(), equalTo(processorTag));
assertThat(renameProcessor.getField().newInstance(Map.of()).execute(), equalTo("old_field"));
assertThat(renameProcessor.getTargetField().newInstance(Map.of()).execute(), equalTo("new_field"));
@ -47,7 +47,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
config.put("target_field", "new_field");
config.put("ignore_missing", true);
String processorTag = randomAlphaOfLength(10);
RenameProcessor renameProcessor = factory.create(null, processorTag, null, config);
RenameProcessor renameProcessor = factory.create(null, processorTag, null, config, null);
assertThat(renameProcessor.getTag(), equalTo(processorTag));
assertThat(renameProcessor.getField().newInstance(Map.of()).execute(), equalTo("old_field"));
assertThat(renameProcessor.getTargetField().newInstance(Map.of()).execute(), equalTo("new_field"));
@ -60,7 +60,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
config.put("target_field", "new_field");
config.put("override", true);
String processorTag = randomAlphaOfLength(10);
RenameProcessor renameProcessor = factory.create(null, processorTag, null, config);
RenameProcessor renameProcessor = factory.create(null, processorTag, null, config, null);
assertThat(renameProcessor.getTag(), equalTo(processorTag));
assertThat(renameProcessor.getField().newInstance(Map.of()).execute(), equalTo("old_field"));
assertThat(renameProcessor.getTargetField().newInstance(Map.of()).execute(), equalTo("new_field"));
@ -71,7 +71,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("target_field", "new_field");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@ -82,7 +82,7 @@ public class RenameProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "old_field");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[target_field] required property is missing"));

2
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RerouteProcessorFactoryTests.java
View file

@ -74,6 +74,6 @@ public class RerouteProcessorFactoryTests extends ESTestCase {
}
private static RerouteProcessor create(Map<String, Object> config) throws Exception {
return new RerouteProcessor.Factory().create(null, null, null, new HashMap<>(config));
return new RerouteProcessor.Factory().create(null, null, null, new HashMap<>(config), null);
}
}

16
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ScriptProcessorFactoryTests.java
View file

@ -53,7 +53,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
Map<String, Object> configMap = new HashMap<>();
String randomType = randomFrom("id", "source");
configMap.put(randomType, "foo");
ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap);
ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap, null);
assertThat(processor.getScript().getLang(), equalTo(randomType.equals("id") ? null : Script.DEFAULT_SCRIPT_LANG));
assertThat(processor.getScript().getType().toString(), equalTo(INGEST_SCRIPT_PARAM_TO_TYPE.get(randomType)));
assertThat(processor.getScript().getParams(), equalTo(Map.of()));
@ -69,7 +69,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
Map<String, Object> randomParams = Map.of(randomAlphaOfLength(10), randomAlphaOfLength(10));
configMap.put(randomType, "foo");
configMap.put("params", randomParams);
ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap);
ScriptProcessor processor = factory.create(null, randomAlphaOfLength(10), null, configMap, null);
assertThat(processor.getScript().getLang(), equalTo(randomType.equals("id") ? null : Script.DEFAULT_SCRIPT_LANG));
assertThat(processor.getScript().getType().toString(), equalTo(INGEST_SCRIPT_PARAM_TO_TYPE.get(randomType)));
assertThat(processor.getScript().getParams(), equalTo(randomParams));
@ -83,7 +83,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
XContentParseException exception = expectThrows(
XContentParseException.class,
() -> factory.create(null, randomAlphaOfLength(10), null, configMap)
() -> factory.create(null, randomAlphaOfLength(10), null, configMap, null)
);
assertThat(exception.getMessage(), containsString("[script] failed to parse field [source]"));
}
@ -94,7 +94,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
IllegalArgumentException exception = expectThrows(
IllegalArgumentException.class,
() -> factory.create(null, randomAlphaOfLength(10), null, configMap)
() -> factory.create(null, randomAlphaOfLength(10), null, configMap, null)
);
assertThat(exception.getMessage(), is("must specify either [source] for an inline script or [id] for a stored script"));
@ -108,7 +108,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
Map<String, Object> configMap = new HashMap<>();
configMap.put("inline", "code");
factory.create(null, randomAlphaOfLength(10), null, configMap);
factory.create(null, randomAlphaOfLength(10), null, configMap, null);
assertWarnings("Deprecated field [inline] used, expected [source] instead");
}
@ -130,7 +130,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
ElasticsearchException exception = expectThrows(
ElasticsearchException.class,
() -> factory.create(null, randomAlphaOfLength(10), null, configMap)
() -> factory.create(null, randomAlphaOfLength(10), null, configMap, null)
);
assertThat(exception.getMessage(), is("compile-time exception"));
@ -151,7 +151,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
Map<String, Object> configMap = new HashMap<>();
configMap.put("source", scriptName);
ScriptProcessor processor = factory.create(null, null, randomAlphaOfLength(10), configMap);
ScriptProcessor processor = factory.create(null, null, randomAlphaOfLength(10), configMap, null);
assertThat(processor.getScript().getLang(), equalTo(Script.DEFAULT_SCRIPT_LANG));
assertThat(processor.getScript().getType(), equalTo(ScriptType.INLINE));
assertThat(processor.getScript().getParams(), equalTo(Map.of()));
@ -167,7 +167,7 @@ public class ScriptProcessorFactoryTests extends ESTestCase {
factory = new ScriptProcessor.Factory(mockedScriptService);
Map<String, Object> configMap = new HashMap<>();
configMap.put("id", "script_name");
ScriptProcessor processor = factory.create(null, null, randomAlphaOfLength(10), configMap);
ScriptProcessor processor = factory.create(null, null, randomAlphaOfLength(10), configMap, null);
assertNull(processor.getScript().getLang());
assertThat(processor.getScript().getType(), equalTo(ScriptType.STORED));
assertThat(processor.getScript().getParams(), equalTo(Map.of()));

27
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SetProcessorFactoryTests.java
View file

@ -37,7 +37,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
config.put("field", "field1");
config.put("value", "value1");
String processorTag = randomAlphaOfLength(10);
SetProcessor setProcessor = factory.create(null, processorTag, null, config);
SetProcessor setProcessor = factory.create(null, processorTag, null, config, null);
assertThat(setProcessor.getTag(), equalTo(processorTag));
assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
assertThat(setProcessor.getValue().copyAndResolve(Map.of()), equalTo("value1"));
@ -52,7 +52,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
config.put("value", "value1");
config.put("override", overrideEnabled);
String processorTag = randomAlphaOfLength(10);
SetProcessor setProcessor = factory.create(null, processorTag, null, config);
SetProcessor setProcessor = factory.create(null, processorTag, null, config, null);
assertThat(setProcessor.getTag(), equalTo(processorTag));
assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
assertThat(setProcessor.getValue().copyAndResolve(Map.of()), equalTo("value1"));
@ -66,7 +66,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
config.put("value", "value1");
config.put("ignore_empty_value", ignoreEmptyValueEnabled);
String processorTag = randomAlphaOfLength(10);
SetProcessor setProcessor = factory.create(null, processorTag, null, config);
SetProcessor setProcessor = factory.create(null, processorTag, null, config, null);
assertThat(setProcessor.getTag(), equalTo(processorTag));
assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
assertThat(setProcessor.getValue().copyAndResolve(Map.of()), equalTo("value1"));
@ -77,7 +77,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("value", "value1");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@ -88,7 +88,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "field1");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[value] required property is missing"));
@ -100,7 +100,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
config.put("field", "field1");
config.put("value", null);
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[value] required property is missing"));
@ -115,7 +115,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
ElasticsearchException exception = expectThrows(
ElasticsearchException.class,
() -> factory.create(null, processorTag, null, config)
() -> factory.create(null, processorTag, null, config, null)
);
assertThat(exception.getMessage(), equalTo("java.lang.RuntimeException: could not compile script"));
assertThat(exception.getMetadata("es.processor_tag").get(0), equalTo(processorTag));
@ -126,7 +126,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
config.put("field", "field1");
config.put("copy_from", "field2");
String processorTag = randomAlphaOfLength(10);
SetProcessor setProcessor = factory.create(null, processorTag, null, config);
SetProcessor setProcessor = factory.create(null, processorTag, null, config, null);
assertThat(setProcessor.getTag(), equalTo(processorTag));
assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo("field1"));
assertThat(setProcessor.getCopyFrom(), equalTo("field2"));
@ -140,7 +140,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
ElasticsearchException exception = expectThrows(
ElasticsearchException.class,
() -> factory.create(null, processorTag, null, config)
() -> factory.create(null, processorTag, null, config, null)
);
assertThat(exception.getMessage(), equalTo("[copy_from] cannot set both `copy_from` and `value` in the same processor"));
}
@ -153,7 +153,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
config.put("value", "value1");
config.put("media_type", expectedMediaType);
String processorTag = randomAlphaOfLength(10);
SetProcessor setProcessor = factory.create(null, processorTag, null, config);
SetProcessor setProcessor = factory.create(null, processorTag, null, config, null);
assertThat(setProcessor.getTag(), equalTo(processorTag));
// invalid media type
@ -165,7 +165,10 @@ public class SetProcessorFactoryTests extends ESTestCase {
config2.put("field", "field1");
config2.put("value", "value1");
config2.put("media_type", expectedMediaType);
ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> factory.create(null, processorTag, null, config2));
ElasticsearchException e = expectThrows(
ElasticsearchException.class,
() -> factory.create(null, processorTag, null, config2, null)
);
assertThat(e.getMessage(), containsString("property does not contain a supported media type [" + expectedMediaType + "]"));
}
@ -176,7 +179,7 @@ public class SetProcessorFactoryTests extends ESTestCase {
config.put("field", "");
config.put("value", "value1");
String processorTag = randomAlphaOfLength(10);
SetProcessor setProcessor = factory.create(null, processorTag, null, config);
SetProcessor setProcessor = factory.create(null, processorTag, null, config, null);
assertThat(setProcessor.getTag(), equalTo(processorTag));
assertThat(setProcessor.getField().newInstance(Map.of()).execute(), equalTo(""));
assertThat(setProcessor.getValue().copyAndResolve(Map.of()), equalTo("value1"));

10
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SortProcessorFactoryTests.java
View file

@ -28,7 +28,7 @@ public class SortProcessorFactoryTests extends ESTestCase {
config.put("field", fieldName);
SortProcessor.Factory factory = new SortProcessor.Factory();
SortProcessor processor = factory.create(null, processorTag, null, config);
SortProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo(fieldName));
assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.ASCENDING));
@ -44,7 +44,7 @@ public class SortProcessorFactoryTests extends ESTestCase {
config.put("order", "desc");
SortProcessor.Factory factory = new SortProcessor.Factory();
SortProcessor processor = factory.create(null, processorTag, null, config);
SortProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo(fieldName));
assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.DESCENDING));
@ -61,7 +61,7 @@ public class SortProcessorFactoryTests extends ESTestCase {
config.put("target_field", targetFieldName);
SortProcessor.Factory factory = new SortProcessor.Factory();
SortProcessor processor = factory.create(null, processorTag, null, config);
SortProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo(fieldName));
assertThat(processor.getOrder(), equalTo(SortProcessor.SortOrder.ASCENDING));
@ -78,7 +78,7 @@ public class SortProcessorFactoryTests extends ESTestCase {
SortProcessor.Factory factory = new SortProcessor.Factory();
try {
factory.create(null, processorTag, null, config);
factory.create(null, processorTag, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[order] Sort direction [invalid] not recognized. Valid values are: [asc, desc]"));
@ -89,7 +89,7 @@ public class SortProcessorFactoryTests extends ESTestCase {
SortProcessor.Factory factory = new SortProcessor.Factory();
Map<String, Object> config = new HashMap<>();
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[field] required property is missing"));

10
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorFactoryTests.java
View file

@ -25,7 +25,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
config.put("field", "field1");
config.put("separator", "\\.");
String processorTag = randomAlphaOfLength(10);
SplitProcessor splitProcessor = factory.create(null, processorTag, null, config);
SplitProcessor splitProcessor = factory.create(null, processorTag, null, config, null);
assertThat(splitProcessor.getTag(), equalTo(processorTag));
assertThat(splitProcessor.getField(), equalTo("field1"));
assertThat(splitProcessor.getSeparator(), equalTo("\\."));
@ -38,7 +38,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("separator", "\\.");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@ -50,7 +50,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "field1");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[separator] required property is missing"));
@ -64,7 +64,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
config.put("separator", "\\.");
config.put("target_field", "target");
String processorTag = randomAlphaOfLength(10);
SplitProcessor splitProcessor = factory.create(null, processorTag, null, config);
SplitProcessor splitProcessor = factory.create(null, processorTag, null, config, null);
assertThat(splitProcessor.getTag(), equalTo(processorTag));
assertThat(splitProcessor.getField(), equalTo("field1"));
assertThat(splitProcessor.getSeparator(), equalTo("\\."));
@ -81,7 +81,7 @@ public class SplitProcessorFactoryTests extends ESTestCase {
config.put("target_field", "target");
config.put("preserve_trailing", true);
String processorTag = randomAlphaOfLength(10);
SplitProcessor splitProcessor = factory.create(null, processorTag, null, config);
SplitProcessor splitProcessor = factory.create(null, processorTag, null, config, null);
assertThat(splitProcessor.getTag(), equalTo(processorTag));
assertThat(splitProcessor.getField(), equalTo("field1"));
assertThat(splitProcessor.getSeparator(), equalTo("\\."));

2
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java
View file

@ -97,7 +97,7 @@ public class SplitProcessorTests extends ESTestCase {
Map<String, Object> splitConfig = new HashMap<>();
splitConfig.put("field", "flags");
splitConfig.put("separator", "\\|");
Processor splitProcessor = (new SplitProcessor.Factory()).create(null, null, null, splitConfig);
Processor splitProcessor = (new SplitProcessor.Factory()).create(null, null, null, splitConfig, null);
Map<String, Object> source = new HashMap<>();
source.put("flags", "new|hot|super|fun|interesting");
IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(source);

6
modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorFactoryTests.java
View file

@ -45,7 +45,7 @@ public class UriPartsProcessorFactoryTests extends ESTestCase {
config.put("ignore_missing", ignoreMissing);
String processorTag = randomAlphaOfLength(10);
UriPartsProcessor uriPartsProcessor = factory.create(null, processorTag, null, config);
UriPartsProcessor uriPartsProcessor = factory.create(null, processorTag, null, config, null);
assertThat(uriPartsProcessor.getTag(), equalTo(processorTag));
assertThat(uriPartsProcessor.getField(), equalTo(field));
assertThat(uriPartsProcessor.getTargetField(), equalTo(targetField));
@ -58,7 +58,7 @@ public class UriPartsProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("value", "value1");
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[field] required property is missing"));
@ -69,7 +69,7 @@ public class UriPartsProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", null);
try {
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
fail("factory create should have failed");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[field] required property is missing"));

2
modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java
View file

@ -811,7 +811,7 @@ public class GeoIpDownloaderIT extends AbstractGeoIpIT {
@Override
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
Map<String, Processor.Factory> procMap = new HashMap<>();
procMap.put(NON_GEO_PROCESSOR_TYPE, (factories, tag, description, config) -> {
procMap.put(NON_GEO_PROCESSOR_TYPE, (factories, tag, description, config, projectId) -> {
readStringProperty(NON_GEO_PROCESSOR_TYPE, tag, config, "randomField");
return new AbstractProcessor(tag, description) {
@Override

11
modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java
View file

@ -74,12 +74,19 @@ public class ReloadingDatabasesWhilePerformingGeoLookupsIT extends ESTestCase {
databaseNodeService.updateDatabase("GeoLite2-City-Test.mmdb", "md5", geoIpTmpDir.resolve("GeoLite2-City-Test.mmdb"));
lazyLoadReaders(databaseNodeService);
final GeoIpProcessor processor1 = (GeoIpProcessor) factory.create(null, "_tag", null, new HashMap<>(Map.of("field", "_field")));
final GeoIpProcessor processor1 = (GeoIpProcessor) factory.create(
null,
"_tag",
null,
new HashMap<>(Map.of("field", "_field")),
null
);
final GeoIpProcessor processor2 = (GeoIpProcessor) factory.create(
null,
"_tag",
null,
new HashMap<>(Map.of("field", "_field", "database_file", "GeoLite2-City-Test.mmdb"))
new HashMap<>(Map.of("field", "_field", "database_file", "GeoLite2-City-Test.mmdb")),
null
);
final AtomicBoolean completed = new AtomicBoolean(false);

4
modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.geoip;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.common.CheckedSupplier;
import org.elasticsearch.common.logging.DeprecationCategory;
import org.elasticsearch.common.logging.DeprecationLogger;
@ -227,7 +228,8 @@ public final class GeoIpProcessor extends AbstractProcessor {
final Map<String, Processor.Factory> registry,
final String processorTag,
final String description,
final Map<String, Object> config
final Map<String, Object> config,
final ProjectId projectId
) throws IOException {
String ipField = readStringProperty(type, processorTag, config, "field");
String targetField = readStringProperty(type, processorTag, config, "target_field", type);

53
modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
View file

@ -95,7 +95,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
config.put("field", "_field");
String processorTag = randomAlphaOfLength(10);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("geoip"));
@ -112,7 +112,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
config.put("ignore_missing", true);
String processorTag = randomAlphaOfLength(10);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("geoip"));
@ -129,7 +129,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
config.put("database_file", "GeoLite2-Country.mmdb");
String processorTag = randomAlphaOfLength(10);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo("_field"));
@ -147,7 +147,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
config.put("database_file", "GeoLite2-ASN.mmdb");
String processorTag = randomAlphaOfLength(10);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo("_field"));
@ -162,7 +162,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("target_field", "_field");
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null);
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("_field"));
assertFalse(processor.isIgnoreMissing());
@ -173,7 +173,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("database_file", "GeoLite2-Country.mmdb");
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null);
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("geoip"));
assertThat(processor.getDatabaseType(), equalTo("GeoLite2-Country"));
@ -190,7 +190,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
asnOnlyProperties.remove(Property.IP);
String asnProperty = RandomPicks.randomFrom(Randomness.get(), asnOnlyProperties).toString();
config.put("properties", List.of(asnProperty));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config, null));
assertThat(
e.getMessage(),
equalTo(
@ -211,7 +211,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
cityOnlyProperties.remove(Property.IP);
String cityProperty = RandomPicks.randomFrom(Randomness.get(), cityOnlyProperties).toString();
config.put("properties", List.of(cityProperty));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config, null));
assertThat(
e.getMessage(),
equalTo("[properties] illegal property value [" + cityProperty + "]. valid values are [IP, ASN, ORGANIZATION_NAME, NETWORK]")
@ -226,7 +226,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("database_file", "does-not-exist.mmdb");
Processor processor = factory.create(null, null, null, config);
Processor processor = factory.create(null, null, null, config, null);
assertThat(processor, instanceOf(GeoIpProcessor.DatabaseUnavailableProcessor.class));
}
@ -237,7 +237,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("database_file", randomFrom(DEFAULT_DATABASES));
Processor processor = factory.create(null, null, null, config);
Processor processor = factory.create(null, null, null, config, null);
assertThat(processor, instanceOf(GeoIpProcessor.DatabaseUnavailableProcessor.class));
}
@ -259,7 +259,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("properties", fieldNames);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null);
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getProperties(), equalTo(properties));
assertFalse(processor.isIgnoreMissing());
@ -271,7 +271,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config1 = new HashMap<>();
config1.put("field", "_field");
config1.put("properties", List.of("invalid"));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1, null));
assertThat(
e.getMessage(),
equalTo(
@ -285,7 +285,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config2 = new HashMap<>();
config2.put("field", "_field");
config2.put("properties", "invalid");
e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config2));
e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config2, null));
assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
}
@ -301,7 +301,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config1 = new HashMap<>();
config1.put("field", "_field");
config1.put("properties", List.of("ip"));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1, null));
assertThat(
e.getMessage(),
equalTo("[database_file] Unsupported database type [some-unsupported-database] for file [GeoLite2-City.mmdb]")
@ -320,7 +320,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config1 = new HashMap<>();
config1.put("field", "_field");
config1.put("properties", List.of("ip"));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1, null));
assertThat(e.getMessage(), equalTo("[database_file] Unsupported database type [null] for file [GeoLite2-City.mmdb]"));
}
@ -336,7 +336,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
config1.put("database_file", "some-ipinfo-database.mmdb");
config1.put("field", "_field");
config1.put("properties", List.of("ip"));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1));
Exception e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config1, null));
assertThat(
e.getMessage(),
equalTo(
@ -358,7 +358,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
config1.put("database_file", "some-custom-database.mmdb");
config1.put("field", "_field");
config1.put("properties", List.of("ip"));
factory.create(null, null, null, config1);
factory.create(null, null, null, config1, null);
assertWarnings(GeoIpProcessor.UNSUPPORTED_DATABASE_DEPRECATION_MESSAGE.replaceAll("\\{}", "some_custom_database.mmdb-City"));
}
@ -393,7 +393,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("database_file", "GeoLite2-City.mmdb");
final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config);
final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config, null);
// these are lazy loaded until first use, so we expect null here
assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-City.mmdb").databaseReader.get());
@ -404,7 +404,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
config = new HashMap<>();
config.put("field", "_field");
config.put("database_file", "GeoLite2-Country.mmdb");
final GeoIpProcessor country = (GeoIpProcessor) factory.create(null, "_tag", null, config);
final GeoIpProcessor country = (GeoIpProcessor) factory.create(null, "_tag", null, config, null);
// these are lazy loaded until first use, so we expect null here
assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-Country.mmdb").databaseReader.get());
@ -415,7 +415,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
config = new HashMap<>();
config.put("field", "_field");
config.put("database_file", "GeoLite2-ASN.mmdb");
final GeoIpProcessor asn = (GeoIpProcessor) factory.create(null, "_tag", null, config);
final GeoIpProcessor asn = (GeoIpProcessor) factory.create(null, "_tag", null, config, null);
// these are lazy loaded until first use, so we expect null here
assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoLite2-ASN.mmdb").databaseReader.get());
@ -462,7 +462,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("database_file", "GeoIP2-City.mmdb");
final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config);
final GeoIpProcessor city = (GeoIpProcessor) factory.create(null, "_tag", null, config, null);
// these are lazy loaded until first use, so we expect null here
assertNull(databaseNodeService.getDatabaseReaderLazyLoader("GeoIP2-City.mmdb").databaseReader.get());
@ -478,7 +478,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("field", randomIdentifier());
config.put("download_database_on_pipeline_creation", randomBoolean());
factory.create(null, null, null, config);
factory.create(null, null, null, config, null);
// Check all the config params were consumed.
assertThat(config, anEmptyMap());
}
@ -498,7 +498,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
config.put("field", "_field");
String processorTag = randomAlphaOfLength(10);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, processorTag, null, config, null);
processor.execute(RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>(Map.of("_field", "89.160.20.128"))));
}
@ -507,7 +507,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(GEOIP_TYPE, databaseNodeService);
Map<String, Object> config = new HashMap<>();
config.put("field", "source_field");
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null);
Map<String, Object> document = Map.of("source_field", "89.160.20.128");
{
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>(document));
@ -558,7 +558,8 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
null,
null,
null,
config
config,
null
);
processor.execute(ingestDocument);
assertThat(ingestDocument.getSourceAndMetadata().get("geoip"), nullValue());
@ -580,7 +581,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
document.put("source_field", "89.160.20.128");
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config);
GeoIpProcessor processor = (GeoIpProcessor) factory.create(null, null, null, config, null);
processor.execute(ingestDocument);
assertThat(ingestDocument.getSourceAndMetadata().get("tags"), nullValue());
Map<?, ?> geoData = (Map<?, ?>) ingestDocument.getSourceAndMetadata().get("geoip");

4
modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest.useragent;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.util.Maps;
import org.elasticsearch.core.UpdateForV10;
@ -191,7 +192,8 @@ public class UserAgentProcessor extends AbstractProcessor {
Map<String, Processor.Factory> factories,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) {
String field = readStringProperty(TYPE, processorTag, config, "field");
String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "user_agent");

27
modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java
View file

@ -74,7 +74,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
UserAgentProcessor processor = factory.create(null, processorTag, null, config);
UserAgentProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("user_agent"));
@ -95,7 +95,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
String processorTag = randomAlphaOfLength(10);
UserAgentProcessor processor = factory.create(null, processorTag, null, config);
UserAgentProcessor processor = factory.create(null, processorTag, null, config, null);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("user_agent"));
@ -113,7 +113,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field");
config.put("target_field", "_target_field");
UserAgentProcessor processor = factory.create(null, null, null, config);
UserAgentProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("_target_field"));
}
@ -125,7 +125,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field");
config.put("regex_file", regexWithoutDevicesFilename);
UserAgentProcessor processor = factory.create(null, null, null, config);
UserAgentProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0));
assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
@ -140,7 +140,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field");
config.put("extract_device_type", extractDeviceType);
UserAgentProcessor processor = factory.create(null, null, null, config);
UserAgentProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.isExtractDeviceType(), equalTo(extractDeviceType));
}
@ -152,7 +152,10 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field");
config.put("regex_file", "does-not-exist.yml");
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> factory.create(null, null, null, config, null)
);
assertThat(e.getMessage(), equalTo("[regex_file] regex file [does-not-exist.yml] doesn't exist (has to exist at node startup)"));
}
@ -172,7 +175,7 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field");
config.put("properties", fieldNames);
UserAgentProcessor processor = factory.create(null, null, null, config);
UserAgentProcessor processor = factory.create(null, null, null, config, null);
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getProperties(), equalTo(properties));
}
@ -184,7 +187,10 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field");
config.put("properties", Collections.singletonList("invalid"));
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> factory.create(null, null, null, config, null)
);
assertThat(
e.getMessage(),
equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " + "ORIGINAL, VERSION]")
@ -198,7 +204,10 @@ public class UserAgentProcessorFactoryTests extends ESTestCase {
config.put("field", "_field");
config.put("properties", "invalid");
ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, null, config));
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> factory.create(null, null, null, config, null)
);
assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]"));
}
}

10
server/src/internalClusterTest/java/org/elasticsearch/index/FinalPipelineIT.java
View file

@ -358,7 +358,7 @@ public class FinalPipelineIT extends ESIntegTestCase {
"final",
getFinal(parameters, randomBoolean()),
"request",
(processorFactories, tag, description, config) -> new AbstractProcessor(tag, description) {
(processorFactories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) {
@Override
public IngestDocument execute(final IngestDocument ingestDocument) throws Exception {
ingestDocument.setFieldValue("request", true);
@ -371,7 +371,7 @@ public class FinalPipelineIT extends ESIntegTestCase {
}
},
"changing_dest",
(processorFactories, tag, description, config) -> new AbstractProcessor(tag, description) {
(processorFactories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) {
@Override
public IngestDocument execute(final IngestDocument ingestDocument) throws Exception {
ingestDocument.setFieldValue(IngestDocument.Metadata.INDEX.getFieldName(), "target");
@ -385,7 +385,7 @@ public class FinalPipelineIT extends ESIntegTestCase {
},
"reroute",
(processorFactories, tag, description, config) -> {
(processorFactories, tag, description, config, projectId) -> {
final String dest = Objects.requireNonNullElse(
ConfigurationUtils.readOptionalStringProperty(description, tag, config, "dest"),
"target"
@ -408,7 +408,7 @@ public class FinalPipelineIT extends ESIntegTestCase {
}
private static Processor.Factory getDefault(Processor.Parameters parameters, boolean async) {
return (factories, tag, description, config) -> new AbstractProcessor(tag, description) {
return (factories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) {
@Override
public void execute(IngestDocument ingestDocument, BiConsumer<IngestDocument, Exception> handler) {
@ -446,7 +446,7 @@ public class FinalPipelineIT extends ESIntegTestCase {
}
private static Processor.Factory getFinal(Processor.Parameters parameters, boolean async) {
return (processorFactories, tag, description, config) -> {
return (processorFactories, tag, description, config, projectId) -> {
final String exists = (String) config.remove("exists");
return new AbstractProcessor(tag, description) {

4
server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestAsyncProcessorIT.java
View file

@ -96,7 +96,7 @@ public class IngestAsyncProcessorIT extends ESSingleNodeTestCase {
@Override
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
return Map.of("test-async", (factories, tag, description, config) -> new AbstractProcessor(tag, description) {
return Map.of("test-async", (factories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) {
@Override
public void execute(IngestDocument ingestDocument, BiConsumer<IngestDocument, Exception> handler) {
@ -129,7 +129,7 @@ public class IngestAsyncProcessorIT extends ESSingleNodeTestCase {
return true;
}
}, "test", (processorFactories, tag, description, config) -> new AbstractProcessor(tag, description) {
}, "test", (processorFactories, tag, description, config, projectId) -> new AbstractProcessor(tag, description) {
@Override
public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
String id = (String) ingestDocument.getSourceAndMetadata().get("_id");

9
server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java
View file

@ -386,11 +386,16 @@ public class IngestClientIT extends ESIntegTestCase {
factories.put(PipelineProcessor.TYPE, new PipelineProcessor.Factory(parameters.ingestService));
factories.put(
"fail",
(processorFactories, tag, description, config) -> new TestProcessor(tag, "fail", description, new RuntimeException())
(processorFactories, tag, description, config, projectId) -> new TestProcessor(
tag,
"fail",
description,
new RuntimeException()
)
);
factories.put(
"onfailure_processor",
(processorFactories, tag, description, config) -> new TestProcessor(tag, "fail", description, document -> {
(processorFactories, tag, description, config, projectId) -> new TestProcessor(tag, "fail", description, document -> {
String onFailurePipeline = document.getFieldValue("_ingest.on_failure_pipeline", String.class);
document.setFieldValue("readme", "pipeline with id [" + onFailurePipeline + "] is a bad pipeline");
})

2
server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestFileSettingsIT.java
View file

@ -262,7 +262,7 @@ public class IngestFileSettingsIT extends ESIntegTestCase {
@Override
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
Map<String, Processor.Factory> processors = new HashMap<>();
processors.put("test", (factories, tag, description, config) -> {
processors.put("test", (factories, tag, description, config, projectId) -> {
String field = (String) config.remove("field");
String value = (String) config.remove("value");
return new FakeProcessor("test", tag, description, (ingestDocument) -> ingestDocument.setFieldValue(field, value));

2
server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestStatsNamesAndTypesIT.java
View file

@ -166,7 +166,7 @@ public class IngestStatsNamesAndTypesIT extends ESIntegTestCase {
@Override
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
Map<String, Processor.Factory> processors = new HashMap<>();
processors.put("set", (factories, tag, description, config) -> {
processors.put("set", (factories, tag, description, config, projectId) -> {
String field = (String) config.remove("field");
String value = (String) config.remove("value");
return new FakeProcessor("set", tag, description, (ingestDocument) -> ingestDocument.setFieldValue(field, value));

12
server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
View file

@ -149,14 +149,20 @@ public class SimulatePipelineRequest extends ActionRequest implements ToXContent
return new Parsed(pipeline, ingestDocumentList, verbose);
}
static Parsed parse(Map<String, Object> config, boolean verbose, IngestService ingestService, RestApiVersion restApiVersion)
throws Exception {
static Parsed parse(
ProjectId projectId,
Map<String, Object> config,
boolean verbose,
IngestService ingestService,
RestApiVersion restApiVersion
) throws Exception {
Map<String, Object> pipelineConfig = ConfigurationUtils.readMap(null, null, config, Fields.PIPELINE);
Pipeline pipeline = Pipeline.create(
SIMULATED_PIPELINE_ID,
pipelineConfig,
ingestService.getProcessorFactories(),
ingestService.getScriptService()
ingestService.getScriptService(),
projectId
);
List<IngestDocument> ingestDocumentList = parseDocs(config, restApiVersion);
return new Parsed(pipeline, ingestDocumentList, verbose);

1
server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java
View file

@ -113,6 +113,7 @@ public class SimulatePipelineTransportAction extends HandledTransportAction<Simu
);
} else {
simulateRequest = SimulatePipelineRequest.parse(
projectId,
source,
request.isVerbose(),
ingestService,

25
server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java
View file

@ -12,6 +12,7 @@ package org.elasticsearch.ingest;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.util.Maps;
@ -487,7 +488,8 @@ public final class ConfigurationUtils {
public static List<Processor> readProcessorConfigs(
List<Map<String, Object>> processorConfigs,
ScriptService scriptService,
Map<String, Processor.Factory> processorFactories
Map<String, Processor.Factory> processorFactories,
ProjectId projectId
) throws Exception {
Exception exception = null;
List<Processor> processors = new ArrayList<>();
@ -498,7 +500,7 @@ public final class ConfigurationUtils {
if (entry.getValue() == null) {
throw newConfigurationException(entry.getKey(), null, null, "processor config cannot be [null]");
} else {
processors.add(readProcessor(processorFactories, scriptService, entry.getKey(), entry.getValue()));
processors.add(readProcessor(processorFactories, scriptService, entry.getKey(), entry.getValue(), projectId));
}
} catch (Exception e) {
exception = ExceptionsHelper.useOrSuppress(exception, e);
@ -575,14 +577,15 @@ public final class ConfigurationUtils {
Map<String, Processor.Factory> processorFactories,
ScriptService scriptService,
String type,
Object config
Object config,
ProjectId projectId
) throws Exception {
if (config instanceof Map) {
return readProcessor(processorFactories, scriptService, type, (Map<String, Object>) config);
return readProcessor(processorFactories, scriptService, type, (Map<String, Object>) config, projectId);
} else if (config instanceof String && "script".equals(type)) {
Map<String, Object> normalizedScript = Maps.newMapWithExpectedSize(1);
normalizedScript.put(ScriptType.INLINE.getParseField().getPreferredName(), config);
return readProcessor(processorFactories, scriptService, type, normalizedScript);
return readProcessor(processorFactories, scriptService, type, normalizedScript, projectId);
} else {
throw newConfigurationException(type, null, null, "property isn't a map, but of type [" + config.getClass().getName() + "]");
}
@ -592,7 +595,8 @@ public final class ConfigurationUtils {
Map<String, Processor.Factory> processorFactories,
ScriptService scriptService,
String type,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY);
String description = ConfigurationUtils.readOptionalStringProperty(null, tag, config, DESCRIPTION_KEY);
@ -607,14 +611,19 @@ public final class ConfigurationUtils {
Pipeline.ON_FAILURE_KEY
);
List<Processor> onFailureProcessors = readProcessorConfigs(onFailureProcessorConfigs, scriptService, processorFactories);
List<Processor> onFailureProcessors = readProcessorConfigs(
onFailureProcessorConfigs,
scriptService,
processorFactories,
projectId
);
if (onFailureProcessorConfigs != null && onFailureProcessors.isEmpty()) {
throw newConfigurationException(type, tag, Pipeline.ON_FAILURE_KEY, "processors list cannot be empty");
}
try {
Processor processor = factory.create(processorFactories, tag, description, config);
Processor processor = factory.create(processorFactories, tag, description, config, projectId);
if (config.isEmpty() == false) {
throw new ElasticsearchParseException(
"processor [{}] doesn't support one or more provided configuration parameters {}",

5
server/src/main/java/org/elasticsearch/ingest/DropProcessor.java
View file

@ -9,6 +9,8 @@
package org.elasticsearch.ingest;
import org.elasticsearch.cluster.metadata.ProjectId;
import java.util.Map;
/**
@ -40,7 +42,8 @@ public final class DropProcessor extends AbstractProcessor {
final Map<String, Processor.Factory> processorFactories,
final String tag,
final String description,
final Map<String, Object> config
final Map<String, Object> config,
final ProjectId projectId
) {
return new DropProcessor(tag, description);
}

25
server/src/main/java/org/elasticsearch/ingest/IngestService.java
View file

@ -372,6 +372,10 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
return scriptService;
}
public ProjectResolver getProjectResolver() {
return projectResolver;
}
/**
* Deletes the pipeline specified by id in the request.
*/
@ -522,7 +526,7 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
}
nodeInfoListener.accept(listener.delegateFailureAndWrap((l, nodeInfos) -> {
validatePipelineRequest(request, nodeInfos);
validatePipelineRequest(projectId, request, nodeInfos);
taskQueue.submitTask(
"put-pipeline-" + request.getId(),
@ -532,14 +536,14 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
}));
}
public void validatePipelineRequest(PutPipelineRequest request, NodesInfoResponse nodeInfos) throws Exception {
public void validatePipelineRequest(ProjectId projectId, PutPipelineRequest request, NodesInfoResponse nodeInfos) throws Exception {
final Map<String, Object> config = XContentHelper.convertToMap(request.getSource(), false, request.getXContentType()).v2();
Map<DiscoveryNode, IngestInfo> ingestInfos = new HashMap<>();
for (NodeInfo nodeInfo : nodeInfos.getNodes()) {
ingestInfos.put(nodeInfo.getNode(), nodeInfo.getInfo(IngestInfo.class));
}
validatePipeline(ingestInfos, request.getId(), config);
validatePipeline(ingestInfos, projectId, request.getId(), config);
}
public static boolean isNoOpPipelineUpdate(ProjectMetadata metadata, PutPipelineRequest request) {
@ -729,8 +733,12 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
}
@UpdateForV10(owner = DATA_MANAGEMENT) // Change deprecation log for special characters in name to a failure
void validatePipeline(Map<DiscoveryNode, IngestInfo> ingestInfos, String pipelineId, Map<String, Object> pipelineConfig)
throws Exception {
void validatePipeline(
Map<DiscoveryNode, IngestInfo> ingestInfos,
ProjectId projectId,
String pipelineId,
Map<String, Object> pipelineConfig
) throws Exception {
if (ingestInfos.isEmpty()) {
throw new IllegalStateException("Ingest info is empty");
}
@ -746,7 +754,7 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
deprecationLogger.critical(DeprecationCategory.API, "pipeline_name_special_chars", e.getMessage());
}
Pipeline pipeline = Pipeline.create(pipelineId, pipelineConfig, processorFactories, scriptService);
Pipeline pipeline = Pipeline.create(pipelineId, pipelineConfig, processorFactories, scriptService, projectId);
List<Exception> exceptions = new ArrayList<>();
for (Processor processor : pipeline.flattenAllProcessors()) {
@ -1394,7 +1402,8 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
newConfiguration.getId(),
newConfiguration.getConfig(false),
processorFactories,
scriptService
scriptService,
projectId
);
newPipelines.put(newConfiguration.getId(), new PipelineHolder(newConfiguration, newPipeline));
@ -1523,7 +1532,7 @@ public class IngestService implements ClusterStateApplier, ReportingService<Inge
public synchronized void reloadPipeline(ProjectId projectId, String id) throws Exception {
var originalPipelines = this.pipelines.getOrDefault(projectId, ImmutableOpenMap.of());
PipelineHolder holder = originalPipelines.get(id);
Pipeline updatedPipeline = Pipeline.create(id, holder.configuration.getConfig(false), processorFactories, scriptService);
Pipeline updatedPipeline = Pipeline.create(id, holder.configuration.getConfig(false), processorFactories, scriptService, projectId);
ImmutableOpenMap<String, PipelineHolder> updatedPipelines = ImmutableOpenMap.builder(originalPipelines)
.fPut(id, new PipelineHolder(holder.configuration, updatedPipeline))
.build();

14
server/src/main/java/org/elasticsearch/ingest/Pipeline.java
View file

@ -10,6 +10,7 @@
package org.elasticsearch.ingest;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.script.ScriptService;
@ -89,19 +90,26 @@ public final class Pipeline {
String id,
Map<String, Object> config,
Map<String, Processor.Factory> processorFactories,
ScriptService scriptService
ScriptService scriptService,
ProjectId projectId
) throws Exception {
String description = ConfigurationUtils.readOptionalStringProperty(null, null, config, DESCRIPTION_KEY);
Integer version = ConfigurationUtils.readIntProperty(null, null, config, VERSION_KEY, null);
Map<String, Object> metadata = ConfigurationUtils.readOptionalMap(null, null, config, META_KEY);
Boolean deprecated = ConfigurationUtils.readOptionalBooleanProperty(null, null, config, DEPRECATED_KEY);
List<Map<String, Object>> processorConfigs = ConfigurationUtils.readList(null, null, config, PROCESSORS_KEY);
List<Processor> processors = ConfigurationUtils.readProcessorConfigs(processorConfigs, scriptService, processorFactories);
List<Processor> processors = ConfigurationUtils.readProcessorConfigs(
processorConfigs,
scriptService,
processorFactories,
projectId
);
List<Map<String, Object>> onFailureProcessorConfigs = ConfigurationUtils.readOptionalList(null, null, config, ON_FAILURE_KEY);
List<Processor> onFailureProcessors = ConfigurationUtils.readProcessorConfigs(
onFailureProcessorConfigs,
scriptService,
processorFactories
processorFactories,
projectId
);
if (config.isEmpty() == false) {
throw new ElasticsearchParseException(

4
server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java
View file

@ -9,6 +9,7 @@
package org.elasticsearch.ingest;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.script.TemplateScript;
import java.util.Map;
@ -93,7 +94,8 @@ public class PipelineProcessor extends AbstractProcessor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
TemplateScript.Factory pipelineTemplate = ConfigurationUtils.readTemplateProperty(
TYPE,

18
server/src/main/java/org/elasticsearch/ingest/Processor.java
View file

@ -10,6 +10,7 @@
package org.elasticsearch.ingest;
import org.elasticsearch.client.internal.Client;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.env.Environment;
import org.elasticsearch.grok.MatcherWatchdog;
@ -84,12 +85,12 @@ public interface Processor {
*
* Override this method to perform additional post-construction validation that should be performed at the rest/transport level.
* If there's an issue with the processor, then indicate that by throwing an exception. See
* {@link IngestService#validatePipeline(Map, String, Map)}} for the call site where there is invoked in a try/catch.
* {@link IngestService#validatePipeline(Map, ProjectId, String, Map)}} for the call site where there is invoked in a try/catch.
*
* An example of where this would be needed is a processor that interacts with external state like the license state -- it may
* be okay to create that processor on day 1 with license state A, but later illegal to create a similar processor on day 2 with
* state B. We want to reject put requests on day 2 (at the rest/transport level), but still allow for restarting nodes in the
* cluster (so we can't throw exceptions from {@link Factory#create(Map, String, String, Map)}).
* cluster (so we can't throw exceptions from {@link Factory#create(Map, String, String, Map, ProjectId)}).
*/
default void extraValidation() throws Exception {}
@ -100,15 +101,22 @@ public interface Processor {
/**
* Creates a processor based on the specified map of maps config.
* @param processorFactories Other processors which may be created inside this processor
*
* @param processorFactories Other processors which may be created inside this processor
* @param tag The tag for the processor
* @param description A short description of what this processor does
* @param config The configuration for the processor
* @param projectId The project for which the processor is created
*
* <b>Note:</b> Implementations are responsible for removing the used configuration keys, so that after
*/
Processor create(Map<String, Factory> processorFactories, String tag, String description, Map<String, Object> config)
throws Exception;
Processor create(
Map<String, Factory> processorFactories,
String tag,
String description,
Map<String, Object> config,
ProjectId projectId
) throws Exception;
}
/**

3
server/src/main/java/org/elasticsearch/ingest/SimulateIngestService.java
View file

@ -57,7 +57,8 @@ public class SimulateIngestService extends IngestService {
pipelineId,
entry.getValue(),
ingestService.getProcessorFactories(),
ingestService.getScriptService()
ingestService.getScriptService(),
ingestService.getProjectResolver().getProjectId()
);
parsedPipelineSubstitutions.put(pipelineId, pipeline);
}

6
server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java
View file

@ -261,7 +261,7 @@ public class SimulateExecutionServiceTests extends ESTestCase {
public void testDropDocument() throws Exception {
TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field", "value"));
Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of());
Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of(), null);
Pipeline pipeline = new Pipeline("_id", "_description", version, null, new CompoundProcessor(processor1, processor2));
CountDownLatch latch = new CountDownLatch(1);
@ -281,7 +281,7 @@ public class SimulateExecutionServiceTests extends ESTestCase {
public void testDropDocumentVerbose() throws Exception {
TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field", "value"));
Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of());
Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of(), null);
Pipeline pipeline = new Pipeline("_id", "_description", version, null, new CompoundProcessor(processor1, processor2));
CountDownLatch latch = new CountDownLatch(1);
@ -304,7 +304,7 @@ public class SimulateExecutionServiceTests extends ESTestCase {
public void testDropDocumentVerboseExtraProcessor() throws Exception {
TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field1", "value"));
Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of());
Processor processor2 = new DropProcessor.Factory().create(Map.of(), null, null, Map.of(), null);
TestProcessor processor3 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field2", "value"));
Pipeline pipeline = new Pipeline("_id", "_description", version, null, new CompoundProcessor(processor1, processor2, processor3));

13
server/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java
View file

@ -60,7 +60,7 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
Pipeline pipeline = new Pipeline(SIMULATED_PIPELINE_ID, null, null, null, pipelineCompoundProcessor);
Map<String, Processor.Factory> registry = Collections.singletonMap(
"mock_processor",
(factories, tag, description, config) -> processor
(factories, tag, description, config, projectId) -> processor
);
ingestService = mock(IngestService.class);
when(ingestService.getPipeline(any(), eq(SIMULATED_PIPELINE_ID))).thenReturn(pipeline);
@ -189,7 +189,9 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
requestContent.put(Fields.PIPELINE, pipelineConfig);
var projectId = randomProjectIdOrDefault();
SimulatePipelineRequest.Parsed actualRequest = SimulatePipelineRequest.parse(
projectId,
requestContent,
false,
ingestService,
@ -256,6 +258,7 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
}
public void testNotValidDocs() {
var projectId = randomProjectIdOrDefault();
Map<String, Object> requestContent = new HashMap<>();
List<Map<String, Object>> docs = new ArrayList<>();
Map<String, Object> pipelineConfig = new HashMap<>();
@ -265,7 +268,7 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
requestContent.put(Fields.PIPELINE, pipelineConfig);
Exception e1 = expectThrows(
IllegalArgumentException.class,
() -> SimulatePipelineRequest.parse(requestContent, false, ingestService, RestApiVersion.current())
() -> SimulatePipelineRequest.parse(projectId, requestContent, false, ingestService, RestApiVersion.current())
);
assertThat(e1.getMessage(), equalTo("must specify at least one document in [docs]"));
@ -276,7 +279,7 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
requestContent.put(Fields.PIPELINE, pipelineConfig);
Exception e2 = expectThrows(
IllegalArgumentException.class,
() -> SimulatePipelineRequest.parse(requestContent, false, ingestService, RestApiVersion.current())
() -> SimulatePipelineRequest.parse(projectId, requestContent, false, ingestService, RestApiVersion.current())
);
assertThat(e2.getMessage(), equalTo("malformed [docs] section, should include an inner object"));
@ -285,7 +288,7 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
requestContent.put(Fields.PIPELINE, pipelineConfig);
Exception e3 = expectThrows(
ElasticsearchParseException.class,
() -> SimulatePipelineRequest.parse(requestContent, false, ingestService, RestApiVersion.current())
() -> SimulatePipelineRequest.parse(projectId, requestContent, false, ingestService, RestApiVersion.current())
);
assertThat(e3.getMessage(), containsString("required property is missing"));
}
@ -358,7 +361,9 @@ public class SimulatePipelineRequestParsingTests extends ESTestCase {
pipelineConfig.put("on_failure", onFailureProcessors);
}
requestContent.put(Fields.PIPELINE, pipelineConfig);
var projectId = randomProjectIdOrDefault();
SimulatePipelineRequest.Parsed actualRequest = SimulatePipelineRequest.parse(
projectId,
requestContent,
false,
ingestService,

55
server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java
View file

@ -11,6 +11,7 @@ package org.elasticsearch.ingest;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.TemplateScript;
import org.elasticsearch.test.ESTestCase;
@ -40,6 +41,7 @@ public class ConfigurationUtilsTests extends ESTestCase {
private final ScriptService scriptService = mock(ScriptService.class);
private final ProjectId projectId = randomProjectIdOrDefault();
private Map<String, Object> config;
@Before
@ -148,7 +150,7 @@ public class ConfigurationUtilsTests extends ESTestCase {
Processor processor = mock(Processor.class);
Map<String, Processor.Factory> registry = Collections.singletonMap(
"test_processor",
(factories, tag, description, config) -> processor
(factories, tag, description, config, projectId) -> processor
);
List<Map<String, Object>> config = new ArrayList<>();
@ -156,7 +158,7 @@ public class ConfigurationUtilsTests extends ESTestCase {
config.add(Collections.singletonMap("test_processor", emptyConfig));
config.add(Collections.singletonMap("test_processor", emptyConfig));
List<Processor> result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry);
List<Processor> result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry, projectId);
assertThat(result.size(), equalTo(2));
assertThat(result.get(0), sameInstance(processor));
assertThat(result.get(1), sameInstance(processor));
@ -169,7 +171,7 @@ public class ConfigurationUtilsTests extends ESTestCase {
config.add(Collections.singletonMap("unknown_processor", unknownTaggedConfig));
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> ConfigurationUtils.readProcessorConfigs(config, scriptService, registry)
() -> ConfigurationUtils.readProcessorConfigs(config, scriptService, registry, projectId)
);
assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]"));
assertThat(e.getMetadata("es.processor_tag"), equalTo(Collections.singletonList("my_unknown")));
@ -186,7 +188,7 @@ public class ConfigurationUtilsTests extends ESTestCase {
config2.add(Collections.singletonMap("second_unknown_processor", secondUnknownTaggedConfig));
e = expectThrows(
ElasticsearchParseException.class,
() -> ConfigurationUtils.readProcessorConfigs(config2, scriptService, registry)
() -> ConfigurationUtils.readProcessorConfigs(config2, scriptService, registry, projectId)
);
assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]"));
assertThat(e.getMetadata("es.processor_tag"), equalTo(Collections.singletonList("my_unknown")));
@ -205,7 +207,7 @@ public class ConfigurationUtilsTests extends ESTestCase {
config3.add(Collections.singletonMap("test_processor", null));
ElasticsearchParseException e3 = expectThrows(
ElasticsearchParseException.class,
() -> ConfigurationUtils.readProcessorConfigs(config3, scriptService, registry)
() -> ConfigurationUtils.readProcessorConfigs(config3, scriptService, registry, projectId)
);
assertThat(e3.getMetadata("es.processor_type"), equalTo(Collections.singletonList("test_processor")));
assertThat(e3.getMessage(), equalTo("processor config cannot be [null]"));
@ -213,15 +215,18 @@ public class ConfigurationUtilsTests extends ESTestCase {
public void testReadProcessorNullDescription() throws Exception {
Processor processor = new TestProcessor("tag", "type", null, (ingestDocument) -> {});
Map<String, Processor.Factory> registry = Collections.singletonMap("test_processor", (factories, tag, description, config) -> {
assertNull(description);
return processor;
});
Map<String, Processor.Factory> registry = Collections.singletonMap(
"test_processor",
(factories, tag, description, config, projectId) -> {
assertNull(description);
return processor;
}
);
List<Map<String, Object>> config = new ArrayList<>();
Map<String, Object> emptyConfig = Collections.emptyMap();
config.add(Collections.singletonMap("test_processor", emptyConfig));
List<Processor> result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry);
List<Processor> result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry, projectId);
assertThat(result.size(), equalTo(1));
assertThat(result.get(0), sameInstance(processor));
}
@ -229,40 +234,46 @@ public class ConfigurationUtilsTests extends ESTestCase {
public void testReadProcessorDescription() throws Exception {
String testDescription = randomAlphaOfLengthBetween(10, 20);
Processor processor = new TestProcessor("tag", "type", testDescription, (ingestDocument) -> {});
Map<String, Processor.Factory> registry = Collections.singletonMap("test_processor", (factories, tag, description, config) -> {
assertThat(description, equalTo(processor.getDescription()));
return processor;
});
Map<String, Processor.Factory> registry = Collections.singletonMap(
"test_processor",
(factories, tag, description, config, projectId) -> {
assertThat(description, equalTo(processor.getDescription()));
return processor;
}
);
List<Map<String, Object>> config = new ArrayList<>();
Map<String, Object> processorConfig = new HashMap<>();
processorConfig.put(ConfigurationUtils.DESCRIPTION_KEY, testDescription);
config.add(Collections.singletonMap("test_processor", processorConfig));
List<Processor> result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry);
List<Processor> result = ConfigurationUtils.readProcessorConfigs(config, scriptService, registry, projectId);
assertThat(result.size(), equalTo(1));
assertThat(result.get(0), sameInstance(processor));
}
public void testReadProcessorFromObjectOrMap() throws Exception {
Processor processor = mock(Processor.class);
Map<String, Processor.Factory> registry = Collections.singletonMap("script", (processorFactories, tag, description, config) -> {
config.clear();
return processor;
});
Map<String, Processor.Factory> registry = Collections.singletonMap(
"script",
(processorFactories, tag, description, config, projectId) -> {
config.clear();
return processor;
}
);
Object emptyConfig = Collections.emptyMap();
Processor processor1 = ConfigurationUtils.readProcessor(registry, scriptService, "script", emptyConfig);
Processor processor1 = ConfigurationUtils.readProcessor(registry, scriptService, "script", emptyConfig, projectId);
assertThat(processor1, sameInstance(processor));
Object inlineScript = "test_script";
Processor processor2 = ConfigurationUtils.readProcessor(registry, scriptService, "script", inlineScript);
Processor processor2 = ConfigurationUtils.readProcessor(registry, scriptService, "script", inlineScript, projectId);
assertThat(processor2, sameInstance(processor));
Object invalidConfig = 12L;
ElasticsearchParseException ex = expectThrows(
ElasticsearchParseException.class,
() -> ConfigurationUtils.readProcessor(registry, scriptService, "unknown_processor", invalidConfig)
() -> ConfigurationUtils.readProcessor(registry, scriptService, "unknown_processor", invalidConfig, projectId)
);
assertThat(ex.getMessage(), equalTo("property isn't a map, but of type [" + invalidConfig.getClass().getName() + "]"));
}

134
server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
View file

@ -135,7 +135,7 @@ public class IngestServiceTests extends ESTestCase {
private static final IngestPlugin DUMMY_PLUGIN = new IngestPlugin() {
@Override
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
return Map.of("foo", (factories, tag, description, config) -> null);
return Map.of("foo", (factories, tag, description, config, projectId) -> null);
}
};
@ -347,11 +347,11 @@ public class IngestServiceTests extends ESTestCase {
public void testInnerUpdatePipelinesValidation() {
Map<String, Processor.Factory> processors = new HashMap<>();
processors.put("fail_validation", (factories, tag, description, config) -> {
processors.put("fail_validation", (factories, tag, description, config, projectId) -> {
// ordinary validation issues happen at processor construction time
throw newConfigurationException("fail_validation", tag, "no_property_name", "validation failure reason");
});
processors.put("fail_extra_validation", (factories, tag, description, config) -> {
processors.put("fail_extra_validation", (factories, tag, description, config, projectId) -> {
// 'extra validation' issues happen post- processor construction time
return new FakeProcessor("fail_extra_validation", tag, description, ingestDocument -> {}) {
@Override
@ -444,16 +444,17 @@ public class IngestServiceTests extends ESTestCase {
PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """
{"processors": [{"set" : {"field": "_field", "value": "_value"}}]}""");
var projectId = randomProjectIdOrDefault();
var pipelineConfig = XContentHelper.convertToMap(putRequest.getSource(), false, putRequest.getXContentType()).v2();
Exception e = expectThrows(
IllegalStateException.class,
() -> ingestService.validatePipeline(Map.of(), putRequest.getId(), pipelineConfig)
() -> ingestService.validatePipeline(Map.of(), projectId, putRequest.getId(), pipelineConfig)
);
assertEquals("Ingest info is empty", e.getMessage());
DiscoveryNode discoveryNode = DiscoveryNodeUtils.create("_node_id", buildNewFakeTransportAddress(), Map.of(), Set.of());
IngestInfo ingestInfo = new IngestInfo(List.of(new ProcessorInfo("set")));
ingestService.validatePipeline(Map.of(discoveryNode, ingestInfo), putRequest.getId(), pipelineConfig);
ingestService.validatePipeline(Map.of(discoveryNode, ingestInfo), projectId, putRequest.getId(), pipelineConfig);
}
public void testValidateNotInUse() {
@ -618,7 +619,7 @@ public class IngestServiceTests extends ESTestCase {
boolean[] externalProperty = new boolean[] { false };
Map<String, Processor.Factory> processorFactories = new HashMap<>();
processorFactories.put("set", (factories, tag, description, config) -> {
processorFactories.put("set", (factories, tag, description, config, projectId) -> {
String field = (String) config.remove("field");
String value = (String) config.remove("value");
if (externalProperty[0]) {
@ -685,7 +686,7 @@ public class IngestServiceTests extends ESTestCase {
);
Map<String, Processor.Factory> processors = new HashMap<>();
processors.put("complexSet", (factories, tag, description, config) -> {
processors.put("complexSet", (factories, tag, description, config, projectId) -> {
String field = (String) config.remove("field");
String value = (String) config.remove("value");
@ -1017,6 +1018,7 @@ public class IngestServiceTests extends ESTestCase {
public void testValidateProcessorTypeOnAllNodes() throws Exception {
IngestService ingestService = createWithProcessors();
var projectId = randomProjectIdOrDefault();
PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """
{
"processors": [
@ -1045,7 +1047,7 @@ public class IngestServiceTests extends ESTestCase {
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> ingestService.validatePipeline(ingestInfos, putRequest.getId(), pipelineConfig)
() -> ingestService.validatePipeline(ingestInfos, projectId, putRequest.getId(), pipelineConfig)
);
assertEquals("Processor type [remove] is not installed on node [" + node2 + "]", e.getMessage());
assertEquals("remove", e.getMetadata("es.processor_type").get(0));
@ -1053,14 +1055,15 @@ public class IngestServiceTests extends ESTestCase {
var pipelineConfig2 = XContentHelper.convertToMap(putRequest.getSource(), false, putRequest.getXContentType()).v2();
ingestInfos.put(node2, new IngestInfo(List.of(new ProcessorInfo("set"), new ProcessorInfo("remove"))));
ingestService.validatePipeline(ingestInfos, putRequest.getId(), pipelineConfig2);
ingestService.validatePipeline(ingestInfos, projectId, putRequest.getId(), pipelineConfig2);
}
public void testValidateConfigurationExceptions() {
IngestService ingestService = createWithProcessors(Map.of("fail_validation", (factories, tag, description, config) -> {
IngestService ingestService = createWithProcessors(Map.of("fail_validation", (factories, tag, description, config, projectId) -> {
// ordinary validation issues happen at processor construction time
throw newConfigurationException("fail_validation", tag, "no_property_name", "validation failure reason");
}));
var projectId = randomProjectIdOrDefault();
PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """
{
"processors": [
@ -1079,22 +1082,30 @@ public class IngestServiceTests extends ESTestCase {
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> ingestService.validatePipeline(ingestInfos, putRequest.getId(), pipelineConfig)
() -> ingestService.validatePipeline(ingestInfos, projectId, putRequest.getId(), pipelineConfig)
);
assertEquals("[no_property_name] validation failure reason", e.getMessage());
assertEquals("fail_validation", e.getMetadata("es.processor_type").get(0));
}
public void testValidateExtraValidationConfigurationExceptions() {
IngestService ingestService = createWithProcessors(Map.of("fail_extra_validation", (factories, tag, description, config) -> {
// 'extra validation' issues happen post- processor construction time
return new FakeProcessor("fail_extra_validation", tag, description, ingestDocument -> {}) {
@Override
public void extraValidation() throws Exception {
throw newConfigurationException("fail_extra_validation", tag, "no_property_name", "extra validation failure reason");
}
};
}));
IngestService ingestService = createWithProcessors(
Map.of("fail_extra_validation", (factories, tag, description, config, projectId) -> {
// 'extra validation' issues happen post- processor construction time
return new FakeProcessor("fail_extra_validation", tag, description, ingestDocument -> {}) {
@Override
public void extraValidation() throws Exception {
throw newConfigurationException(
"fail_extra_validation",
tag,
"no_property_name",
"extra validation failure reason"
);
}
};
})
);
var projectId = randomProjectIdOrDefault();
PutPipelineRequest putRequest = putJsonPipelineRequest("pipeline-id", """
{
"processors": [
@ -1113,13 +1124,14 @@ public class IngestServiceTests extends ESTestCase {
ElasticsearchParseException e = expectThrows(
ElasticsearchParseException.class,
() -> ingestService.validatePipeline(ingestInfos, putRequest.getId(), pipelineConfig)
() -> ingestService.validatePipeline(ingestInfos, projectId, putRequest.getId(), pipelineConfig)
);
assertEquals("[no_property_name] extra validation failure reason", e.getMessage());
assertEquals("fail_extra_validation", e.getMetadata("es.processor_type").get(0));
}
public void testValidatePipelineName() throws Exception {
var projectId = randomProjectIdOrDefault();
IngestService ingestService = createWithProcessors();
for (Character badChar : List.of('\\', '/', '*', '?', '"', '<', '>', '|', ' ', ',')) {
PutPipelineRequest putRequest = new PutPipelineRequest(
@ -1135,7 +1147,7 @@ public class IngestServiceTests extends ESTestCase {
Map<DiscoveryNode, IngestInfo> ingestInfos = new HashMap<>();
ingestInfos.put(node1, new IngestInfo(List.of(new ProcessorInfo("set"))));
final String name = randomAlphaOfLength(5) + badChar + randomAlphaOfLength(5);
ingestService.validatePipeline(ingestInfos, name, pipelineConfig);
ingestService.validatePipeline(ingestInfos, projectId, name, pipelineConfig);
assertCriticalWarnings(
"Pipeline name ["
+ name
@ -1147,7 +1159,7 @@ public class IngestServiceTests extends ESTestCase {
public void testExecuteIndexPipelineExistsButFailedParsing() {
IngestService ingestService = createWithProcessors(
Map.of("mock", (factories, tag, description, config) -> new AbstractProcessor("mock", "description") {
Map.of("mock", (factories, tag, description, config, projectId) -> new AbstractProcessor("mock", "description") {
@Override
public IngestDocument execute(IngestDocument ingestDocument) {
throw new IllegalStateException("error");
@ -1213,7 +1225,7 @@ public class IngestServiceTests extends ESTestCase {
public void testExecuteBulkPipelineDoesNotExist() {
IngestService ingestService = createWithProcessors(
Map.of("mock", (factories, tag, description, config) -> mockCompoundProcessor())
Map.of("mock", (factories, tag, description, config, projectId) -> mockCompoundProcessor())
);
PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
@ -1267,7 +1279,7 @@ public class IngestServiceTests extends ESTestCase {
public void testExecuteSuccess() {
IngestService ingestService = createWithProcessors(
Map.of("mock", (factories, tag, description, config) -> mockCompoundProcessor())
Map.of("mock", (factories, tag, description, config, projectId) -> mockCompoundProcessor())
);
PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
var projectId = randomProjectIdOrDefault();
@ -1304,7 +1316,7 @@ public class IngestServiceTests extends ESTestCase {
IngestService ingestService = createWithProcessors(
Map.of(
"set",
(factories, tag, description, config) -> new FakeProcessor(
(factories, tag, description, config, projectId) -> new FakeProcessor(
"set",
"",
"",
@ -1380,7 +1392,7 @@ public class IngestServiceTests extends ESTestCase {
public void testExecutePropagateAllMetadataUpdates() throws Exception {
final CompoundProcessor processor = mockCompoundProcessor();
IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> processor));
IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config, projectId) -> processor));
PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
var projectId = randomProjectIdOrDefault();
ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
@ -1454,9 +1466,9 @@ public class IngestServiceTests extends ESTestCase {
IngestService ingestService = createWithProcessors(
Map.of(
"mock",
(factories, tag, description, config) -> processor,
(factories, tag, description, config, projectId) -> processor,
"set",
(factories, tag, description, config) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail())
(factories, tag, description, config, projectId) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail())
)
);
PutPipelineRequest putRequest1 = putJsonPipelineRequest("_id1", "{\"processors\": [{\"mock\" : {}}]}");
@ -1526,7 +1538,9 @@ public class IngestServiceTests extends ESTestCase {
List.of(processor),
List.of(new CompoundProcessor(onFailureProcessor))
);
IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> compoundProcessor));
IngestService ingestService = createWithProcessors(
Map.of("mock", (factories, tag, description, config, projectId) -> compoundProcessor)
);
PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
var projectId = randomProjectIdOrDefault();
ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
@ -1572,7 +1586,9 @@ public class IngestServiceTests extends ESTestCase {
List.of(processor),
List.of(new CompoundProcessor(false, processors, onFailureProcessors))
);
IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> compoundProcessor));
IngestService ingestService = createWithProcessors(
Map.of("mock", (factories, tag, description, config, projectId) -> compoundProcessor)
);
PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
var projectId = randomProjectIdOrDefault();
ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
@ -1650,7 +1666,7 @@ public class IngestServiceTests extends ESTestCase {
handler.accept(null, error);
return null;
}).when(processor).execute(any(), any());
IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> processor));
IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config, projectId) -> processor));
PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
var projectId = randomProjectIdOrDefault();
ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
@ -1691,9 +1707,9 @@ public class IngestServiceTests extends ESTestCase {
IngestService ingestService = createWithProcessors(
Map.of(
"mock",
(factories, tag, description, config) -> processor,
(factories, tag, description, config, projectId) -> processor,
"set",
(factories, tag, description, config) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail())
(factories, tag, description, config, projectId) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail())
)
);
PutPipelineRequest putRequest1 = putJsonPipelineRequest("_id1", "{\"processors\": [{\"mock\" : {}}]}");
@ -1742,9 +1758,9 @@ public class IngestServiceTests extends ESTestCase {
IngestService ingestService = createWithProcessors(
Map.of(
"mock",
(factories, tag, description, config) -> processor,
(factories, tag, description, config, projectId) -> processor,
"set",
(factories, tag, description, config) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail())
(factories, tag, description, config, projectId) -> new FakeProcessor("set", "", "", (ingestDocument) -> fail())
)
);
PutPipelineRequest putRequest1 = putJsonPipelineRequest("_id1", "{\"processors\": [{\"mock\" : {}}]}");
@ -1799,7 +1815,9 @@ public class IngestServiceTests extends ESTestCase {
List.of(processor),
List.of(new CompoundProcessor(false, processors, onFailureProcessors))
);
IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> compoundProcessor));
IngestService ingestService = createWithProcessors(
Map.of("mock", (factories, tag, description, config, projectId) -> compoundProcessor)
);
PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
var projectId = randomProjectIdOrDefault();
ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
@ -1875,7 +1893,7 @@ public class IngestServiceTests extends ESTestCase {
handler.accept(null, error);
return null;
}).when(processor).execute(any(), any());
IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config) -> processor));
IngestService ingestService = createWithProcessors(Map.of("mock", (factories, tag, description, config, projectId) -> processor));
PutPipelineRequest putRequest = putJsonPipelineRequest("_id", "{\"processors\": [{\"mock\" : {}}]}");
var projectId = randomProjectIdOrDefault();
ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
@ -1939,7 +1957,7 @@ public class IngestServiceTests extends ESTestCase {
return null;
}).when(processor).execute(any(), any());
Map<String, Processor.Factory> map = Maps.newMapWithExpectedSize(2);
map.put("mock", (factories, tag, description, config) -> processor);
map.put("mock", (factories, tag, description, config, projectId) -> processor);
IngestService ingestService = createWithProcessors(map);
PutPipelineRequest putRequest = putJsonPipelineRequest("_id", """
@ -2007,14 +2025,20 @@ public class IngestServiceTests extends ESTestCase {
IngestService ingestService = createWithProcessors(
Map.of(
"pipeline",
(factories, tag, description, config) -> new PipelineProcessor(tag, description, (params) -> new TemplateScript(params) {
@Override
public String execute() {
return "_id3";
} // this pipeline processor will always execute the '_id3' processor
}, false, pipelineIngestService),
(factories, tag, description, config, projectId) -> new PipelineProcessor(
tag,
description,
(params) -> new TemplateScript(params) {
@Override
public String execute() {
return "_id3";
} // this pipeline processor will always execute the '_id3' processor
},
false,
pipelineIngestService
),
"mock",
(factories, tag, description, config) -> processor
(factories, tag, description, config, projectId) -> processor
)
);
@ -2116,8 +2140,8 @@ public class IngestServiceTests extends ESTestCase {
return null;
}).when(processorFailure).execute(any(IngestDocument.class), any());
Map<String, Processor.Factory> map = Maps.newMapWithExpectedSize(2);
map.put("mock", (factories, tag, description, config) -> processor);
map.put("failure-mock", (factories, tag, description, config) -> processorFailure);
map.put("mock", (factories, tag, description, config, projectId) -> processor);
map.put("failure-mock", (factories, tag, description, config, projectId) -> processorFailure);
map.put("drop", new DropProcessor.Factory());
IngestService ingestService = createWithProcessors(map);
@ -2324,7 +2348,7 @@ public class IngestServiceTests extends ESTestCase {
public void testExecuteWithDrop() {
Map<String, Processor.Factory> factories = new HashMap<>();
factories.put("drop", new DropProcessor.Factory());
factories.put("mock", (processorFactories, tag, description, config) -> new Processor() {
factories.put("mock", (processorFactories, tag, description, config, projectId) -> new Processor() {
@Override
public IngestDocument execute(final IngestDocument ingestDocument) {
throw new AssertionError("Document should have been dropped but reached this processor");
@ -2397,7 +2421,7 @@ public class IngestServiceTests extends ESTestCase {
IngestPlugin testPlugin = new IngestPlugin() {
@Override
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
return Map.of("test", (factories, tag, description, config) -> {
return Map.of("test", (factories, tag, description, config, projectId) -> {
assertThat(counter.compareAndSet(1, 2), is(true));
return new FakeProcessor("test", tag, description, ingestDocument -> {});
});
@ -2438,7 +2462,7 @@ public class IngestServiceTests extends ESTestCase {
AtomicReference<Object> reference = new AtomicReference<>();
Consumer<IngestDocument> executor = doc -> reference.set(doc.getFieldValueAsBytes("data"));
final IngestService ingestService = createWithProcessors(
Map.of("foo", (factories, tag, description, config) -> new FakeProcessor("foo", tag, description, executor))
Map.of("foo", (factories, tag, description, config, projectId) -> new FakeProcessor("foo", tag, description, executor))
);
var projectId = randomProjectIdOrDefault();
@ -2481,9 +2505,9 @@ public class IngestServiceTests extends ESTestCase {
IngestService ingestService = createWithProcessors(
Map.of(
"mock",
(factories, tag, description, config) -> mockCompoundProcessor(),
(factories, tag, description, config, projectId) -> mockCompoundProcessor(),
"set",
(factories, tag, description, config) -> new FakeProcessor(
(factories, tag, description, config, projectId) -> new FakeProcessor(
"set",
tag,
description,
@ -3206,12 +3230,12 @@ public class IngestServiceTests extends ESTestCase {
private static IngestService createWithProcessors() {
Map<String, Processor.Factory> processors = new HashMap<>();
processors.put("set", (factories, tag, description, config) -> {
processors.put("set", (factories, tag, description, config, projectId) -> {
String field = (String) config.remove("field");
String value = (String) config.remove("value");
return new FakeProcessor("set", tag, description, (ingestDocument) -> ingestDocument.setFieldValue(field, value));
});
processors.put("remove", (factories, tag, description, config) -> {
processors.put("remove", (factories, tag, description, config, projectId) -> {
String field = (String) config.remove("field");
return new WrappingProcessorImpl("remove", tag, description, (ingestDocument -> ingestDocument.removeField(field))) {
};

20
server/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java
View file

@ -10,6 +10,7 @@
package org.elasticsearch.ingest;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.test.ESTestCase;
@ -26,6 +27,7 @@ import static org.mockito.Mockito.mock;
public class PipelineFactoryTests extends ESTestCase {
private final ProjectId projectId = randomProjectIdOrDefault();
private final Integer version = randomBoolean() ? randomInt() : null;
private final String versionString = version != null ? Integer.toString(version) : null;
private final ScriptService scriptService = mock(ScriptService.class);
@ -45,7 +47,7 @@ public class PipelineFactoryTests extends ESTestCase {
pipelineConfig.put(Pipeline.DEPRECATED_KEY, deprecated);
pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig0), Map.of("test", processorConfig1)));
Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService);
Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null);
assertThat(pipeline.getId(), equalTo("_id"));
assertThat(pipeline.getDescription(), equalTo("_description"));
assertThat(pipeline.getVersion(), equalTo(version));
@ -65,7 +67,7 @@ public class PipelineFactoryTests extends ESTestCase {
pipelineConfig.put(Pipeline.META_KEY, metadata);
}
try {
Pipeline.create("_id", pipelineConfig, Map.of(), scriptService);
Pipeline.create("_id", pipelineConfig, Map.of(), scriptService, null);
fail("should fail, missing required [processors] field");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[processors] required property is missing"));
@ -80,7 +82,7 @@ public class PipelineFactoryTests extends ESTestCase {
pipelineConfig.put(Pipeline.META_KEY, metadata);
}
pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of());
Pipeline pipeline = Pipeline.create("_id", pipelineConfig, null, scriptService);
Pipeline pipeline = Pipeline.create("_id", pipelineConfig, null, scriptService, null);
assertThat(pipeline.getId(), equalTo("_id"));
assertThat(pipeline.getDescription(), equalTo("_description"));
assertThat(pipeline.getVersion(), equalTo(version));
@ -98,7 +100,7 @@ public class PipelineFactoryTests extends ESTestCase {
pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig)));
pipelineConfig.put(Pipeline.ON_FAILURE_KEY, List.of(Map.of("test", processorConfig)));
Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService);
Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null);
assertThat(pipeline.getId(), equalTo("_id"));
assertThat(pipeline.getDescription(), equalTo("_description"));
assertThat(pipeline.getVersion(), equalTo(version));
@ -121,7 +123,7 @@ public class PipelineFactoryTests extends ESTestCase {
Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
Exception e = expectThrows(
ElasticsearchParseException.class,
() -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService)
() -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null)
);
assertThat(e.getMessage(), equalTo("pipeline [_id] cannot have an empty on_failure option defined"));
}
@ -139,7 +141,7 @@ public class PipelineFactoryTests extends ESTestCase {
Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
Exception e = expectThrows(
ElasticsearchParseException.class,
() -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService)
() -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null)
);
assertThat(e.getMessage(), equalTo("[on_failure] processors list cannot be empty"));
}
@ -157,7 +159,7 @@ public class PipelineFactoryTests extends ESTestCase {
}
pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig)));
Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService);
Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null);
assertThat(pipeline.getId(), equalTo("_id"));
assertThat(pipeline.getDescription(), equalTo("_description"));
assertThat(pipeline.getVersion(), equalTo(version));
@ -182,7 +184,7 @@ public class PipelineFactoryTests extends ESTestCase {
Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
Exception e = expectThrows(
ElasticsearchParseException.class,
() -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService)
() -> Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null)
);
assertThat(e.getMessage(), equalTo("processor [test] doesn't support one or more provided configuration parameters [unused]"));
}
@ -199,7 +201,7 @@ public class PipelineFactoryTests extends ESTestCase {
}
pipelineConfig.put(Pipeline.PROCESSORS_KEY, List.of(Map.of("test", processorConfig)));
Map<String, Processor.Factory> processorRegistry = Map.of("test", new TestProcessor.Factory());
Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService);
Pipeline pipeline = Pipeline.create("_id", pipelineConfig, processorRegistry, scriptService, null);
assertThat(pipeline.getId(), equalTo("_id"));
assertThat(pipeline.getDescription(), equalTo("_description"));
assertThat(pipeline.getVersion(), equalTo(version));

19
server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java
View file

@ -61,7 +61,7 @@ public class PipelineProcessorTests extends ESTestCase {
PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService);
Map<String, Object> config = new HashMap<>();
config.put("name", pipelineId);
factory.create(Map.of(), null, null, config).execute(testIngestDocument, (result, e) -> {});
factory.create(Map.of(), null, null, config, null).execute(testIngestDocument, (result, e) -> {});
assertIngestDocument(testIngestDocument, safeGet(invoked));
}
@ -72,7 +72,7 @@ public class PipelineProcessorTests extends ESTestCase {
Map<String, Object> config = new HashMap<>();
config.put("name", "missingPipelineId");
IllegalStateException[] e = new IllegalStateException[1];
factory.create(Map.of(), null, null, config).execute(testIngestDocument, (result, e1) -> e[0] = (IllegalStateException) e1);
factory.create(Map.of(), null, null, config, null).execute(testIngestDocument, (result, e1) -> e[0] = (IllegalStateException) e1);
assertEquals("Pipeline processor configured for non-existent pipeline [missingPipelineId]", e[0].getMessage());
}
@ -86,7 +86,7 @@ public class PipelineProcessorTests extends ESTestCase {
var r = new IngestDocument[1];
var e = new Exception[1];
var processor = factory.create(Map.of(), null, null, config);
var processor = factory.create(Map.of(), null, null, config, null);
processor.execute(testIngestDocument, (result, e1) -> {
r[0] = result;
e[0] = e1;
@ -108,7 +108,7 @@ public class PipelineProcessorTests extends ESTestCase {
null,
null,
null,
new CompoundProcessor(factory.create(Map.of(), null, null, outerConfig))
new CompoundProcessor(factory.create(Map.of(), null, null, outerConfig, null))
);
Map<String, Object> innerConfig = new HashMap<>();
innerConfig.put("name", outerPipelineId);
@ -117,13 +117,14 @@ public class PipelineProcessorTests extends ESTestCase {
null,
null,
null,
new CompoundProcessor(factory.create(Map.of(), null, null, innerConfig))
new CompoundProcessor(factory.create(Map.of(), null, null, innerConfig, null))
);
when(ingestService.getPipeline(outerPipelineId)).thenReturn(outer);
when(ingestService.getPipeline(innerPipelineId)).thenReturn(inner);
outerConfig.put("name", innerPipelineId);
ElasticsearchException[] e = new ElasticsearchException[1];
factory.create(Map.of(), null, null, outerConfig).execute(testIngestDocument, (result, e1) -> e[0] = (ElasticsearchException) e1);
factory.create(Map.of(), null, null, outerConfig, null)
.execute(testIngestDocument, (result, e1) -> e[0] = (ElasticsearchException) e1);
assertEquals("Cycle detected for pipeline: inner", e[0].getRootCause().getMessage());
}
@ -136,7 +137,7 @@ public class PipelineProcessorTests extends ESTestCase {
PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService);
Pipeline inner = new Pipeline(innerPipelineId, null, null, null, new CompoundProcessor());
when(ingestService.getPipeline(innerPipelineId)).thenReturn(inner);
Processor outerProc = factory.create(Map.of(), null, null, outerConfig);
Processor outerProc = factory.create(Map.of(), null, null, outerConfig, null);
outerProc.execute(testIngestDocument, (result, e) -> {});
outerProc.execute(testIngestDocument, (result, e) -> {});
}
@ -150,11 +151,11 @@ public class PipelineProcessorTests extends ESTestCase {
Map<String, Object> pipeline1ProcessorConfig = new HashMap<>();
pipeline1ProcessorConfig.put("name", pipeline2Id);
PipelineProcessor pipeline1Processor = factory.create(Map.of(), null, null, pipeline1ProcessorConfig);
PipelineProcessor pipeline1Processor = factory.create(Map.of(), null, null, pipeline1ProcessorConfig, null);
Map<String, Object> pipeline2ProcessorConfig = new HashMap<>();
pipeline2ProcessorConfig.put("name", pipeline3Id);
PipelineProcessor pipeline2Processor = factory.create(Map.of(), null, null, pipeline2ProcessorConfig);
PipelineProcessor pipeline2Processor = factory.create(Map.of(), null, null, pipeline2ProcessorConfig, null);
LongSupplier relativeTimeProvider = mock(LongSupplier.class);
when(relativeTimeProvider.getAsLong()).thenReturn(0L);

13
server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java
View file

@ -12,6 +12,7 @@ package org.elasticsearch.ingest;
import org.elasticsearch.action.bulk.FailureStoreMetrics;
import org.elasticsearch.action.bulk.SimulateBulkRequest;
import org.elasticsearch.client.internal.Client;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.cluster.project.TestProjectResolvers;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.bytes.BytesArray;
@ -48,21 +49,21 @@ public class SimulateIngestServiceTests extends ESTestCase {
Map<String, Processor.Factory> processors = new HashMap<>();
processors.put(
"processor1",
(factories, tag, description, config) -> new FakeProcessor("processor1", tag, description, ingestDocument -> {}) {
(factories, tag, description, config, projectId) -> new FakeProcessor("processor1", tag, description, ingestDocument -> {}) {
}
);
processors.put(
"processor2",
(factories, tag, description, config) -> new FakeProcessor("processor2", tag, description, ingestDocument -> {}) {
(factories, tag, description, config, projectId) -> new FakeProcessor("processor2", tag, description, ingestDocument -> {}) {
}
);
processors.put(
"processor3",
(factories, tag, description, config) -> new FakeProcessor("processor3", tag, description, ingestDocument -> {}) {
(factories, tag, description, config, projectId) -> new FakeProcessor("processor3", tag, description, ingestDocument -> {}) {
}
);
IngestService ingestService = createWithProcessors(processors);
final var projectId = randomProjectIdOrDefault();
IngestService ingestService = createWithProcessors(projectId, processors);
ingestService.innerUpdatePipelines(projectId, ingestMetadata);
{
// First we make sure that if there are no substitutions that we get our original pipeline back:
@ -113,7 +114,7 @@ public class SimulateIngestServiceTests extends ESTestCase {
}
}
private static IngestService createWithProcessors(Map<String, Processor.Factory> processors) {
private static IngestService createWithProcessors(ProjectId projectId, Map<String, Processor.Factory> processors) {
Client client = mock(Client.class);
ThreadPool threadPool = mock(ThreadPool.class);
when(threadPool.generic()).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE);
@ -134,7 +135,7 @@ public class SimulateIngestServiceTests extends ESTestCase {
client,
null,
FailureStoreMetrics.NOOP,
TestProjectResolvers.singleProjectOnly()
TestProjectResolvers.singleProject(projectId)
);
}
}

30
server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java
View file

@ -306,7 +306,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
})));
when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig, null);
CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@ -375,7 +375,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
null,
new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()),
scriptService,
factory.create(Map.of(), "pipeline1", null, pipelineConfig2)
factory.create(Map.of(), "pipeline1", null, pipelineConfig2, null)
),
new TestProcessor(ingestDocument -> {
ingestDocument.setFieldValue(key3, randomInt());
@ -389,7 +389,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1);
when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), "pipeline0", null, pipelineConfig0);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), "pipeline0", null, pipelineConfig0, null);
CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@ -468,7 +468,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
null,
new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Map.of()),
scriptService,
factory.create(Map.of(), null, null, pipelineConfig2)
factory.create(Map.of(), null, null, pipelineConfig2, null)
),
new TestProcessor(ingestDocument -> {
ingestDocument.setFieldValue(key3, randomInt());
@ -482,7 +482,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1);
when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig0);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig0, null);
CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@ -545,7 +545,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
));
when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig, null);
CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@ -608,7 +608,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
);
when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig, null);
CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@ -650,7 +650,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
null,
null,
null,
new CompoundProcessor(factory.create(Map.of(), null, null, pipelineConfig2))
new CompoundProcessor(factory.create(Map.of(), null, null, pipelineConfig2, null))
);
Pipeline pipeline2 = new Pipeline(
@ -658,13 +658,13 @@ public class TrackingResultProcessorTests extends ESTestCase {
null,
null,
null,
new CompoundProcessor(factory.create(Map.of(), null, null, pipelineConfig1))
new CompoundProcessor(factory.create(Map.of(), null, null, pipelineConfig1, null))
);
when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1);
when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig0);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig0, null);
CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@ -694,7 +694,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
null,
null,
null,
new CompoundProcessor(factory.create(Map.of(), null, null, nextPipelineConfig))
new CompoundProcessor(factory.create(Map.of(), null, null, nextPipelineConfig, null))
);
when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
}
@ -708,7 +708,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
String firstPipelineId = "pipeline0";
Map<String, Object> firstPipelineConfig = new HashMap<>();
firstPipelineConfig.put("name", firstPipelineId);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, firstPipelineConfig);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, firstPipelineConfig, null);
CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@ -747,7 +747,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
null,
null,
null,
new CompoundProcessor(factory.create(Map.of(), null, null, nextPipelineConfig))
new CompoundProcessor(factory.create(Map.of(), null, null, nextPipelineConfig, null))
);
when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
}
@ -761,7 +761,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
String firstPipelineId = "pipeline0";
Map<String, Object> firstPipelineConfig = new HashMap<>();
firstPipelineConfig.put("name", firstPipelineId);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, firstPipelineConfig);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, firstPipelineConfig, null);
CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
@ -789,7 +789,7 @@ public class TrackingResultProcessorTests extends ESTestCase {
PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService);
String key1 = randomAlphaOfLength(10);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig);
PipelineProcessor pipelineProcessor = factory.create(Map.of(), null, null, pipelineConfig, null);
Pipeline pipeline = new Pipeline(pipelineId, null, null, null, new CompoundProcessor(new TestProcessor(ingestDocument -> {
ingestDocument.setFieldValue(key1, randomInt());
})));

2
test/framework/src/main/java/org/elasticsearch/ingest/IngestTestPlugin.java
View file

@ -20,7 +20,7 @@ import java.util.Map;
public class IngestTestPlugin extends Plugin implements IngestPlugin {
@Override
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
return Map.of("test", (factories, tag, description, config) -> new TestProcessor("id", "test", "description", doc -> {
return Map.of("test", (factories, tag, description, config, projectId) -> new TestProcessor("id", "test", "description", doc -> {
doc.setFieldValue("processed", true);
if (doc.hasField("fail") && doc.getFieldValue("fail", Boolean.class)) {
throw new IllegalArgumentException("test processor failed");

5
test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java
View file

@ -9,6 +9,8 @@
package org.elasticsearch.ingest;
import org.elasticsearch.cluster.metadata.ProjectId;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
@ -102,7 +104,8 @@ public class TestProcessor implements Processor {
Map<String, Processor.Factory> registry,
String processorTag,
String description,
Map<String, Object> config
Map<String, Object> config,
ProjectId projectId
) throws Exception {
return new TestProcessor(processorTag, "test-processor", description, ingestDocument -> {});
}

3
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/validation/SourceDestValidatorTests.java
View file

@ -599,7 +599,8 @@ public class SourceDestValidatorTests extends ESTestCase {
Arrays.asList(Collections.singletonMap("test", processorConfig0), Collections.singletonMap("test", processorConfig1))
);
Map<String, Processor.Factory> processorRegistry = Collections.singletonMap("test", new TestProcessor.Factory());
Pipeline pipeline = Pipeline.create("missing-pipeline", pipelineConfig, processorRegistry, null);
var projectId = randomProjectIdOrDefault();
Pipeline pipeline = Pipeline.create("missing-pipeline", pipelineConfig, processorRegistry, null, projectId);
when(ingestService.getPipeline("missing-pipeline")).thenReturn(pipeline);
assertValidation(

2
x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichCache.java
View file

@ -13,6 +13,7 @@ import org.elasticsearch.common.cache.Cache;
import org.elasticsearch.common.cache.CacheBuilder;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.util.Maps;
import org.elasticsearch.core.FixForMultiProject;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.xpack.core.enrich.action.EnrichStatsAction;
@ -89,6 +90,7 @@ public final class EnrichCache {
* @param searchResponseFetcher The function used to compute the value to be put in the cache, if there is no value in the cache already
* @param listener A listener to be notified of the value in the cache
*/
@FixForMultiProject(description = "The enrich cache will currently leak data between projects. We need to either disable or fix it.")
public void computeIfAbsent(
String enrichIndex,
Object lookupValue,

8
x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyExecutor.java
View file

@ -18,6 +18,7 @@ import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskRequest;
import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse;
import org.elasticsearch.client.internal.Client;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
@ -115,19 +116,20 @@ public class EnrichPolicyExecutor {
}
public void runPolicyLocally(
ProjectId projectId,
ExecuteEnrichPolicyTask task,
String policyName,
String enrichIndexName,
ActionListener<ExecuteEnrichPolicyStatus> listener
) {
try {
EnrichPolicy policy = EnrichStore.getPolicy(policyName, clusterService.state().metadata().getProject());
EnrichPolicy policy = EnrichStore.getPolicy(policyName, clusterService.state().metadata().getProject(projectId));
if (policy == null) {
throw new ResourceNotFoundException("policy [{}] does not exist", policyName);
}
task.setStatus(new ExecuteEnrichPolicyStatus(ExecuteEnrichPolicyStatus.PolicyPhases.SCHEDULED));
var policyRunner = createPolicyRunner(policyName, policy, enrichIndexName, task);
var policyRunner = createPolicyRunner(projectId, policyName, policy, enrichIndexName, task);
threadPool.executor(ThreadPool.Names.GENERIC)
.execute(ActionRunnable.wrap(ActionListener.assertOnce(listener), policyRunner::run));
} catch (Exception e) {
@ -209,12 +211,14 @@ public class EnrichPolicyExecutor {
}
private EnrichPolicyRunner createPolicyRunner(
ProjectId projectId,
String policyName,
EnrichPolicy policy,
String enrichIndexName,
ExecuteEnrichPolicyTask task
) {
return new EnrichPolicyRunner(
projectId,
policyName,
policy,
task,

8
x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyReindexPipeline.java
View file

@ -12,7 +12,7 @@ import org.elasticsearch.action.ingest.PutPipelineRequest;
import org.elasticsearch.action.ingest.PutPipelineTransportAction;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.internal.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.ProjectMetadata;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.ingest.IngestMetadata;
import org.elasticsearch.ingest.PipelineConfiguration;
@ -47,11 +47,11 @@ public class EnrichPolicyReindexPipeline {
/**
* Checks if the current version of the pipeline definition is installed in the cluster
* @param clusterState The cluster state to check
* @param project The project metadata to check
* @return true if a pipeline exists that is compatible with this version of Enrich, false otherwise
*/
static boolean exists(ClusterState clusterState) {
final IngestMetadata ingestMetadata = clusterState.getMetadata().getProject().custom(IngestMetadata.TYPE);
static boolean exists(ProjectMetadata project) {
final IngestMetadata ingestMetadata = project.custom(IngestMetadata.TYPE);
// we ensure that we both have the pipeline and its version represents the current (or later) version
if (ingestMetadata != null) {
final PipelineConfiguration pipeline = ingestMetadata.getPipelines().get(pipelineName());

20
x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPolicyRunner.java
View file

@ -40,10 +40,11 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.internal.Client;
import org.elasticsearch.client.internal.FilterClient;
import org.elasticsearch.client.internal.OriginSettingClient;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.cluster.metadata.ProjectMetadata;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.UUIDs;
@ -102,6 +103,7 @@ public class EnrichPolicyRunner {
*/
static final TimeValue ENRICH_MASTER_REQUEST_TIMEOUT = TimeValue.THIRTY_SECONDS;
private final ProjectId projectId;
private final String policyName;
private final EnrichPolicy policy;
private final ExecuteEnrichPolicyTask task;
@ -114,6 +116,7 @@ public class EnrichPolicyRunner {
private final int maxForceMergeAttempts;
EnrichPolicyRunner(
ProjectId projectId,
String policyName,
EnrichPolicy policy,
ExecuteEnrichPolicyTask task,
@ -125,6 +128,7 @@ public class EnrichPolicyRunner {
int fetchSize,
int maxForceMergeAttempts
) {
this.projectId = projectId;
this.policyName = Objects.requireNonNull(policyName);
this.policy = Objects.requireNonNull(policy);
this.task = Objects.requireNonNull(task);
@ -480,7 +484,7 @@ public class EnrichPolicyRunner {
private void prepareReindexOperation(ActionListener<AcknowledgedResponse> listener) {
// Check to make sure that the enrich pipeline exists, and create it if it is missing.
if (EnrichPolicyReindexPipeline.exists(clusterService.state())) {
if (EnrichPolicyReindexPipeline.exists(clusterService.state().getMetadata().getProject(projectId))) {
listener.onResponse(null);
} else {
EnrichPolicyReindexPipeline.create(enrichOriginClient(), listener);
@ -700,8 +704,8 @@ public class EnrichPolicyRunner {
* and recreated with invalid mappings/data. We validate that the mapping exists and that it contains the expected meta fields on it to
* guard against accidental removal and recreation during policy execution.
*/
private void validateIndexBeforePromotion(String destinationIndexName, ClusterState clusterState) {
IndexMetadata destinationIndex = clusterState.metadata().getProject().index(destinationIndexName);
private void validateIndexBeforePromotion(String destinationIndexName, ProjectMetadata project) {
IndexMetadata destinationIndex = project.index(destinationIndexName);
if (destinationIndex == null) {
throw new IndexNotFoundException(
"was not able to promote it as part of executing enrich policy [" + policyName + "]",
@ -749,12 +753,12 @@ public class EnrichPolicyRunner {
String enrichIndexBase = EnrichPolicy.getBaseName(policyName);
logger.debug("Policy [{}]: Promoting new enrich index [{}] to alias [{}]", policyName, enrichIndexName, enrichIndexBase);
GetAliasesRequest aliasRequest = new GetAliasesRequest(ENRICH_MASTER_REQUEST_TIMEOUT, enrichIndexBase);
ClusterState clusterState = clusterService.state();
validateIndexBeforePromotion(enrichIndexName, clusterState);
String[] concreteIndices = indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(clusterState, aliasRequest);
final var project = clusterService.state().metadata().getProject(projectId);
validateIndexBeforePromotion(enrichIndexName, project);
String[] concreteIndices = indexNameExpressionResolver.concreteIndexNamesWithSystemIndexAccess(project, aliasRequest);
String[] aliases = aliasRequest.aliases();
IndicesAliasesRequest aliasToggleRequest = new IndicesAliasesRequest(ENRICH_MASTER_REQUEST_TIMEOUT, ENRICH_MASTER_REQUEST_TIMEOUT);
String[] indices = clusterState.metadata().getProject().findAliases(aliases, concreteIndices).keySet().toArray(new String[0]);
String[] indices = project.findAliases(aliases, concreteIndices).keySet().toArray(new String[0]);
if (indices.length > 0) {
aliasToggleRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove().indices(indices).alias(enrichIndexBase));
}

26
x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java
View file

@ -14,6 +14,8 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexAbstraction;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.ProjectId;
import org.elasticsearch.cluster.metadata.ProjectMetadata;
import org.elasticsearch.common.geo.Orientation;
import org.elasticsearch.common.geo.ShapeRelation;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
@ -50,20 +52,26 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer<Cluste
}
@Override
public Processor create(Map<String, Processor.Factory> processorFactories, String tag, String description, Map<String, Object> config)
throws Exception {
public Processor create(
Map<String, Processor.Factory> processorFactories,
String tag,
String description,
Map<String, Object> config,
ProjectId projectId
) throws Exception {
final String policyName = ConfigurationUtils.readStringProperty(TYPE, tag, config, "policy_name");
final String indexAlias = EnrichPolicy.getBaseName(policyName);
if (metadata == null) {
throw new IllegalStateException("enrich processor factory has not yet been initialized with cluster state");
}
IndexAbstraction indexAbstraction = metadata.getProject().getIndicesLookup().get(indexAlias);
final var project = metadata.getProject(projectId);
IndexAbstraction indexAbstraction = project.getIndicesLookup().get(indexAlias);
if (indexAbstraction == null) {
throw new IllegalArgumentException("no enrich index exists for policy with name [" + policyName + "]");
}
assert indexAbstraction.getType() == IndexAbstraction.Type.ALIAS;
assert indexAbstraction.getIndices().size() == 1;
IndexMetadata imd = metadata.getProject().index(indexAbstraction.getIndices().get(0));
IndexMetadata imd = project.index(indexAbstraction.getIndices().get(0));
Map<String, Object> mappingAsMap = imd.mapping().sourceAsMap();
String policyType = (String) XContentMapValues.extractValue(
@ -80,7 +88,7 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer<Cluste
if (maxMatches < 1 || maxMatches > 128) {
throw ConfigurationUtils.newConfigurationException(TYPE, tag, "max_matches", "should be between 1 and 128");
}
var searchRunner = createSearchRunner(indexAlias, client, enrichCache);
var searchRunner = createSearchRunner(project, indexAlias);
switch (policyType) {
case EnrichPolicy.MATCH_TYPE:
case EnrichPolicy.RANGE_TYPE:
@ -125,12 +133,12 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer<Cluste
metadata = state.getMetadata();
}
private SearchRunner createSearchRunner(String indexAlias, Client client, EnrichCache enrichCache) {
private SearchRunner createSearchRunner(ProjectMetadata project, String indexAlias) {
Client originClient = new OriginSettingClient(client, ENRICH_ORIGIN);
return (value, maxMatches, reqSupplier, handler) -> {
// intentionally non-locking for simplicity...it's OK if we re-put the same key/value in the cache during a race condition.
enrichCache.computeIfAbsent(
getEnrichIndexKey(indexAlias),
getEnrichIndexKey(project, indexAlias),
value,
maxMatches,
(searchResponseActionListener) -> originClient.execute(
@ -143,8 +151,8 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer<Cluste
};
}
private String getEnrichIndexKey(String indexAlias) {
IndexAbstraction ia = metadata.getProject().getIndicesLookup().get(indexAlias);
private String getEnrichIndexKey(ProjectMetadata project, String indexAlias) {
IndexAbstraction ia = project.getIndicesLookup().get(indexAlias);
if (ia == null) {
throw new IndexNotFoundException("no generated enrich index [" + indexAlias + "]");
}

Some files were not shown because too many files have changed in this diff Show more