diff --git a/docs/reference/ingest.asciidoc b/docs/reference/ingest.asciidoc index 08f79f5b797d..62da9f4b7d66 100644 --- a/docs/reference/ingest.asciidoc +++ b/docs/reference/ingest.asciidoc @@ -191,6 +191,7 @@ The API returns transformed documents: "doc": { "_index": "_index", "_id": "_id", + "_version": "-3", "_source": { "my-keyword-field": "foo" }, @@ -203,6 +204,7 @@ The API returns transformed documents: "doc": { "_index": "_index", "_id": "_id", + "_version": "-3", "_source": { "my-keyword-field": "bar" }, diff --git a/docs/reference/ingest/apis/simulate-pipeline.asciidoc b/docs/reference/ingest/apis/simulate-pipeline.asciidoc index eb79db434fb2..bb7c34338313 100644 --- a/docs/reference/ingest/apis/simulate-pipeline.asciidoc +++ b/docs/reference/ingest/apis/simulate-pipeline.asciidoc @@ -179,6 +179,7 @@ The API returns the following response: "doc": { "_id": "id", "_index": "index", + "_version": "-3", "_source": { "field2": "_value", "foo": "bar" @@ -192,6 +193,7 @@ The API returns the following response: "doc": { "_id": "id", "_index": "index", + "_version": "-3", "_source": { "field2": "_value", "foo": "rab" @@ -256,6 +258,7 @@ The API returns the following response: "doc": { "_id": "id", "_index": "index", + "_version": "-3", "_source": { "field2": "_value", "foo": "bar" @@ -269,6 +272,7 @@ The API returns the following response: "doc": { "_id": "id", "_index": "index", + "_version": "-3", "_source": { "field2": "_value", "foo": "rab" @@ -350,6 +354,7 @@ The API returns the following response: "doc" : { "_index" : "index", "_id" : "id", + "_version": "-3", "_source" : { "field2" : "_value2", "foo" : "bar" @@ -366,6 +371,7 @@ The API returns the following response: "doc" : { "_index" : "index", "_id" : "id", + "_version": "-3", "_source" : { "field3" : "_value3", "field2" : "_value2", @@ -387,6 +393,7 @@ The API returns the following response: "doc" : { "_index" : "index", "_id" : "id", + "_version": "-3", "_source" : { "field2" : "_value2", "foo" : "rab" @@ -403,6 +410,7 @@ The API returns the following response: "doc" : { "_index" : "index", "_id" : "id", + "_version": "-3", "_source" : { "field3" : "_value3", "field2" : "_value2", diff --git a/docs/reference/ingest/processors/date-index-name.asciidoc b/docs/reference/ingest/processors/date-index-name.asciidoc index a25e954224a9..77898413a37d 100644 --- a/docs/reference/ingest/processors/date-index-name.asciidoc +++ b/docs/reference/ingest/processors/date-index-name.asciidoc @@ -111,6 +111,7 @@ and the result: "doc" : { "_id" : "_id", "_index" : "", + "_version" : "-3", "_source" : { "date1" : "2016-04-25T12:02:01.789Z" }, diff --git a/docs/reference/ingest/processors/fingerprint.asciidoc b/docs/reference/ingest/processors/fingerprint.asciidoc index ebdf1a2c8908..0b585db0e7f6 100644 --- a/docs/reference/ingest/processors/fingerprint.asciidoc +++ b/docs/reference/ingest/processors/fingerprint.asciidoc @@ -86,4 +86,4 @@ Which produces the following result: ] } ---- -// TESTRESPONSE[s/\.\.\./"_index":"_index","_id":"_id","_ingest":{"timestamp":$body.docs.0.doc._ingest.timestamp},/] +// TESTRESPONSE[s/\.\.\./"_index":"_index","_id":"_id","_version":"-3","_ingest":{"timestamp":$body.docs.0.doc._ingest.timestamp},/] diff --git a/docs/reference/ingest/processors/grok.asciidoc b/docs/reference/ingest/processors/grok.asciidoc index 3188e0b2bbfb..207e43a44272 100644 --- a/docs/reference/ingest/processors/grok.asciidoc +++ b/docs/reference/ingest/processors/grok.asciidoc @@ -70,6 +70,7 @@ This pipeline will insert these named captures as new fields within the document "doc": { "_index": "_index", "_id": "_id", + "_version": "-3", "_source" : { "duration" : 0.043, "request" : "/index.html", @@ -166,6 +167,7 @@ response: "doc": { "_index": "_index", "_id": "_id", + "_version": "-3", "_source": { "message": "I love burmese cats!", "pet": "burmese" @@ -225,6 +227,7 @@ POST _ingest/pipeline/_simulate "doc": { "_index": "_index", "_id": "_id", + "_version": "-3", "_source": { "message": "I love burmese cats!", "pet": "burmese" diff --git a/docs/reference/ingest/processors/network-direction.asciidoc b/docs/reference/ingest/processors/network-direction.asciidoc index ae820365a67f..b2e4257d990c 100644 --- a/docs/reference/ingest/processors/network-direction.asciidoc +++ b/docs/reference/ingest/processors/network-direction.asciidoc @@ -115,5 +115,5 @@ Which produces the following result: ] } ---- -// TESTRESPONSE[s/\.\.\./"_index":"_index","_id":"_id","_ingest":{"timestamp":$body.docs.0.doc._ingest.timestamp},/] +// TESTRESPONSE[s/\.\.\./"_index":"_index","_id":"_id","_version":"-3","_ingest":{"timestamp":$body.docs.0.doc._ingest.timestamp},/] // NOTCONSOLE diff --git a/docs/reference/ingest/processors/registered-domain.asciidoc b/docs/reference/ingest/processors/registered-domain.asciidoc index d8481caf0c85..a0701f07e313 100644 --- a/docs/reference/ingest/processors/registered-domain.asciidoc +++ b/docs/reference/ingest/processors/registered-domain.asciidoc @@ -78,4 +78,4 @@ Which produces the following result: ] } ---- -// TESTRESPONSE[s/\.\.\./"_index":"_index","_id":"_id","_ingest":{"timestamp":$body.docs.0.doc._ingest.timestamp},/] +// TESTRESPONSE[s/\.\.\./"_index":"_index","_id":"_id","_version":"-3","_ingest":{"timestamp":$body.docs.0.doc._ingest.timestamp},/] diff --git a/docs/reference/ingest/processors/script.asciidoc b/docs/reference/ingest/processors/script.asciidoc index 72a18db475a2..68d565f813be 100644 --- a/docs/reference/ingest/processors/script.asciidoc +++ b/docs/reference/ingest/processors/script.asciidoc @@ -98,7 +98,7 @@ The processor produces: ] } ---- -// TESTRESPONSE[s/\.\.\./"_index":"_index","_id":"_id","_ingest":{"timestamp":$body.docs.0.doc._ingest.timestamp},/] +// TESTRESPONSE[s/\.\.\./"_index":"_index","_id":"_id","_version":"-3","_ingest":{"timestamp":$body.docs.0.doc._ingest.timestamp},/] [discrete] @@ -158,4 +158,4 @@ The processor changes the document's `_index` to `fr-catalog` from ] } ---- -// TESTRESPONSE[s/\.\.\./"_id":"_id","_ingest":{"timestamp":$body.docs.0.doc._ingest.timestamp},/] +// TESTRESPONSE[s/\.\.\./"_id":"_id","_version":"-3","_ingest":{"timestamp":$body.docs.0.doc._ingest.timestamp},/] diff --git a/docs/reference/ingest/processors/set.asciidoc b/docs/reference/ingest/processors/set.asciidoc index 4d735d12a174..e5c331c11571 100644 --- a/docs/reference/ingest/processors/set.asciidoc +++ b/docs/reference/ingest/processors/set.asciidoc @@ -72,6 +72,7 @@ Result: "doc" : { "_index" : "_index", "_id" : "_id", + "_version" : "-3", "_source" : { "host" : { "os" : { @@ -127,6 +128,7 @@ Result: "doc" : { "_index" : "_index", "_id" : "_id", + "_version" : "-3", "_source" : { "bar": ["foo1", "foo2"], "foo": ["foo1", "foo2"] diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java index a8c0de2cfd69..a79580d743a0 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java @@ -37,7 +37,7 @@ public class DateIndexNameProcessorTests extends ESTestCase { IngestDocument document = new IngestDocument( "_index", "_id", - null, + 1, null, null, Collections.singletonMap("_field", "2016-04-25T12:24:20.101Z") @@ -59,7 +59,7 @@ public class DateIndexNameProcessorTests extends ESTestCase { IngestDocument document = new IngestDocument( "_index", "_id", - null, + 1, null, null, Collections.singletonMap("_field", (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024") @@ -78,11 +78,11 @@ public class DateIndexNameProcessorTests extends ESTestCase { "m", "yyyyMMdd" ); - IngestDocument document = new IngestDocument("_index", "_id", null, null, null, Collections.singletonMap("_field", "1000500")); + IngestDocument document = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("_field", "1000500")); dateProcessor.execute(document); assertThat(document.getSourceAndMetadata().get("_index"), equalTo("")); - document = new IngestDocument("_index", "_id", null, null, null, Collections.singletonMap("_field", 1000500L)); + document = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("_field", 1000500L)); dateProcessor.execute(document); assertThat(document.getSourceAndMetadata().get("_index"), equalTo("")); } @@ -97,7 +97,7 @@ public class DateIndexNameProcessorTests extends ESTestCase { "m", "yyyyMMdd" ); - IngestDocument document = new IngestDocument("_index", "_id", null, null, null, Collections.singletonMap("_field", "1000.5")); + IngestDocument document = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("_field", "1000.5")); dateProcessor.execute(document); assertThat(document.getSourceAndMetadata().get("_index"), equalTo("")); } @@ -118,7 +118,7 @@ public class DateIndexNameProcessorTests extends ESTestCase { indexNameFormat ); - IngestDocument document = new IngestDocument("_index", "_id", null, null, null, Collections.singletonMap("_field", date)); + IngestDocument document = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("_field", date)); dateProcessor.execute(document); assertThat( diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java index c15cc80ecbcc..ec2762ebfe53 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DissectProcessorTests.java @@ -32,7 +32,7 @@ public class DissectProcessorTests extends ESTestCase { IngestDocument ingestDocument = new IngestDocument( "_index", "_id", - null, + 1, null, null, Collections.singletonMap("message", "foo,bar,baz") @@ -48,7 +48,7 @@ public class DissectProcessorTests extends ESTestCase { IngestDocument ingestDocument = new IngestDocument( "_index", "_id", - null, + 1, null, null, MapBuilder.newMapBuilder().put("message", "foo,bar,baz").put("a", "willgetstompped").map() @@ -65,7 +65,7 @@ public class DissectProcessorTests extends ESTestCase { IngestDocument ingestDocument = new IngestDocument( "_index", "_id", - null, + 1, null, null, Collections.singletonMap("message", "foo bar,,,,,,,baz nope:notagain 😊 🐇 🙃") @@ -90,7 +90,7 @@ public class DissectProcessorTests extends ESTestCase { IngestDocument ingestDocument = new IngestDocument( "_index", "_id", - null, + 1, null, null, Collections.singletonMap("message", "foo:bar,baz") diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java index 82ace16b0223..0d96e4c6680b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java @@ -41,7 +41,7 @@ public class ForEachProcessorTests extends ESTestCase { values.add("foo"); values.add("bar"); values.add("baz"); - IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, Collections.singletonMap("values", values)); + IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("values", values)); ForEachProcessor processor = new ForEachProcessor("_tag", null, "values", new AsyncUpperCaseProcessor("_ingest._value"), false); execProcessor(processor, ingestDocument, (result, e) -> {}); @@ -60,7 +60,7 @@ public class ForEachProcessorTests extends ESTestCase { IngestDocument ingestDocument = new IngestDocument( "_index", "_id", - null, + 1, null, null, Collections.singletonMap("values", Arrays.asList("a", "b", "c")) @@ -104,7 +104,7 @@ public class ForEachProcessorTests extends ESTestCase { List> values = new ArrayList<>(); values.add(new HashMap<>()); values.add(new HashMap<>()); - IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, Collections.singletonMap("values", values)); + IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("values", values)); TestProcessor innerProcessor = new TestProcessor(id -> { id.setFieldValue("_ingest._value.index", id.getSourceAndMetadata().get("_index")); @@ -131,7 +131,7 @@ public class ForEachProcessorTests extends ESTestCase { document.put("values", values); document.put("flat_values", new ArrayList<>()); document.put("other", "value"); - IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, document); + IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, document); ForEachProcessor processor = new ForEachProcessor( "_tag", @@ -182,7 +182,7 @@ public class ForEachProcessorTests extends ESTestCase { int numValues = randomIntBetween(1, 10000); List values = IntStream.range(0, numValues).mapToObj(i -> "").collect(Collectors.toList()); - IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, Collections.singletonMap("values", values)); + IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("values", values)); ForEachProcessor processor = new ForEachProcessor("_tag", null, "values", innerProcessor, false); execProcessor(processor, ingestDocument, (result, e) -> {}); @@ -198,7 +198,7 @@ public class ForEachProcessorTests extends ESTestCase { values.add("string"); values.add(1); values.add(null); - IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, Collections.singletonMap("values", values)); + IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("values", values)); TemplateScript.Factory template = new TestTemplateService.MockTemplateScript.Factory("errors"); @@ -232,7 +232,7 @@ public class ForEachProcessorTests extends ESTestCase { Map source = new HashMap<>(); source.put("_value", "new_value"); source.put("values", values); - IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, source); + IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, source); TestProcessor processor = new TestProcessor( doc -> doc.setFieldValue("_ingest._value", doc.getFieldValue("_source._value", String.class)) @@ -262,7 +262,7 @@ public class ForEachProcessorTests extends ESTestCase { value.put("values2", innerValues); values.add(value); - IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, Collections.singletonMap("values1", values)); + IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Collections.singletonMap("values1", values)); TestProcessor testProcessor = new TestProcessor( doc -> doc.setFieldValue("_ingest._value", doc.getFieldValue("_ingest._value", String.class).toUpperCase(Locale.ENGLISH)) @@ -291,7 +291,7 @@ public class ForEachProcessorTests extends ESTestCase { Map innerMap3 = Map.of("foo3", 7, "bar3", 8, "baz3", 9, "otherKey", 42); Map outerMap = Map.of("foo", innerMap1, "bar", innerMap2, "baz", innerMap3); - IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, Map.of("field", outerMap)); + IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of("field", outerMap)); List visitedKeys = new ArrayList<>(); List visitedValues = new ArrayList<>(); @@ -337,7 +337,7 @@ public class ForEachProcessorTests extends ESTestCase { } public void testIgnoreMissing() { - IngestDocument originalIngestDocument = new IngestDocument("_index", "_id", null, null, null, Collections.emptyMap()); + IngestDocument originalIngestDocument = new IngestDocument("_index", "_id", 1, null, null, Collections.emptyMap()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); TestProcessor testProcessor = new TestProcessor(doc -> {}); ForEachProcessor processor = new ForEachProcessor("_tag", null, "_ingest._value", testProcessor, true); @@ -347,7 +347,7 @@ public class ForEachProcessorTests extends ESTestCase { } public void testAppendingToTheSameField() { - IngestDocument originalIngestDocument = new IngestDocument("_index", "_id", null, null, null, Map.of("field", List.of("a", "b"))); + IngestDocument originalIngestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of("field", List.of("a", "b"))); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); TestProcessor testProcessor = new TestProcessor(id -> id.appendFieldValue("field", "a")); ForEachProcessor processor = new ForEachProcessor("_tag", null, "field", testProcessor, true); @@ -358,7 +358,7 @@ public class ForEachProcessorTests extends ESTestCase { } public void testRemovingFromTheSameField() { - IngestDocument originalIngestDocument = new IngestDocument("_index", "_id", null, null, null, Map.of("field", List.of("a", "b"))); + IngestDocument originalIngestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of("field", List.of("a", "b"))); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); TestProcessor testProcessor = new TestProcessor(id -> id.removeField("field.0")); ForEachProcessor processor = new ForEachProcessor("_tag", null, "field", testProcessor, true); @@ -370,7 +370,7 @@ public class ForEachProcessorTests extends ESTestCase { public void testMapIteration() { Map mapValue = Map.of("foo", 1, "bar", 2, "baz", 3); - IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, Map.of("field", mapValue)); + IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of("field", mapValue)); List encounteredKeys = new ArrayList<>(); List encounteredValues = new ArrayList<>(); @@ -399,7 +399,7 @@ public class ForEachProcessorTests extends ESTestCase { public void testRemovalOfMapKey() { Map mapValue = Map.of("foo", 1, "bar", 2, "baz", 3); - IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, Map.of("field", mapValue)); + IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of("field", mapValue)); List encounteredKeys = new ArrayList<>(); List encounteredValues = new ArrayList<>(); @@ -428,7 +428,7 @@ public class ForEachProcessorTests extends ESTestCase { Map innerMap3 = Map.of("foo3", 7, "bar3", 8, "baz3", 9, "otherKey", 42); Map outerMap = Map.of("foo", innerMap1, "bar", innerMap2, "baz", innerMap3); - IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, Map.of("field", outerMap)); + IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1, null, null, Map.of("field", outerMap)); List visitedKeys = new ArrayList<>(); List visitedValues = new ArrayList<>(); diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java index 3afd8d55fdcb..fffd1343a339 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java @@ -96,8 +96,8 @@ public class ReloadingDatabasesWhilePerformingGeoLookupsIT extends ESTestCase { IngestDocument document1 = new IngestDocument( "index", "id", - "routing", 1L, + "routing", VersionType.EXTERNAL, Map.of("_field", "89.160.20.128") ); @@ -106,8 +106,8 @@ public class ReloadingDatabasesWhilePerformingGeoLookupsIT extends ESTestCase { IngestDocument document2 = new IngestDocument( "index", "id", - "routing", 1L, + "routing", VersionType.EXTERNAL, Map.of("_field", "89.160.20.128") ); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index f3616d1b83c7..0f97f212c42c 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -305,7 +305,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase { } final Map field = Collections.singletonMap("_field", "1.1.1.1"); - final IngestDocument document = new IngestDocument("index", "id", "routing", 1L, VersionType.EXTERNAL, field); + final IngestDocument document = new IngestDocument("index", "id", 1L, "routing", VersionType.EXTERNAL, field); Map config = new HashMap<>(); config.put("field", "_field"); @@ -368,7 +368,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase { } final Map field = Collections.singletonMap("_field", "1.1.1.1"); - final IngestDocument document = new IngestDocument("index", "id", "routing", 1L, VersionType.EXTERNAL, field); + final IngestDocument document = new IngestDocument("index", "id", 1L, "routing", VersionType.EXTERNAL, field); Map config = new HashMap<>(); config.put("field", "_field"); diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java index a77785333bc7..724f5c2d51be 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/IngestDocumentMustacheIT.java @@ -22,7 +22,7 @@ public class IngestDocumentMustacheIT extends AbstractScriptTestCase { public void testAccessMetadataViaTemplate() { Map document = new HashMap<>(); document.put("foo", "bar"); - IngestDocument ingestDocument = new IngestDocument("index", "id", null, null, null, document); + IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); ingestDocument.setFieldValue(compile("field1"), ValueSource.wrap("1 {{foo}}", scriptService)); assertThat(ingestDocument.getFieldValue("field1", String.class), equalTo("1 bar")); @@ -37,7 +37,7 @@ public class IngestDocumentMustacheIT extends AbstractScriptTestCase { innerObject.put("baz", "hello baz"); innerObject.put("qux", Collections.singletonMap("fubar", "hello qux and fubar")); document.put("foo", innerObject); - IngestDocument ingestDocument = new IngestDocument("index", "id", null, null, null, document); + IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); ingestDocument.setFieldValue(compile("field1"), ValueSource.wrap("1 {{foo.bar}} {{foo.baz}} {{foo.qux.fubar}}", scriptService)); assertThat(ingestDocument.getFieldValue("field1", String.class), equalTo("1 hello bar hello baz hello qux and fubar")); @@ -57,7 +57,7 @@ public class IngestDocumentMustacheIT extends AbstractScriptTestCase { list.add(value); list.add(null); document.put("list2", list); - IngestDocument ingestDocument = new IngestDocument("index", "id", null, null, null, document); + IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); ingestDocument.setFieldValue(compile("field1"), ValueSource.wrap("1 {{list1.0}} {{list2.0}}", scriptService)); assertThat(ingestDocument.getFieldValue("field1", String.class), equalTo("1 foo {field=value}")); } @@ -67,7 +67,7 @@ public class IngestDocumentMustacheIT extends AbstractScriptTestCase { Map ingestMap = new HashMap<>(); ingestMap.put("timestamp", "bogus_timestamp"); document.put("_ingest", ingestMap); - IngestDocument ingestDocument = new IngestDocument("index", "id", null, null, null, document); + IngestDocument ingestDocument = new IngestDocument("index", "id", 1, null, null, document); ingestDocument.setFieldValue( compile("ingest_timestamp"), ValueSource.wrap("{{_ingest.timestamp}} and {{_source._ingest.timestamp}}", scriptService) diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java index 15cf58ee3a66..c93ef3073196 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/java/org/elasticsearch/ingest/ValueSourceMustacheIT.java @@ -55,7 +55,7 @@ public class ValueSourceMustacheIT extends AbstractScriptTestCase { } public void testAccessSourceViaTemplate() { - IngestDocument ingestDocument = new IngestDocument("marvel", "id", null, null, null, new HashMap<>()); + IngestDocument ingestDocument = new IngestDocument("marvel", "id", 1, null, null, new HashMap<>()); assertThat(ingestDocument.hasField("marvel"), is(false)); ingestDocument.setFieldValue(compile("{{_index}}"), ValueSource.wrap("{{_index}}", scriptService)); assertThat(ingestDocument.getFieldValue("marvel", String.class), equalTo("marvel")); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java index 028a5920c0be..97ac5d2ba7e6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/ingest/IngestClientIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.internal.Requests; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; @@ -112,7 +113,7 @@ public class IngestClientIT extends ESIntegTestCase { source.put("foo", "bar"); source.put("fail", false); source.put("processed", true); - IngestDocument ingestDocument = new IngestDocument("index", "id", null, null, null, source); + IngestDocument ingestDocument = new IngestDocument("index", "id", Versions.MATCH_ANY, null, null, source); assertThat(simulateDocumentBaseResult.getIngestDocument().getSourceAndMetadata(), equalTo(ingestDocument.getSourceAndMetadata())); assertThat(simulateDocumentBaseResult.getFailure(), nullValue()); diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java index 081175819f5d..c5395118e5a5 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.VersionType; @@ -178,7 +179,7 @@ public class SimulatePipelineRequest extends ActionRequest implements ToXContent "[types removal] specifying _type in pipeline simulation requests is deprecated" ); } - Long version = null; + long version = Versions.MATCH_ANY; if (dataMap.containsKey(Metadata.VERSION.getFieldName())) { String versionValue = ConfigurationUtils.readOptionalStringOrLongProperty( null, @@ -187,7 +188,7 @@ public class SimulatePipelineRequest extends ActionRequest implements ToXContent Metadata.VERSION.getFieldName() ); if (versionValue != null) { - version = Long.valueOf(versionValue); + version = Long.parseLong(versionValue); } else { throw new IllegalArgumentException("[_version] cannot be null"); } @@ -198,7 +199,7 @@ public class SimulatePipelineRequest extends ActionRequest implements ToXContent ConfigurationUtils.readStringProperty(null, null, dataMap, Metadata.VERSION_TYPE.getFieldName()) ); } - IngestDocument ingestDocument = new IngestDocument(index, id, routing, version, versionType, document); + IngestDocument ingestDocument = new IngestDocument(index, id, version, routing, versionType, document); if (dataMap.containsKey(Metadata.IF_SEQ_NO.getFieldName())) { String ifSeqNoValue = ConfigurationUtils.readOptionalStringOrLongProperty( null, diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index 2fb862dc46cc..c5fb380e066c 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -58,18 +58,16 @@ public final class IngestDocument { private boolean doNoSelfReferencesCheck = false; - public IngestDocument(String index, String id, String routing, Long version, VersionType versionType, Map source) { + public IngestDocument(String index, String id, long version, String routing, VersionType versionType, Map source) { // source + at max 5 extra fields this.sourceAndMetadata = Maps.newMapWithExpectedSize(source.size() + 5); this.sourceAndMetadata.putAll(source); this.sourceAndMetadata.put(Metadata.INDEX.getFieldName(), index); this.sourceAndMetadata.put(Metadata.ID.getFieldName(), id); + this.sourceAndMetadata.put(Metadata.VERSION.getFieldName(), version); if (routing != null) { this.sourceAndMetadata.put(Metadata.ROUTING.getFieldName(), routing); } - if (version != null) { - sourceAndMetadata.put(Metadata.VERSION.getFieldName(), version); - } if (versionType != null) { sourceAndMetadata.put(Metadata.VERSION_TYPE.getFieldName(), VersionType.toString(versionType)); } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 7963ec225c67..67594d96ec71 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -883,10 +883,10 @@ public class IngestService implements ClusterStateApplier, ReportingService sourceAsMap = indexRequest.sourceAsMap(); - IngestDocument ingestDocument = new IngestDocument(index, id, routing, version, versionType, sourceAsMap); + IngestDocument ingestDocument = new IngestDocument(index, id, version, routing, versionType, sourceAsMap); ingestDocument.executePipeline(pipeline, (result, e) -> { long ingestTimeInNanos = System.nanoTime() - startTimeInNanos; totalMetrics.postIngest(ingestTimeInNanos); diff --git a/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java b/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java index 8037cf10bc24..0a40080e56c8 100644 --- a/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/ingest/SimulateExecutionServiceTests.java @@ -329,7 +329,7 @@ public class SimulateExecutionServiceTests extends ESTestCase { int numDocs = randomIntBetween(1, 64); List documents = new ArrayList<>(numDocs); for (int id = 0; id < numDocs; id++) { - documents.add(new IngestDocument("_index", Integer.toString(id), null, 0L, VersionType.INTERNAL, new HashMap<>())); + documents.add(new IngestDocument("_index", Integer.toString(id), 0L, null, VersionType.INTERNAL, new HashMap<>())); } Processor processor1 = new AbstractProcessor(null, null) { diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java index 7e339fb46b8c..0212a221154c 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java @@ -83,7 +83,7 @@ public class IngestDocumentTests extends ESTestCase { DoubleStream.generate(ESTestCase::randomDouble).limit(randomInt(1000)).toArray() } ); - ingestDocument = new IngestDocument("index", "id", null, null, null, document); + ingestDocument = new IngestDocument("index", "id", 1, null, null, document); } public void testSimpleGetFieldValue() { @@ -797,23 +797,23 @@ public class IngestDocumentTests extends ESTestCase { public void testRemoveField() { ingestDocument.removeField("foo"); - assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(9)); + assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(10)); assertThat(ingestDocument.getSourceAndMetadata().containsKey("foo"), equalTo(false)); ingestDocument.removeField("_index"); - assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(8)); + assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(9)); assertThat(ingestDocument.getSourceAndMetadata().containsKey("_index"), equalTo(false)); ingestDocument.removeField("_source.fizz"); - assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(7)); + assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(8)); assertThat(ingestDocument.getSourceAndMetadata().containsKey("fizz"), equalTo(false)); assertThat(ingestDocument.getIngestMetadata().size(), equalTo(1)); ingestDocument.removeField("_ingest.timestamp"); - assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(7)); + assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(8)); assertThat(ingestDocument.getIngestMetadata().size(), equalTo(0)); } public void testRemoveInnerField() { ingestDocument.removeField("fizz.buzz"); - assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(10)); + assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(11)); assertThat(ingestDocument.getSourceAndMetadata().get("fizz"), instanceOf(Map.class)); @SuppressWarnings("unchecked") Map map = (Map) ingestDocument.getSourceAndMetadata().get("fizz"); @@ -822,17 +822,17 @@ public class IngestDocumentTests extends ESTestCase { ingestDocument.removeField("fizz.foo_null"); assertThat(map.size(), equalTo(2)); - assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(10)); + assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(11)); assertThat(ingestDocument.getSourceAndMetadata().containsKey("fizz"), equalTo(true)); ingestDocument.removeField("fizz.1"); assertThat(map.size(), equalTo(1)); - assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(10)); + assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(11)); assertThat(ingestDocument.getSourceAndMetadata().containsKey("fizz"), equalTo(true)); ingestDocument.removeField("fizz.list"); assertThat(map.size(), equalTo(0)); - assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(10)); + assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(11)); assertThat(ingestDocument.getSourceAndMetadata().containsKey("fizz"), equalTo(true)); } @@ -868,7 +868,7 @@ public class IngestDocumentTests extends ESTestCase { public void testRemoveIngestObject() { ingestDocument.removeField("_ingest"); - assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(9)); + assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(10)); assertThat(ingestDocument.getSourceAndMetadata().containsKey("_ingest"), equalTo(false)); } @@ -890,7 +890,7 @@ public class IngestDocumentTests extends ESTestCase { public void testListRemoveField() { ingestDocument.removeField("list.0.field"); - assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(10)); + assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(11)); assertThat(ingestDocument.getSourceAndMetadata().containsKey("list"), equalTo(true)); Object object = ingestDocument.getSourceAndMetadata().get("list"); assertThat(object, instanceOf(List.class)); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 45cd93b6f9e8..667d38bd1381 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -2170,7 +2170,7 @@ public class IngestServiceTests extends ESTestCase { return argThat(new IngestDocumentMatcher("_index", "_type", "_id", -3L, VersionType.INTERNAL, source)); } - private IngestDocument eqIndexTypeId(final Long version, final VersionType versionType, final Map source) { + private IngestDocument eqIndexTypeId(final long version, final VersionType versionType, final Map source) { return argThat(new IngestDocumentMatcher("_index", "_type", "_id", version, versionType, source)); } @@ -2220,11 +2220,11 @@ public class IngestServiceTests extends ESTestCase { private final IngestDocument ingestDocument; IngestDocumentMatcher(String index, String type, String id, Map source) { - this.ingestDocument = new IngestDocument(index, id, null, null, null, source); + this.ingestDocument = new IngestDocument(index, id, 1, null, null, source); } - IngestDocumentMatcher(String index, String type, String id, Long version, VersionType versionType, Map source) { - this.ingestDocument = new IngestDocument(index, id, null, version, versionType, source); + IngestDocumentMatcher(String index, String type, String id, long version, VersionType versionType, Map source) { + this.ingestDocument = new IngestDocument(index, id, version, null, versionType, source); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java b/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java index 6a63979a1284..03f26537a82b 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/RandomDocumentPicks.java @@ -136,7 +136,7 @@ public final class RandomDocumentPicks { if (random.nextBoolean()) { routing = randomString(random); } - return new IngestDocument(index, id, routing, version, versionType, source); + return new IngestDocument(index, id, version, routing, versionType, source); } public static Map randomSource(Random random) { diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java index 3588e8edf6e2..857a8a9f66f6 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java @@ -288,8 +288,8 @@ public class EnrichProcessorFactoryTests extends ESTestCase { IngestDocument ingestDocument = new IngestDocument( "_index", "_id", - "_routing", 1L, + "_routing", VersionType.INTERNAL, Map.of("domain", "elastic.co") ); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java index 1fa623007680..5642e685a592 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java @@ -96,8 +96,8 @@ public class GeoMatchProcessorTests extends ESTestCase { IngestDocument ingestDocument = new IngestDocument( "_index", "_id", - "_routing", 1L, + "_routing", VersionType.INTERNAL, Map.of("location", fieldValue) ); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java index 398a478b2818..230518d3d050 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java @@ -51,8 +51,8 @@ public class MatchProcessorTests extends ESTestCase { IngestDocument ingestDocument = new IngestDocument( "_index", "_id", - "_routing", 1L, + "_routing", VersionType.INTERNAL, Map.of("domain", "elastic.co") ); @@ -106,8 +106,8 @@ public class MatchProcessorTests extends ESTestCase { IngestDocument ingestDocument = new IngestDocument( "_index", "_id", - "_routing", 1L, + "_routing", VersionType.INTERNAL, Map.of("domain", "elastic.com") ); @@ -153,8 +153,8 @@ public class MatchProcessorTests extends ESTestCase { IngestDocument ingestDocument = new IngestDocument( "_index", "_id", - "_routing", 1L, + "_routing", VersionType.INTERNAL, Map.of("domain", "elastic.com") ); @@ -201,7 +201,7 @@ public class MatchProcessorTests extends ESTestCase { "domain", 1 ); - IngestDocument ingestDocument = new IngestDocument("_index", "_id", "_routing", 1L, VersionType.INTERNAL, Map.of()); + IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1L, "_routing", VersionType.INTERNAL, Map.of()); assertThat(ingestDocument.getSourceAndMetadata().size(), equalTo(5)); IngestDocument[] holder = new IngestDocument[1]; @@ -222,7 +222,7 @@ public class MatchProcessorTests extends ESTestCase { "domain", 1 ); - IngestDocument ingestDocument = new IngestDocument("_index", "_id", "_routing", 1L, VersionType.INTERNAL, Map.of()); + IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1L, "_routing", VersionType.INTERNAL, Map.of()); IngestDocument[] resultHolder = new IngestDocument[1]; Exception[] exceptionHolder = new Exception[1]; processor.execute(ingestDocument, (result, e) -> { @@ -306,7 +306,7 @@ public class MatchProcessorTests extends ESTestCase { "domain", 1 ); - IngestDocument ingestDocument = new IngestDocument("_index", "_id", "_routing", 1L, VersionType.INTERNAL, Map.of("domain", 2)); + IngestDocument ingestDocument = new IngestDocument("_index", "_id", 1L, "_routing", VersionType.INTERNAL, Map.of("domain", 2)); // Execute IngestDocument[] holder = new IngestDocument[1]; @@ -346,8 +346,8 @@ public class MatchProcessorTests extends ESTestCase { IngestDocument ingestDocument = new IngestDocument( "_index", "_id", - "_routing", 1L, + "_routing", VersionType.INTERNAL, Map.of("domain", List.of("1", "2")) );