From 0cf2ebb02f6476f21be0a72a909280e526aaf6af Mon Sep 17 00:00:00 2001 From: Carlos Delgado <6339205+carlosdelest@users.noreply.github.com> Date: Thu, 9 Jan 2025 08:38:18 +0100 Subject: [PATCH 01/33] Vector rescoring - Simplify code for k == null (#118997) --- .../VectorSimilarityFloatValueSource.java | 11 +---- .../search/vectors/KnnVectorQueryBuilder.java | 27 +++++------- .../search/vectors/RescoreKnnVectorQuery.java | 23 +++-------- .../vectors/DenseVectorFieldTypeTests.java | 11 ++--- .../vectors/RescoreKnnVectorQueryTests.java | 41 +++++-------------- 5 files changed, 34 insertions(+), 79 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorSimilarityFloatValueSource.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorSimilarityFloatValueSource.java index 74a7dbe168e6..80cbec1e2f6c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorSimilarityFloatValueSource.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/VectorSimilarityFloatValueSource.java @@ -18,8 +18,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.DoubleValues; import org.apache.lucene.search.DoubleValuesSource; import org.apache.lucene.search.IndexSearcher; -import org.elasticsearch.search.profile.query.QueryProfiler; -import org.elasticsearch.search.vectors.QueryProfilerProvider; import java.io.IOException; import java.util.Arrays; @@ -29,12 +27,11 @@ import java.util.Objects; * DoubleValuesSource that is used to calculate scores according to a similarity function for a KnnFloatVectorField, using the * original vector values stored in the index */ -public class VectorSimilarityFloatValueSource extends DoubleValuesSource implements QueryProfilerProvider { +public class VectorSimilarityFloatValueSource extends DoubleValuesSource { private final String field; private final float[] target; private final VectorSimilarityFunction vectorSimilarityFunction; - private long vectorOpsCount; public VectorSimilarityFloatValueSource(String field, float[] target, VectorSimilarityFunction vectorSimilarityFunction) { this.field = field; @@ -52,7 +49,6 @@ public class VectorSimilarityFloatValueSource extends DoubleValuesSource impleme return new DoubleValues() { @Override public double doubleValue() throws IOException { - vectorOpsCount++; return vectorSimilarityFunction.compare(target, vectorValues.vectorValue(iterator.index())); } @@ -73,11 +69,6 @@ public class VectorSimilarityFloatValueSource extends DoubleValuesSource impleme return this; } - @Override - public void profile(QueryProfiler queryProfiler) { - queryProfiler.addVectorOpsCount(vectorOpsCount); - } - @Override public int hashCode() { return Objects.hash(field, Arrays.hashCode(target), vectorSimilarityFunction); diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index a65757cc2587..4a6dbab39914 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -487,11 +487,6 @@ public class KnnVectorQueryBuilder extends AbstractQueryBuilder rescoredDocs = Arrays.stream(docs.scoreDocs) .collect(Collectors.toMap(scoreDoc -> scoreDoc.doc, scoreDoc -> scoreDoc.score)); - assertThat(rescoredDocs.size(), equalTo(adjustedK)); + assertThat(rescoredDocs.size(), equalTo(k)); Collection rescoredScores = new HashSet<>(rescoredDocs.values()); @@ -113,7 +99,7 @@ public class RescoreKnnVectorQueryTests extends ESTestCase { assertThat(rescoredDocs.size(), equalTo(0)); // Check top scoring docs are contained in rescored docs - for (int i = 0; i < adjustedK; i++) { + for (int i = 0; i < k; i++) { Float topScore = topK.poll(); if (rescoredScores.contains(topScore) == false) { fail("Top score " + topScore + " not contained in rescored doc scores " + rescoredScores); @@ -124,7 +110,9 @@ public class RescoreKnnVectorQueryTests extends ESTestCase { } public void testProfiling() throws Exception { + int numDocs = randomIntBetween(10, 100); int numDims = randomIntBetween(5, 100); + int k = randomIntBetween(1, numDocs - 1); try (Directory d = newDirectory()) { addRandomDocuments(numDocs, d, numDims); @@ -132,13 +120,13 @@ public class RescoreKnnVectorQueryTests extends ESTestCase { try (IndexReader reader = DirectoryReader.open(d)) { float[] queryVector = randomVector(numDims); - checkProfiling(queryVector, reader, new MatchAllDocsQuery()); - checkProfiling(queryVector, reader, new MockQueryProfilerProvider(randomIntBetween(1, 100))); + checkProfiling(k, numDocs, queryVector, reader, new MatchAllDocsQuery()); + checkProfiling(k, numDocs, queryVector, reader, new MockQueryProfilerProvider(randomIntBetween(1, 100))); } } } - private void checkProfiling(float[] queryVector, IndexReader reader, Query innerQuery) throws IOException { + private void checkProfiling(int k, int numDocs, float[] queryVector, IndexReader reader, Query innerQuery) throws IOException { RescoreKnnVectorQuery rescoreKnnVectorQuery = new RescoreKnnVectorQuery( FIELD_NAME, queryVector, @@ -229,13 +217,4 @@ public class RescoreKnnVectorQueryTests extends ESTestCase { w.forceMerge(1); } } - - @ParametersFactory - public static Iterable parameters() { - List params = new ArrayList<>(); - params.add(new Object[] { true }); - params.add(new Object[] { false }); - - return params; - } } From b2f290f102f8051fdd4a6e1bbaf08df4b6ec6770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20R=C3=BChsen?= Date: Thu, 9 Jan 2025 08:40:08 +0100 Subject: [PATCH 02/33] [Profiling] Remove legacy aggregation_field (#119770) --- .../action/GetFlameGraphActionIT.java | 1 - .../action/GetStackTracesActionIT.java | 11 +--- .../action/GetTopNFunctionsActionIT.java | 7 +-- .../action/GetStackTracesRequest.java | 40 +------------ .../xpack/profiling/action/SubGroup.java | 44 +++++--------- .../profiling/action/SubGroupCollector.java | 14 ++--- .../action/TransportGetStackTracesAction.java | 12 +--- .../action/GetStackTracesRequestTests.java | 60 +------------------ .../action/GetTopNFunctionsResponseTests.java | 2 +- .../profiling/action/ResamplerTests.java | 5 -- .../action/SubGroupCollectorTests.java | 6 +- .../xpack/profiling/action/SubGroupTests.java | 30 ++-------- .../profiling/action/TopNFunctionTests.java | 4 +- 13 files changed, 41 insertions(+), 195 deletions(-) diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetFlameGraphActionIT.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetFlameGraphActionIT.java index db343b62c5a1..49a5cfa7ca06 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetFlameGraphActionIT.java +++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetFlameGraphActionIT.java @@ -22,7 +22,6 @@ public class GetFlameGraphActionIT extends ProfilingTestCase { null, null, null, - null, null ); GetFlamegraphResponse response = client().execute(GetFlamegraphAction.INSTANCE, request).get(); diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetStackTracesActionIT.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetStackTracesActionIT.java index 4b3a4fb0108f..4a9f1146fe95 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetStackTracesActionIT.java +++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetStackTracesActionIT.java @@ -28,7 +28,6 @@ public class GetStackTracesActionIT extends ProfilingTestCase { null, null, null, - null, null ); request.setAdjustSampleCount(true); @@ -68,8 +67,7 @@ public class GetStackTracesActionIT extends ProfilingTestCase { null, null, null, - "service.name", - null, + new String[] { "service.name" }, null, null, null, @@ -117,8 +115,7 @@ public class GetStackTracesActionIT extends ProfilingTestCase { // also match an index that does not contain stacktrace ids to ensure it is ignored new String[] { "apm-test-*", "apm-legacy-test-*" }, "transaction.profiler_stack_trace_ids", - "transaction.name", - null, + new String[] { "transaction.name" }, null, null, null, @@ -168,7 +165,6 @@ public class GetStackTracesActionIT extends ProfilingTestCase { null, null, null, - null, null ); // ensures consistent results in the random sampler aggregation that is used internally @@ -219,7 +215,6 @@ public class GetStackTracesActionIT extends ProfilingTestCase { null, null, null, - null, null ); GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get(); @@ -242,7 +237,6 @@ public class GetStackTracesActionIT extends ProfilingTestCase { null, null, null, - null, null ); GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get(); @@ -265,7 +259,6 @@ public class GetStackTracesActionIT extends ProfilingTestCase { null, null, null, - null, null ); GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get(); diff --git a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsActionIT.java b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsActionIT.java index c6250dae4d64..4a3e200af673 100644 --- a/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsActionIT.java +++ b/x-pack/plugin/profiling/src/internalClusterTest/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsActionIT.java @@ -25,7 +25,6 @@ public class GetTopNFunctionsActionIT extends ProfilingTestCase { null, null, null, - null, null ); request.setAdjustSampleCount(true); @@ -42,8 +41,7 @@ public class GetTopNFunctionsActionIT extends ProfilingTestCase { null, null, null, - "service.name", - null, + new String[] { "service.name" }, null, null, null, @@ -70,8 +68,7 @@ public class GetTopNFunctionsActionIT extends ProfilingTestCase { // also match an index that does not contain stacktrace ids to ensure it is ignored new String[] { "apm-test-*", "apm-legacy-test-*" }, "transaction.profiler_stack_trace_ids", - "transaction.name", - null, + new String[] { "transaction.name" }, null, null, null, diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesRequest.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesRequest.java index fb780a3f6f98..82acaa0020ad 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesRequest.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesRequest.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; @@ -43,8 +42,6 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque public static final ParseField LIMIT_FIELD = new ParseField("limit"); public static final ParseField INDICES_FIELD = new ParseField("indices"); public static final ParseField STACKTRACE_IDS_FIELD = new ParseField("stacktrace_ids_field"); - @UpdateForV9(owner = UpdateForV9.Owner.PROFILING) // Remove this BWC layer and allow only AGGREGATION_FIELDS - public static final ParseField AGGREGATION_FIELD = new ParseField("aggregation_field"); public static final ParseField AGGREGATION_FIELDS = new ParseField("aggregation_fields"); public static final ParseField REQUESTED_DURATION_FIELD = new ParseField("requested_duration"); public static final ParseField AWS_COST_FACTOR_FIELD = new ParseField("aws_cost_factor"); @@ -62,8 +59,6 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque private String[] indices; private boolean userProvidedIndices; private String stackTraceIdsField; - @UpdateForV9(owner = UpdateForV9.Owner.PROFILING) // Remove this BWC layer and allow only aggregationFields - private String aggregationField; private String[] aggregationFields; private Double requestedDuration; private Double awsCostFactor; @@ -83,7 +78,7 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque private Integer shardSeed; public GetStackTracesRequest() { - this(null, null, null, null, null, null, null, null, null, null, null, null, null, null); + this(null, null, null, null, null, null, null, null, null, null, null, null, null); } public GetStackTracesRequest( @@ -94,7 +89,6 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque QueryBuilder query, String[] indices, String stackTraceIdsField, - String aggregationField, String[] aggregationFields, Double customCO2PerKWH, Double customDatacenterPUE, @@ -110,7 +104,6 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque this.indices = indices; this.userProvidedIndices = indices != null && indices.length > 0; this.stackTraceIdsField = stackTraceIdsField; - this.aggregationField = aggregationField; this.aggregationFields = aggregationFields; this.customCO2PerKWH = customCO2PerKWH; this.customDatacenterPUE = customDatacenterPUE; @@ -184,12 +177,8 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque return stackTraceIdsField; } - public String getAggregationField() { - return aggregationField; - } - public String[] getAggregationFields() { - return aggregationField != null ? new String[] { aggregationField } : aggregationFields; + return aggregationFields; } public boolean hasAggregationFields() { @@ -197,10 +186,6 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque return f != null && f.length > 0; } - public boolean isLegacyAggregationField() { - return aggregationField != null; - } - public boolean isAdjustSampleCount() { return Boolean.TRUE.equals(adjustSampleCount); } @@ -237,8 +222,6 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque this.limit = parser.intValue(); } else if (STACKTRACE_IDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { this.stackTraceIdsField = parser.text(); - } else if (AGGREGATION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - this.aggregationField = parser.text(); } else if (REQUESTED_DURATION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { this.requestedDuration = parser.doubleValue(); } else if (AWS_COST_FACTOR_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { @@ -322,17 +305,6 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque ); } } - if (aggregationField != null && aggregationFields != null) { - validationException = addValidationError( - "[" - + AGGREGATION_FIELD.getPreferredName() - + "] must not be set when [" - + AGGREGATION_FIELDS.getPreferredName() - + "] is also set", - validationException - ); - - } if (aggregationFields != null) { // limit so we avoid an explosion of buckets if (aggregationFields.length < 1 || aggregationFields.length > 2) { @@ -348,13 +320,6 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque } - if (aggregationField != null && aggregationField.isBlank()) { - validationException = addValidationError( - "[" + AGGREGATION_FIELD.getPreferredName() + "] must be non-empty", - validationException - ); - } - validationException = requirePositive(SAMPLE_SIZE_FIELD, sampleSize, validationException); validationException = requirePositive(LIMIT_FIELD, limit, validationException); validationException = requirePositive(REQUESTED_DURATION_FIELD, requestedDuration, validationException); @@ -386,7 +351,6 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque StringBuilder sb = new StringBuilder(); appendField(sb, "indices", indices); appendField(sb, "stacktrace_ids_field", stackTraceIdsField); - appendField(sb, "aggregation_field", aggregationField); appendField(sb, "aggregation_fields", aggregationFields); appendField(sb, "sample_size", sampleSize); appendField(sb, "limit", limit); diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroup.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroup.java index 513ec8fe1bc4..9c953eea0496 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroup.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroup.java @@ -20,23 +20,21 @@ public class SubGroup implements ToXContentFragment { private final String name; private Long count; @UpdateForV9(owner = UpdateForV9.Owner.PROFILING) // remove legacy XContent rendering - private final boolean renderLegacyXContent; private final Map subgroups; - public static SubGroup root(String name, boolean renderLegacyXContent) { - return new SubGroup(name, null, renderLegacyXContent, new HashMap<>()); + public static SubGroup root(String name) { + return new SubGroup(name, null, new HashMap<>()); } - public SubGroup(String name, Long count, boolean renderLegacyXContent, Map subgroups) { + public SubGroup(String name, Long count, Map subgroups) { this.name = name; this.count = count; - this.renderLegacyXContent = renderLegacyXContent; this.subgroups = subgroups; } public SubGroup addCount(String name, long count) { if (this.subgroups.containsKey(name) == false) { - this.subgroups.put(name, new SubGroup(name, count, renderLegacyXContent, new HashMap<>())); + this.subgroups.put(name, new SubGroup(name, count, new HashMap<>())); } else { SubGroup s = this.subgroups.get(name); s.count += count; @@ -46,7 +44,7 @@ public class SubGroup implements ToXContentFragment { public SubGroup getOrAddChild(String name) { if (subgroups.containsKey(name) == false) { - this.subgroups.put(name, new SubGroup(name, null, renderLegacyXContent, new HashMap<>())); + this.subgroups.put(name, new SubGroup(name, null, new HashMap<>())); } return this.subgroups.get(name); } @@ -65,32 +63,22 @@ public class SubGroup implements ToXContentFragment { for (Map.Entry subGroup : subgroups.entrySet()) { copy.put(subGroup.getKey(), subGroup.getValue().copy()); } - return new SubGroup(name, count, renderLegacyXContent, copy); + return new SubGroup(name, count, copy); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - if (renderLegacyXContent) { - // This assumes that we only have one level of sub groups - if (subgroups != null && subgroups.isEmpty() == false) { - for (SubGroup subgroup : subgroups.values()) { - builder.field(subgroup.name, subgroup.count); - } - } - return builder; - } else { - builder.startObject(name); - // only the root node has no count - if (count != null) { - builder.field("count", count); - } - if (subgroups != null && subgroups.isEmpty() == false) { - for (SubGroup subgroup : subgroups.values()) { - subgroup.toXContent(builder, params); - } - } - return builder.endObject(); + builder.startObject(name); + // only the root node has no count + if (count != null) { + builder.field("count", count); } + if (subgroups != null && subgroups.isEmpty() == false) { + for (SubGroup subgroup : subgroups.values()) { + subgroup.toXContent(builder, params); + } + } + return builder.endObject(); } @Override diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroupCollector.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroupCollector.java index 63491a63243d..6eaa346be3e4 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroupCollector.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/SubGroupCollector.java @@ -26,21 +26,15 @@ public final class SubGroupCollector { private static final Logger log = LogManager.getLogger(SubGroupCollector.class); private final String[] aggregationFields; - private final boolean legacyAggregationField; - public static SubGroupCollector attach( - AbstractAggregationBuilder parentAggregation, - String[] aggregationFields, - boolean legacyAggregationField - ) { - SubGroupCollector c = new SubGroupCollector(aggregationFields, legacyAggregationField); + public static SubGroupCollector attach(AbstractAggregationBuilder parentAggregation, String[] aggregationFields) { + SubGroupCollector c = new SubGroupCollector(aggregationFields); c.addAggregations(parentAggregation); return c; } - private SubGroupCollector(String[] aggregationFields, boolean legacyAggregationField) { + private SubGroupCollector(String[] aggregationFields) { this.aggregationFields = aggregationFields; - this.legacyAggregationField = legacyAggregationField; } private boolean hasAggregationFields() { @@ -68,7 +62,7 @@ public final class SubGroupCollector { void collectResults(Bucket bucket, TraceEvent event) { if (hasAggregationFields()) { if (event.subGroups == null) { - event.subGroups = SubGroup.root(aggregationFields[0], legacyAggregationField); + event.subGroups = SubGroup.root(aggregationFields[0]); } collectInternal(bucket.getAggregations(), event.subGroups, 0); } diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java index f447f67b4cdd..982410e8a534 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/TransportGetStackTracesAction.java @@ -255,11 +255,7 @@ public class TransportGetStackTracesAction extends TransportAction validationErrors = request.validate().validationErrors(); @@ -367,7 +361,6 @@ public class GetStackTracesRequestTests extends ESTestCase { null, null, null, - null, null ); assertNull("Expecting no validation errors", request.validate()); @@ -387,7 +380,6 @@ public class GetStackTracesRequestTests extends ESTestCase { null, null, null, - null, null ); List validationErrors = request.validate().validationErrors(); @@ -409,7 +401,6 @@ public class GetStackTracesRequestTests extends ESTestCase { null, null, null, - null, null ); List validationErrors = request.validate().validationErrors(); @@ -417,50 +408,6 @@ public class GetStackTracesRequestTests extends ESTestCase { assertEquals("[stacktrace_ids_field] is mandatory", validationErrors.get(0)); } - public void testValidateEmptyAggregationField() { - GetStackTracesRequest request = new GetStackTracesRequest( - null, - 1.0d, - 1.0d, - 1.0d, - null, - new String[] { randomAlphaOfLength(5) }, - randomAlphaOfLength(5), - "", - null, - null, - null, - null, - null, - null - ); - List validationErrors = request.validate().validationErrors(); - assertEquals(1, validationErrors.size()); - assertEquals("[aggregation_field] must be non-empty", validationErrors.get(0)); - } - - public void testValidateAggregationFieldAndAggregationFields() { - GetStackTracesRequest request = new GetStackTracesRequest( - null, - 1.0d, - 1.0d, - 1.0d, - null, - new String[] { randomAlphaOfLength(5) }, - randomAlphaOfLength(5), - "transaction.name", - new String[] { "transaction.name", "service.name" }, - null, - null, - null, - null, - null - ); - List validationErrors = request.validate().validationErrors(); - assertEquals(1, validationErrors.size()); - assertEquals("[aggregation_field] must not be set when [aggregation_fields] is also set", validationErrors.get(0)); - } - public void testValidateAggregationFieldsContainsTooFewElements() { GetStackTracesRequest request = new GetStackTracesRequest( null, @@ -470,7 +417,6 @@ public class GetStackTracesRequestTests extends ESTestCase { null, new String[] { randomAlphaOfLength(5) }, randomAlphaOfLength(5), - null, new String[] {}, null, null, @@ -492,7 +438,6 @@ public class GetStackTracesRequestTests extends ESTestCase { null, new String[] { randomAlphaOfLength(5) }, randomAlphaOfLength(5), - null, new String[] { "application", "service", "transaction" }, null, null, @@ -514,7 +459,6 @@ public class GetStackTracesRequestTests extends ESTestCase { null, new String[] { randomAlphaOfLength(5) }, randomAlphaOfLength(5), - null, new String[] { "service", "service" }, null, null, @@ -540,7 +484,6 @@ public class GetStackTracesRequestTests extends ESTestCase { null, null, null, - null, null ); String[] indices = request.indices(); @@ -562,7 +505,6 @@ public class GetStackTracesRequestTests extends ESTestCase { null, null, null, - null, null ); String[] indices = request.indices(); diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsResponseTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsResponseTests.java index 24f8ea85212d..cf8742b5ddf6 100644 --- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsResponseTests.java +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/GetTopNFunctionsResponseTests.java @@ -90,7 +90,7 @@ public class GetTopNFunctionsResponseTests extends ESTestCase { 22.0d, 12.0d, 120.0d, - SubGroup.root("transaction.name", false).addCount("basket", 7L) + SubGroup.root("transaction.name").addCount("basket", 7L) ); GetTopNFunctionsResponse response = new GetTopNFunctionsResponse(1, 10, 2.2d, 12.0d, List.of(topNFunction)); response.toXContent(actualResponse, ToXContent.EMPTY_PARAMS); diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/ResamplerTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/ResamplerTests.java index fec9704dc8c0..c2537edab6bb 100644 --- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/ResamplerTests.java +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/ResamplerTests.java @@ -43,7 +43,6 @@ public class ResamplerTests extends ESTestCase { null, null, null, - null, null ); request.setAdjustSampleCount(false); @@ -73,7 +72,6 @@ public class ResamplerTests extends ESTestCase { null, null, null, - null, null ); request.setAdjustSampleCount(true); @@ -103,7 +101,6 @@ public class ResamplerTests extends ESTestCase { null, null, null, - null, null ); request.setAdjustSampleCount(false); @@ -136,7 +133,6 @@ public class ResamplerTests extends ESTestCase { null, null, null, - null, null ); @@ -166,7 +162,6 @@ public class ResamplerTests extends ESTestCase { null, null, null, - null, null ); request.setAdjustSampleCount(true); diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupCollectorTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupCollectorTests.java index 5d6022f32276..73164ed9069d 100644 --- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupCollectorTests.java +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupCollectorTests.java @@ -21,7 +21,7 @@ public class SubGroupCollectorTests extends ESTestCase { TermsAggregationBuilder stackTraces = new TermsAggregationBuilder("stacktraces").field("stacktrace.id"); TraceEvent traceEvent = new TraceEvent("1"); - SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[0], false); + SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[0]); assertTrue("Sub aggregations attached", stackTraces.getSubAggregations().isEmpty()); SubGroupCollector.Bucket currentStackTrace = bucket("1", 5); @@ -34,7 +34,7 @@ public class SubGroupCollectorTests extends ESTestCase { TermsAggregationBuilder stackTraces = new TermsAggregationBuilder("stacktraces").field("stacktrace.id"); TraceEvent traceEvent = new TraceEvent("1"); - SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[] { "service.name", "transaction.name" }, false); + SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[] { "service.name", "transaction.name" }); assertFalse("No sub aggregations attached", stackTraces.getSubAggregations().isEmpty()); StaticAgg services = new StaticAgg(); @@ -73,7 +73,7 @@ public class SubGroupCollectorTests extends ESTestCase { TermsAggregationBuilder stackTraces = new TermsAggregationBuilder("stacktraces").field("stacktrace.id"); TraceEvent traceEvent = new TraceEvent("1"); - SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[] { "service.name" }, false); + SubGroupCollector collector = SubGroupCollector.attach(stackTraces, new String[] { "service.name" }); assertFalse("No sub aggregations attached", stackTraces.getSubAggregations().isEmpty()); StaticAgg services1 = new StaticAgg(); diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupTests.java index c571d7c03c25..0d85b62be1c8 100644 --- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupTests.java +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/SubGroupTests.java @@ -34,25 +34,7 @@ public class SubGroupTests extends ESTestCase { XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType); actualRequest.startObject(); - SubGroup g = SubGroup.root("transaction.name", false).addCount("basket", 7L); - g.toXContent(actualRequest, ToXContent.EMPTY_PARAMS); - actualRequest.endObject(); - - assertToXContentEquivalent(BytesReference.bytes(expectedRequest), BytesReference.bytes(actualRequest), contentType); - } - - public void testRenderLegacyXContent() throws IOException { - XContentType contentType = randomFrom(XContentType.values()); - // tag::noformat - XContentBuilder expectedRequest = XContentFactory.contentBuilder(contentType) - .startObject() - .field("basket", 7L) - .endObject(); - // end::noformat - - XContentBuilder actualRequest = XContentFactory.contentBuilder(contentType); - actualRequest.startObject(); - SubGroup g = SubGroup.root("transaction.name", true).addCount("basket", 7L); + SubGroup g = SubGroup.root("transaction.name").addCount("basket", 7L); g.toXContent(actualRequest, ToXContent.EMPTY_PARAMS); actualRequest.endObject(); @@ -60,8 +42,8 @@ public class SubGroupTests extends ESTestCase { } public void testMergeNoCommonRoot() { - SubGroup root1 = SubGroup.root("transaction.name", false); - SubGroup root2 = SubGroup.root("service.name", false); + SubGroup root1 = SubGroup.root("transaction.name"); + SubGroup root2 = SubGroup.root("service.name"); SubGroup toMerge = root1.copy(); @@ -71,7 +53,7 @@ public class SubGroupTests extends ESTestCase { } public void testMergeIdenticalTree() { - SubGroup g = SubGroup.root("transaction.name", false); + SubGroup g = SubGroup.root("transaction.name"); g.addCount("basket", 5L); g.addCount("checkout", 7L); @@ -84,11 +66,11 @@ public class SubGroupTests extends ESTestCase { } public void testMergeMixedTree() { - SubGroup g1 = SubGroup.root("transaction.name", false); + SubGroup g1 = SubGroup.root("transaction.name"); g1.addCount("basket", 5L); g1.addCount("checkout", 7L); - SubGroup g2 = SubGroup.root("transaction.name", false); + SubGroup g2 = SubGroup.root("transaction.name"); g2.addCount("catalog", 8L); g2.addCount("basket", 5L); g2.addCount("checkout", 2L); diff --git a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TopNFunctionTests.java b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TopNFunctionTests.java index 76379adcd3b8..b63cca21f9b4 100644 --- a/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TopNFunctionTests.java +++ b/x-pack/plugin/profiling/src/test/java/org/elasticsearch/xpack/profiling/action/TopNFunctionTests.java @@ -81,7 +81,7 @@ public class TopNFunctionTests extends ESTestCase { 22.0d, 12.0d, 120.0d, - SubGroup.root("transaction.name", false).addCount("basket", 7L) + SubGroup.root("transaction.name").addCount("basket", 7L) ); topNFunction.toXContent(actualRequest, ToXContent.EMPTY_PARAMS); @@ -116,7 +116,7 @@ public class TopNFunctionTests extends ESTestCase { 4.0d, 23.2d, 12.0d, - SubGroup.root("transaction.name", false).addCount("checkout", 4L).addCount("basket", 12L) + SubGroup.root("transaction.name").addCount("checkout", 4L).addCount("basket", 12L) ); EqualsHashCodeTestUtils.checkEqualsAndHashCode(topNFunction, (TopNFunction::copy)); } From 16d8aaf3865588a323b04d281985bd8daf569daf Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Thu, 9 Jan 2025 08:57:47 +0100 Subject: [PATCH 03/33] Strengthen encryption for elasticsearch-keystore tool to AES 256 (#119749) The latest format version (7) of KeyStoreWrapper will be using a 256 bits key for the AES cipher. Closes #ES-10419 --- .../cli/keystore/KeyStoreWrapperTests.java | 44 ++++++++++------- .../format-v6-elasticsearch.keystore | Bin 0 -> 216 bytes docs/changelog/119749.yaml | 5 ++ .../common/settings/KeyStoreWrapper.java | 46 ++++++++++++------ 4 files changed, 63 insertions(+), 32 deletions(-) create mode 100644 distribution/tools/keystore-cli/src/test/resources/format-v6-elasticsearch.keystore create mode 100644 docs/changelog/119749.yaml diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java index 38bb7d592f7c..5ab27bac3998 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/cli/keystore/KeyStoreWrapperTests.java @@ -58,6 +58,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; public class KeyStoreWrapperTests extends ESTestCase { @@ -436,17 +437,8 @@ public class KeyStoreWrapperTests extends ESTestCase { public void testLegacyV3() throws GeneralSecurityException, IOException { assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm()); final Path configDir = createTempDir(); - final Path keystore = configDir.resolve("elasticsearch.keystore"); - try ( - InputStream is = KeyStoreWrapperTests.class.getResourceAsStream("/format-v3-elasticsearch.keystore"); - OutputStream os = Files.newOutputStream(keystore) - ) { - final byte[] buffer = new byte[4096]; - int readBytes; - while ((readBytes = is.read(buffer)) > 0) { - os.write(buffer, 0, readBytes); - } - } + copyKeyStoreFromResourceToConfigDir(configDir, "/format-v3-elasticsearch.keystore"); + final KeyStoreWrapper wrapper = KeyStoreWrapper.load(configDir); assertNotNull(wrapper); wrapper.decrypt(new char[0]); @@ -460,9 +452,31 @@ public class KeyStoreWrapperTests extends ESTestCase { public void testLegacyV5() throws GeneralSecurityException, IOException { final Path configDir = createTempDir(); + copyKeyStoreFromResourceToConfigDir(configDir, "/format-v5-with-password-elasticsearch.keystore"); + + final KeyStoreWrapper wrapper = KeyStoreWrapper.load(configDir); + assertNotNull(wrapper); + wrapper.decrypt("keystorepassword".toCharArray()); + assertThat(wrapper.getFormatVersion(), equalTo(5)); + assertThat(wrapper.getSettingNames(), equalTo(Set.of("keystore.seed"))); + } + + public void testLegacyV6() throws GeneralSecurityException, IOException { + final Path configDir = createTempDir(); + copyKeyStoreFromResourceToConfigDir(configDir, "/format-v6-elasticsearch.keystore"); + + final KeyStoreWrapper wrapper = KeyStoreWrapper.load(configDir); + assertNotNull(wrapper); + wrapper.decrypt("keystorepassword".toCharArray()); + assertThat(wrapper.getFormatVersion(), equalTo(6)); + assertThat(wrapper.getSettingNames(), equalTo(Set.of("keystore.seed", "string"))); + assertThat(wrapper.getString("string"), equalTo("value")); + } + + private void copyKeyStoreFromResourceToConfigDir(Path configDir, String name) throws IOException { final Path keystore = configDir.resolve("elasticsearch.keystore"); try ( - InputStream is = KeyStoreWrapperTests.class.getResourceAsStream("/format-v5-with-password-elasticsearch.keystore"); + InputStream is = KeyStoreWrapperTests.class.getResourceAsStream(name); // OutputStream os = Files.newOutputStream(keystore) ) { final byte[] buffer = new byte[4096]; @@ -471,11 +485,6 @@ public class KeyStoreWrapperTests extends ESTestCase { os.write(buffer, 0, readBytes); } } - final KeyStoreWrapper wrapper = KeyStoreWrapper.load(configDir); - assertNotNull(wrapper); - wrapper.decrypt("keystorepassword".toCharArray()); - assertThat(wrapper.getFormatVersion(), equalTo(5)); - assertThat(wrapper.getSettingNames(), equalTo(Set.of("keystore.seed"))); } public void testSerializationNewlyCreated() throws Exception { @@ -487,6 +496,7 @@ public class KeyStoreWrapperTests extends ESTestCase { wrapper.writeTo(out); final KeyStoreWrapper fromStream = new KeyStoreWrapper(out.bytes().streamInput()); + assertThat(fromStream.getFormatVersion(), is(KeyStoreWrapper.CURRENT_VERSION)); assertThat(fromStream.getSettingNames(), hasSize(2)); assertThat(fromStream.getSettingNames(), containsInAnyOrder("string_setting", "keystore.seed")); diff --git a/distribution/tools/keystore-cli/src/test/resources/format-v6-elasticsearch.keystore b/distribution/tools/keystore-cli/src/test/resources/format-v6-elasticsearch.keystore new file mode 100644 index 0000000000000000000000000000000000000000..0f680cc013563e3b1841070919b8e093a25759ca GIT binary patch literal 216 zcmcD&o+B=nnv+;ul9^nbnpl*ap_iRnSzMA|l*+)sz{a=)NIC#Y+Dw1H^olTsHf8<+PYxg|MxTO!e^|TC+Mx_ zS#jZ9#*;R??>+%u%hlg)Im;muRX)dwzs*z9a=-1&`DZ0#4yCSJaH3dlC1=yZzXvoX OzW^Eu2FqW5V+H^riC9|z literal 0 HcmV?d00001 diff --git a/docs/changelog/119749.yaml b/docs/changelog/119749.yaml new file mode 100644 index 000000000000..aa2b16ceda5e --- /dev/null +++ b/docs/changelog/119749.yaml @@ -0,0 +1,5 @@ +pr: 119749 +summary: Strengthen encryption for elasticsearch-keystore tool to AES 256 +area: Infra/CLI +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java index defaddb25eb4..a192dda84c4c 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java +++ b/server/src/main/java/org/elasticsearch/common/settings/KeyStoreWrapper.java @@ -120,7 +120,8 @@ public class KeyStoreWrapper implements SecureSettings { /** The version where lucene directory API changed from BE to LE. */ public static final int LE_VERSION = 5; public static final int HIGHER_KDF_ITERATION_COUNT_VERSION = 6; - public static final int CURRENT_VERSION = HIGHER_KDF_ITERATION_COUNT_VERSION; + public static final int CIPHER_KEY_BITS_256_VERSION = 7; + public static final int CURRENT_VERSION = CIPHER_KEY_BITS_256_VERSION; /** The algorithm used to derive the cipher key from a password. */ private static final String KDF_ALGO = "PBKDF2WithHmacSHA512"; @@ -128,14 +129,8 @@ public class KeyStoreWrapper implements SecureSettings { /** The number of iterations to derive the cipher key. */ private static final int KDF_ITERS = 210000; - /** - * The number of bits for the cipher key. - * - * Note: The Oracle JDK 8 ships with a limited JCE policy that restricts key length for AES to 128 bits. - * This can be increased to 256 bits once minimum java 9 is the minimum java version. - * See http://www.oracle.com/technetwork/java/javase/terms/readme/jdk9-readme-3852447.html#jce - * */ - private static final int CIPHER_KEY_BITS = 128; + /** The number of bits for the cipher key (256 bits are supported as of Java 9).*/ + private static final int CIPHER_KEY_BITS = 256; /** The number of bits for the GCM tag. */ private static final int GCM_TAG_BITS = 128; @@ -156,6 +151,7 @@ public class KeyStoreWrapper implements SecureSettings { // 4: remove distinction between string/files, ES 6.8/7.1 // 5: Lucene directory API changed to LE, ES 8.0 // 6: increase KDF iteration count, ES 8.14 + // 7: increase cipher key length to 256 bits, ES 9.0 /** The metadata format version used to read the current keystore wrapper. */ private final int formatVersion; @@ -318,8 +314,9 @@ public class KeyStoreWrapper implements SecureSettings { return hasPassword; } - private static Cipher createCipher(int opmode, char[] password, byte[] salt, byte[] iv, int kdfIters) throws GeneralSecurityException { - PBEKeySpec keySpec = new PBEKeySpec(password, salt, kdfIters, CIPHER_KEY_BITS); + private static Cipher createCipher(int opmode, char[] password, byte[] salt, byte[] iv, int kdfIters, int cipherKeyBits) + throws GeneralSecurityException { + PBEKeySpec keySpec = new PBEKeySpec(password, salt, kdfIters, cipherKeyBits); SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(KDF_ALGO); SecretKey secretKey; try { @@ -343,6 +340,11 @@ public class KeyStoreWrapper implements SecureSettings { return formatVersion < HIGHER_KDF_ITERATION_COUNT_VERSION ? 10000 : KDF_ITERS; } + private static int getCipherKeyBitsForVersion(int formatVersion) { + // cipher key length was increased in version 7; it was 128 bits in previous versions + return formatVersion < CIPHER_KEY_BITS_256_VERSION ? 128 : CIPHER_KEY_BITS; + } + /** * Decrypts the underlying keystore data. * @@ -371,7 +373,14 @@ public class KeyStoreWrapper implements SecureSettings { throw new SecurityException("Keystore has been corrupted or tampered with", e); } - Cipher cipher = createCipher(Cipher.DECRYPT_MODE, password, salt, iv, getKdfIterationCountForVersion(formatVersion)); + Cipher cipher = createCipher( + Cipher.DECRYPT_MODE, + password, + salt, + iv, + getKdfIterationCountForVersion(formatVersion), + getCipherKeyBitsForVersion(formatVersion) + ); try ( ByteArrayInputStream bytesStream = new ByteArrayInputStream(encryptedBytes); CipherInputStream cipherStream = new CipherInputStream(bytesStream, cipher); @@ -409,11 +418,12 @@ public class KeyStoreWrapper implements SecureSettings { } /** Encrypt the keystore entries and return the encrypted data. */ - private byte[] encrypt(char[] password, byte[] salt, byte[] iv, int kdfIterationCount) throws GeneralSecurityException, IOException { + private byte[] encrypt(char[] password, byte[] salt, byte[] iv, int kdfIterationCount, int cipherKeyBits) + throws GeneralSecurityException, IOException { assert isLoaded(); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - Cipher cipher = createCipher(Cipher.ENCRYPT_MODE, password, salt, iv, kdfIterationCount); + Cipher cipher = createCipher(Cipher.ENCRYPT_MODE, password, salt, iv, kdfIterationCount, cipherKeyBits); try ( CipherOutputStream cipherStream = new CipherOutputStream(bytes, cipher); DataOutputStream output = new DataOutputStream(cipherStream) @@ -456,7 +466,13 @@ public class KeyStoreWrapper implements SecureSettings { byte[] iv = new byte[12]; random.nextBytes(iv); // encrypted data - byte[] encryptedBytes = encrypt(password, salt, iv, getKdfIterationCountForVersion(CURRENT_VERSION)); + byte[] encryptedBytes = encrypt( + password, + salt, + iv, + getKdfIterationCountForVersion(CURRENT_VERSION), + getCipherKeyBitsForVersion(CURRENT_VERSION) + ); // size of data block output.writeInt(4 + salt.length + 4 + iv.length + 4 + encryptedBytes.length); From 169cb44b3f0c29802efe7a02f8ef8b4491518e13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenzo=20Dematt=C3=A9?= Date: Thu, 9 Jan 2025 09:03:20 +0100 Subject: [PATCH 04/33] Add getSubject -> current method replacement patcher (#119779) --- .../hdfs/patch/HdfsClassPatcher.java | 6 +- .../hdfs/patch/SubjectGetSubjectPatcher.java | 60 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java index 6636b3944596..732c55929454 100644 --- a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java +++ b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/HdfsClassPatcher.java @@ -27,7 +27,11 @@ public class HdfsClassPatcher { "org/apache/hadoop/util/ShutdownHookManager.class", ShutdownHookManagerPatcher::new, "org/apache/hadoop/util/Shell.class", - ShellPatcher::new + ShellPatcher::new, + "org/apache/hadoop/security/UserGroupInformation.class", + SubjectGetSubjectPatcher::new, + "org/apache/hadoop/security/authentication/client/KerberosAuthenticator.class", + SubjectGetSubjectPatcher::new ); public static void main(String[] args) throws Exception { diff --git a/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java new file mode 100644 index 000000000000..3fb8a23be794 --- /dev/null +++ b/plugins/repository-hdfs/hadoop-client-api/src/patcher/java/org/elasticsearch/hdfs/patch/SubjectGetSubjectPatcher.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.hdfs.patch; + +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Type; + +import static org.objectweb.asm.Opcodes.ASM9; +import static org.objectweb.asm.Opcodes.INVOKESTATIC; +import static org.objectweb.asm.Opcodes.POP; + +class SubjectGetSubjectPatcher extends ClassVisitor { + SubjectGetSubjectPatcher(ClassWriter classWriter) { + super(ASM9, classWriter); + } + + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + return new ReplaceCallMethodVisitor(super.visitMethod(access, name, descriptor, signature, exceptions), name, access, descriptor); + } + + /** + * Replaces calls to Subject.getSubject(context); with calls to Subject.current(); + */ + private static class ReplaceCallMethodVisitor extends MethodVisitor { + private static final String SUBJECT_CLASS_INTERNAL_NAME = "javax/security/auth/Subject"; + private static final String METHOD_NAME = "getSubject"; + + ReplaceCallMethodVisitor(MethodVisitor methodVisitor, String name, int access, String descriptor) { + super(ASM9, methodVisitor); + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { + if (opcode == INVOKESTATIC && SUBJECT_CLASS_INTERNAL_NAME.equals(owner) && METHOD_NAME.equals(name)) { + // Get rid of the extra arg on the stack + mv.visitInsn(POP); + // Call Subject.current() + mv.visitMethodInsn( + INVOKESTATIC, + SUBJECT_CLASS_INTERNAL_NAME, + "current", + Type.getMethodDescriptor(Type.getObjectType(SUBJECT_CLASS_INTERNAL_NAME)), + false + ); + } else { + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + } + } + } +} From 7804a25b8ed4019b3a3c073745d0dd4c1dd465e9 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Thu, 9 Jan 2025 09:09:37 +0000 Subject: [PATCH 05/33] Refactor QueryBuilderResolver Rewrite Logic (#119740) * Refactor QueryBuilderResolver Rewrite Logic This commit improves the rewrite logic by switching to reference comparison for termination checks. While the existing implementation functions correctly, the rewrite contract is designed to compare references rather than performing a full object comparison, which is unnecessary. Additionally, this change guarantees that only a single rewrite pass is executed per query builder. * avoid getting the value --------- Co-authored-by: Elastic Machine --- .../xpack/esql/session/QueryBuilderResolver.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java index b6424c5f7fa5..69c95dfe9f40 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java @@ -157,15 +157,12 @@ public class QueryBuilderResolver { Map results = new HashMap<>(); boolean hasChanged = false; - for (FullTextFunction func : queryBuilderMap.keySet()) { - var initial = queryBuilderMap.get(func); - var rewritten = Rewriteable.rewrite(initial, ctx, false); + for (var entry : queryBuilderMap.entrySet()) { + var initial = entry.getValue(); + var rewritten = initial.rewrite(ctx); + hasChanged |= rewritten != initial; - if (rewritten.equals(initial) == false) { - hasChanged = true; - } - - results.put(func, rewritten); + results.put(entry.getKey(), rewritten); } return hasChanged ? new FullTextFunctionsRewritable(results) : this; From 412214435564f508a6b6c61866f3a36ee2cefe24 Mon Sep 17 00:00:00 2001 From: Ioana Tagirta Date: Thu, 9 Jan 2025 10:10:51 +0100 Subject: [PATCH 06/33] Enable query rewrites on the coordinator for ESQL (#119667) * Enable query rewrites on the coordinator for ESQL * [CI] Auto commit changes from spotless --------- Co-authored-by: elasticsearchmachine --- .../xpack/esql/session/QueryBuilderResolver.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java index 69c95dfe9f40..ab4a9b02e41f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/QueryBuilderResolver.java @@ -16,7 +16,6 @@ import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.search.SearchService; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; @@ -59,12 +58,6 @@ public class QueryBuilderResolver { ActionListener listener, BiConsumer> callback ) { - // TODO: remove once SEMANTIC_TEXT_TYPE is enabled outside of snapshots - if (false == EsqlCapabilities.Cap.SEMANTIC_TEXT_TYPE.isEnabled()) { - callback.accept(plan, listener); - return; - } - if (plan.optimized() == false) { listener.onFailure(new IllegalStateException("Expected optimized plan before query builder rewrite.")); return; From 996a4f8e7d2cd488e100ff38228ae806e93a9f87 Mon Sep 17 00:00:00 2001 From: Liam Thompson <32779855+leemthompo@users.noreply.github.com> Date: Thu, 9 Jan 2025 11:01:46 +0100 Subject: [PATCH 07/33] Update rejected-requests.asciidoc (#119547) (#119673) I believe this is a typo, as in our 8.16.1 cluster this field appears to be called `combined_coordinating_and_primary` Co-authored-by: Ian Lee --- .../troubleshooting/common-issues/rejected-requests.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc b/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc index 34ef388f2e3c..83f6962a219e 100644 --- a/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc +++ b/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc @@ -69,7 +69,7 @@ These stats are cumulative from node startup. Indexing pressure rejections appear as an `EsRejectedExecutionException`, and indicate that they were rejected due -to `coordinating_and_primary_bytes`, `coordinating`, `primary`, or `replica`. +to `combined_coordinating_and_primary`, `coordinating`, `primary`, or `replica`. These errors are often related to <>, <> sizing, or the ingest target's @@ -86,4 +86,4 @@ of diagnosing indexing pressure rejections. If {es} regularly rejects requests and other tasks, your cluster likely has high CPU usage or high JVM memory pressure. For tips, see <> and -<>. \ No newline at end of file +<>. From e0cefb8ff0662943f91d2fd0245a6c57309a984a Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Thu, 9 Jan 2025 11:02:51 +0100 Subject: [PATCH 08/33] Revert "Restrict Connector APIs to manage/monitor_connector privileges (#119389)" (#119833) This reverts commit c88eef308c9a32f80fb5f86025bd962d2f7a8c1a. --- docs/changelog/119389.yaml | 5 - x-pack/plugin/ent-search/qa/rest/roles.yml | 8 +- .../connector/ConnectorIndexService.java | 242 ++++++++---------- .../action/ConnectorActionRequest.java | 18 +- .../action/DeleteConnectorAction.java | 11 +- .../connector/action/GetConnectorAction.java | 2 +- .../connector/action/ListConnectorAction.java | 2 +- .../connector/action/PostConnectorAction.java | 2 +- .../connector/action/PutConnectorAction.java | 5 +- .../UpdateConnectorActiveFilteringAction.java | 2 +- .../action/UpdateConnectorApiKeyIdAction.java | 2 +- .../UpdateConnectorConfigurationAction.java | 2 +- .../action/UpdateConnectorErrorAction.java | 2 +- .../action/UpdateConnectorFeaturesAction.java | 2 +- .../UpdateConnectorFilteringAction.java | 2 +- ...ateConnectorFilteringValidationAction.java | 2 +- .../UpdateConnectorIndexNameAction.java | 2 +- .../action/UpdateConnectorLastSeenAction.java | 2 +- .../UpdateConnectorLastSyncStatsAction.java | 2 +- .../action/UpdateConnectorNameAction.java | 2 +- .../action/UpdateConnectorNativeAction.java | 2 +- .../action/UpdateConnectorPipelineAction.java | 2 +- .../UpdateConnectorSchedulingAction.java | 2 +- .../UpdateConnectorServiceTypeAction.java | 2 +- .../action/UpdateConnectorStatusAction.java | 2 +- .../syncjob/ConnectorSyncJobIndexService.java | 29 +-- .../action/CancelConnectorSyncJobAction.java | 2 +- .../action/CheckInConnectorSyncJobAction.java | 2 +- .../action/ClaimConnectorSyncJobAction.java | 2 +- .../action/ConnectorSyncJobActionRequest.java | 17 +- .../action/DeleteConnectorSyncJobAction.java | 2 +- .../action/GetConnectorSyncJobAction.java | 2 +- .../action/ListConnectorSyncJobsAction.java | 2 +- .../action/PostConnectorSyncJobAction.java | 11 +- .../UpdateConnectorSyncJobErrorAction.java | 2 +- ...eConnectorSyncJobIngestionStatsAction.java | 2 +- .../xpack/security/operator/Constants.java | 60 ++--- 37 files changed, 230 insertions(+), 230 deletions(-) delete mode 100644 docs/changelog/119389.yaml diff --git a/docs/changelog/119389.yaml b/docs/changelog/119389.yaml deleted file mode 100644 index 267eaa345b9f..000000000000 --- a/docs/changelog/119389.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 119389 -summary: Restrict Connector APIs to manage/monitor_connector privileges -area: Extract&Transform -type: feature -issues: [] diff --git a/x-pack/plugin/ent-search/qa/rest/roles.yml b/x-pack/plugin/ent-search/qa/rest/roles.yml index 661ba482c636..d32f05b7b749 100644 --- a/x-pack/plugin/ent-search/qa/rest/roles.yml +++ b/x-pack/plugin/ent-search/qa/rest/roles.yml @@ -4,12 +4,13 @@ admin: - manage_behavioral_analytics - manage - monitor - - manage_connector indices: - names: [ # indices and search applications "test-*", "another-test-search-application", + ".elastic-connectors-v1", + ".elastic-connectors-sync-jobs-v1" ] privileges: [ "manage", "write", "read" ] @@ -19,7 +20,6 @@ user: - manage_api_key - read_connector_secrets - write_connector_secrets - - monitor_connector indices: - names: [ "test-index1", @@ -27,7 +27,9 @@ user: "test-search-application-1", "test-search-application-with-aggs", "test-search-application-with-list", - "test-search-application-with-list-invalid" + "test-search-application-with-list-invalid", + ".elastic-connectors-v1", + ".elastic-connectors-sync-jobs-v1" ] privileges: [ "read" ] diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index 14d6c0e10368..d5d2159d8f37 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -24,7 +24,6 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Client; -import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.common.Strings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -78,15 +77,13 @@ import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.fromXContentBytesConnectorFiltering; import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.sortFilteringRulesByOrder; import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX; -import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; /** * A service that manages persistent {@link Connector} configurations. */ public class ConnectorIndexService { - // The client to interact with the system index (internal user). - private final Client clientWithOrigin; + private final Client client; public static final String CONNECTOR_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN; @@ -94,7 +91,7 @@ public class ConnectorIndexService { * @param client A client for executing actions on the connector index */ public ConnectorIndexService(Client client) { - this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN); + this.client = client; } /** @@ -140,7 +137,7 @@ public class ConnectorIndexService { indexRequest = indexRequest.id(connectorId); } - clientWithOrigin.index( + client.index( indexRequest, listener.delegateFailureAndWrap( (ll, indexResponse) -> ll.onResponse( @@ -204,7 +201,7 @@ public class ConnectorIndexService { try { final GetRequest getRequest = new GetRequest(CONNECTOR_INDEX_NAME).id(connectorId).realtime(true); - clientWithOrigin.get(getRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, getResponse) -> { + client.get(getRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, getResponse) -> { if (getResponse.isExists() == false) { l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -238,23 +235,17 @@ public class ConnectorIndexService { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); try { - clientWithOrigin.delete( - deleteRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, deleteResponse) -> { - if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - if (shouldDeleteSyncJobs) { - new ConnectorSyncJobIndexService(clientWithOrigin).deleteAllSyncJobsByConnectorId( - connectorId, - l.map(r -> deleteResponse) - ); - } else { - l.onResponse(deleteResponse); - } - }) - ); + client.delete(deleteRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, deleteResponse) -> { + if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + if (shouldDeleteSyncJobs) { + new ConnectorSyncJobIndexService(client).deleteAllSyncJobsByConnectorId(connectorId, l.map(r -> deleteResponse)); + } else { + l.onResponse(deleteResponse); + } + })); } catch (Exception e) { listener.onFailure(e); } @@ -288,7 +279,7 @@ public class ConnectorIndexService { .fetchSource(true) .sort(Connector.INDEX_NAME_FIELD.getPreferredName(), SortOrder.ASC); final SearchRequest req = new SearchRequest(CONNECTOR_INDEX_NAME).source(source); - clientWithOrigin.search(req, new ActionListener<>() { + client.search(req, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { try { @@ -463,7 +454,7 @@ public class ConnectorIndexService { return; } - clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -500,16 +491,13 @@ public class ConnectorIndexService { } }) ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - }) - ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + })); } catch (Exception e) { listener.onFailure(e); } @@ -531,16 +519,13 @@ public class ConnectorIndexService { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - }) - ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + })); } catch (Exception e) { listener.onFailure(e); } @@ -561,16 +546,13 @@ public class ConnectorIndexService { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(Map.of(Connector.FILTERING_FIELD.getPreferredName(), filtering)) ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - }) - ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + })); } catch (Exception e) { listener.onFailure(e); } @@ -591,16 +573,13 @@ public class ConnectorIndexService { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(Map.of(Connector.FEATURES_FIELD.getPreferredName(), features)) ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - }) - ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + })); } catch (Exception e) { listener.onFailure(e); } @@ -656,16 +635,13 @@ public class ConnectorIndexService { .source(Map.of(Connector.FILTERING_FIELD.getPreferredName(), List.of(connectorFilteringWithUpdatedDraft))) ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - ll.onResponse(updateResponse); - }) - ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + ll.onResponse(updateResponse); + })); })); } catch (Exception e) { @@ -707,7 +683,7 @@ public class ConnectorIndexService { .source(Map.of(Connector.FILTERING_FIELD.getPreferredName(), List.of(activatedConnectorFiltering))) ); - clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -764,7 +740,7 @@ public class ConnectorIndexService { .source(Map.of(Connector.FILTERING_FIELD.getPreferredName(), List.of(activatedConnectorFiltering))) ); - clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); return; @@ -792,16 +768,13 @@ public class ConnectorIndexService { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(Map.of(Connector.LAST_SEEN_FIELD.getPreferredName(), Instant.now())) ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - }) - ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + })); } catch (Exception e) { listener.onFailure(e); } @@ -822,16 +795,13 @@ public class ConnectorIndexService { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - }) - ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + })); } catch (Exception e) { listener.onFailure(e); } @@ -895,16 +865,13 @@ public class ConnectorIndexService { ) ) ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - ll.onResponse(updateResponse); - }) - ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + ll.onResponse(updateResponse); + })); })); } catch (Exception e) { listener.onFailure(e); @@ -927,16 +894,13 @@ public class ConnectorIndexService { .source(Map.of(Connector.PIPELINE_FIELD.getPreferredName(), request.getPipeline())) .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - }) - ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + })); } catch (Exception e) { listener.onFailure(e); } @@ -995,7 +959,7 @@ public class ConnectorIndexService { } }) ); - clientWithOrigin.update( + client.update( updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (lll, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { @@ -1028,16 +992,13 @@ public class ConnectorIndexService { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(Map.of(Connector.SCHEDULING_FIELD.getPreferredName(), request.getScheduling())) ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - }) - ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + })); } catch (Exception e) { listener.onFailure(e); } @@ -1073,7 +1034,7 @@ public class ConnectorIndexService { ) ); - clientWithOrigin.update( + client.update( updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (updateListener, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { @@ -1116,7 +1077,7 @@ public class ConnectorIndexService { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(Map.of(Connector.STATUS_FIELD.getPreferredName(), request.getStatus())) ); - clientWithOrigin.update( + client.update( updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (updateListener, updateResponse) -> { if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { @@ -1144,16 +1105,13 @@ public class ConnectorIndexService { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .source(request.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)) ); - clientWithOrigin.update( - updateRequest, - new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { - if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - l.onResponse(updateResponse); - }) - ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + l.onResponse(updateResponse); + })); } catch (Exception e) { listener.onFailure(e); } @@ -1223,7 +1181,7 @@ public class ConnectorIndexService { final SearchSourceBuilder searchSource = new SearchSourceBuilder().query(boolFilterQueryBuilder); final SearchRequest searchRequest = new SearchRequest(CONNECTOR_INDEX_NAME).source(searchSource); - clientWithOrigin.search(searchRequest, new ActionListener<>() { + client.search(searchRequest, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { boolean indexNameIsInUse = searchResponse.getHits().getTotalHits().value() > 0L; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java index 723ff12b3b1e..66f347bc4dbb 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java @@ -9,9 +9,12 @@ package org.elasticsearch.xpack.application.connector.action; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.indices.InvalidIndexNameException; +import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import java.io.IOException; @@ -19,9 +22,10 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX; /** - * Abstract base class for action requests targeting the connectors index. + * Abstract base class for action requests targeting the connectors index. Implements {@link org.elasticsearch.action.IndicesRequest} + * to ensure index-level privilege support. This class defines the connectors index as the target for all derived action requests. */ -public abstract class ConnectorActionRequest extends ActionRequest { +public abstract class ConnectorActionRequest extends ActionRequest implements IndicesRequest { public ConnectorActionRequest() { super(); @@ -74,4 +78,14 @@ public abstract class ConnectorActionRequest extends ActionRequest { } return validationException; } + + @Override + public String[] indices() { + return new String[] { ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN }; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.lenientExpandHidden(); + } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java index 5d98f9703ece..930068a2a46e 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/DeleteConnectorAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import java.io.IOException; import java.util.Objects; @@ -27,7 +28,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg public class DeleteConnectorAction { - public static final String NAME = "cluster:admin/xpack/connector/delete"; + public static final String NAME = "indices:data/write/xpack/connector/delete"; public static final ActionType INSTANCE = new ActionType<>(NAME); private DeleteConnectorAction() {/* no instances */} @@ -70,6 +71,14 @@ public class DeleteConnectorAction { return deleteSyncJobs; } + @Override + public String[] indices() { + // When deleting a connector, corresponding sync jobs can also be deleted + return new String[] { + ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN, + ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN }; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java index a976e97adc0c..2edd47b1fce3 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java @@ -28,7 +28,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg public class GetConnectorAction { - public static final String NAME = "cluster:admin/xpack/connector/get"; + public static final String NAME = "indices:data/read/xpack/connector/get"; public static final ActionType INSTANCE = new ActionType<>(NAME); private GetConnectorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java index c5de4f054535..e543d805b709 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java @@ -34,7 +34,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr public class ListConnectorAction { - public static final String NAME = "cluster:admin/xpack/connector/list"; + public static final String NAME = "indices:data/read/xpack/connector/list"; public static final ActionType INSTANCE = new ActionType<>(NAME); private ListConnectorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java index b5087634a77f..b1c38637298c 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java @@ -25,7 +25,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr public class PostConnectorAction { - public static final String NAME = "cluster:admin/xpack/connector/post"; + public static final String NAME = "indices:data/write/xpack/connector/post"; public static final ActionType INSTANCE = new ActionType<>(NAME); private PostConnectorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java index c5922ebfafe1..f3e8ed6b6e76 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java @@ -9,6 +9,7 @@ package org.elasticsearch.xpack.application.connector.action; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -26,12 +27,12 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr public class PutConnectorAction { - public static final String NAME = "cluster:admin/xpack/connector/put"; + public static final String NAME = "indices:data/write/xpack/connector/put"; public static final ActionType INSTANCE = new ActionType<>(NAME); private PutConnectorAction() {/* no instances */} - public static class Request extends ConnectorActionRequest implements ToXContentObject { + public static class Request extends ConnectorActionRequest implements IndicesRequest, ToXContentObject { @Nullable private final String connectorId; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java index a7bd9c238233..7b4ce08ef832 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorActiveFilteringAction.java @@ -22,7 +22,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; public class UpdateConnectorActiveFilteringAction { - public static final String NAME = "cluster:admin/xpack/connector/update_filtering/activate"; + public static final String NAME = "indices:data/write/xpack/connector/update_filtering/activate"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorActiveFilteringAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java index e76c5191a58a..7f726f21ce22 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorApiKeyIdAction.java @@ -27,7 +27,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr public class UpdateConnectorApiKeyIdAction { - public static final String NAME = "cluster:admin/xpack/connector/update_api_key_id"; + public static final String NAME = "indices:data/write/xpack/connector/update_api_key_id"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorApiKeyIdAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java index 6948667fa735..5d36c5f886ea 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorConfigurationAction.java @@ -32,7 +32,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr public class UpdateConnectorConfigurationAction { - public static final String NAME = "cluster:admin/xpack/connector/update_configuration"; + public static final String NAME = "indices:data/write/xpack/connector/update_configuration"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorConfigurationAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java index a8c2b334cfee..3e506fc835f6 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorErrorAction.java @@ -27,7 +27,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg public class UpdateConnectorErrorAction { - public static final String NAME = "cluster:admin/xpack/connector/update_error"; + public static final String NAME = "indices:data/write/xpack/connector/update_error"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorErrorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java index 4bd51794c1f9..56656855583a 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFeaturesAction.java @@ -27,7 +27,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr public class UpdateConnectorFeaturesAction { - public static final String NAME = "cluster:admin/xpack/connector/update_features"; + public static final String NAME = "indices:data/write/xpack/connector/update_features"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorFeaturesAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java index e527e79161f2..660956b2e9d7 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringAction.java @@ -32,7 +32,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr public class UpdateConnectorFilteringAction { - public static final String NAME = "cluster:admin/xpack/connector/update_filtering"; + public static final String NAME = "indices:data/write/xpack/connector/update_filtering"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorFilteringAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java index a22b08152f50..92291506d071 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorFilteringValidationAction.java @@ -27,7 +27,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg public class UpdateConnectorFilteringValidationAction { - public static final String NAME = "cluster:admin/xpack/connector/update_filtering/draft_validation"; + public static final String NAME = "indices:data/write/xpack/connector/update_filtering/draft_validation"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorFilteringValidationAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java index ddc98d4756dc..e7840e1f84fa 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorIndexNameAction.java @@ -27,7 +27,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg public class UpdateConnectorIndexNameAction { - public static final String NAME = "cluster:admin/xpack/connector/update_index_name"; + public static final String NAME = "indices:data/write/xpack/connector/update_index_name"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorIndexNameAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java index deae10d901c1..f72938ec8dba 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSeenAction.java @@ -23,7 +23,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; public class UpdateConnectorLastSeenAction { - public static final String NAME = "cluster:admin/xpack/connector/update_last_seen"; + public static final String NAME = "indices:data/write/xpack/connector/update_last_seen"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorLastSeenAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java index 029e261e51fa..ae3be3801786 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorLastSyncStatsAction.java @@ -32,7 +32,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr public class UpdateConnectorLastSyncStatsAction { - public static final String NAME = "cluster:admin/xpack/connector/update_last_sync_stats"; + public static final String NAME = "indices:data/write/xpack/connector/update_last_sync_stats"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorLastSyncStatsAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java index 5a63bb106747..bbc1f992b48e 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNameAction.java @@ -27,7 +27,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr public class UpdateConnectorNameAction { - public static final String NAME = "cluster:admin/xpack/connector/update_name"; + public static final String NAME = "indices:data/write/xpack/connector/update_name"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorNameAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java index d20b535d5cc5..7b3f2e4577f4 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorNativeAction.java @@ -26,7 +26,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg public class UpdateConnectorNativeAction { - public static final String NAME = "cluster:admin/xpack/connector/update_native"; + public static final String NAME = "indices:data/write/xpack/connector/update_native"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorNativeAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java index 2e738033c338..e58d614f4ef2 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorPipelineAction.java @@ -27,7 +27,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg public class UpdateConnectorPipelineAction { - public static final String NAME = "cluster:admin/xpack/connector/update_pipeline"; + public static final String NAME = "indices:data/write/xpack/connector/update_pipeline"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorPipelineAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java index 65cfd645ea60..578639f065a0 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorSchedulingAction.java @@ -32,7 +32,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg public class UpdateConnectorSchedulingAction { - public static final String NAME = "cluster:admin/xpack/connector/update_scheduling"; + public static final String NAME = "indices:data/write/xpack/connector/update_scheduling"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorSchedulingAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java index 1d4df12e10eb..de07a6db21ba 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorServiceTypeAction.java @@ -26,7 +26,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg public class UpdateConnectorServiceTypeAction { - public static final String NAME = "cluster:admin/xpack/connector/update_service_type"; + public static final String NAME = "indices:data/write/xpack/connector/update_service_type"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorServiceTypeAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java index 79f097db2dec..aebaa0afb905 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/UpdateConnectorStatusAction.java @@ -28,7 +28,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr public class UpdateConnectorStatusAction { - public static final String NAME = "cluster:admin/xpack/connector/update_status"; + public static final String NAME = "indices:data/write/xpack/connector/update_status"; public static final ActionType INSTANCE = new ActionType<>(NAME); public UpdateConnectorStatusAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java index f46d915a7123..ce6f7f0dbf2b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java @@ -28,7 +28,6 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Client; -import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.common.Strings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.engine.DocumentMissingException; @@ -69,7 +68,6 @@ import java.util.stream.Stream; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.application.connector.ConnectorIndexService.CONNECTOR_INDEX_NAME; -import static org.elasticsearch.xpack.core.ClientHelper.CONNECTORS_ORIGIN; /** * A service that manages persistent {@link ConnectorSyncJob} configurations. @@ -78,8 +76,7 @@ public class ConnectorSyncJobIndexService { private static final Long ZERO = 0L; - // The client to interact with the system index (internal user). - private final Client clientWithOrigin; + private final Client client; public static final String CONNECTOR_SYNC_JOB_INDEX_NAME = ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN; @@ -87,7 +84,7 @@ public class ConnectorSyncJobIndexService { * @param client A client for executing actions on the connectors sync jobs index. */ public ConnectorSyncJobIndexService(Client client) { - this.clientWithOrigin = new OriginSettingClient(client, CONNECTORS_ORIGIN); + this.client = client; } /** @@ -152,7 +149,7 @@ public class ConnectorSyncJobIndexService { indexRequest.source(syncJob.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - clientWithOrigin.index( + client.index( indexRequest, l.delegateFailureAndWrap( (ll, indexResponse) -> ll.onResponse(new PostConnectorSyncJobAction.Response(indexResponse.getId())) @@ -178,7 +175,7 @@ public class ConnectorSyncJobIndexService { .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); try { - clientWithOrigin.delete( + client.delete( deleteRequest, new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(connectorSyncJobId, listener, (l, deleteResponse) -> { if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { @@ -208,7 +205,7 @@ public class ConnectorSyncJobIndexService { ).doc(Map.of(ConnectorSyncJob.LAST_SEEN_FIELD.getPreferredName(), newLastSeen)); try { - clientWithOrigin.update( + client.update( updateRequest, new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(connectorSyncJobId, listener, (l, updateResponse) -> { if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { @@ -233,7 +230,7 @@ public class ConnectorSyncJobIndexService { final GetRequest getRequest = new GetRequest(CONNECTOR_SYNC_JOB_INDEX_NAME).id(connectorSyncJobId).realtime(true); try { - clientWithOrigin.get( + client.get( getRequest, new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(connectorSyncJobId, listener, (l, getResponse) -> { if (getResponse.isExists() == false) { @@ -309,7 +306,7 @@ public class ConnectorSyncJobIndexService { WriteRequest.RefreshPolicy.IMMEDIATE ).doc(syncJobFieldsToUpdate); - clientWithOrigin.update( + client.update( updateRequest, new DelegatingIndexNotFoundOrDocumentMissingActionListener<>( connectorSyncJobId, @@ -358,7 +355,7 @@ public class ConnectorSyncJobIndexService { final SearchRequest searchRequest = new SearchRequest(CONNECTOR_SYNC_JOB_INDEX_NAME).source(searchSource); - clientWithOrigin.search(searchRequest, new ActionListener<>() { + client.search(searchRequest, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { try { @@ -477,7 +474,7 @@ public class ConnectorSyncJobIndexService { ).doc(fieldsToUpdate); try { - clientWithOrigin.update( + client.update( updateRequest, new DelegatingIndexNotFoundOrDocumentMissingActionListener<>(syncJobId, listener, (l, updateResponse) -> { if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { @@ -504,7 +501,7 @@ public class ConnectorSyncJobIndexService { final GetRequest request = new GetRequest(CONNECTOR_INDEX_NAME, connectorId); - clientWithOrigin.get(request, new ActionListener<>() { + client.get(request, new ActionListener<>() { @Override public void onResponse(GetResponse response) { final boolean connectorDoesNotExist = response.isExists() == false; @@ -597,7 +594,7 @@ public class ConnectorSyncJobIndexService { ) ); - clientWithOrigin.update( + client.update( updateRequest, new DelegatingIndexNotFoundOrDocumentMissingActionListener<>( connectorSyncJobId, @@ -632,7 +629,7 @@ public class ConnectorSyncJobIndexService { ) ).setRefresh(true).setIndicesOptions(IndicesOptions.fromOptions(true, true, false, false)); - clientWithOrigin.execute(DeleteByQueryAction.INSTANCE, deleteByQueryRequest, listener.delegateFailureAndWrap((l, r) -> { + client.execute(DeleteByQueryAction.INSTANCE, deleteByQueryRequest, listener.delegateFailureAndWrap((l, r) -> { final List bulkDeleteFailures = r.getBulkFailures(); if (bulkDeleteFailures.isEmpty() == false) { l.onFailure( @@ -684,7 +681,7 @@ public class ConnectorSyncJobIndexService { WriteRequest.RefreshPolicy.IMMEDIATE ).doc(document); - clientWithOrigin.update( + client.update( updateRequest, new DelegatingIndexNotFoundOrDocumentMissingActionListener<>( connectorSyncJobId, diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java index 658d48e9752d..160eda3aeef7 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CancelConnectorSyncJobAction.java @@ -28,7 +28,7 @@ import static org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyn public class CancelConnectorSyncJobAction { - public static final String NAME = "cluster:admin/xpack/connector/sync_job/cancel"; + public static final String NAME = "indices:data/write/xpack/connector/sync_job/cancel"; public static final ActionType INSTANCE = new ActionType(NAME); private CancelConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java index ae44813354eb..aa6dea50de46 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/CheckInConnectorSyncJobAction.java @@ -28,7 +28,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg public class CheckInConnectorSyncJobAction { - public static final String NAME = "cluster:admin/xpack/connector/sync_job/check_in"; + public static final String NAME = "indices:data/write/xpack/connector/sync_job/check_in"; public static final ActionType INSTANCE = new ActionType<>(NAME); private CheckInConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java index 84e3183c2830..b108116a5e68 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java @@ -31,7 +31,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr public class ClaimConnectorSyncJobAction { public static final ParseField CONNECTOR_SYNC_JOB_ID_FIELD = new ParseField("connector_sync_job_id"); - public static final String NAME = "cluster:admin/xpack/connector/sync_job/claim"; + public static final String NAME = "indices:data/write/xpack/connector/sync_job/claim"; public static final ActionType INSTANCE = new ActionType<>(NAME); private ClaimConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java index 11a3334aed4e..bb83fd78151d 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ConnectorSyncJobActionRequest.java @@ -8,14 +8,19 @@ package org.elasticsearch.xpack.application.connector.syncjob.action; import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import java.io.IOException; /** * Abstract base class for action requests targeting the connector sync job index. + * Implements {@link org.elasticsearch.action.IndicesRequest} to ensure index-level privilege support. + * This class defines the connectors sync job index as the target for all derived action requests. */ -public abstract class ConnectorSyncJobActionRequest extends ActionRequest { +public abstract class ConnectorSyncJobActionRequest extends ActionRequest implements IndicesRequest { public ConnectorSyncJobActionRequest() { super(); @@ -24,4 +29,14 @@ public abstract class ConnectorSyncJobActionRequest extends ActionRequest { public ConnectorSyncJobActionRequest(StreamInput in) throws IOException { super(in); } + + @Override + public String[] indices() { + return new String[] { ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN }; + } + + @Override + public IndicesOptions indicesOptions() { + return IndicesOptions.lenientExpandHidden(); + } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java index 935aa1fd6b7d..c9c84ead2e4b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/DeleteConnectorSyncJobAction.java @@ -28,7 +28,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg public class DeleteConnectorSyncJobAction { - public static final String NAME = "cluster:admin/xpack/connector/sync_job/delete"; + public static final String NAME = "indices:data/write/xpack/connector/sync_job/delete"; public static final ActionType INSTANCE = new ActionType<>(NAME); private DeleteConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java index 485e516c80c3..7ff82b9881fb 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobAction.java @@ -29,7 +29,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg public class GetConnectorSyncJobAction { - public static final String NAME = "cluster:admin/xpack/connector/sync_job/get"; + public static final String NAME = "indices:data/read/xpack/connector/sync_job/get"; public static final ActionType INSTANCE = new ActionType<>(NAME); private GetConnectorSyncJobAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java index 04b765f3b568..5b34e05ae37f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsAction.java @@ -33,7 +33,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr public class ListConnectorSyncJobsAction { - public static final String NAME = "cluster:admin/xpack/connector/sync_job/list"; + public static final String NAME = "indices:data/read/xpack/connector/sync_job/list"; public static final ActionType INSTANCE = new ActionType<>(NAME); private ListConnectorSyncJobsAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java index 0d17a6dba6c3..8c1d24e466da 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/PostConnectorSyncJobAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xpack.application.connector.Connector; +import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTriggerMethod; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobType; @@ -31,7 +32,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr public class PostConnectorSyncJobAction { - public static final String NAME = "cluster:admin/xpack/connector/sync_job/post"; + public static final String NAME = "indices:data/write/xpack/connector/sync_job/post"; public static final ActionType INSTANCE = new ActionType<>(NAME); private PostConnectorSyncJobAction() {/* no instances */} @@ -139,6 +140,14 @@ public class PostConnectorSyncJobAction { public int hashCode() { return Objects.hash(id, jobType, triggerMethod); } + + @Override + public String[] indices() { + // Creating a new sync job requires reading from connector index + return new String[] { + ConnectorTemplateRegistry.CONNECTOR_SYNC_JOBS_INDEX_NAME_PATTERN, + ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN }; + } } public static class Response extends ActionResponse implements ToXContentObject { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java index 0d5f57c202d9..2235ba7cfe72 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobErrorAction.java @@ -28,7 +28,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg public class UpdateConnectorSyncJobErrorAction { - public static final String NAME = "cluster:admin/xpack/connector/sync_job/update_error"; + public static final String NAME = "indices:data/write/xpack/connector/sync_job/update_error"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorSyncJobErrorAction() {/* no instances */} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java index c890ca0d69bc..0fd9b6dec818 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/UpdateConnectorSyncJobIngestionStatsAction.java @@ -35,7 +35,7 @@ import static org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyn public class UpdateConnectorSyncJobIngestionStatsAction { - public static final String NAME = "cluster:admin/xpack/connector/sync_job/update_stats"; + public static final String NAME = "indices:data/write/xpack/connector/sync_job/update_stats"; public static final ActionType INSTANCE = new ActionType<>(NAME); private UpdateConnectorSyncJobIngestionStatsAction() {/* no instances */} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 82bad85fa34d..62687e42b091 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -134,40 +134,40 @@ public class Constants { "cluster:admin/xpack/ccr/auto_follow_pattern/put", "cluster:admin/xpack/ccr/pause_follow", "cluster:admin/xpack/ccr/resume_follow", - "cluster:admin/xpack/connector/delete", - "cluster:admin/xpack/connector/get", - "cluster:admin/xpack/connector/list", - "cluster:admin/xpack/connector/post", - "cluster:admin/xpack/connector/put", - "cluster:admin/xpack/connector/update_api_key_id", - "cluster:admin/xpack/connector/update_configuration", - "cluster:admin/xpack/connector/update_error", - "cluster:admin/xpack/connector/update_features", - "cluster:admin/xpack/connector/update_filtering", - "cluster:admin/xpack/connector/update_filtering/activate", - "cluster:admin/xpack/connector/update_filtering/draft_validation", - "cluster:admin/xpack/connector/update_index_name", - "cluster:admin/xpack/connector/update_last_seen", - "cluster:admin/xpack/connector/update_last_sync_stats", - "cluster:admin/xpack/connector/update_name", - "cluster:admin/xpack/connector/update_native", - "cluster:admin/xpack/connector/update_pipeline", - "cluster:admin/xpack/connector/update_scheduling", - "cluster:admin/xpack/connector/update_service_type", - "cluster:admin/xpack/connector/update_status", + "indices:data/write/xpack/connector/delete", + "indices:data/read/xpack/connector/get", + "indices:data/read/xpack/connector/list", + "indices:data/write/xpack/connector/post", + "indices:data/write/xpack/connector/put", + "indices:data/write/xpack/connector/update_api_key_id", + "indices:data/write/xpack/connector/update_configuration", + "indices:data/write/xpack/connector/update_error", + "indices:data/write/xpack/connector/update_features", + "indices:data/write/xpack/connector/update_filtering", + "indices:data/write/xpack/connector/update_filtering/activate", + "indices:data/write/xpack/connector/update_filtering/draft_validation", + "indices:data/write/xpack/connector/update_index_name", + "indices:data/write/xpack/connector/update_last_seen", + "indices:data/write/xpack/connector/update_last_sync_stats", + "indices:data/write/xpack/connector/update_name", + "indices:data/write/xpack/connector/update_native", + "indices:data/write/xpack/connector/update_pipeline", + "indices:data/write/xpack/connector/update_scheduling", + "indices:data/write/xpack/connector/update_service_type", + "indices:data/write/xpack/connector/update_status", "cluster:admin/xpack/connector/secret/delete", "cluster:admin/xpack/connector/secret/get", "cluster:admin/xpack/connector/secret/post", "cluster:admin/xpack/connector/secret/put", - "cluster:admin/xpack/connector/sync_job/cancel", - "cluster:admin/xpack/connector/sync_job/check_in", - "cluster:admin/xpack/connector/sync_job/claim", - "cluster:admin/xpack/connector/sync_job/delete", - "cluster:admin/xpack/connector/sync_job/get", - "cluster:admin/xpack/connector/sync_job/list", - "cluster:admin/xpack/connector/sync_job/post", - "cluster:admin/xpack/connector/sync_job/update_error", - "cluster:admin/xpack/connector/sync_job/update_stats", + "indices:data/write/xpack/connector/sync_job/cancel", + "indices:data/write/xpack/connector/sync_job/check_in", + "indices:data/write/xpack/connector/sync_job/claim", + "indices:data/write/xpack/connector/sync_job/delete", + "indices:data/read/xpack/connector/sync_job/get", + "indices:data/read/xpack/connector/sync_job/list", + "indices:data/write/xpack/connector/sync_job/post", + "indices:data/write/xpack/connector/sync_job/update_error", + "indices:data/write/xpack/connector/sync_job/update_stats", "cluster:admin/xpack/deprecation/info", "cluster:admin/xpack/deprecation/nodes/info", "cluster:admin/xpack/enrich/delete", From a2a4e7cacf7b229a5d30cb428511cca55af97082 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 9 Jan 2025 10:38:47 +0000 Subject: [PATCH 09/33] Remove the 8.15 transport conditions for event ingested range setting (#119502) --- .../cluster/ClusterStateDiffIT.java | 3 +- .../TransportSimulateIndexTemplateAction.java | 6 +- .../action/shard/ShardStateAction.java | 2 +- .../cluster/metadata/IndexMetadata.java | 31 +--------- .../metadata/MetadataCreateIndexService.java | 11 +--- .../metadata/MetadataIndexStateService.java | 4 +- .../MetadataIndexTemplateService.java | 2 +- .../snapshots/RestoreService.java | 29 ++------- .../reroute/ClusterRerouteResponseTests.java | 3 +- .../CanMatchPreFilterSearchPhaseTests.java | 5 +- .../cluster/ClusterStateTests.java | 2 +- .../MetadataCreateIndexServiceTests.java | 41 +------------ .../ClusterSerializationTests.java | 60 +------------------ .../ClusterStateCreationUtils.java | 7 +-- 14 files changed, 27 insertions(+), 179 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index e8e4eb756246..15b4a557b2b8 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -9,7 +9,6 @@ package org.elasticsearch.cluster; -import org.elasticsearch.TransportVersion; import org.elasticsearch.Version; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -565,7 +564,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase { settingsBuilder.put(randomSettings(Settings.EMPTY)).put(IndexMetadata.SETTING_VERSION_CREATED, randomWriteVersion()); builder.settings(settingsBuilder); builder.numberOfShards(randomIntBetween(1, 10)).numberOfReplicas(randomInt(10)); - builder.eventIngestedRange(IndexLongFieldRange.UNKNOWN, TransportVersion.current()); + builder.eventIngestedRange(IndexLongFieldRange.UNKNOWN); int aliasCount = randomInt(10); for (int i = 0; i < aliasCount; i++) { builder.putAlias(randomAlias()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 5f98852148ed..d3d557b598b3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -214,8 +214,7 @@ public class TransportSimulateIndexTemplateAction extends TransportMasterNodeRea .build(); final IndexMetadata indexMetadata = IndexMetadata.builder(indexName) - // handle mixed-cluster states by passing in minTransportVersion to reset event.ingested range to UNKNOWN if an older version - .eventIngestedRange(getEventIngestedRange(indexName, simulatedState), simulatedState.getMinTransportVersion()) + .eventIngestedRange(getEventIngestedRange(indexName, simulatedState)) .settings(dummySettings) .build(); return ClusterState.builder(simulatedState) @@ -304,8 +303,7 @@ public class TransportSimulateIndexTemplateAction extends TransportMasterNodeRea dummySettings.put(templateSettings); final IndexMetadata indexMetadata = IndexMetadata.builder(indexName) - // handle mixed-cluster states by passing in minTransportVersion to reset event.ingested range to UNKNOWN if an older version - .eventIngestedRange(getEventIngestedRange(indexName, simulatedState), simulatedState.getMinTransportVersion()) + .eventIngestedRange(getEventIngestedRange(indexName, simulatedState)) .settings(dummySettings) .build(); diff --git a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java index bd438d66549a..ed6ca57d67b2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java +++ b/server/src/main/java/org/elasticsearch/cluster/action/shard/ShardStateAction.java @@ -754,7 +754,7 @@ public class ShardStateAction { metadataBuilder.put( IndexMetadata.builder(metadataBuilder.getSafe(updatedTimeRangesEntry.getKey())) .timestampRange(timeRanges.timestampRange()) - .eventIngestedRange(timeRanges.eventIngestedRange(), maybeUpdatedState.getMinTransportVersion()) + .eventIngestedRange(timeRanges.eventIngestedRange()) ); } maybeUpdatedState = ClusterState.builder(maybeUpdatedState).metadata(metadataBuilder).build(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 056292177646..b2e0233463bf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -1703,12 +1703,7 @@ public class IndexMetadata implements Diffable, ToXContentFragmen out.writeOptionalDouble(indexWriteLoadForecast); out.writeOptionalLong(shardSizeInBytesForecast); } - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { - eventIngestedRange.writeTo(out); - } else { - assert eventIngestedRange == IndexLongFieldRange.UNKNOWN - : "eventIngestedRange should be UNKNOWN until all nodes are on the new version but is " + eventIngestedRange; - } + eventIngestedRange.writeTo(out); } @Override @@ -1813,11 +1808,7 @@ public class IndexMetadata implements Diffable, ToXContentFragmen builder.indexWriteLoadForecast(in.readOptionalDouble()); builder.shardSizeInBytesForecast(in.readOptionalLong()); } - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { - builder.eventIngestedRange(IndexLongFieldRange.readFrom(in), in.getTransportVersion()); - } else { - builder.eventIngestedRange(IndexLongFieldRange.UNKNOWN, in.getTransportVersion()); - } + builder.eventIngestedRange(IndexLongFieldRange.readFrom(in)); return builder.build(true); } @@ -2180,28 +2171,12 @@ public class IndexMetadata implements Diffable, ToXContentFragmen return this; } - // only for use within this class file where minClusterTransportVersion is not known (e.g., IndexMetadataDiff.apply) - Builder eventIngestedRange(IndexLongFieldRange eventIngestedRange) { + public Builder eventIngestedRange(IndexLongFieldRange eventIngestedRange) { assert eventIngestedRange != null : "eventIngestedRange cannot be null"; this.eventIngestedRange = eventIngestedRange; return this; } - public Builder eventIngestedRange(IndexLongFieldRange eventIngestedRange, TransportVersion minClusterTransportVersion) { - assert eventIngestedRange != null : "eventIngestedRange cannot be null"; - assert minClusterTransportVersion != null || eventIngestedRange == IndexLongFieldRange.UNKNOWN - : "eventIngestedRange must be UNKNOWN when minClusterTransportVersion is null, but minClusterTransportVersion: " - + minClusterTransportVersion - + "; eventIngestedRange = " - + eventIngestedRange; - if (minClusterTransportVersion != null && minClusterTransportVersion.before(TransportVersions.V_8_15_0)) { - this.eventIngestedRange = IndexLongFieldRange.UNKNOWN; - } else { - this.eventIngestedRange = eventIngestedRange; - } - return this; - } - public Builder stats(IndexMetadataStats stats) { this.stats = stats; return this; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index a99822c9de00..7fc8a8693dcd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -71,7 +71,6 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.indices.IndexCreationException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexNameException; @@ -529,8 +528,7 @@ public class MetadataCreateIndexService { temporaryIndexMeta.getSettings(), temporaryIndexMeta.getRoutingNumShards(), sourceMetadata, - temporaryIndexMeta.isSystem(), - currentState.getMinTransportVersion() + temporaryIndexMeta.isSystem() ); } catch (Exception e) { logger.info("failed to build index metadata [{}]", request.index()); @@ -1340,15 +1338,10 @@ public class MetadataCreateIndexService { Settings indexSettings, int routingNumShards, @Nullable IndexMetadata sourceMetadata, - boolean isSystem, - TransportVersion minClusterTransportVersion + boolean isSystem ) { IndexMetadata.Builder indexMetadataBuilder = createIndexMetadataBuilder(indexName, sourceMetadata, indexSettings, routingNumShards); indexMetadataBuilder.system(isSystem); - if (minClusterTransportVersion.before(TransportVersions.V_8_15_0)) { - // promote to UNKNOWN for older versions since they don't know how to handle event.ingested in cluster state - indexMetadataBuilder.eventIngestedRange(IndexLongFieldRange.UNKNOWN, minClusterTransportVersion); - } // now, update the mappings with the actual source Map mappingsMetadata = new HashMap<>(); DocumentMapper docMapper = documentMapperSupplier.get(); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java index 95d1c37ec41a..f4318eb017eb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java @@ -885,7 +885,7 @@ public class MetadataIndexStateService { final IndexMetadata.Builder updatedMetadata = IndexMetadata.builder(indexMetadata).state(IndexMetadata.State.CLOSE); metadata.put( updatedMetadata.timestampRange(IndexLongFieldRange.NO_SHARDS) - .eventIngestedRange(IndexLongFieldRange.NO_SHARDS, currentState.getMinTransportVersion()) + .eventIngestedRange(IndexLongFieldRange.NO_SHARDS) .settingsVersion(indexMetadata.getSettingsVersion() + 1) .settings(Settings.builder().put(indexMetadata.getSettings()).put(VERIFIED_BEFORE_CLOSE_SETTING.getKey(), true)) ); @@ -1133,7 +1133,7 @@ public class MetadataIndexStateService { .settingsVersion(indexMetadata.getSettingsVersion() + 1) .settings(updatedSettings) .timestampRange(IndexLongFieldRange.NO_SHARDS) - .eventIngestedRange(IndexLongFieldRange.NO_SHARDS, currentState.getMinTransportVersion()) + .eventIngestedRange(IndexLongFieldRange.NO_SHARDS) .build(); // The index might be closed because we couldn't import it due to an old incompatible diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 7f8b87d2d3f4..569526ca1d9b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1784,7 +1784,7 @@ public class MetadataIndexTemplateService { .put( IndexMetadata.builder(temporaryIndexName) // necessary to pass asserts in ClusterState constructor - .eventIngestedRange(IndexLongFieldRange.UNKNOWN, state.getMinTransportVersion()) + .eventIngestedRange(IndexLongFieldRange.UNKNOWN) .settings(finalResolvedSettings) ) .build() diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index 38e72ee87403..f260c7aad30e 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -11,7 +11,6 @@ package org.elasticsearch.snapshots; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; import org.elasticsearch.action.support.IndicesOptions; @@ -1399,11 +1398,7 @@ public final class RestoreService implements ClusterStateApplier { currentState.metadata() ); - final IndexMetadata.Builder indexMdBuilder = restoreToCreateNewIndex( - snapshotIndexMetadata, - renamedIndexName, - currentState.getMinTransportVersion() - ); + final IndexMetadata.Builder indexMdBuilder = restoreToCreateNewIndex(snapshotIndexMetadata, renamedIndexName); if (request.includeAliases() == false && snapshotIndexMetadata.getAliases().isEmpty() == false && isSystemIndex(snapshotIndexMetadata) == false) { @@ -1421,11 +1416,7 @@ public final class RestoreService implements ClusterStateApplier { } else { // Index exists and it's closed - open it in metadata and start recovery validateExistingClosedIndex(currentIndexMetadata, snapshotIndexMetadata, renamedIndexName, partial); - final IndexMetadata.Builder indexMdBuilder = restoreOverClosedIndex( - snapshotIndexMetadata, - currentIndexMetadata, - currentState.getMinTransportVersion() - ); + final IndexMetadata.Builder indexMdBuilder = restoreOverClosedIndex(snapshotIndexMetadata, currentIndexMetadata); if (request.includeAliases() == false && isSystemIndex(snapshotIndexMetadata) == false) { // Remove all snapshot aliases @@ -1800,11 +1791,7 @@ public final class RestoreService implements ClusterStateApplier { return convertedIndexMetadataBuilder.build(); } - private static IndexMetadata.Builder restoreToCreateNewIndex( - IndexMetadata snapshotIndexMetadata, - String renamedIndexName, - TransportVersion minClusterTransportVersion - ) { + private static IndexMetadata.Builder restoreToCreateNewIndex(IndexMetadata snapshotIndexMetadata, String renamedIndexName) { return IndexMetadata.builder(snapshotIndexMetadata) .state(IndexMetadata.State.OPEN) .index(renamedIndexName) @@ -1812,14 +1799,10 @@ public final class RestoreService implements ClusterStateApplier { Settings.builder().put(snapshotIndexMetadata.getSettings()).put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) ) .timestampRange(IndexLongFieldRange.NO_SHARDS) - .eventIngestedRange(IndexLongFieldRange.NO_SHARDS, minClusterTransportVersion); + .eventIngestedRange(IndexLongFieldRange.NO_SHARDS); } - private static IndexMetadata.Builder restoreOverClosedIndex( - IndexMetadata snapshotIndexMetadata, - IndexMetadata currentIndexMetadata, - TransportVersion minTransportVersion - ) { + private static IndexMetadata.Builder restoreOverClosedIndex(IndexMetadata snapshotIndexMetadata, IndexMetadata currentIndexMetadata) { final IndexMetadata.Builder indexMdBuilder = IndexMetadata.builder(snapshotIndexMetadata) .state(IndexMetadata.State.OPEN) .version(Math.max(snapshotIndexMetadata.getVersion(), 1 + currentIndexMetadata.getVersion())) @@ -1828,7 +1811,7 @@ public final class RestoreService implements ClusterStateApplier { .settingsVersion(Math.max(snapshotIndexMetadata.getSettingsVersion(), 1 + currentIndexMetadata.getSettingsVersion())) .aliasesVersion(Math.max(snapshotIndexMetadata.getAliasesVersion(), 1 + currentIndexMetadata.getAliasesVersion())) .timestampRange(IndexLongFieldRange.NO_SHARDS) - .eventIngestedRange(IndexLongFieldRange.NO_SHARDS, minTransportVersion) + .eventIngestedRange(IndexLongFieldRange.NO_SHARDS) .index(currentIndexMetadata.getIndex().getName()) .settings( Settings.builder() diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index aada157e8ca8..a73e9635a53b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.action.admin.cluster.reroute; -import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; @@ -347,7 +346,7 @@ public class ClusterRerouteResponseTests extends ESTestCase { .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) .build() ) - .eventIngestedRange(IndexLongFieldRange.UNKNOWN, TransportVersion.current()) + .eventIngestedRange(IndexLongFieldRange.UNKNOWN) .build(), false ) diff --git a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java index 69872b5e4b54..1460270c4829 100644 --- a/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java @@ -10,7 +10,6 @@ package org.elasticsearch.action.search; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.search.CanMatchNodeResponse.ResponseOrFailure; @@ -1462,7 +1461,7 @@ public class CanMatchPreFilterSearchPhaseTests extends ESTestCase { indexMetadataBuilder.timestampRange(timestampRange); fields.put(index, new DateFieldRangeInfo(new DateFieldMapper.DateFieldType(fieldName), null, null, null)); } else if (fieldName.equals(IndexMetadata.EVENT_INGESTED_FIELD_NAME)) { - indexMetadataBuilder.eventIngestedRange(timestampRange, TransportVersion.current()); + indexMetadataBuilder.eventIngestedRange(timestampRange); fields.put(index, new DateFieldRangeInfo(null, null, new DateFieldMapper.DateFieldType(fieldName), null)); } @@ -1502,7 +1501,7 @@ public class CanMatchPreFilterSearchPhaseTests extends ESTestCase { .numberOfShards(1) .numberOfReplicas(0) .timestampRange(tsTimestampRange) - .eventIngestedRange(eventIngestedTimestampRange, TransportVersion.current()); + .eventIngestedRange(eventIngestedTimestampRange); Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata()).put(indexMetadataBuilder); clusterState = ClusterState.builder(clusterState).metadata(metadataBuilder).build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 668aea70c23f..5c1aed9457e8 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -1115,7 +1115,7 @@ public class ClusterStateTests extends ESTestCase { .putRolloverInfo(new RolloverInfo("rolloveAlias", new ArrayList<>(), 1L)) .stats(new IndexMetadataStats(IndexWriteLoad.builder(1).build(), 120, 1)) .indexWriteLoadForecast(8.0) - .eventIngestedRange(IndexLongFieldRange.UNKNOWN, TransportVersions.V_8_0_0) + .eventIngestedRange(IndexLongFieldRange.UNKNOWN) .build(); return ClusterState.builder(ClusterName.DEFAULT) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 633a678733f0..3623683532c5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -13,7 +13,6 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceAlreadyExistsException; -import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.Alias; @@ -1258,16 +1257,7 @@ public class MetadataCreateIndexServiceTests extends ESTestCase { Settings indexSettings = indexSettings(IndexVersion.current(), 1, 0).build(); List aliases = List.of(AliasMetadata.builder("alias1").build()); - IndexMetadata indexMetadata = buildIndexMetadata( - "test", - aliases, - () -> null, - indexSettings, - 4, - sourceIndexMetadata, - false, - TransportVersion.current() - ); + IndexMetadata indexMetadata = buildIndexMetadata("test", aliases, () -> null, indexSettings, 4, sourceIndexMetadata, false); assertThat(indexMetadata.getAliases().size(), is(1)); assertThat(indexMetadata.getAliases().keySet().iterator().next(), is("alias1")); @@ -1276,35 +1266,6 @@ public class MetadataCreateIndexServiceTests extends ESTestCase { assertThat(indexMetadata.getEventIngestedRange(), equalTo(IndexLongFieldRange.NO_SHARDS)); } - public void testBuildIndexMetadataWithTransportVersionBeforeEventIngestedRangeAdded() { - IndexMetadata sourceIndexMetadata = IndexMetadata.builder("parent") - .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build()) - .numberOfShards(1) - .numberOfReplicas(0) - .primaryTerm(0, 3L) - .build(); - - Settings indexSettings = indexSettings(IndexVersion.current(), 1, 0).build(); - List aliases = List.of(AliasMetadata.builder("alias1").build()); - IndexMetadata indexMetadata = buildIndexMetadata( - "test", - aliases, - () -> null, - indexSettings, - 4, - sourceIndexMetadata, - false, - TransportVersions.V_8_0_0 - ); - - assertThat(indexMetadata.getAliases().size(), is(1)); - assertThat(indexMetadata.getAliases().keySet().iterator().next(), is("alias1")); - assertThat("The source index primary term must be used", indexMetadata.primaryTerm(0), is(3L)); - assertThat(indexMetadata.getTimestampRange(), equalTo(IndexLongFieldRange.NO_SHARDS)); - // on versions before event.ingested was added to cluster state, it should default to UNKNOWN, not NO_SHARDS - assertThat(indexMetadata.getEventIngestedRange(), equalTo(IndexLongFieldRange.UNKNOWN)); - } - public void testGetIndexNumberOfRoutingShardsWithNullSourceIndex() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) diff --git a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java index 293d74f27ae3..0c7af66a7190 100644 --- a/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java @@ -51,7 +51,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -70,7 +69,7 @@ public class ClusterSerializationTests extends ESAllocationTestCase { .settings(settings(IndexVersion.current())) .numberOfShards(10) .numberOfReplicas(1) - .eventIngestedRange(eventIngestedRangeInput, TransportVersions.V_8_15_0); + .eventIngestedRange(eventIngestedRangeInput); ClusterStateTestRecord result = createAndSerializeClusterState(indexMetadataBuilder, TransportVersion.current()); @@ -86,63 +85,6 @@ public class ClusterSerializationTests extends ESAllocationTestCase { } } - public void testClusterStateSerializationWithTimestampRangesWithOlderTransportVersion() throws Exception { - TransportVersion versionBeforeEventIngestedInClusterState = randomFrom( - TransportVersions.V_8_0_0, - TransportVersionUtils.getPreviousVersion(TransportVersions.V_8_15_0) - ); - { - IndexLongFieldRange eventIngestedRangeInput = randomFrom( - IndexLongFieldRange.UNKNOWN, - IndexLongFieldRange.NO_SHARDS, - IndexLongFieldRange.EMPTY, - IndexLongFieldRange.NO_SHARDS.extendWithShardRange(0, 1, ShardLongFieldRange.of(100000, 200000)) - ); - - IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder("test") - .settings(settings(IndexVersion.current())) - .numberOfShards(10) - .numberOfReplicas(1) - .eventIngestedRange(eventIngestedRangeInput, versionBeforeEventIngestedInClusterState); - - ClusterStateTestRecord result = createAndSerializeClusterState(indexMetadataBuilder, versionBeforeEventIngestedInClusterState); - - assertThat(result.serializedClusterState().getClusterName().value(), equalTo(result.clusterState().getClusterName().value())); - assertThat(result.serializedClusterState().routingTable().toString(), equalTo(result.clusterState().routingTable().toString())); - - IndexLongFieldRange eventIngestedRangeOutput = result.serializedClusterState() - .getMetadata() - .index("test") - .getEventIngestedRange(); - // should always come back as UNKNOWN when an older transport version is passed in - assertSame(IndexLongFieldRange.UNKNOWN, eventIngestedRangeOutput); - } - { - // UNKNOWN is the only allowed state for event.ingested range in older versions, so this serialization test should fail - IndexLongFieldRange eventIngestedRangeInput = randomFrom( - IndexLongFieldRange.NO_SHARDS, - IndexLongFieldRange.EMPTY, - IndexLongFieldRange.NO_SHARDS.extendWithShardRange(0, 1, ShardLongFieldRange.of(100000, 200000)) - ); - - IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder("test") - .settings(settings(IndexVersion.current())) - .numberOfShards(10) - .numberOfReplicas(1) - .eventIngestedRange(eventIngestedRangeInput, TransportVersion.current()); - - AssertionError assertionError = expectThrows( - AssertionError.class, - () -> createAndSerializeClusterState(indexMetadataBuilder, versionBeforeEventIngestedInClusterState) - ); - - assertThat( - assertionError.getMessage(), - containsString("eventIngestedRange should be UNKNOWN until all nodes are on the new version") - ); - } - } - /** * @param clusterState original ClusterState created by helper method * @param serializedClusterState serialized version of the clusterState diff --git a/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java index a11291e846ec..950c54ddb1d2 100644 --- a/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java @@ -10,7 +10,6 @@ package org.elasticsearch.action.support.replication; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -151,7 +150,7 @@ public class ClusterStateCreationUtils { .settings(indexSettings(IndexVersion.current(), 1, numberOfReplicas).put(SETTING_CREATION_DATE, System.currentTimeMillis())) .primaryTerm(0, primaryTerm) .timestampRange(timeFieldRange) - .eventIngestedRange(timeFieldRange, timeFieldRange == IndexLongFieldRange.UNKNOWN ? null : TransportVersions.V_8_15_0) + .eventIngestedRange(timeFieldRange) .build(); IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); @@ -284,7 +283,7 @@ public class ClusterStateCreationUtils { .settings( indexSettings(IndexVersion.current(), numberOfPrimaries, 0).put(SETTING_CREATION_DATE, System.currentTimeMillis()) ) - .eventIngestedRange(IndexLongFieldRange.UNKNOWN, randomFrom(TransportVersions.V_8_0_0, TransportVersions.V_8_15_0)) + .eventIngestedRange(IndexLongFieldRange.UNKNOWN) .build(); IndexRoutingTable.Builder indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex()); @@ -390,7 +389,7 @@ public class ClusterStateCreationUtils { ) ) .timestampRange(IndexLongFieldRange.UNKNOWN) - .eventIngestedRange(IndexLongFieldRange.UNKNOWN, null) + .eventIngestedRange(IndexLongFieldRange.UNKNOWN) .build(); metadataBuilder.put(indexMetadata, false).generateClusterUuidIfNeeded(); IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(indexMetadata.getIndex()); From d18e3293ebbb00ebdb111c6d1228e2d904f967b8 Mon Sep 17 00:00:00 2001 From: Pooya Salehi Date: Thu, 9 Jan 2025 12:32:40 +0100 Subject: [PATCH 10/33] Issue S3 web identity token refresh call with sufficient permissions (#119748) Closes #119747 --- docs/changelog/119748.yaml | 6 ++++++ .../java/org/elasticsearch/repositories/s3/S3Service.java | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/119748.yaml diff --git a/docs/changelog/119748.yaml b/docs/changelog/119748.yaml new file mode 100644 index 000000000000..8b29fb7c1a64 --- /dev/null +++ b/docs/changelog/119748.yaml @@ -0,0 +1,6 @@ +pr: 119748 +summary: Issue S3 web identity token refresh call with sufficient permissions +area: Snapshot/Restore +type: bug +issues: + - 119747 diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index 1a66f5782fc0..a8a6986ccbb7 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -434,7 +434,7 @@ class S3Service implements Closeable { public void onFileChanged(Path file) { if (file.equals(webIdentityTokenFileSymlink)) { LOGGER.debug("WS web identity token file [{}] changed, updating credentials", file); - credentialsProvider.refresh(); + SocketAccess.doPrivilegedVoid(credentialsProvider::refresh); } } }); From 39c2dd8df14869a5367146e2c1975d3f04ac7205 Mon Sep 17 00:00:00 2001 From: Moritz Mack Date: Thu, 9 Jan 2025 13:01:08 +0100 Subject: [PATCH 11/33] Add deprecation check for tracing.apm.* settings for upgrade assistant (#119773) All usages of tracing.apm.* settings will be presented as a single deprecation issue. Relates to #ES-10293 --- .../xpack/deprecation/DeprecationChecks.java | 3 +- .../deprecation/NodeDeprecationChecks.java | 82 +++++++++++++++++-- .../NodeDeprecationChecksTests.java | 23 ++++++ 3 files changed, 100 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index f9b2cc5afe3a..0b9a538d505c 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -87,7 +87,8 @@ public class DeprecationChecks { NodeDeprecationChecks::checkLifecyleStepMasterTimeoutSetting, NodeDeprecationChecks::checkEqlEnabledSetting, NodeDeprecationChecks::checkNodeAttrData, - NodeDeprecationChecks::checkWatcherBulkConcurrentRequestsSetting + NodeDeprecationChecks::checkWatcherBulkConcurrentRequestsSetting, + NodeDeprecationChecks::checkTracingApmSettings ); static List> INDEX_SETTINGS_CHECKS = List.of( diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java index 349762456cd3..b6fff5a82f0c 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecks.java @@ -34,6 +34,9 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.elasticsearch.common.settings.Setting.Property.Deprecated; +import static org.elasticsearch.common.settings.Setting.Property.NodeScope; +import static org.elasticsearch.common.settings.Setting.Property.OperatorDynamic; import static org.elasticsearch.xpack.core.security.authc.RealmSettings.RESERVED_REALM_AND_DOMAIN_NAME_PREFIX; public class NodeDeprecationChecks { @@ -95,15 +98,13 @@ public class NodeDeprecationChecks { return null; } final String removedSettingKey = removedSetting.getKey(); - Object removedSettingValue = removedSetting.exists(clusterSettings) - ? removedSetting.get(clusterSettings) - : removedSetting.get(nodeSettings); - String value; - if (removedSettingValue instanceof TimeValue) { - value = ((TimeValue) removedSettingValue).getStringRep(); + // read setting to force the deprecation warning + if (removedSetting.exists(clusterSettings)) { + removedSetting.get(clusterSettings); } else { - value = removedSettingValue.toString(); + removedSetting.get(nodeSettings); } + final String message = String.format(Locale.ROOT, "Setting [%s] is deprecated", removedSettingKey); final String details = additionalDetailMessage == null ? String.format(Locale.ROOT, "Remove the [%s] setting.", removedSettingKey) @@ -113,6 +114,45 @@ public class NodeDeprecationChecks { return new DeprecationIssue(deprecationLevel, message, url, details, false, meta); } + static DeprecationIssue checkMultipleRemovedSettings( + final Settings clusterSettings, + final Settings nodeSettings, + final List> removedSettings, + final String url, + String additionalDetailMessage, + DeprecationIssue.Level deprecationLevel + ) { + + var removedSettingsRemaining = removedSettings.stream().filter(s -> s.exists(clusterSettings) || s.exists(nodeSettings)).toList(); + if (removedSettingsRemaining.isEmpty()) { + return null; + } + if (removedSettingsRemaining.size() == 1) { + Setting removedSetting = removedSettingsRemaining.get(0); + return checkRemovedSetting(clusterSettings, nodeSettings, removedSetting, url, additionalDetailMessage, deprecationLevel); + } + + // read settings to force the deprecation warning + removedSettingsRemaining.forEach(s -> { + if (s.exists(clusterSettings)) { + s.get(clusterSettings); + } else { + s.get(nodeSettings); + } + }); + + var removedSettingKeysRemaining = removedSettingsRemaining.stream().map(Setting::getKey).sorted().toList(); + final String message = String.format(Locale.ROOT, "Settings %s are deprecated", removedSettingKeysRemaining); + final String details = additionalDetailMessage == null + ? String.format(Locale.ROOT, "Remove each setting in %s.", removedSettingKeysRemaining) + : String.format(Locale.ROOT, "Remove each setting in %s. %s", removedSettingKeysRemaining, additionalDetailMessage); + + var canAutoRemoveSettings = removedSettingsRemaining.stream() + .allMatch(s -> s.exists(clusterSettings) && s.exists(nodeSettings) == false); + var meta = createMetaMapForRemovableSettings(canAutoRemoveSettings, removedSettingKeysRemaining); + return new DeprecationIssue(deprecationLevel, message, url, details, false, meta); + } + static DeprecationIssue checkMultipleDataPaths( Settings nodeSettings, PluginsAndModules plugins, @@ -944,4 +984,32 @@ public class NodeDeprecationChecks { DeprecationIssue.Level.WARNING ); } + + static DeprecationIssue checkTracingApmSettings( + final Settings settings, + final PluginsAndModules pluginsAndModules, + final ClusterState clusterState, + final XPackLicenseState licenseState + ) { + String url = "https://ela.st/es-deprecation-9-tracing-apm-settings"; + Setting.Property[] properties = { NodeScope, OperatorDynamic, Deprecated }; + List> tracingApmSettings = List.of( + Setting.prefixKeySetting("tracing.apm.agent.", key -> Setting.simpleString(key, properties)), + Setting.stringListSetting("tracing.apm.names.include", properties), + Setting.stringListSetting("tracing.apm.names.exclude", properties), + Setting.stringListSetting("tracing.apm.sanitize_field_names", properties), + Setting.boolSetting("tracing.apm.enabled", false, properties), + SecureSetting.secureString("tracing.apm.api_key", null, Deprecated), + SecureSetting.secureString("tracing.apm.secret_token", null, Deprecated) + ); + return checkMultipleRemovedSettings( + clusterState.metadata().settings(), + settings, + tracingApmSettings, + url, + "[tracing.apm.*] settings are no longer accepted as of 9.0.0" + + " and should be replaced by [telemetry.*] or [telemetry.tracing.*] settings.", + DeprecationIssue.Level.CRITICAL + ); + } } diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java index 9d89a3cbe328..3aaee0e5cdb5 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodeDeprecationChecksTests.java @@ -75,6 +75,29 @@ public class NodeDeprecationChecksTests extends ESTestCase { assertThat(issue.getUrl(), equalTo("https://removed-setting.example.com")); } + public void testMultipleRemovedSettings() { + final Settings clusterSettings = Settings.EMPTY; + final Settings nodeSettings = Settings.builder() + .put("node.removed_setting1", "value") + .put("node.removed_setting2", "value") + .build(); + final Setting removedSetting1 = Setting.simpleString("node.removed_setting1"); + final Setting removedSetting2 = Setting.simpleString("node.removed_setting2"); + final DeprecationIssue issue = NodeDeprecationChecks.checkMultipleRemovedSettings( + clusterSettings, + nodeSettings, + shuffledList(List.of(removedSetting1, removedSetting2)), + "https://removed-setting.example.com", + "Some detail.", + DeprecationIssue.Level.CRITICAL + ); + assertThat(issue, not(nullValue())); + assertThat(issue.getLevel(), equalTo(DeprecationIssue.Level.CRITICAL)); + assertThat(issue.getMessage(), equalTo("Settings [node.removed_setting1, node.removed_setting2] are deprecated")); + assertThat(issue.getDetails(), equalTo("Remove each setting in [node.removed_setting1, node.removed_setting2]. Some detail.")); + assertThat(issue.getUrl(), equalTo("https://removed-setting.example.com")); + } + public void testMultipleDataPaths() { final Settings settings = Settings.builder().putList("path.data", Arrays.asList("d1", "d2")).build(); final XPackLicenseState licenseState = new XPackLicenseState(() -> 0); From 750a0ab84682478f500348e1bd14a27a1c8866bc Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Thu, 9 Jan 2025 13:20:28 +0100 Subject: [PATCH 12/33] [Connector API] Support soft-deletes of connectors (#118669) * [Connector API] Add interface for soft-deletes * Define connector deleted system index * Got soft-delete logic working * Add unit tests * Add yaml e2e test and attempt to update permissions * Fix permissions * Update docs * Fix docs * Update docs/changelog/118282.yaml * Change logic * Fix tests * Remove unnecessary privilege from yaml rest test * Update changelog * Update docs/changelog/118669.yaml * Adapt yaml tests * Undo changes to muted-tests.yml * Fix compilation issue after other PR got merged * Exclude soft-deleted connector from checks about index_name already in use * Update docs/reference/connector/apis/get-connector-api.asciidoc Co-authored-by: Tim Grein * Update rest-api-spec/src/main/resources/rest-api-spec/api/connector.list.json Co-authored-by: Tim Grein * Adapt comments, add connector wire serializing test * Introduce new transport versions for passing the delete flag * Get rid of wire serialisation, use include_deleted instead of deleted flag * Remove unused import * Final tweaks * Adapt variable name in rest layer --------- Co-authored-by: Elastic Machine Co-authored-by: Tim Grein --- docs/changelog/118669.yaml | 5 + .../apis/delete-connector-api.asciidoc | 4 +- .../connector/apis/get-connector-api.asciidoc | 3 + .../apis/list-connectors-api.asciidoc | 3 + .../rest-api-spec/api/connector.get.json | 7 + .../rest-api-spec/api/connector.list.json | 5 + .../elastic-connectors-mappings.json | 3 + .../entsearch/connector/10_connector_put.yml | 24 +++ .../entsearch/connector/20_connector_list.yml | 144 +++++++++++++++++- .../connector/30_connector_delete.yml | 27 ++++ .../application/connector/Connector.java | 91 ++++------- .../connector/ConnectorIndexService.java | 98 +++++++----- .../connector/action/GetConnectorAction.java | 28 ++-- .../connector/action/ListConnectorAction.java | 35 +++-- .../action/RestGetConnectorAction.java | 3 +- .../action/RestListConnectorAction.java | 4 +- .../action/TransportGetConnectorAction.java | 18 +-- .../action/TransportListConnectorAction.java | 14 +- .../connector/syncjob/ConnectorSyncJob.java | 48 +----- .../connector/ConnectorIndexServiceTests.java | 86 ++++++++++- .../application/connector/ConnectorTests.java | 33 ---- ...ectorActionRequestBWCSerializingTests.java | 43 ------ ...ectorActionRequestBWCSerializingTests.java | 58 ------- ...ctorSecretResponseBWCSerializingTests.java | 8 - ...ctorSecretResponseBWCSerializingTests.java | 8 - .../syncjob/ConnectorSyncJobTests.java | 29 ---- ...cJobActionResponseBWCSerializingTests.java | 8 - ...JobsActionResponseBWCSerializingTests.java | 8 - 28 files changed, 444 insertions(+), 401 deletions(-) create mode 100644 docs/changelog/118669.yaml delete mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java delete mode 100644 x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java diff --git a/docs/changelog/118669.yaml b/docs/changelog/118669.yaml new file mode 100644 index 000000000000..4e0d10aaac81 --- /dev/null +++ b/docs/changelog/118669.yaml @@ -0,0 +1,5 @@ +pr: 118669 +summary: "[Connector API] Support soft-deletes of connectors" +area: Extract&Transform +type: feature +issues: [] diff --git a/docs/reference/connector/apis/delete-connector-api.asciidoc b/docs/reference/connector/apis/delete-connector-api.asciidoc index b36a99bc2d8c..f161a3c3b593 100644 --- a/docs/reference/connector/apis/delete-connector-api.asciidoc +++ b/docs/reference/connector/apis/delete-connector-api.asciidoc @@ -6,14 +6,14 @@ beta::[] + .New API reference [sidebar] -- For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs]. -- -Removes a connector and associated sync jobs. -This is a destructive action that is not recoverable. +Soft-deletes a connector and removes associated sync jobs. Note: this action doesn't delete any API key, ingest pipeline or data index associated with the connector. These need to be removed manually. diff --git a/docs/reference/connector/apis/get-connector-api.asciidoc b/docs/reference/connector/apis/get-connector-api.asciidoc index cff13539f80c..c8cbae668c26 100644 --- a/docs/reference/connector/apis/get-connector-api.asciidoc +++ b/docs/reference/connector/apis/get-connector-api.asciidoc @@ -33,6 +33,9 @@ To get started with Connector APIs, check out <`:: (Required, string) +`include_deleted`:: +(Optional, boolean) A flag indicating whether to also return connectors that have been soft-deleted. Defaults to `false`. + [[get-connector-api-response-codes]] ==== {api-response-codes-title} diff --git a/docs/reference/connector/apis/list-connectors-api.asciidoc b/docs/reference/connector/apis/list-connectors-api.asciidoc index 5cc099a6b67e..d334e5d92c23 100644 --- a/docs/reference/connector/apis/list-connectors-api.asciidoc +++ b/docs/reference/connector/apis/list-connectors-api.asciidoc @@ -47,6 +47,9 @@ To get started with Connector APIs, check out <A {@link ConnectorStatus} indicating the current status of the connector. *
  • A sync cursor, used for incremental syncs.
    • *
  • A boolean flag 'syncNow', which, when set, triggers an immediate synchronization operation.
    • + *
  • A boolean flag 'isDeleted', when set indicates that connector has been soft-deleted.
    • * */ -public class Connector implements NamedWriteable, ToXContentObject { +public class Connector implements ToXContentObject { public static final String NAME = Connector.class.getName().toUpperCase(Locale.ROOT); @@ -106,6 +104,7 @@ public class Connector implements NamedWriteable, ToXContentObject { @Nullable private final Object syncCursor; private final boolean syncNow; + private final boolean isDeleted; /** * Constructor for Connector. @@ -132,6 +131,7 @@ public class Connector implements NamedWriteable, ToXContentObject { * @param status Current status of the connector. * @param syncCursor Position or state indicating the current point of synchronization. * @param syncNow Flag indicating whether an immediate synchronization is requested. + * @param isDeleted Flag indicating whether connector has been soft-deleted. */ private Connector( String connectorId, @@ -155,7 +155,8 @@ public class Connector implements NamedWriteable, ToXContentObject { String serviceType, ConnectorStatus status, Object syncCursor, - boolean syncNow + boolean syncNow, + Boolean isDeleted ) { this.connectorId = connectorId; this.apiKeyId = apiKeyId; @@ -179,31 +180,7 @@ public class Connector implements NamedWriteable, ToXContentObject { this.status = status; this.syncCursor = syncCursor; this.syncNow = syncNow; - } - - public Connector(StreamInput in) throws IOException { - this.connectorId = in.readOptionalString(); - this.apiKeyId = in.readOptionalString(); - this.apiKeySecretId = in.readOptionalString(); - this.configuration = in.readMap(ConnectorConfiguration::new); - this.customScheduling = in.readMap(ConnectorCustomSchedule::new); - this.description = in.readOptionalString(); - this.error = in.readOptionalString(); - this.features = in.readOptionalWriteable(ConnectorFeatures::new); - this.filtering = in.readOptionalCollectionAsList(ConnectorFiltering::new); - this.syncJobFiltering = in.readOptionalWriteable(FilteringRules::new); - this.indexName = in.readOptionalString(); - this.isNative = in.readBoolean(); - this.language = in.readOptionalString(); - this.lastSeen = in.readOptionalInstant(); - this.syncInfo = in.readOptionalWriteable(ConnectorSyncInfo::new); - this.name = in.readOptionalString(); - this.pipeline = in.readOptionalWriteable(ConnectorIngestPipeline::new); - this.scheduling = in.readOptionalWriteable(ConnectorScheduling::new); - this.serviceType = in.readOptionalString(); - this.status = in.readEnum(ConnectorStatus.class); - this.syncCursor = in.readGenericValue(); - this.syncNow = in.readBoolean(); + this.isDeleted = isDeleted; } public static final ParseField ID_FIELD = new ParseField("id"); @@ -226,6 +203,7 @@ public class Connector implements NamedWriteable, ToXContentObject { public static final ParseField STATUS_FIELD = new ParseField("status"); public static final ParseField SYNC_CURSOR_FIELD = new ParseField("sync_cursor"); static final ParseField SYNC_NOW_FIELD = new ParseField("sync_now"); + public static final ParseField IS_DELETED_FIELD = new ParseField("deleted"); @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -265,7 +243,8 @@ public class Connector implements NamedWriteable, ToXContentObject { .setServiceType((String) args[i++]) .setStatus((ConnectorStatus) args[i++]) .setSyncCursor(args[i++]) - .setSyncNow((Boolean) args[i]) + .setSyncNow((Boolean) args[i++]) + .setIsDeleted((Boolean) args[i]) .build(); } ); @@ -357,6 +336,7 @@ public class Connector implements NamedWriteable, ToXContentObject { ); PARSER.declareObjectOrNull(optionalConstructorArg(), (p, c) -> p.map(), null, SYNC_CURSOR_FIELD); PARSER.declareBoolean(optionalConstructorArg(), SYNC_NOW_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), IS_DELETED_FIELD); } public static Connector fromXContentBytes(BytesReference source, String docId, XContentType xContentType) { @@ -433,6 +413,7 @@ public class Connector implements NamedWriteable, ToXContentObject { builder.field(STATUS_FIELD.getPreferredName(), status.toString()); } builder.field(SYNC_NOW_FIELD.getPreferredName(), syncNow); + builder.field(IS_DELETED_FIELD.getPreferredName(), isDeleted); } @Override @@ -445,32 +426,6 @@ public class Connector implements NamedWriteable, ToXContentObject { return builder; } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeOptionalString(connectorId); - out.writeOptionalString(apiKeyId); - out.writeOptionalString(apiKeySecretId); - out.writeMap(configuration, StreamOutput::writeWriteable); - out.writeMap(customScheduling, StreamOutput::writeWriteable); - out.writeOptionalString(description); - out.writeOptionalString(error); - out.writeOptionalWriteable(features); - out.writeOptionalCollection(filtering); - out.writeOptionalWriteable(syncJobFiltering); - out.writeOptionalString(indexName); - out.writeBoolean(isNative); - out.writeOptionalString(language); - out.writeOptionalInstant(lastSeen); - out.writeOptionalWriteable(syncInfo); - out.writeOptionalString(name); - out.writeOptionalWriteable(pipeline); - out.writeOptionalWriteable(scheduling); - out.writeOptionalString(serviceType); - out.writeEnum(status); - out.writeGenericValue(syncCursor); - out.writeBoolean(syncNow); - } - public String getConnectorId() { return connectorId; } @@ -559,6 +514,10 @@ public class Connector implements NamedWriteable, ToXContentObject { return syncNow; } + public boolean isDeleted() { + return isDeleted; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -566,6 +525,7 @@ public class Connector implements NamedWriteable, ToXContentObject { Connector connector = (Connector) o; return isNative == connector.isNative && syncNow == connector.syncNow + && isDeleted == connector.isDeleted && Objects.equals(connectorId, connector.connectorId) && Objects.equals(apiKeyId, connector.apiKeyId) && Objects.equals(apiKeySecretId, connector.apiKeySecretId) @@ -612,15 +572,11 @@ public class Connector implements NamedWriteable, ToXContentObject { serviceType, status, syncCursor, - syncNow + syncNow, + isDeleted ); } - @Override - public String getWriteableName() { - return NAME; - } - public static class Builder { private String connectorId; @@ -645,6 +601,7 @@ public class Connector implements NamedWriteable, ToXContentObject { private ConnectorStatus status = ConnectorStatus.CREATED; private Object syncCursor; private boolean syncNow; + private boolean isDeleted; public Builder setConnectorId(String connectorId) { this.connectorId = connectorId; @@ -756,6 +713,11 @@ public class Connector implements NamedWriteable, ToXContentObject { return this; } + public Builder setIsDeleted(Boolean isDeleted) { + this.isDeleted = Objects.requireNonNullElse(isDeleted, false); + return this; + } + public Connector build() { return new Connector( connectorId, @@ -779,7 +741,8 @@ public class Connector implements NamedWriteable, ToXContentObject { serviceType, status, syncCursor, - syncNow + syncNow, + isDeleted ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java index d5d2159d8f37..4c0b4cf814a0 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java @@ -13,9 +13,6 @@ import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DelegatingActionListener; import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; @@ -194,10 +191,13 @@ public class ConnectorIndexService { /** * Gets the {@link Connector} from the underlying index. * - * @param connectorId The id of the connector object. - * @param listener The action listener to invoke on response/failure. + * @param connectorId The id of the connector object. + * @param includeDeleted If false, returns only the non-deleted connector with the matching ID; + * if true, returns the connector with the matching ID. + * @param listener The action listener to invoke on response/failure. */ - public void getConnector(String connectorId, ActionListener listener) { + public void getConnector(String connectorId, boolean includeDeleted, ActionListener listener) { + try { final GetRequest getRequest = new GetRequest(CONNECTOR_INDEX_NAME).id(connectorId).realtime(true); @@ -212,6 +212,12 @@ public class ConnectorIndexService { .setResultMap(getResponse.getSourceAsMap()) .build(); + boolean connectorIsSoftDeleted = includeDeleted == false && isConnectorDeleted(connector); + + if (connectorIsSoftDeleted) { + l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } l.onResponse(connector); } catch (Exception e) { listener.onFailure(e); @@ -223,33 +229,40 @@ public class ConnectorIndexService { } /** - * Deletes the {@link Connector} and the related instances of {@link ConnectorSyncJob} in the underlying index. + * Soft deletes the {@link Connector} and optionally removes the related instances of {@link ConnectorSyncJob} in the underlying index. * * @param connectorId The id of the {@link Connector}. * @param shouldDeleteSyncJobs The flag indicating if {@link ConnectorSyncJob} should also be deleted. * @param listener The action listener to invoke on response/failure. */ - public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, ActionListener listener) { - - final DeleteRequest deleteRequest = new DeleteRequest(CONNECTOR_INDEX_NAME).id(connectorId) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, ActionListener listener) { try { - client.delete(deleteRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, deleteResponse) -> { - if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { - l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); - return; - } - if (shouldDeleteSyncJobs) { - new ConnectorSyncJobIndexService(client).deleteAllSyncJobsByConnectorId(connectorId, l.map(r -> deleteResponse)); - } else { - l.onResponse(deleteResponse); - } + // ensure that if connector is soft-deleted, deleting it again results in 404 + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { + final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).setRefreshPolicy( + WriteRequest.RefreshPolicy.IMMEDIATE + ) + .doc( + new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX) + .id(connectorId) + .source(Map.of(Connector.IS_DELETED_FIELD.getPreferredName(), true)) + ); + client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> { + if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) { + ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId))); + return; + } + if (shouldDeleteSyncJobs) { + new ConnectorSyncJobIndexService(client).deleteAllSyncJobsByConnectorId(connectorId, ll.map(r -> updateResponse)); + } else { + ll.onResponse(updateResponse); + } + })); })); } catch (Exception e) { listener.onFailure(e); } - } /** @@ -261,6 +274,7 @@ public class ConnectorIndexService { * @param connectorNames Filter connectors by connector names, if provided. * @param serviceTypes Filter connectors by service types, if provided. * @param searchQuery Apply a wildcard search on index name, connector name, and description, if provided. + * @param includeDeleted If false, filters to include only non-deleted connectors; if true, no filter is applied. * @param listener Invoked with search results or upon failure. */ public void listConnectors( @@ -270,12 +284,13 @@ public class ConnectorIndexService { List connectorNames, List serviceTypes, String searchQuery, + boolean includeDeleted, ActionListener listener ) { try { final SearchSourceBuilder source = new SearchSourceBuilder().from(from) .size(size) - .query(buildListQuery(indexNames, connectorNames, serviceTypes, searchQuery)) + .query(buildListQuery(indexNames, connectorNames, serviceTypes, searchQuery, includeDeleted)) .fetchSource(true) .sort(Connector.INDEX_NAME_FIELD.getPreferredName(), SortOrder.ASC); final SearchRequest req = new SearchRequest(CONNECTOR_INDEX_NAME).source(source); @@ -311,13 +326,15 @@ public class ConnectorIndexService { * @param connectorNames List of connector names for filtering, or null/empty to skip. * @param serviceTypes List of connector service types for filtering, or null/empty to skip. * @param searchQuery Search query for wildcard filtering on index name, connector name, and description, or null/empty to skip. + * @param includeDeleted If false, filters to include only non-deleted connectors; if true, no filter is applied. * @return A {@link QueryBuilder} customized based on provided filters. */ private QueryBuilder buildListQuery( List indexNames, List connectorNames, List serviceTypes, - String searchQuery + String searchQuery, + boolean includeDeleted ) { boolean filterByIndexNames = indexNames != null && indexNames.isEmpty() == false; boolean filterByConnectorNames = indexNames != null && connectorNames.isEmpty() == false; @@ -349,7 +366,12 @@ public class ConnectorIndexService { ); } } - return usesFilter ? boolFilterQueryBuilder : new MatchAllQueryBuilder(); + + if (includeDeleted == false) { + boolFilterQueryBuilder.mustNot(new TermQueryBuilder(Connector.IS_DELETED_FIELD.getPreferredName(), true)); + } + + return boolFilterQueryBuilder; } /** @@ -368,7 +390,7 @@ public class ConnectorIndexService { Map configurationValues = request.getConfigurationValues(); String connectorId = request.getConnectorId(); - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).setRefreshPolicy( WriteRequest.RefreshPolicy.IMMEDIATE @@ -600,7 +622,7 @@ public class ConnectorIndexService { ActionListener listener ) { try { - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { List connectorFilteringList = fromXContentBytesConnectorFiltering( connector.getSourceRef(), XContentType.JSON @@ -660,7 +682,7 @@ public class ConnectorIndexService { FilteringValidationInfo validation, ActionListener listener ) { - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { try { List connectorFilteringList = fromXContentBytesConnectorFiltering( connector.getSourceRef(), @@ -704,7 +726,7 @@ public class ConnectorIndexService { * @param listener Listener to respond to a successful response or an error. */ public void activateConnectorDraftFiltering(String connectorId, ActionListener listener) { - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { try { List connectorFilteringList = fromXContentBytesConnectorFiltering( connector.getSourceRef(), @@ -819,7 +841,7 @@ public class ConnectorIndexService { String connectorId = request.getConnectorId(); boolean isNative = request.isNative(); - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { String indexName = getConnectorIndexNameFromSearchResult(connector); @@ -930,7 +952,7 @@ public class ConnectorIndexService { return; } - getConnector(connectorId, l.delegateFailure((ll, connector) -> { + getConnector(connectorId, false, l.delegateFailure((ll, connector) -> { Boolean isNativeConnector = getConnectorIsNativeFlagFromSearchResult(connector); Boolean doesNotHaveContentPrefix = indexName != null && isValidManagedConnectorIndexName(indexName) == false; @@ -1013,7 +1035,7 @@ public class ConnectorIndexService { public void updateConnectorServiceType(UpdateConnectorServiceTypeAction.Request request, ActionListener listener) { try { String connectorId = request.getConnectorId(); - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { ConnectorStatus prevStatus = getConnectorStatusFromSearchResult(connector); ConnectorStatus newStatus = prevStatus == ConnectorStatus.CREATED @@ -1060,7 +1082,7 @@ public class ConnectorIndexService { try { String connectorId = request.getConnectorId(); ConnectorStatus newStatus = request.getStatus(); - getConnector(connectorId, listener.delegateFailure((l, connector) -> { + getConnector(connectorId, false, listener.delegateFailure((l, connector) -> { ConnectorStatus prevStatus = getConnectorStatusFromSearchResult(connector); @@ -1138,6 +1160,11 @@ public class ConnectorIndexService { return (Map) searchResult.getResultMap().get(Connector.CONFIGURATION_FIELD.getPreferredName()); } + private boolean isConnectorDeleted(ConnectorSearchResult searchResult) { + Boolean isDeletedFlag = (Boolean) searchResult.getResultMap().get(Connector.IS_DELETED_FIELD.getPreferredName()); + return Boolean.TRUE.equals(isDeletedFlag); + } + private static ConnectorIndexService.ConnectorResult mapSearchResponseToConnectorList(SearchResponse response) { final List connectorResults = Arrays.stream(response.getHits().getHits()) .map(ConnectorIndexService::hitToConnector) @@ -1157,7 +1184,7 @@ public class ConnectorIndexService { /** * This method determines if any documents in the connector index have the same index name as the one specified, - * excluding the document with the given _id if it is provided. + * excluding the docs marked as deleted (soft-deleted) and document with the given _id if it is provided. * * @param indexName The name of the index to check for existence in the connector index. * @param connectorId The ID of the {@link Connector} to exclude from the search. Can be null if no document should be excluded. @@ -1173,6 +1200,9 @@ public class ConnectorIndexService { boolFilterQueryBuilder.must().add(new TermQueryBuilder(Connector.INDEX_NAME_FIELD.getPreferredName(), indexName)); + // exclude soft-deleted connectors + boolFilterQueryBuilder.mustNot(new TermQueryBuilder(Connector.IS_DELETED_FIELD.getPreferredName(), true)); + // If we know the connector _id, exclude this from search query if (connectorId != null) { boolFilterQueryBuilder.mustNot(new IdsQueryBuilder().addIds(connectorId)); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java index 2edd47b1fce3..82f6cf77def5 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/GetConnectorAction.java @@ -10,6 +10,7 @@ package org.elasticsearch.xpack.application.connector.action; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -25,6 +26,7 @@ import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; public class GetConnectorAction { @@ -36,22 +38,25 @@ public class GetConnectorAction { public static class Request extends ConnectorActionRequest implements ToXContentObject { private final String connectorId; + private final Boolean includeDeleted; private static final ParseField CONNECTOR_ID_FIELD = new ParseField("connector_id"); - public Request(String connectorId) { - this.connectorId = connectorId; - } + private static final ParseField INCLUDE_DELETED_FIELD = new ParseField("include_deleted"); - public Request(StreamInput in) throws IOException { - super(in); - this.connectorId = in.readString(); + public Request(String connectorId, Boolean includeDeleted) { + this.connectorId = connectorId; + this.includeDeleted = includeDeleted; } public String getConnectorId() { return connectorId; } + public Boolean getIncludeDeleted() { + return includeDeleted; + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; @@ -65,8 +70,7 @@ public class GetConnectorAction { @Override public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(connectorId); + TransportAction.localOnly(); } @Override @@ -74,12 +78,12 @@ public class GetConnectorAction { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Request request = (Request) o; - return Objects.equals(connectorId, request.connectorId); + return Objects.equals(connectorId, request.connectorId) && Objects.equals(includeDeleted, request.includeDeleted); } @Override public int hashCode() { - return Objects.hash(connectorId); + return Objects.hash(connectorId, includeDeleted); } @Override @@ -87,6 +91,7 @@ public class GetConnectorAction { builder.startObject(); { builder.field(CONNECTOR_ID_FIELD.getPreferredName(), connectorId); + builder.field(INCLUDE_DELETED_FIELD.getPreferredName(), includeDeleted); } builder.endObject(); return builder; @@ -95,11 +100,12 @@ public class GetConnectorAction { private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "get_connector_request", false, - (p) -> new Request((String) p[0]) + (p) -> new Request((String) p[0], (Boolean) p[1]) ); static { PARSER.declareString(constructorArg(), CONNECTOR_ID_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), INCLUDE_DELETED_FIELD); } public static Request parse(XContentParser parser) { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java index e543d805b709..1220c3c58fbb 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ListConnectorAction.java @@ -10,6 +10,7 @@ package org.elasticsearch.xpack.application.connector.action; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -46,33 +47,28 @@ public class ListConnectorAction { private final List connectorNames; private final List connectorServiceTypes; private final String connectorSearchQuery; + private final Boolean includeDeleted; private static final ParseField PAGE_PARAMS_FIELD = new ParseField("pageParams"); private static final ParseField INDEX_NAMES_FIELD = new ParseField("index_names"); private static final ParseField NAMES_FIELD = new ParseField("names"); private static final ParseField SEARCH_QUERY_FIELD = new ParseField("query"); - - public Request(StreamInput in) throws IOException { - super(in); - this.pageParams = new PageParams(in); - this.indexNames = in.readOptionalStringCollectionAsList(); - this.connectorNames = in.readOptionalStringCollectionAsList(); - this.connectorServiceTypes = in.readOptionalStringCollectionAsList(); - this.connectorSearchQuery = in.readOptionalString(); - } + private static final ParseField INCLUDE_DELETED_FIELD = new ParseField("include_deleted"); public Request( PageParams pageParams, List indexNames, List connectorNames, List serviceTypes, - String connectorSearchQuery + String connectorSearchQuery, + Boolean includeDeleted ) { this.pageParams = pageParams; this.indexNames = indexNames; this.connectorNames = connectorNames; this.connectorServiceTypes = serviceTypes; this.connectorSearchQuery = connectorSearchQuery; + this.includeDeleted = includeDeleted; } public PageParams getPageParams() { @@ -95,6 +91,10 @@ public class ListConnectorAction { return connectorSearchQuery; } + public Boolean getIncludeDeleted() { + return includeDeleted; + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; @@ -114,12 +114,7 @@ public class ListConnectorAction { @Override public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - pageParams.writeTo(out); - out.writeOptionalStringCollection(indexNames); - out.writeOptionalStringCollection(connectorNames); - out.writeOptionalStringCollection(connectorServiceTypes); - out.writeOptionalString(connectorSearchQuery); + TransportAction.localOnly(); } @Override @@ -131,7 +126,8 @@ public class ListConnectorAction { && Objects.equals(indexNames, request.indexNames) && Objects.equals(connectorNames, request.connectorNames) && Objects.equals(connectorServiceTypes, request.connectorServiceTypes) - && Objects.equals(connectorSearchQuery, request.connectorSearchQuery); + && Objects.equals(connectorSearchQuery, request.connectorSearchQuery) + && Objects.equals(includeDeleted, request.includeDeleted); } @Override @@ -147,7 +143,8 @@ public class ListConnectorAction { (List) p[1], (List) p[2], (List) p[3], - (String) p[4] + (String) p[4], + (Boolean) p[5] ) ); @@ -157,6 +154,7 @@ public class ListConnectorAction { PARSER.declareStringArray(optionalConstructorArg(), NAMES_FIELD); PARSER.declareStringArray(optionalConstructorArg(), Connector.SERVICE_TYPE_FIELD); PARSER.declareString(optionalConstructorArg(), SEARCH_QUERY_FIELD); + PARSER.declareBoolean(optionalConstructorArg(), INCLUDE_DELETED_FIELD); } public static ListConnectorAction.Request parse(XContentParser parser) { @@ -172,6 +170,7 @@ public class ListConnectorAction { builder.field(NAMES_FIELD.getPreferredName(), connectorNames); builder.field(Connector.SERVICE_TYPE_FIELD.getPreferredName(), connectorServiceTypes); builder.field(SEARCH_QUERY_FIELD.getPreferredName(), connectorSearchQuery); + builder.field(INCLUDE_DELETED_FIELD.getPreferredName(), includeDeleted); } builder.endObject(); return builder; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java index 8d3d5914ca69..81960db46ad6 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestGetConnectorAction.java @@ -36,7 +36,8 @@ public class RestGetConnectorAction extends BaseRestHandler { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { - GetConnectorAction.Request request = new GetConnectorAction.Request(restRequest.param(CONNECTOR_ID_PARAM)); + Boolean includeDeleted = restRequest.paramAsBoolean("include_deleted", false); + GetConnectorAction.Request request = new GetConnectorAction.Request(restRequest.param(CONNECTOR_ID_PARAM), includeDeleted); return channel -> client.execute(GetConnectorAction.INSTANCE, request, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java index 765f6eca1229..df3371846291 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/RestListConnectorAction.java @@ -43,13 +43,15 @@ public class RestListConnectorAction extends BaseRestHandler { List connectorNames = List.of(restRequest.paramAsStringArray("connector_name", new String[0])); List serviceTypes = List.of(restRequest.paramAsStringArray("service_type", new String[0])); String searchQuery = restRequest.param("query"); + Boolean includeDeleted = restRequest.paramAsBoolean("include_deleted", false); ListConnectorAction.Request request = new ListConnectorAction.Request( new PageParams(from, size), indexNames, connectorNames, serviceTypes, - searchQuery + searchQuery, + includeDeleted ); return channel -> client.execute(ListConnectorAction.INSTANCE, request, new RestToXContentListener<>(channel)); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java index e0beeb9b1951..98891427ab84 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportGetConnectorAction.java @@ -9,7 +9,7 @@ package org.elasticsearch.xpack.application.connector.action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; @@ -17,23 +17,21 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.application.connector.ConnectorIndexService; -public class TransportGetConnectorAction extends HandledTransportAction { +public class TransportGetConnectorAction extends TransportAction { protected final ConnectorIndexService connectorIndexService; @Inject public TransportGetConnectorAction(TransportService transportService, ActionFilters actionFilters, Client client) { - super( - GetConnectorAction.NAME, - transportService, - actionFilters, - GetConnectorAction.Request::new, - EsExecutors.DIRECT_EXECUTOR_SERVICE - ); + super(GetConnectorAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.connectorIndexService = new ConnectorIndexService(client); } @Override protected void doExecute(Task task, GetConnectorAction.Request request, ActionListener listener) { - connectorIndexService.getConnector(request.getConnectorId(), listener.map(GetConnectorAction.Response::new)); + connectorIndexService.getConnector( + request.getConnectorId(), + request.getIncludeDeleted(), + listener.map(GetConnectorAction.Response::new) + ); } } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java index 8ff180aa6189..408aa6fc1240 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/TransportListConnectorAction.java @@ -9,7 +9,7 @@ package org.elasticsearch.xpack.application.connector.action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.injection.guice.Inject; @@ -18,18 +18,12 @@ import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.application.connector.ConnectorIndexService; import org.elasticsearch.xpack.core.action.util.PageParams; -public class TransportListConnectorAction extends HandledTransportAction { +public class TransportListConnectorAction extends TransportAction { protected final ConnectorIndexService connectorIndexService; @Inject public TransportListConnectorAction(TransportService transportService, ActionFilters actionFilters, Client client) { - super( - ListConnectorAction.NAME, - transportService, - actionFilters, - ListConnectorAction.Request::new, - EsExecutors.DIRECT_EXECUTOR_SERVICE - ); + super(ListConnectorAction.NAME, actionFilters, transportService.getTaskManager(), EsExecutors.DIRECT_EXECUTOR_SERVICE); this.connectorIndexService = new ConnectorIndexService(client); } @@ -44,6 +38,8 @@ public class TransportListConnectorAction extends HandledTransportAction new ListConnectorAction.Response(r.connectors(), r.totalResults())) ); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java index 4aabb9e1af66..5ba45ea4757b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java @@ -10,9 +10,6 @@ package org.elasticsearch.xpack.application.connector.syncjob; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.ConstructingObjectParser; @@ -65,7 +62,7 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr *
  • The hostname of the worker to run the sync job.
    • * */ -public class ConnectorSyncJob implements Writeable, ToXContentObject { +public class ConnectorSyncJob implements ToXContentObject { static final ParseField CANCELATION_REQUESTED_AT_FIELD = new ParseField("cancelation_requested_at"); @@ -213,27 +210,6 @@ public class ConnectorSyncJob implements Writeable, ToXContentObject { this.workerHostname = workerHostname; } - public ConnectorSyncJob(StreamInput in) throws IOException { - this.cancelationRequestedAt = in.readOptionalInstant(); - this.canceledAt = in.readOptionalInstant(); - this.completedAt = in.readOptionalInstant(); - this.connector = in.readNamedWriteable(Connector.class); - this.createdAt = in.readInstant(); - this.deletedDocumentCount = in.readLong(); - this.error = in.readOptionalString(); - this.id = in.readString(); - this.indexedDocumentCount = in.readLong(); - this.indexedDocumentVolume = in.readLong(); - this.jobType = in.readEnum(ConnectorSyncJobType.class); - this.lastSeen = in.readOptionalInstant(); - this.metadata = in.readMap(StreamInput::readString, StreamInput::readGenericValue); - this.startedAt = in.readOptionalInstant(); - this.status = in.readEnum(ConnectorSyncStatus.class); - this.totalDocumentCount = in.readOptionalLong(); - this.triggerMethod = in.readEnum(ConnectorSyncJobTriggerMethod.class); - this.workerHostname = in.readOptionalString(); - } - @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "connector_sync_job", @@ -548,28 +524,6 @@ public class ConnectorSyncJob implements Writeable, ToXContentObject { return builder; } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeOptionalInstant(cancelationRequestedAt); - out.writeOptionalInstant(canceledAt); - out.writeOptionalInstant(completedAt); - out.writeNamedWriteable(connector); - out.writeInstant(createdAt); - out.writeLong(deletedDocumentCount); - out.writeOptionalString(error); - out.writeString(id); - out.writeLong(indexedDocumentCount); - out.writeLong(indexedDocumentVolume); - out.writeEnum(jobType); - out.writeOptionalInstant(lastSeen); - out.writeMap(metadata, StreamOutput::writeString, StreamOutput::writeGenericValue); - out.writeOptionalInstant(startedAt); - out.writeEnum(status); - out.writeOptionalLong(totalDocumentCount); - out.writeEnum(triggerMethod); - out.writeOptionalString(workerHostname); - } - public boolean equals(Object other) { if (this == other) return true; if (other == null || getClass() != other.getClass()) return false; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java index 28d4fe0956d0..6189db51df18 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorIndexServiceTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; @@ -112,13 +111,66 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase { } String connectorIdToDelete = connectorIds.get(0); - DeleteResponse resp = awaitDeleteConnector(connectorIdToDelete, false); + UpdateResponse resp = awaitDeleteConnector(connectorIdToDelete, false); assertThat(resp.status(), equalTo(RestStatus.OK)); expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorIdToDelete)); expectThrows(ResourceNotFoundException.class, () -> awaitDeleteConnector(connectorIdToDelete, false)); } + public void testDeleteConnector_expectSoftDeletion() throws Exception { + int numConnectors = 5; + List connectorIds = new ArrayList<>(); + List connectors = new ArrayList<>(); + for (int i = 0; i < numConnectors; i++) { + Connector connector = ConnectorTestUtils.getRandomConnector(); + ConnectorCreateActionResponse resp = awaitCreateConnector(null, connector); + connectorIds.add(resp.getId()); + connectors.add(connector); + } + + String connectorIdToDelete = connectorIds.get(0); + UpdateResponse resp = awaitDeleteConnector(connectorIdToDelete, false); + assertThat(resp.status(), equalTo(RestStatus.OK)); + expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorIdToDelete)); + + expectThrows(ResourceNotFoundException.class, () -> awaitDeleteConnector(connectorIdToDelete, false)); + + Connector softDeletedConnector = awaitGetSoftDeletedConnector(connectorIdToDelete); + assertThat(softDeletedConnector.getConnectorId(), equalTo(connectorIdToDelete)); + assertThat(softDeletedConnector.getServiceType(), equalTo(connectors.get(0).getServiceType())); + } + + public void testDeleteConnector_expectSoftDeletionMultipleConnectors() throws Exception { + int numConnectors = 5; + List connectorIds = new ArrayList<>(); + for (int i = 0; i < numConnectors; i++) { + Connector connector = ConnectorTestUtils.getRandomConnector(); + ConnectorCreateActionResponse resp = awaitCreateConnector(null, connector); + connectorIds.add(resp.getId()); + } + + // Delete all of them + for (int i = 0; i < numConnectors; i++) { + String connectorIdToDelete = connectorIds.get(i); + UpdateResponse resp = awaitDeleteConnector(connectorIdToDelete, false); + assertThat(resp.status(), equalTo(RestStatus.OK)); + } + + // Connectors were deleted from main index + for (int i = 0; i < numConnectors; i++) { + String connectorId = connectorIds.get(i); + expectThrows(ResourceNotFoundException.class, () -> awaitGetConnector(connectorId)); + } + + // Soft deleted connectors available in system index + for (int i = 0; i < numConnectors; i++) { + String connectorId = connectorIds.get(i); + Connector softDeletedConnector = awaitGetSoftDeletedConnector(connectorId); + assertThat(softDeletedConnector.getConnectorId(), equalTo(connectorId)); + } + } + public void testUpdateConnectorConfiguration_FullConfiguration() throws Exception { Connector connector = ConnectorTestUtils.getRandomConnector(); String connectorId = randomUUID(); @@ -897,13 +949,13 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase { assertThat(updateApiKeyIdRequest.getApiKeySecretId(), equalTo(indexedConnector.getApiKeySecretId())); } - private DeleteResponse awaitDeleteConnector(String connectorId, boolean deleteConnectorSyncJobs) throws Exception { + private UpdateResponse awaitDeleteConnector(String connectorId, boolean deleteConnectorSyncJobs) throws Exception { CountDownLatch latch = new CountDownLatch(1); - final AtomicReference resp = new AtomicReference<>(null); + final AtomicReference resp = new AtomicReference<>(null); final AtomicReference exc = new AtomicReference<>(null); connectorIndexService.deleteConnector(connectorId, deleteConnectorSyncJobs, new ActionListener<>() { @Override - public void onResponse(DeleteResponse deleteResponse) { + public void onResponse(UpdateResponse deleteResponse) { resp.set(deleteResponse); latch.countDown(); } @@ -956,11 +1008,19 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase { return resp.get(); } + private Connector awaitGetSoftDeletedConnector(String connectorId) throws Exception { + return awaitGetConnector(connectorId, true); + } + private Connector awaitGetConnector(String connectorId) throws Exception { + return awaitGetConnector(connectorId, false); + } + + private Connector awaitGetConnector(String connectorId, Boolean isDeleted) throws Exception { CountDownLatch latch = new CountDownLatch(1); final AtomicReference resp = new AtomicReference<>(null); final AtomicReference exc = new AtomicReference<>(null); - connectorIndexService.getConnector(connectorId, new ActionListener<>() { + connectorIndexService.getConnector(connectorId, isDeleted, new ActionListener<>() { @Override public void onResponse(ConnectorSearchResult connectorResult) { // Serialize the sourceRef to Connector class for unit tests @@ -994,11 +1054,23 @@ public class ConnectorIndexServiceTests extends ESSingleNodeTestCase { List names, List serviceTypes, String searchQuery + ) throws Exception { + return awaitListConnector(from, size, indexNames, names, serviceTypes, searchQuery, false); + } + + private ConnectorIndexService.ConnectorResult awaitListConnector( + int from, + int size, + List indexNames, + List names, + List serviceTypes, + String searchQuery, + Boolean isDeleted ) throws Exception { CountDownLatch latch = new CountDownLatch(1); final AtomicReference resp = new AtomicReference<>(null); final AtomicReference exc = new AtomicReference<>(null); - connectorIndexService.listConnectors(from, size, indexNames, names, serviceTypes, searchQuery, new ActionListener<>() { + connectorIndexService.listConnectors(from, size, indexNames, names, serviceTypes, searchQuery, isDeleted, new ActionListener<>() { @Override public void onResponse(ConnectorIndexService.ConnectorResult result) { resp.set(result); diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java index bcb647d978ab..bb37a2b1e80b 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/ConnectorTests.java @@ -9,43 +9,20 @@ package org.elasticsearch.xpack.application.connector; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; -import org.junit.Before; import java.io.IOException; -import java.util.List; -import static java.util.Collections.emptyList; import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; import static org.hamcrest.CoreMatchers.equalTo; public class ConnectorTests extends ESTestCase { - private NamedWriteableRegistry namedWriteableRegistry; - - @Before - public void registerNamedObjects() { - SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); - - List namedWriteables = searchModule.getNamedWriteables(); - namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); - } - - public final void testRandomSerialization() throws IOException { - for (int runs = 0; runs < 10; runs++) { - Connector testInstance = ConnectorTestUtils.getRandomConnector(); - assertTransportSerialization(testInstance); - } - } - public void testToXContent() throws IOException { String connectorId = "test-connector"; String content = XContentHelper.stripWhitespace(""" @@ -383,14 +360,4 @@ public class ConnectorTests extends ESTestCase { assertToXContentEquivalent(originalBytes, toXContent(parsed, XContentType.JSON, humanReadable), XContentType.JSON); assertThat(parsed.getApiKeySecretId(), equalTo(null)); } - - private void assertTransportSerialization(Connector testInstance) throws IOException { - Connector deserializedInstance = copyInstance(testInstance); - assertNotSame(testInstance, deserializedInstance); - assertThat(testInstance, equalTo(deserializedInstance)); - } - - private Connector copyInstance(Connector instance) throws IOException { - return copyWriteable(instance, namedWriteableRegistry, Connector::new); - } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java deleted file mode 100644 index 2b8e8735fa2c..000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/GetConnectorActionRequestBWCSerializingTests.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.test.AbstractBWCSerializationTestCase; -import org.elasticsearch.xcontent.XContentParser; - -import java.io.IOException; - -public class GetConnectorActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase { - - @Override - protected Writeable.Reader instanceReader() { - return GetConnectorAction.Request::new; - } - - @Override - protected GetConnectorAction.Request createTestInstance() { - return new GetConnectorAction.Request(randomAlphaOfLengthBetween(1, 10)); - } - - @Override - protected GetConnectorAction.Request mutateInstance(GetConnectorAction.Request instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected GetConnectorAction.Request doParseInstance(XContentParser parser) throws IOException { - return GetConnectorAction.Request.parse(parser); - } - - @Override - protected GetConnectorAction.Request mutateInstanceForVersion(GetConnectorAction.Request instance, TransportVersion version) { - return new GetConnectorAction.Request(instance.getConnectorId()); - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java deleted file mode 100644 index 3390ece073e5..000000000000 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/action/ListConnectorActionRequestBWCSerializingTests.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.application.connector.action; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.test.AbstractBWCSerializationTestCase; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.application.EnterpriseSearchModuleTestUtils; -import org.elasticsearch.xpack.core.action.util.PageParams; - -import java.io.IOException; -import java.util.List; - -public class ListConnectorActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase { - @Override - protected Writeable.Reader instanceReader() { - return ListConnectorAction.Request::new; - } - - @Override - protected ListConnectorAction.Request createTestInstance() { - PageParams pageParams = EnterpriseSearchModuleTestUtils.randomPageParams(); - return new ListConnectorAction.Request( - pageParams, - List.of(generateRandomStringArray(10, 10, false)), - List.of(generateRandomStringArray(10, 10, false)), - List.of(generateRandomStringArray(10, 10, false)), - randomAlphaOfLengthBetween(3, 10) - ); - } - - @Override - protected ListConnectorAction.Request mutateInstance(ListConnectorAction.Request instance) throws IOException { - return randomValueOtherThan(instance, this::createTestInstance); - } - - @Override - protected ListConnectorAction.Request doParseInstance(XContentParser parser) throws IOException { - return ListConnectorAction.Request.parse(parser); - } - - @Override - protected ListConnectorAction.Request mutateInstanceForVersion(ListConnectorAction.Request instance, TransportVersion version) { - return new ListConnectorAction.Request( - instance.getPageParams(), - instance.getIndexNames(), - instance.getConnectorNames(), - instance.getConnectorServiceTypes(), - instance.getConnectorSearchQuery() - ); - } -} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretResponseBWCSerializingTests.java index 964c5e15d845..5523ad4f50fe 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/DeleteConnectorSecretResponseBWCSerializingTests.java @@ -8,22 +8,14 @@ package org.elasticsearch.xpack.application.connector.secrets.action; import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsTestUtils; import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; import java.io.IOException; -import java.util.List; public class DeleteConnectorSecretResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase { - @Override - public NamedWriteableRegistry getNamedWriteableRegistry() { - return new NamedWriteableRegistry(List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new))); - } - @Override protected Writeable.Reader instanceReader() { return DeleteConnectorSecretResponse::new; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretResponseBWCSerializingTests.java index 4448024814df..c5ea4cbc8e55 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/secrets/action/GetConnectorSecretResponseBWCSerializingTests.java @@ -8,22 +8,14 @@ package org.elasticsearch.xpack.application.connector.secrets.action; import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.secrets.ConnectorSecretsTestUtils; import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; import java.io.IOException; -import java.util.List; public class GetConnectorSecretResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase { - @Override - public NamedWriteableRegistry getNamedWriteableRegistry() { - return new NamedWriteableRegistry(List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new))); - } - @Override protected Writeable.Reader instanceReader() { return GetConnectorSecretResponse::new; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java index ed3338c715bd..449dd258359c 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTests.java @@ -8,17 +8,14 @@ package org.elasticsearch.xpack.application.connector.syncjob; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus; -import org.junit.Before; import java.io.IOException; import java.time.Instant; -import java.util.List; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -26,22 +23,6 @@ import static org.hamcrest.Matchers.notNullValue; public class ConnectorSyncJobTests extends ESTestCase { - private NamedWriteableRegistry namedWriteableRegistry; - - @Before - public void registerNamedObjects() { - namedWriteableRegistry = new NamedWriteableRegistry( - List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new)) - ); - } - - public final void testRandomSerialization() throws IOException { - for (int run = 0; run < 10; run++) { - ConnectorSyncJob syncJob = ConnectorSyncJobTestUtils.getRandomConnectorSyncJob(); - assertTransportSerialization(syncJob); - } - } - public void testFromXContent_WithAllFields_AllSet() throws IOException { String content = XContentHelper.stripWhitespace(""" { @@ -332,14 +313,4 @@ public class ConnectorSyncJobTests extends ESTestCase { ConnectorSyncJob.syncJobConnectorFromXContentBytes(new BytesArray(content), null, XContentType.JSON); } - - private void assertTransportSerialization(ConnectorSyncJob testInstance) throws IOException { - ConnectorSyncJob deserializedInstance = copyInstance(testInstance); - assertNotSame(testInstance, deserializedInstance); - assertThat(testInstance, equalTo(deserializedInstance)); - } - - private ConnectorSyncJob copyInstance(ConnectorSyncJob instance) throws IOException { - return copyWriteable(instance, namedWriteableRegistry, ConnectorSyncJob::new); - } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java index 00f6e7cf57fc..ff915090f3c2 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/GetConnectorSyncJobActionResponseBWCSerializingTests.java @@ -8,23 +8,15 @@ package org.elasticsearch.xpack.application.connector.syncjob.action; import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; import java.io.IOException; -import java.util.List; public class GetConnectorSyncJobActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< GetConnectorSyncJobAction.Response> { - @Override - public NamedWriteableRegistry getNamedWriteableRegistry() { - return new NamedWriteableRegistry(List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new))); - } - @Override protected Writeable.Reader instanceReader() { return GetConnectorSyncJobAction.Response::new; diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionResponseBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionResponseBWCSerializingTests.java index bc7b6320dddb..ead5975d590a 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionResponseBWCSerializingTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ListConnectorSyncJobsActionResponseBWCSerializingTests.java @@ -8,23 +8,15 @@ package org.elasticsearch.xpack.application.connector.syncjob.action; import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; import java.io.IOException; -import java.util.List; public class ListConnectorSyncJobsActionResponseBWCSerializingTests extends AbstractBWCWireSerializationTestCase< ListConnectorSyncJobsAction.Response> { - @Override - protected NamedWriteableRegistry getNamedWriteableRegistry() { - return new NamedWriteableRegistry(List.of(new NamedWriteableRegistry.Entry(Connector.class, Connector.NAME, Connector::new))); - } - @Override protected Writeable.Reader instanceReader() { return ListConnectorSyncJobsAction.Response::new; From 8330c86e7c081589adf1312a75f8b63b72d9201b Mon Sep 17 00:00:00 2001 From: Jan Kuipers <148754765+jan-elastic@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:05:04 +0100 Subject: [PATCH 13/33] Fix(?) and unmute DatafeedJobsRestIT. (#119839) * Fix(?) and unmute DatafeedJobsRestIT. * [CI] Auto commit changes from spotless --------- Co-authored-by: elasticsearchmachine --- muted-tests.yml | 5 ---- .../ml/integration/DatafeedJobsRestIT.java | 25 +++++++++---------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index b49cd0a32bfe..d07bafb468e1 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -84,9 +84,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/115816 - class: org.elasticsearch.xpack.application.connector.ConnectorIndexServiceTests issue: https://github.com/elastic/elasticsearch/issues/116087 -- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT - method: testLookbackWithIndicesOptions - issue: https://github.com/elastic/elasticsearch/issues/116127 - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=transform/transforms_start_stop/Test start already started transform} issue: https://github.com/elastic/elasticsearch/issues/98802 @@ -152,8 +149,6 @@ tests: - class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT method: test {p0=search.highlight/50_synthetic_source/text multi unified from vectors} issue: https://github.com/elastic/elasticsearch/issues/117815 -- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsRestIT - issue: https://github.com/elastic/elasticsearch/issues/111319 - class: org.elasticsearch.xpack.esql.plugin.ClusterRequestTests method: testFallbackIndicesOptions issue: https://github.com/elastic/elasticsearch/issues/117937 diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java index 2fc9a80ae367..804033ef531b 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java @@ -37,7 +37,6 @@ import java.util.stream.Collectors; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; public class DatafeedJobsRestIT extends ESRestTestCase { @@ -503,12 +502,12 @@ public class DatafeedJobsRestIT extends ESRestTestCase { }"""); client().performRequest(createJobRequest); String datafeedId = jobId + "-datafeed"; - new DatafeedBuilder(datafeedId, jobId, "*hidden-*").setIndicesOptions(""" + new DatafeedBuilder(datafeedId, jobId, "hidden-*").setIndicesOptions(""" {"expand_wildcards": ["all"],"allow_no_indices": true}""").build(); StringBuilder bulk = new StringBuilder(); - Request createGeoData = new Request("PUT", "/.hidden-index"); + Request createGeoData = new Request("PUT", "/hidden-index"); createGeoData.setJsonEntity(""" { "mappings": { @@ -528,23 +527,23 @@ public class DatafeedJobsRestIT extends ESRestTestCase { client().performRequest(createGeoData); bulk.append(""" - {"index": {"_index": ".hidden-index", "_id": 1}} + {"index": {"_index": "hidden-index", "_id": 1}} {"time":"2016-06-01T00:00:00Z","value": 1000} - {"index": {"_index": ".hidden-index", "_id": 2}} + {"index": {"_index": "hidden-index", "_id": 2}} {"time":"2016-06-01T00:05:00Z","value":1500} - {"index": {"_index": ".hidden-index", "_id": 3}} + {"index": {"_index": "hidden-index", "_id": 3}} {"time":"2016-06-01T00:10:00Z","value":1600} - {"index": {"_index": ".hidden-index", "_id": 4}} + {"index": {"_index": "hidden-index", "_id": 4}} {"time":"2016-06-01T00:15:00Z","value":100} - {"index": {"_index": ".hidden-index", "_id": 5}} + {"index": {"_index": "hidden-index", "_id": 5}} {"time":"2016-06-01T00:20:00Z","value":1} - {"index": {"_index": ".hidden-index", "_id": 6}} + {"index": {"_index": "hidden-index", "_id": 6}} {"time":"2016-06-01T00:25:00Z","value":1500} - {"index": {"_index": ".hidden-index", "_id": 7}} + {"index": {"_index": "hidden-index", "_id": 7}} {"time":"2016-06-01T00:30:00Z","value":1500} - {"index": {"_index": ".hidden-index", "_id": 8}} + {"index": {"_index": "hidden-index", "_id": 8}} {"time":"2016-06-01T00:40:00Z","value":2100} - {"index": {"_index": ".hidden-index", "_id": 9}} + {"index": {"_index": "hidden-index", "_id": 9}} {"time":"2016-06-01T00:41:00Z","value":0} """); bulkIndex(bulk.toString()); @@ -1802,7 +1801,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase { bulkRequest.addParameter("refresh", "true"); bulkRequest.addParameter("pretty", null); String bulkResponse = EntityUtils.toString(client().performRequest(bulkRequest).getEntity()); - assertThat(bulkResponse, not(containsString("\"errors\": false"))); + assertThat(bulkResponse, containsString("\"errors\" : false")); } private Response createJobAndDataFeed(String jobId, String datafeedId) throws IOException { From 750459ef07adecf35bea6ff1cdbae1202ac410fd Mon Sep 17 00:00:00 2001 From: Max Hniebergall <137079448+maxhniebergall@users.noreply.github.com> Date: Thu, 9 Jan 2025 08:20:02 -0500 Subject: [PATCH 14/33] [Inference API] Fix bug checking for e5 or reranker default IDs (#119797) * Fix bug checking for non-elser defaults * Update docs/changelog/119797.yaml * fix typo --- docs/changelog/119797.yaml | 5 +++++ .../services/elasticsearch/ElasticsearchInternalService.java | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/119797.yaml diff --git a/docs/changelog/119797.yaml b/docs/changelog/119797.yaml new file mode 100644 index 000000000000..992c2078e0ca --- /dev/null +++ b/docs/changelog/119797.yaml @@ -0,0 +1,5 @@ +pr: 119797 +summary: "[Inference API] Fix bug checking for e5 or reranker default IDs" +area: Machine Learning +type: bug +issues: [] diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java index 39ad729b1699..2931f2e23f12 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elasticsearch/ElasticsearchInternalService.java @@ -133,7 +133,7 @@ public class ElasticsearchInternalService extends BaseElasticsearchInternalServi Map config, ActionListener modelListener ) { - if (inferenceEntityId.equals(DEFAULT_ELSER_ID)) { + if (isDefaultId(inferenceEntityId)) { modelListener.onFailure( new ElasticsearchStatusException( "[{}] is a reserved inference Id. Cannot create a new inference endpoint with a reserved Id", From 55da5220496674b9fd4e94d08bd2a7ad9d59b2fa Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Thu, 9 Jan 2025 07:28:53 -0600 Subject: [PATCH 15/33] Removing AwaitsFix annotation for fixed DataStreamsUpgradeIT test (#119809) --- .../java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java index e0294ba3a6e2..094ca9304a69 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java @@ -174,7 +174,6 @@ public class DataStreamsUpgradeIT extends AbstractUpgradeTestCase { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/119713") public void testUpgradeDataStream() throws Exception { String dataStreamName = "reindex_test_data_stream"; int numRollovers = randomIntBetween(0, 5); From f66ce0516158cc665ae1f7c10b5ebe5a84e262f8 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 10 Jan 2025 00:41:15 +1100 Subject: [PATCH 16/33] Mute org.elasticsearch.index.rankeval.RankEvalRequestTests testSerialization #119859 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index d07bafb468e1..4698bb2311dd 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -260,6 +260,9 @@ tests: - class: org.elasticsearch.script.mustache.SearchTemplateRequestTests method: testSerialization issue: https://github.com/elastic/elasticsearch/issues/119822 +- class: org.elasticsearch.index.rankeval.RankEvalRequestTests + method: testSerialization + issue: https://github.com/elastic/elasticsearch/issues/119859 # Examples: # From c04dd692be78ca3f5b9999e8751e97052fdf92bd Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 10 Jan 2025 00:41:30 +1100 Subject: [PATCH 17/33] Mute org.elasticsearch.index.rankeval.RankEvalRequestTests testEqualsAndHashcode #119860 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 4698bb2311dd..39c188b28333 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -263,6 +263,9 @@ tests: - class: org.elasticsearch.index.rankeval.RankEvalRequestTests method: testSerialization issue: https://github.com/elastic/elasticsearch/issues/119859 +- class: org.elasticsearch.index.rankeval.RankEvalRequestTests + method: testEqualsAndHashcode + issue: https://github.com/elastic/elasticsearch/issues/119860 # Examples: # From 330ff2fbaa27b0070e3c5cdf9f19155d1551b9c4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 10 Jan 2025 00:41:41 +1100 Subject: [PATCH 18/33] Mute org.elasticsearch.index.rankeval.RankEvalRequestTests testConcurrentSerialization #119861 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index 39c188b28333..e125cb72dc4d 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -266,6 +266,9 @@ tests: - class: org.elasticsearch.index.rankeval.RankEvalRequestTests method: testEqualsAndHashcode issue: https://github.com/elastic/elasticsearch/issues/119860 +- class: org.elasticsearch.index.rankeval.RankEvalRequestTests + method: testConcurrentSerialization + issue: https://github.com/elastic/elasticsearch/issues/119861 # Examples: # From 5294b9795245257a922dafe2fd20f3f693f8fd08 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com> Date: Fri, 10 Jan 2025 00:41:53 +1100 Subject: [PATCH 19/33] Mute org.elasticsearch.index.rankeval.RankEvalRequestTests testConcurrentEquals #119862 --- muted-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index e125cb72dc4d..69112af1ece1 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -269,6 +269,9 @@ tests: - class: org.elasticsearch.index.rankeval.RankEvalRequestTests method: testConcurrentSerialization issue: https://github.com/elastic/elasticsearch/issues/119861 +- class: org.elasticsearch.index.rankeval.RankEvalRequestTests + method: testConcurrentEquals + issue: https://github.com/elastic/elasticsearch/issues/119862 # Examples: # From 230acb8ac5d8bb0d583ebf111c3308875edca737 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Thu, 9 Jan 2025 08:56:57 -0500 Subject: [PATCH 20/33] Resolve/cluster should mark remotes as not connected when a security exception is thrown (#119793) Fixes two bugs in _resolve/cluster. First, the code that detects older clusters versions and does a fallback to the _resolve/index endpoint was using an outdated string match for error detection. That has been adjusted. Second, upon security exceptions, the _resolve/cluster endpoint was marking the clusters as connected: true, under the assumption that all security exceptions related to cross cluster calls and remote index access were coming from the remote cluster, but that is not always the case. Some cross-cluster security violations can be detected on the local querying cluster after issuing the remoteClient.execute call but before the transport layer actually sends the request remotely. So we now mark the connected status as false for all ElasticsearchSecurityException cases. End user docs have been updated with this information. --- docs/changelog/119793.yaml | 6 +++++ .../indices/resolve-cluster.asciidoc | 15 +++++++++-- .../resolve/ResolveClusterActionRequest.java | 26 ++++++++++--------- .../TransportResolveClusterAction.java | 11 +++++--- ...teClusterSecurityRCS1ResolveClusterIT.java | 11 +++++--- ...teClusterSecurityRCS2ResolveClusterIT.java | 11 +++++--- 6 files changed, 55 insertions(+), 25 deletions(-) create mode 100644 docs/changelog/119793.yaml diff --git a/docs/changelog/119793.yaml b/docs/changelog/119793.yaml new file mode 100644 index 000000000000..80330c25c2f3 --- /dev/null +++ b/docs/changelog/119793.yaml @@ -0,0 +1,6 @@ +pr: 119793 +summary: Resolve/cluster should mark remotes as not connected when a security exception + is thrown +area: CCS +type: bug +issues: [] diff --git a/docs/reference/indices/resolve-cluster.asciidoc b/docs/reference/indices/resolve-cluster.asciidoc index e623cb983e86..a0583f4d7bea 100644 --- a/docs/reference/indices/resolve-cluster.asciidoc +++ b/docs/reference/indices/resolve-cluster.asciidoc @@ -28,8 +28,19 @@ For each cluster in the index expression, information is returned about: 3. whether there are any indices, aliases or data streams on that cluster that match the index expression 4. whether the search is likely to have errors returned when you do the {ccs} (including any - authorization errors if your user does not have permission to query the index) -5. cluster version information, including the Elasticsearch server version + authorization errors if your user does not have permission to query a remote cluster or + the indices on that cluster) +5. (in some cases) cluster version information, including the Elasticsearch server version + +[TIP] +==== +Whenever a security exception is returned for a remote cluster, that remote +will always be marked as connected=false in the response, since your user does not have +permissions to access that cluster (or perhaps the remote index) you are querying. +Once the proper security permissions are obtained, then you can rely on the `connected` field +in the response to determine whether the remote cluster is available and ready for querying. +==== + //// [source,console] diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java index 9c5b6097b11b..ebc9b0fea1be 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveClusterActionRequest.java @@ -9,12 +9,14 @@ package org.elasticsearch.action.admin.indices.resolve; +import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.tasks.CancellableTask; @@ -30,6 +32,7 @@ import java.util.Objects; public class ResolveClusterActionRequest extends ActionRequest implements IndicesRequest.Replaceable { public static final IndicesOptions DEFAULT_INDICES_OPTIONS = IndicesOptions.strictExpandOpen(); + public static final String TRANSPORT_VERSION_ERROR_MESSAGE_PREFIX = "ResolveClusterAction requires at least version"; private String[] names; /* @@ -65,12 +68,7 @@ public class ResolveClusterActionRequest extends ActionRequest implements Indice public ResolveClusterActionRequest(StreamInput in) throws IOException { super(in); if (in.getTransportVersion().before(TransportVersions.V_8_13_0)) { - throw new UnsupportedOperationException( - "ResolveClusterAction requires at least version " - + TransportVersions.V_8_13_0.toReleaseVersion() - + " but was " - + in.getTransportVersion().toReleaseVersion() - ); + throw new UnsupportedOperationException(createVersionErrorMessage(in.getTransportVersion())); } this.names = in.readStringArray(); this.indicesOptions = IndicesOptions.readIndicesOptions(in); @@ -81,17 +79,21 @@ public class ResolveClusterActionRequest extends ActionRequest implements Indice public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); if (out.getTransportVersion().before(TransportVersions.V_8_13_0)) { - throw new UnsupportedOperationException( - "ResolveClusterAction requires at least version " - + TransportVersions.V_8_13_0.toReleaseVersion() - + " but was " - + out.getTransportVersion().toReleaseVersion() - ); + throw new UnsupportedOperationException(createVersionErrorMessage(out.getTransportVersion())); } out.writeStringArray(names); indicesOptions.writeIndicesOptions(out); } + private String createVersionErrorMessage(TransportVersion versionFound) { + return Strings.format( + "%s %s but was %s", + TRANSPORT_VERSION_ERROR_MESSAGE_PREFIX, + TransportVersions.V_8_13_0.toReleaseVersion(), + versionFound.toReleaseVersion() + ); + } + @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java index e3e737595cac..50dbaf33d2e4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/TransportResolveClusterAction.java @@ -51,7 +51,6 @@ import static org.elasticsearch.action.search.TransportSearchHelper.checkCCSVers public class TransportResolveClusterAction extends HandledTransportAction { private static final Logger logger = LogManager.getLogger(TransportResolveClusterAction.class); - private static final String TRANSPORT_VERSION_ERROR_MESSAGE = "ResolveClusterAction requires at least Transport Version"; public static final String NAME = "indices:admin/resolve/cluster"; public static final ActionType TYPE = new ActionType<>(NAME); @@ -175,7 +174,13 @@ public class TransportResolveClusterAction extends HandledTransportAction remoteClusterResponse = (Map) responseMap.get("my_remote_cluster"); - assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true)); + // with security exceptions, the remote should be marked as connected=false, since you can't tell whether a security + // exception comes from the local cluster (intercepted) or the remote + assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false)); assertThat((String) remoteClusterResponse.get("error"), containsString("unauthorized for user [remote_search_user]")); // TEST CASE 2: Query cluster -> add user role and user on remote cluster and try resolve again @@ -171,7 +173,7 @@ public class RemoteClusterSecurityRCS1ResolveClusterIT extends AbstractRemoteClu Map responseMap = responseAsMap(response); assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue()); Map remoteClusterResponse = (Map) responseMap.get("my_remote_cluster"); - assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true)); + assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false)); assertThat((String) remoteClusterResponse.get("error"), containsString("unauthorized for user [remote_search_user]")); assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [secretindex]")); } @@ -183,7 +185,7 @@ public class RemoteClusterSecurityRCS1ResolveClusterIT extends AbstractRemoteClu Map responseMap = responseAsMap(response); assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue()); Map remoteClusterResponse = (Map) responseMap.get("my_remote_cluster"); - assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true)); + assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false)); assertThat((String) remoteClusterResponse.get("error"), containsString("unauthorized for user [remote_search_user]")); assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [doesnotexist]")); } @@ -195,6 +197,7 @@ public class RemoteClusterSecurityRCS1ResolveClusterIT extends AbstractRemoteClu Map responseMap = responseAsMap(response); assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue()); Map remoteClusterResponse = (Map) responseMap.get("my_remote_cluster"); + // with IndexNotFoundExceptions, we know that error came from the remote cluster, so we can mark the remote as connected=true assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true)); assertThat((String) remoteClusterResponse.get("error"), containsString("no such index [index99]")); } @@ -210,7 +213,7 @@ public class RemoteClusterSecurityRCS1ResolveClusterIT extends AbstractRemoteClu Map responseMap = responseAsMap(response); assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue()); Map remoteClusterResponse = (Map) responseMap.get("my_remote_cluster"); - assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true)); + assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false)); assertThat((String) remoteClusterResponse.get("error"), containsString("unauthorized for user [remote_search_user]")); assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [secretindex]")); } diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRCS2ResolveClusterIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRCS2ResolveClusterIT.java index a3bc56dafce9..a1f897b5b050 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRCS2ResolveClusterIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityRCS2ResolveClusterIT.java @@ -180,7 +180,9 @@ public class RemoteClusterSecurityRCS2ResolveClusterIT extends AbstractRemoteClu assertLocalMatching(responseMap); Map remoteClusterResponse = (Map) responseMap.get("my_remote_cluster"); - assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true)); + // with security exceptions, the remote should be marked as connected=false, since you can't tell whether a security + // exception comes from the local cluster (intercepted) or the remote + assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false)); assertThat((String) remoteClusterResponse.get("error"), containsString("is unauthorized for user")); assertThat( (String) remoteClusterResponse.get("error"), @@ -261,7 +263,7 @@ public class RemoteClusterSecurityRCS2ResolveClusterIT extends AbstractRemoteClu Map responseMap = responseAsMap(response); assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue()); Map remoteClusterResponse = (Map) responseMap.get("my_remote_cluster"); - assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true)); + assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false)); assertThat((String) remoteClusterResponse.get("error"), containsString("is unauthorized for user")); assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [secretindex]")); } @@ -273,7 +275,7 @@ public class RemoteClusterSecurityRCS2ResolveClusterIT extends AbstractRemoteClu Map responseMap = responseAsMap(response); assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue()); Map remoteClusterResponse = (Map) responseMap.get("my_remote_cluster"); - assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true)); + assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false)); assertThat((String) remoteClusterResponse.get("error"), containsString("is unauthorized for user")); assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [doesnotexist]")); } @@ -285,6 +287,7 @@ public class RemoteClusterSecurityRCS2ResolveClusterIT extends AbstractRemoteClu Map responseMap = responseAsMap(response); assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue()); Map remoteClusterResponse = (Map) responseMap.get("my_remote_cluster"); + // with IndexNotFoundExceptions, we know that error came from the remote cluster, so we can mark the remote as connected=true assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true)); assertThat((Boolean) remoteClusterResponse.get("skip_unavailable"), equalTo(false)); assertThat((String) remoteClusterResponse.get("error"), containsString("no such index [index99]")); @@ -301,7 +304,7 @@ public class RemoteClusterSecurityRCS2ResolveClusterIT extends AbstractRemoteClu Map responseMap = responseAsMap(response); assertThat(responseMap.get(LOCAL_CLUSTER_NAME_REPRESENTATION), nullValue()); Map remoteClusterResponse = (Map) responseMap.get("my_remote_cluster"); - assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(true)); + assertThat((Boolean) remoteClusterResponse.get("connected"), equalTo(false)); assertThat((String) remoteClusterResponse.get("error"), containsString("is unauthorized for user")); assertThat((String) remoteClusterResponse.get("error"), containsString("on indices [secretindex]")); } From d37e1bd14d578cd2d24ef9847f61a340dbecfa9a Mon Sep 17 00:00:00 2001 From: shainaraskas <58563081+shainaraskas@users.noreply.github.com> Date: Thu, 9 Jan 2025 09:15:00 -0500 Subject: [PATCH 21/33] Fix broken anchors (#119802) --- docs/reference/modules/discovery/publishing.asciidoc | 2 +- docs/reference/modules/gateway.asciidoc | 1 + docs/reference/modules/network.asciidoc | 2 +- docs/reference/modules/node.asciidoc | 2 ++ docs/reference/node-roles.asciidoc | 2 +- docs/reference/setup/advanced-configuration.asciidoc | 3 ++- docs/reference/setup/important-settings.asciidoc | 1 + 7 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/reference/modules/discovery/publishing.asciidoc b/docs/reference/modules/discovery/publishing.asciidoc index 22fed9528e61..2bb031e6a986 100644 --- a/docs/reference/modules/discovery/publishing.asciidoc +++ b/docs/reference/modules/discovery/publishing.asciidoc @@ -77,7 +77,7 @@ and latency of the network interconnections between all nodes in the cluster. You must therefore ensure that the storage and networking available to the nodes in your cluster are good enough to meet your performance goals. -[[dangling-indices]] +[[dangling-index]] ==== Dangling indices When a node joins the cluster, if it finds any shards stored in its local diff --git a/docs/reference/modules/gateway.asciidoc b/docs/reference/modules/gateway.asciidoc index 4716effb4308..35a5ef6ddec0 100644 --- a/docs/reference/modules/gateway.asciidoc +++ b/docs/reference/modules/gateway.asciidoc @@ -1,6 +1,7 @@ [[modules-gateway]] === Local gateway settings +[[dangling-indices]] The local gateway stores the cluster state and shard data across full cluster restarts. diff --git a/docs/reference/modules/network.asciidoc b/docs/reference/modules/network.asciidoc index 2ea4dcb9b18f..5095d73058d7 100644 --- a/docs/reference/modules/network.asciidoc +++ b/docs/reference/modules/network.asciidoc @@ -287,7 +287,7 @@ include::network/tracers.asciidoc[] include::network/threading.asciidoc[] -[[readiness-tcp-port]] +[[tcp-readiness-port]] ==== TCP readiness port preview::[] diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc index e8dd995623a1..f0cb3eecdf39 100644 --- a/docs/reference/modules/node.asciidoc +++ b/docs/reference/modules/node.asciidoc @@ -46,6 +46,8 @@ The following additional roles are available: * `voting_only` +[[coordinating-only-node]]If you leave `node.roles` unset, then the node is considered to be a <>. + [IMPORTANT] ==== If you set `node.roles`, ensure you specify every node role your cluster needs. diff --git a/docs/reference/node-roles.asciidoc b/docs/reference/node-roles.asciidoc index e8c1d9143a38..296c76e6dba9 100644 --- a/docs/reference/node-roles.asciidoc +++ b/docs/reference/node-roles.asciidoc @@ -359,7 +359,7 @@ node.roles: [ ingest ] ---- [discrete] -[[coordinating-only-node]] +[[coordinating-only-node-role]] ==== Coordinating only node If you take away the ability to be able to handle master duties, to hold data, diff --git a/docs/reference/setup/advanced-configuration.asciidoc b/docs/reference/setup/advanced-configuration.asciidoc index 73b210ea559b..ff80b51f0408 100644 --- a/docs/reference/setup/advanced-configuration.asciidoc +++ b/docs/reference/setup/advanced-configuration.asciidoc @@ -19,6 +19,7 @@ CAUTION: Setting your own JVM options is generally not recommended and could neg impact performance and stability. Using the {es}-provided defaults is recommended in most circumstances. +[[readiness-tcp-port]] NOTE: Do not modify the root `jvm.options` file. Use files in `jvm.options.d/` instead. [[jvm-options-syntax]] @@ -153,7 +154,7 @@ options. We do not recommend using `ES_JAVA_OPTS` in production. NOTE: If you are running {es} as a Windows service, you can change the heap size using the service manager. See <>. -[[heap-dump-path]] +[[heap-dump-path-setting]] include::important-settings/heap-dump-path.asciidoc[leveloffset=-1] [[gc-logging]] diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc index 26f9c79cb669..b822ee9b3f90 100644 --- a/docs/reference/setup/important-settings.asciidoc +++ b/docs/reference/setup/important-settings.asciidoc @@ -41,6 +41,7 @@ include::important-settings/discovery-settings.asciidoc[] include::important-settings/heap-size.asciidoc[] +[[heap-dump-path]] include::important-settings/heap-dump-path.asciidoc[] include::important-settings/gc-logging.asciidoc[] From 16c7ec6a82f8de4a45a992b5422873a1d3ea0929 Mon Sep 17 00:00:00 2001 From: Dianna Hohensee Date: Thu, 9 Jan 2025 09:34:19 -0500 Subject: [PATCH 22/33] Add comments and refactor method names in balancer stats code (#119613) This change is adding comments and renaming classes/methods to improve understandability. Relates ES-10341 --- .../elasticsearch/cluster/ClusterModule.java | 15 +++++--- .../allocation/AllocationStatsService.java | 22 +++++++---- .../allocation/NodeAllocationStats.java | 9 +++++ ...eAllocationStatsAndWeightsCalculator.java} | 38 +++++++++++++------ .../allocator/BalancedShardsAllocator.java | 24 +++++++----- .../allocation/allocator/DesiredBalance.java | 2 +- .../allocator/DesiredBalanceMetrics.java | 23 +++++++++-- .../allocator/DesiredBalanceReconciler.java | 24 ++++++------ .../DesiredBalanceShardsAllocator.java | 10 ++--- .../allocation/allocator/WeightFunction.java | 6 +-- .../AllocationStatsServiceTests.java | 6 +-- .../cluster/ESAllocationTestCase.java | 28 +++++++------- 12 files changed, 130 insertions(+), 77 deletions(-) rename server/src/main/java/org/elasticsearch/cluster/routing/allocation/{NodeAllocationStatsProvider.java => NodeAllocationStatsAndWeightsCalculator.java} (76%) diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java index c2da33f8f413..efa20a41518c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -33,7 +33,7 @@ import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.AllocationService.RerouteStrategy; import org.elasticsearch.cluster.routing.allocation.AllocationStatsService; import org.elasticsearch.cluster.routing.allocation.ExistingShardsAllocator; -import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsProvider; +import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator; import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceShardsAllocator; @@ -139,7 +139,10 @@ public class ClusterModule extends AbstractModule { this.clusterPlugins = clusterPlugins; this.deciderList = createAllocationDeciders(settings, clusterService.getClusterSettings(), clusterPlugins); this.allocationDeciders = new AllocationDeciders(deciderList); - var nodeAllocationStatsProvider = new NodeAllocationStatsProvider(writeLoadForecaster, clusterService.getClusterSettings()); + var nodeAllocationStatsAndWeightsCalculator = new NodeAllocationStatsAndWeightsCalculator( + writeLoadForecaster, + clusterService.getClusterSettings() + ); this.shardsAllocator = createShardsAllocator( settings, clusterService.getClusterSettings(), @@ -149,7 +152,7 @@ public class ClusterModule extends AbstractModule { this::reconcile, writeLoadForecaster, telemetryProvider, - nodeAllocationStatsProvider + nodeAllocationStatsAndWeightsCalculator ); this.clusterService = clusterService; this.indexNameExpressionResolver = new IndexNameExpressionResolver(threadPool.getThreadContext(), systemIndices); @@ -167,7 +170,7 @@ public class ClusterModule extends AbstractModule { clusterService, clusterInfoService, shardsAllocator, - nodeAllocationStatsProvider + nodeAllocationStatsAndWeightsCalculator ); this.telemetryProvider = telemetryProvider; } @@ -409,7 +412,7 @@ public class ClusterModule extends AbstractModule { DesiredBalanceReconcilerAction reconciler, WriteLoadForecaster writeLoadForecaster, TelemetryProvider telemetryProvider, - NodeAllocationStatsProvider nodeAllocationStatsProvider + NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator ) { Map> allocators = new HashMap<>(); allocators.put(BALANCED_ALLOCATOR, () -> new BalancedShardsAllocator(clusterSettings, writeLoadForecaster)); @@ -422,7 +425,7 @@ public class ClusterModule extends AbstractModule { clusterService, reconciler, telemetryProvider, - nodeAllocationStatsProvider + nodeAllocationStatsAndWeightsCalculator ) ); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java index b98e9050d2b4..dfcc4da9dcc5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsService.java @@ -19,35 +19,41 @@ import java.util.Map; import java.util.function.Supplier; import java.util.stream.Collectors; +/** + * Exposes cluster allocation metrics. Constructs {@link NodeAllocationStats} per node, on demand. + */ public class AllocationStatsService { private final ClusterService clusterService; private final ClusterInfoService clusterInfoService; private final Supplier desiredBalanceSupplier; - private final NodeAllocationStatsProvider nodeAllocationStatsProvider; + private final NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator; public AllocationStatsService( ClusterService clusterService, ClusterInfoService clusterInfoService, ShardsAllocator shardsAllocator, - NodeAllocationStatsProvider nodeAllocationStatsProvider + NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator ) { this.clusterService = clusterService; this.clusterInfoService = clusterInfoService; - this.nodeAllocationStatsProvider = nodeAllocationStatsProvider; + this.nodeAllocationStatsAndWeightsCalculator = nodeAllocationStatsAndWeightsCalculator; this.desiredBalanceSupplier = shardsAllocator instanceof DesiredBalanceShardsAllocator allocator ? allocator::getDesiredBalance : () -> null; } + /** + * Returns a map of node IDs to node allocation stats. + */ public Map stats() { - var state = clusterService.state(); - var stats = nodeAllocationStatsProvider.stats( - state.metadata(), - state.getRoutingNodes(), + var clusterState = clusterService.state(); + var nodesStatsAndWeights = nodeAllocationStatsAndWeightsCalculator.nodesAllocationStatsAndWeights( + clusterState.metadata(), + clusterState.getRoutingNodes(), clusterInfoService.getClusterInfo(), desiredBalanceSupplier.get() ); - return stats.entrySet() + return nodesStatsAndWeights.entrySet() .stream() .collect( Collectors.toMap( diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStats.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStats.java index f7483a073479..5ad6e7b7817c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStats.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStats.java @@ -18,6 +18,15 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +/** + * Point-in-time allocation stats for a particular node. + * + * @param shards count of shards on this node. + * @param undesiredShards count of shards that we want to move off of this node. + * @param forecastedIngestLoad the predicted near future total ingest load on this node. + * @param forecastedDiskUsage the predicted near future total disk usage on this node. + * @param currentDiskUsage the current total disk usage on this node. + */ public record NodeAllocationStats( int shards, int undesiredShards, diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsProvider.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsAndWeightsCalculator.java similarity index 76% rename from server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsProvider.java rename to server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsAndWeightsCalculator.java index 8368f5916ef9..f793ab8f9eb7 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsProvider.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/NodeAllocationStatsAndWeightsCalculator.java @@ -24,7 +24,10 @@ import org.elasticsearch.core.Nullable; import java.util.Map; -public class NodeAllocationStatsProvider { +/** + * Calculates the allocation weights and usage stats for each node: see {@link NodeAllocationStatsAndWeight} for details. + */ +public class NodeAllocationStatsAndWeightsCalculator { private final WriteLoadForecaster writeLoadForecaster; private volatile float indexBalanceFactor; @@ -32,7 +35,10 @@ public class NodeAllocationStatsProvider { private volatile float writeLoadBalanceFactor; private volatile float diskUsageBalanceFactor; - public record NodeAllocationAndClusterBalanceStats( + /** + * Node shard allocation stats and the total node weight. + */ + public record NodeAllocationStatsAndWeight( int shards, int undesiredShards, double forecastedIngestLoad, @@ -41,7 +47,7 @@ public class NodeAllocationStatsProvider { float currentNodeWeight ) {} - public NodeAllocationStatsProvider(WriteLoadForecaster writeLoadForecaster, ClusterSettings clusterSettings) { + public NodeAllocationStatsAndWeightsCalculator(WriteLoadForecaster writeLoadForecaster, ClusterSettings clusterSettings) { this.writeLoadForecaster = writeLoadForecaster; clusterSettings.initializeAndWatch(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING, value -> this.shardBalanceFactor = value); clusterSettings.initializeAndWatch(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING, value -> this.indexBalanceFactor = value); @@ -55,7 +61,10 @@ public class NodeAllocationStatsProvider { ); } - public Map stats( + /** + * Returns a map of node IDs to {@link NodeAllocationStatsAndWeight}. + */ + public Map nodesAllocationStatsAndWeights( Metadata metadata, RoutingNodes routingNodes, ClusterInfo clusterInfo, @@ -66,7 +75,7 @@ public class NodeAllocationStatsProvider { var avgWriteLoadPerNode = WeightFunction.avgWriteLoadPerNode(writeLoadForecaster, metadata, routingNodes); var avgDiskUsageInBytesPerNode = WeightFunction.avgDiskUsageInBytesPerNode(clusterInfo, metadata, routingNodes); - var stats = Maps.newMapWithExpectedSize(routingNodes.size()); + var nodeAllocationStatsAndWeights = Maps.newMapWithExpectedSize(routingNodes.size()); for (RoutingNode node : routingNodes) { int shards = 0; int undesiredShards = 0; @@ -75,9 +84,10 @@ public class NodeAllocationStatsProvider { long currentDiskUsage = 0; for (ShardRouting shardRouting : node) { if (shardRouting.relocating()) { + // Skip the shard if it is moving off this node. The node running recovery will count it. continue; } - shards++; + ++shards; IndexMetadata indexMetadata = metadata.getIndexSafe(shardRouting.index()); if (isDesiredAllocation(desiredBalance, shardRouting) == false) { undesiredShards++; @@ -86,9 +96,8 @@ public class NodeAllocationStatsProvider { forecastedWriteLoad += writeLoadForecaster.getForecastedWriteLoad(indexMetadata).orElse(0.0); forecastedDiskUsage += Math.max(indexMetadata.getForecastedShardSizeInBytes().orElse(0), shardSize); currentDiskUsage += shardSize; - } - float currentNodeWeight = weightFunction.nodeWeight( + float currentNodeWeight = weightFunction.calculateNodeWeight( shards, avgShardsPerNode, forecastedWriteLoad, @@ -96,10 +105,12 @@ public class NodeAllocationStatsProvider { currentDiskUsage, avgDiskUsageInBytesPerNode ); - stats.put( + nodeAllocationStatsAndWeights.put( node.nodeId(), - new NodeAllocationAndClusterBalanceStats( + new NodeAllocationStatsAndWeight( shards, + // It's part of a public API contract for an 'undesired_shards' field that -1 will be returned if an allocator other + // than the desired balance allocator is used. desiredBalance != null ? undesiredShards : -1, forecastedWriteLoad, forecastedDiskUsage, @@ -109,10 +120,13 @@ public class NodeAllocationStatsProvider { ); } - return stats; + return nodeAllocationStatsAndWeights; } - private static boolean isDesiredAllocation(DesiredBalance desiredBalance, ShardRouting shardRouting) { + /** + * Checks whether a shard is currently allocated to a node that is wanted by the desired balance decision. + */ + private static boolean isDesiredAllocation(@Nullable DesiredBalance desiredBalance, ShardRouting shardRouting) { if (desiredBalance == null) { return true; } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index 8dd1f14564ce..0ce160523463 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -112,6 +112,7 @@ public class BalancedShardsAllocator implements ShardsAllocator { Property.NodeScope ); + // TODO: deduplicate these fields, use the fields in NodeAllocationStatsAndWeightsCalculator instead. private volatile float indexBalanceFactor; private volatile float shardBalanceFactor; private volatile float writeLoadBalanceFactor; @@ -168,7 +169,7 @@ public class BalancedShardsAllocator implements ShardsAllocator { Map nodeLevelWeights = new HashMap<>(); for (var entry : balancer.nodes.entrySet()) { var node = entry.getValue(); - var nodeWeight = weightFunction.nodeWeight( + var nodeWeight = weightFunction.calculateNodeWeight( node.numShards(), balancer.avgShardsPerNode(), node.writeLoad(), @@ -263,7 +264,7 @@ public class BalancedShardsAllocator implements ShardsAllocator { private final RoutingAllocation allocation; private final RoutingNodes routingNodes; private final Metadata metadata; - private final WeightFunction weight; + private final WeightFunction weightFunction; private final float threshold; private final float avgShardsPerNode; @@ -272,12 +273,17 @@ public class BalancedShardsAllocator implements ShardsAllocator { private final Map nodes; private final NodeSorter sorter; - private Balancer(WriteLoadForecaster writeLoadForecaster, RoutingAllocation allocation, WeightFunction weight, float threshold) { + private Balancer( + WriteLoadForecaster writeLoadForecaster, + RoutingAllocation allocation, + WeightFunction weightFunction, + float threshold + ) { this.writeLoadForecaster = writeLoadForecaster; this.allocation = allocation; this.routingNodes = allocation.routingNodes(); this.metadata = allocation.metadata(); - this.weight = weight; + this.weightFunction = weightFunction; this.threshold = threshold; avgShardsPerNode = WeightFunction.avgShardPerNode(metadata, routingNodes); avgWriteLoadPerNode = WeightFunction.avgWriteLoadPerNode(writeLoadForecaster, metadata, routingNodes); @@ -355,7 +361,7 @@ public class BalancedShardsAllocator implements ShardsAllocator { * to sort based on an index. */ private NodeSorter newNodeSorter() { - return new NodeSorter(nodesArray(), weight, this); + return new NodeSorter(nodesArray(), weightFunction, this); } /** @@ -435,7 +441,7 @@ public class BalancedShardsAllocator implements ShardsAllocator { // balance the shard, if a better node can be found final String idxName = shard.getIndexName(); - final float currentWeight = weight.weight(this, currentNode, idxName); + final float currentWeight = weightFunction.calculateNodeWeightWithIndex(this, currentNode, idxName); final AllocationDeciders deciders = allocation.deciders(); Type rebalanceDecisionType = Type.NO; ModelNode targetNode = null; @@ -451,7 +457,7 @@ public class BalancedShardsAllocator implements ShardsAllocator { // this is a comparison of the number of shards on this node to the number of shards // that should be on each node on average (both taking the cluster as a whole into account // as well as shards per index) - final float nodeWeight = weight.weight(this, node, idxName); + final float nodeWeight = weightFunction.calculateNodeWeightWithIndex(this, node, idxName); // if the node we are examining has a worse (higher) weight than the node the shard is // assigned to, then there is no way moving the shard to the node with the worse weight // can make the balance of the cluster better, so we check for that here @@ -1028,7 +1034,7 @@ public class BalancedShardsAllocator implements ShardsAllocator { } // weight of this index currently on the node - float currentWeight = weight.weight(this, node, shard.getIndexName()); + float currentWeight = weightFunction.calculateNodeWeightWithIndex(this, node, shard.getIndexName()); // moving the shard would not improve the balance, and we are not in explain mode, so short circuit if (currentWeight > minWeight && explain == false) { continue; @@ -1319,7 +1325,7 @@ public class BalancedShardsAllocator implements ShardsAllocator { } public float weight(ModelNode node) { - return function.weight(balancer, node, index); + return function.calculateNodeWeightWithIndex(balancer, node, index); } public float minWeightDelta() { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalance.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalance.java index 406ca72868a4..0745c45b2183 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalance.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalance.java @@ -21,7 +21,7 @@ import java.util.Objects; * * @param assignments a set of the (persistent) node IDs to which each {@link ShardId} should be allocated * @param weightsPerNode The node weights calculated based on - * {@link org.elasticsearch.cluster.routing.allocation.allocator.WeightFunction#nodeWeight} + * {@link org.elasticsearch.cluster.routing.allocation.allocator.WeightFunction#calculateNodeWeight} */ public record DesiredBalance( long lastConvergedIndex, diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java index 9f6487bdc8ab..fddf9267cdbb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceMetrics.java @@ -10,7 +10,7 @@ package org.elasticsearch.cluster.routing.allocation.allocator; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsProvider.NodeAllocationAndClusterBalanceStats; +import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator.NodeAllocationStatsAndWeight; import org.elasticsearch.telemetry.metric.DoubleWithAttributes; import org.elasticsearch.telemetry.metric.LongWithAttributes; import org.elasticsearch.telemetry.metric.MeterRegistry; @@ -20,6 +20,12 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +/** + * Maintains balancer metrics and makes them accessible to the {@link MeterRegistry} and APM reporting. Metrics are updated + * ({@link #updateMetrics}) or cleared ({@link #zeroAllMetrics}) as a result of cluster events and the metrics will be pulled for reporting + * via the MeterRegistry implementation. Only the master node reports metrics: see {@link #setNodeIsMaster}. When + * {@link #nodeIsMaster} is false, empty values are returned such that MeterRegistry ignores the metrics for reporting purposes. + */ public class DesiredBalanceMetrics { public record AllocationStats(long unassignedShards, long totalAllocations, long undesiredAllocationsExcludingShuttingDownNodes) {} @@ -69,13 +75,14 @@ public class DesiredBalanceMetrics { private volatile long undesiredAllocations; private final AtomicReference> weightStatsPerNodeRef = new AtomicReference<>(Map.of()); - private final AtomicReference> allocationStatsPerNodeRef = - new AtomicReference<>(Map.of()); + private final AtomicReference> allocationStatsPerNodeRef = new AtomicReference<>( + Map.of() + ); public void updateMetrics( AllocationStats allocationStats, Map weightStatsPerNode, - Map nodeAllocationStats + Map nodeAllocationStats ) { assert allocationStats != null : "allocation stats cannot be null"; assert weightStatsPerNode != null : "node balance weight stats cannot be null"; @@ -170,6 +177,10 @@ public class DesiredBalanceMetrics { ); } + /** + * When {@link #nodeIsMaster} is set to true, the server will report APM metrics registered in this file. When set to false, empty + * values will be returned such that no APM metrics are sent from this server. + */ public void setNodeIsMaster(boolean nodeIsMaster) { this.nodeIsMaster = nodeIsMaster; } @@ -339,6 +350,10 @@ public class DesiredBalanceMetrics { return List.of(); } + /** + * Sets all the internal class fields to zero/empty. Typically used in conjunction with {@link #setNodeIsMaster}. + * This is best-effort because it is possible for {@link #updateMetrics} to race with this method. + */ public void zeroAllMetrics() { unassignedShards = 0; totalAllocations = 0; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java index 2ee905634f76..83b370c1a792 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java @@ -20,8 +20,8 @@ import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.UnassignedInfo.AllocationStatus; -import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsProvider; -import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsProvider.NodeAllocationAndClusterBalanceStats; +import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator; +import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator.NodeAllocationStatsAndWeight; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceMetrics.AllocationStats; import org.elasticsearch.cluster.routing.allocation.decider.Decision; @@ -75,13 +75,13 @@ public class DesiredBalanceReconciler { private final NodeAllocationOrdering allocationOrdering = new NodeAllocationOrdering(); private final NodeAllocationOrdering moveOrdering = new NodeAllocationOrdering(); private final DesiredBalanceMetrics desiredBalanceMetrics; - private final NodeAllocationStatsProvider nodeAllocationStatsProvider; + private final NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator; public DesiredBalanceReconciler( ClusterSettings clusterSettings, ThreadPool threadPool, DesiredBalanceMetrics desiredBalanceMetrics, - NodeAllocationStatsProvider nodeAllocationStatsProvider + NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator ) { this.desiredBalanceMetrics = desiredBalanceMetrics; this.undesiredAllocationLogInterval = new FrequencyCappedAction( @@ -93,7 +93,7 @@ public class DesiredBalanceReconciler { UNDESIRED_ALLOCATIONS_LOG_THRESHOLD_SETTING, value -> this.undesiredAllocationsLogThreshold = value ); - this.nodeAllocationStatsProvider = nodeAllocationStatsProvider; + this.nodeAllocationStatsAndWeightsCalculator = nodeAllocationStatsAndWeightsCalculator; } public void reconcile(DesiredBalance desiredBalance, RoutingAllocation allocation) { @@ -159,20 +159,22 @@ public class DesiredBalanceReconciler { } private void updateDesireBalanceMetrics(AllocationStats allocationStats) { - var stats = nodeAllocationStatsProvider.stats( + var nodesStatsAndWeights = nodeAllocationStatsAndWeightsCalculator.nodesAllocationStatsAndWeights( allocation.metadata(), allocation.routingNodes(), allocation.clusterInfo(), desiredBalance ); - Map nodeAllocationStats = new HashMap<>(stats.size()); - for (var entry : stats.entrySet()) { - var node = allocation.nodes().get(entry.getKey()); + Map filteredNodeAllocationStatsAndWeights = new HashMap<>( + nodesStatsAndWeights.size() + ); + for (var nodeStatsAndWeight : nodesStatsAndWeights.entrySet()) { + var node = allocation.nodes().get(nodeStatsAndWeight.getKey()); if (node != null) { - nodeAllocationStats.put(node, entry.getValue()); + filteredNodeAllocationStatsAndWeights.put(node, nodeStatsAndWeight.getValue()); } } - desiredBalanceMetrics.updateMetrics(allocationStats, desiredBalance.weightsPerNode(), nodeAllocationStats); + desiredBalanceMetrics.updateMetrics(allocationStats, desiredBalance.weightsPerNode(), filteredNodeAllocationStatsAndWeights); } private boolean allocateUnassignedInvariant() { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java index 8408386b8da5..5be26f0b3e8c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java @@ -18,7 +18,7 @@ import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.AllocationService.RerouteStrategy; -import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsProvider; +import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.RoutingExplanations; import org.elasticsearch.cluster.routing.allocation.ShardAllocationDecision; @@ -89,7 +89,7 @@ public class DesiredBalanceShardsAllocator implements ShardsAllocator { ClusterService clusterService, DesiredBalanceReconcilerAction reconciler, TelemetryProvider telemetryProvider, - NodeAllocationStatsProvider nodeAllocationStatsProvider + NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator ) { this( delegateAllocator, @@ -98,7 +98,7 @@ public class DesiredBalanceShardsAllocator implements ShardsAllocator { new DesiredBalanceComputer(clusterSettings, threadPool, delegateAllocator), reconciler, telemetryProvider, - nodeAllocationStatsProvider + nodeAllocationStatsAndWeightsCalculator ); } @@ -109,7 +109,7 @@ public class DesiredBalanceShardsAllocator implements ShardsAllocator { DesiredBalanceComputer desiredBalanceComputer, DesiredBalanceReconcilerAction reconciler, TelemetryProvider telemetryProvider, - NodeAllocationStatsProvider nodeAllocationStatsProvider + NodeAllocationStatsAndWeightsCalculator nodeAllocationStatsAndWeightsCalculator ) { this.desiredBalanceMetrics = new DesiredBalanceMetrics(telemetryProvider.getMeterRegistry()); this.delegateAllocator = delegateAllocator; @@ -120,7 +120,7 @@ public class DesiredBalanceShardsAllocator implements ShardsAllocator { clusterService.getClusterSettings(), threadPool, desiredBalanceMetrics, - nodeAllocationStatsProvider + nodeAllocationStatsAndWeightsCalculator ); this.desiredBalanceComputation = new ContinuousComputation<>(threadPool.generic()) { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/WeightFunction.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/WeightFunction.java index 7203a92b147f..66660ed7a803 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/WeightFunction.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/WeightFunction.java @@ -58,9 +58,9 @@ public class WeightFunction { theta3 = diskUsageBalance / sum; } - float weight(BalancedShardsAllocator.Balancer balancer, BalancedShardsAllocator.ModelNode node, String index) { + float calculateNodeWeightWithIndex(BalancedShardsAllocator.Balancer balancer, BalancedShardsAllocator.ModelNode node, String index) { final float weightIndex = node.numShards(index) - balancer.avgShardsPerNode(index); - final float nodeWeight = nodeWeight( + final float nodeWeight = calculateNodeWeight( node.numShards(), balancer.avgShardsPerNode(), node.writeLoad(), @@ -71,7 +71,7 @@ public class WeightFunction { return nodeWeight + theta1 * weightIndex; } - public float nodeWeight( + public float calculateNodeWeight( int nodeNumShards, float avgShardsPerNode, double nodeWriteLoad, diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsServiceTests.java index 35f178046465..2a4d91437df0 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationStatsServiceTests.java @@ -84,7 +84,7 @@ public class AllocationStatsServiceTests extends ESAllocationTestCase { clusterService, () -> clusterInfo, createShardAllocator(), - new NodeAllocationStatsProvider(TEST_WRITE_LOAD_FORECASTER, ClusterSettings.createBuiltInClusterSettings()) + new NodeAllocationStatsAndWeightsCalculator(TEST_WRITE_LOAD_FORECASTER, ClusterSettings.createBuiltInClusterSettings()) ); assertThat( service.stats(), @@ -125,7 +125,7 @@ public class AllocationStatsServiceTests extends ESAllocationTestCase { clusterService, EmptyClusterInfoService.INSTANCE, createShardAllocator(), - new NodeAllocationStatsProvider(TEST_WRITE_LOAD_FORECASTER, ClusterSettings.createBuiltInClusterSettings()) + new NodeAllocationStatsAndWeightsCalculator(TEST_WRITE_LOAD_FORECASTER, ClusterSettings.createBuiltInClusterSettings()) ); assertThat( service.stats(), @@ -182,7 +182,7 @@ public class AllocationStatsServiceTests extends ESAllocationTestCase { ); } }, - new NodeAllocationStatsProvider(TEST_WRITE_LOAD_FORECASTER, ClusterSettings.createBuiltInClusterSettings()) + new NodeAllocationStatsAndWeightsCalculator(TEST_WRITE_LOAD_FORECASTER, ClusterSettings.createBuiltInClusterSettings()) ); assertThat( service.stats(), diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java index 75cd6da44724..132dd4b11946 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/ESAllocationTestCase.java @@ -25,7 +25,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.FailedShard; -import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsProvider; +import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.WriteLoadForecaster; import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; @@ -437,18 +437,16 @@ public abstract class ESAllocationTestCase extends ESTestCase { } } - protected static final NodeAllocationStatsProvider EMPTY_NODE_ALLOCATION_STATS = new NodeAllocationStatsProvider( - WriteLoadForecaster.DEFAULT, - createBuiltInClusterSettings() - ) { - @Override - public Map stats( - Metadata metadata, - RoutingNodes routingNodes, - ClusterInfo clusterInfo, - @Nullable DesiredBalance desiredBalance - ) { - return Map.of(); - } - }; + protected static final NodeAllocationStatsAndWeightsCalculator EMPTY_NODE_ALLOCATION_STATS = + new NodeAllocationStatsAndWeightsCalculator(WriteLoadForecaster.DEFAULT, createBuiltInClusterSettings()) { + @Override + public Map nodesAllocationStatsAndWeights( + Metadata metadata, + RoutingNodes routingNodes, + ClusterInfo clusterInfo, + @Nullable DesiredBalance desiredBalance + ) { + return Map.of(); + } + }; } From 0ca32f5dcc4a1361cbc3d7217754fd0afe89bbbe Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 9 Jan 2025 15:35:38 +0100 Subject: [PATCH 23/33] Faster and shorter SearchPhaseController.reduceQueryPhase (#119855) We can avoid one list copy and some indirection by collecting to a list of non-null query responses instead of non-null generic responses right away (this also avoids the unfortunate reassignment of a method parameter). Also, this method is fairly long, this at least removes all redundant local variables and as a result a bit of computation in some cases. --- .../action/search/SearchPhaseController.java | 78 ++++++++----------- 1 file changed, 34 insertions(+), 44 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index b118c2560925..69e7fba4dd0d 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -551,9 +551,8 @@ public final class SearchPhaseController { assert numReducePhases >= 0 : "num reduce phases must be >= 0 but was: " + numReducePhases; numReducePhases++; // increment for this phase if (queryResults.isEmpty()) { // early terminate we have nothing to reduce - final TotalHits totalHits = topDocsStats.getTotalHits(); return new ReducedQueryPhase( - totalHits, + topDocsStats.getTotalHits(), topDocsStats.fetchHits, topDocsStats.getMaxScore(), false, @@ -570,8 +569,7 @@ public final class SearchPhaseController { true ); } - int total = queryResults.size(); - final Collection nonNullResults = new ArrayList<>(); + final List nonNullResults = new ArrayList<>(); boolean hasSuggest = false; boolean hasProfileResults = false; for (SearchPhaseResult queryResult : queryResults) { @@ -581,12 +579,11 @@ public final class SearchPhaseController { } hasSuggest |= res.suggest() != null; hasProfileResults |= res.hasProfileResults(); - nonNullResults.add(queryResult); + nonNullResults.add(res); } - queryResults = nonNullResults; - validateMergeSortValueFormats(queryResults); - if (queryResults.isEmpty()) { - var ex = new IllegalStateException("must have at least one non-empty search result, got 0 out of " + total); + validateMergeSortValueFormats(nonNullResults); + if (nonNullResults.isEmpty()) { + var ex = new IllegalStateException("must have at least one non-empty search result, got 0 out of " + queryResults.size()); assert false : ex; throw ex; } @@ -594,13 +591,12 @@ public final class SearchPhaseController { // count the total (we use the query result provider here, since we might not get any hits (we scrolled past them)) final Map>> groupedSuggestions = hasSuggest ? new HashMap<>() : Collections.emptyMap(); final Map profileShardResults = hasProfileResults - ? Maps.newMapWithExpectedSize(queryResults.size()) + ? Maps.newMapWithExpectedSize(nonNullResults.size()) : Collections.emptyMap(); int from = 0; int size = 0; DocValueFormat[] sortValueFormats = null; - for (SearchPhaseResult entry : queryResults) { - QuerySearchResult result = entry.queryResult(); + for (QuerySearchResult result : nonNullResults) { from = result.from(); // sorted queries can set the size to 0 if they have enough competitive hits. size = Math.max(result.size(), size); @@ -611,8 +607,7 @@ public final class SearchPhaseController { if (hasSuggest) { assert result.suggest() != null; for (Suggestion> suggestion : result.suggest()) { - List> suggestionList = groupedSuggestions.computeIfAbsent(suggestion.getName(), s -> new ArrayList<>()); - suggestionList.add(suggestion); + groupedSuggestions.computeIfAbsent(suggestion.getName(), s -> new ArrayList<>()).add(suggestion); if (suggestion instanceof CompletionSuggestion completionSuggestion) { completionSuggestion.setShardIndex(result.getShardIndex()); } @@ -620,53 +615,48 @@ public final class SearchPhaseController { } assert bufferedTopDocs.isEmpty() || result.hasConsumedTopDocs() : "firstResult has no aggs but we got non null buffered aggs?"; if (hasProfileResults) { - String key = result.getSearchShardTarget().toString(); - profileShardResults.put(key, result.consumeProfileResult()); + profileShardResults.put(result.getSearchShardTarget().toString(), result.consumeProfileResult()); } } - final Suggest reducedSuggest; - final List reducedCompletionSuggestions; - if (groupedSuggestions.isEmpty()) { - reducedSuggest = null; - reducedCompletionSuggestions = Collections.emptyList(); - } else { - reducedSuggest = new Suggest(Suggest.reduce(groupedSuggestions)); - reducedCompletionSuggestions = reducedSuggest.filter(CompletionSuggestion.class); - } - final InternalAggregations aggregations = bufferedAggs == null - ? null - : InternalAggregations.topLevelReduceDelayable( - bufferedAggs, - performFinalReduce ? aggReduceContextBuilder.forFinalReduction() : aggReduceContextBuilder.forPartialReduction() - ); - final SearchProfileResultsBuilder profileBuilder = profileShardResults.isEmpty() - ? null - : new SearchProfileResultsBuilder(profileShardResults); + final Suggest reducedSuggest = groupedSuggestions.isEmpty() ? null : new Suggest(Suggest.reduce(groupedSuggestions)); final SortedTopDocs sortedTopDocs; if (queryPhaseRankCoordinatorContext == null) { - sortedTopDocs = sortDocs(isScrollRequest, bufferedTopDocs, from, size, reducedCompletionSuggestions); - } else { - ScoreDoc[] rankedDocs = queryPhaseRankCoordinatorContext.rankQueryPhaseResults( - queryResults.stream().map(SearchPhaseResult::queryResult).toList(), - topDocsStats + sortedTopDocs = sortDocs( + isScrollRequest, + bufferedTopDocs, + from, + size, + reducedSuggest == null ? Collections.emptyList() : reducedSuggest.filter(CompletionSuggestion.class) + ); + } else { + sortedTopDocs = new SortedTopDocs( + queryPhaseRankCoordinatorContext.rankQueryPhaseResults(nonNullResults, topDocsStats), + false, + null, + null, + null, + 0 ); - sortedTopDocs = new SortedTopDocs(rankedDocs, false, null, null, null, 0); size = sortedTopDocs.scoreDocs.length; // we need to reset from here as pagination and result trimming has already taken place // within the `QueryPhaseRankCoordinatorContext#rankQueryPhaseResults` and we don't want // to apply it again in the `getHits` method. from = 0; } - final TotalHits totalHits = topDocsStats.getTotalHits(); return new ReducedQueryPhase( - totalHits, + topDocsStats.getTotalHits(), topDocsStats.fetchHits, topDocsStats.getMaxScore(), topDocsStats.timedOut, topDocsStats.terminatedEarly, reducedSuggest, - aggregations, - profileBuilder, + bufferedAggs == null + ? null + : InternalAggregations.topLevelReduceDelayable( + bufferedAggs, + performFinalReduce ? aggReduceContextBuilder.forFinalReduction() : aggReduceContextBuilder.forPartialReduction() + ), + profileShardResults.isEmpty() ? null : new SearchProfileResultsBuilder(profileShardResults), sortedTopDocs, sortValueFormats, queryPhaseRankCoordinatorContext, From 844594aec41d8f4dcf69f46cd2015c3c52720b41 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 9 Jan 2025 09:46:02 -0500 Subject: [PATCH 24/33] ESQL: Fewer test cases for conversions (#119774) This applies that pattern from #119678 to the `convert` tests. It doesn't buy us any speed, sadly. This is because conversions are *unary* and we rarely get much bonus from unary functions for this. Still, we do marginally reduce the number of test cases which is good for gradle: ``` 13862 -> 11948 (14%) ``` --- .../function/scalar/package-info.java | 1 + ...ErrorsForCasesWithoutExamplesTestCase.java | 8 +++ .../scalar/convert/FromBase64ErrorTests.java | 37 +++++++++++++ .../scalar/convert/FromBase64Tests.java | 2 +- .../scalar/convert/ToBase64ErrorTests.java | 37 +++++++++++++ .../scalar/convert/ToBase64Tests.java | 2 +- .../scalar/convert/ToBooleanErrorTests.java | 37 +++++++++++++ .../scalar/convert/ToBooleanTests.java | 2 +- .../convert/ToCartesianPointErrorTests.java | 37 +++++++++++++ .../scalar/convert/ToCartesianPointTests.java | 2 +- .../convert/ToCartesianShapeErrorTests.java | 37 +++++++++++++ .../scalar/convert/ToCartesianShapeTests.java | 2 +- .../scalar/convert/ToDateNanosErrorTests.java | 44 ++++++++++++++++ .../ToDateNanosSerializationTests.java | 19 +++++++ .../scalar/convert/ToDateNanosTests.java | 6 +-- .../convert/ToDatePeriodErrorTests.java | 37 +++++++++++++ .../scalar/convert/ToDatePeriodTests.java | 4 +- .../scalar/convert/ToDatetimeErrorTests.java | 37 +++++++++++++ .../scalar/convert/ToDatetimeTests.java | 2 +- .../scalar/convert/ToDegreesErrorTests.java | 37 +++++++++++++ .../scalar/convert/ToDegreesTests.java | 2 +- .../scalar/convert/ToDoubleErrorTests.java | 44 ++++++++++++++++ .../scalar/convert/ToDoubleTests.java | 6 +-- .../scalar/convert/ToGeoPointErrorTests.java | 37 +++++++++++++ .../scalar/convert/ToGeoPointTests.java | 2 +- .../scalar/convert/ToGeoShapeErrorTests.java | 37 +++++++++++++ .../scalar/convert/ToGeoShapeTests.java | 2 +- .../scalar/convert/ToIPErrorTests.java | 37 +++++++++++++ .../function/scalar/convert/ToIPTests.java | 4 +- .../scalar/convert/ToIntegerErrorTests.java | 39 ++++++++++++++ .../scalar/convert/ToIntegerTests.java | 6 +-- .../scalar/convert/ToLongErrorTests.java | 44 ++++++++++++++++ .../function/scalar/convert/ToLongTests.java | 6 +-- .../scalar/convert/ToRadiansErrorTests.java | 37 +++++++++++++ .../scalar/convert/ToRadiansTests.java | 2 +- .../scalar/convert/ToStringErrorTests.java | 52 +++++++++++++++++++ .../scalar/convert/ToStringTests.java | 5 +- .../convert/ToTimeDurationErrorTests.java | 37 +++++++++++++ .../scalar/convert/ToTimeDurationTests.java | 4 +- .../convert/ToUnsignedLongErrorTests.java | 37 +++++++++++++ .../scalar/convert/ToUnsignedLongTests.java | 2 +- .../convert/ToVersionLongErrorTests.java | 37 +++++++++++++ .../scalar/convert/ToVersionTests.java | 2 +- 43 files changed, 826 insertions(+), 45 deletions(-) create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64ErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64ErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosSerializationTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongErrorTests.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionLongErrorTests.java diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java index eccc7ee4672c..d0b646a622b6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java @@ -131,6 +131,7 @@ * the moment, but it's good to extend {@code AbstractScalarFunctionTestCase}. All of * these tests are parameterized and expect to spend some time finding good parameters. * Also add serialization tests that extend {@code AbstractExpressionSerializationTests<>}. + * And also add type error tests that extends {@code ErrorsForCasesWithoutExamplesTestCase}. * *
  • * Once you are happy with the tests run the auto formatter: diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ErrorsForCasesWithoutExamplesTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ErrorsForCasesWithoutExamplesTestCase.java index e2ad9e81939d..7269abad0729 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ErrorsForCasesWithoutExamplesTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/ErrorsForCasesWithoutExamplesTestCase.java @@ -69,6 +69,14 @@ public abstract class ErrorsForCasesWithoutExamplesTestCase extends ESTestCase { checked++; } logger.info("checked {} signatures", checked); + assertNumberOfCheckedSignatures(checked); + } + + /** + * Assert the number of checked signature. Generally shouldn't be overridden but + * can be to assert that, for example, there weren't any unsupported signatures. + */ + protected void assertNumberOfCheckedSignatures(int checked) { assertThat("didn't check any signatures", checked, greaterThan(0)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64ErrorTests.java new file mode 100644 index 000000000000..390995781d34 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64ErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class FromBase64ErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(FromBase64Tests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new FromBase64(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java index 60901e2a8214..8f518d60cd47 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromBase64Tests.java @@ -47,7 +47,7 @@ public class FromBase64Tests extends AbstractScalarFunctionTestCase { })); } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64ErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64ErrorTests.java new file mode 100644 index 000000000000..362105003924 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64ErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToBase64ErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToBase64Tests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToBase64(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java index 6e6ff7bf52fc..dddf27a9ff56 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBase64Tests.java @@ -48,7 +48,7 @@ public class ToBase64Tests extends AbstractScalarFunctionTestCase { })); } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanErrorTests.java new file mode 100644 index 000000000000..3556a09db231 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToBooleanErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToBooleanTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToBoolean(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "boolean or numeric or string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java index 73837c23ff64..93a833f29cd0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToBooleanTests.java @@ -80,7 +80,7 @@ public class ToBooleanTests extends AbstractScalarFunctionTestCase { emptyList() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "boolean or numeric or string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointErrorTests.java new file mode 100644 index 000000000000..1933a1ba739a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToCartesianPointErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToCartesianPointTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToCartesianPoint(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "cartesian_point or string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java index 9548b0476f3c..a4c31445f86b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianPointTests.java @@ -72,7 +72,7 @@ public class ToCartesianPointTests extends AbstractScalarFunctionTestCase { ); } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "cartesian_point or string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeErrorTests.java new file mode 100644 index 000000000000..226bf623305e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToCartesianShapeErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToCartesianShapeTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToCartesianShape(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "cartesian_point or cartesian_shape or string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java index 5784c3559b10..9f8eb52de950 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToCartesianShapeTests.java @@ -73,7 +73,7 @@ public class ToCartesianShapeTests extends AbstractScalarFunctionTestCase { ); } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "cartesian_point or cartesian_shape or string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosErrorTests.java new file mode 100644 index 000000000000..b3a59c9f39ac --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosErrorTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToDateNanosErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToDateNanosTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToDateNanos(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo( + typeErrorMessage( + false, + validPerPosition, + signature, + (v, p) -> "date_nanos or datetime or double or long or string or unsigned_long" + ) + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosSerializationTests.java new file mode 100644 index 000000000000..4fb39bb8d8da --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosSerializationTests.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractUnaryScalarSerializationTests; + +public class ToDateNanosSerializationTests extends AbstractUnaryScalarSerializationTests { + @Override + protected ToDateNanos create(Source source, Expression child) { + return new ToDateNanos(source, child); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java index 7459abf29410..dc523bc24721 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanosTests.java @@ -132,11 +132,7 @@ public class ToDateNanosTests extends AbstractScalarFunctionTestCase { : ("failed to parse date field [" + bytesRef.utf8ToString() + "] with format [strict_date_optional_time_nanos]")) ) ); - return parameterSuppliersFromTypedDataWithDefaultChecks( - true, - suppliers, - (v, p) -> "date_nanos or datetime or double or long or string or unsigned_long" - ); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodErrorTests.java new file mode 100644 index 000000000000..3998fabb1574 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToDatePeriodErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToDatePeriodTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToDatePeriod(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "date_period or string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java index ef71f404eda0..9abbfbd61c1a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java @@ -71,9 +71,7 @@ public class ToDatePeriodTests extends AbstractScalarFunctionTestCase { })); } } - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), (v, p) -> "date_period or string") - ); + return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeErrorTests.java new file mode 100644 index 000000000000..e0ee9a84dde5 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToDatetimeErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToDatetimeTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToDatetime(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "date_nanos or datetime or numeric or string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java index 43b889baf530..98c6e0ea8adc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java @@ -178,7 +178,7 @@ public class ToDatetimeTests extends AbstractScalarFunctionTestCase { ) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "date_nanos or datetime or numeric or string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } private static String randomDateString(long from, long to) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesErrorTests.java new file mode 100644 index 000000000000..c19802a3887d --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToDegreesErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToDegreesTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToDegrees(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "numeric")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java index 4d52a470edb8..cc780624b80a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDegreesTests.java @@ -89,7 +89,7 @@ public class ToDegreesTests extends AbstractScalarFunctionTestCase { ) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleErrorTests.java new file mode 100644 index 000000000000..091193b147c5 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleErrorTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToDoubleErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToDoubleTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToDouble(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo( + typeErrorMessage( + false, + validPerPosition, + signature, + (v, p) -> "boolean or counter_double or counter_integer or counter_long or datetime or numeric or string" + ) + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java index 6fcd61bf15d0..c7e6b4f0f380 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDoubleTests.java @@ -141,11 +141,7 @@ public class ToDoubleTests extends AbstractScalarFunctionTestCase { List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks( - true, - suppliers, - (v, p) -> "boolean or counter_double or counter_integer or counter_long or datetime or numeric or string" - ); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointErrorTests.java new file mode 100644 index 000000000000..13b585a0478f --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToGeoPointErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToGeoPointTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToGeoPoint(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "geo_point or string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java index 445c4b8f15d1..5fdd83d853c7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoPointTests.java @@ -66,7 +66,7 @@ public class ToGeoPointTests extends AbstractScalarFunctionTestCase { ); } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "geo_point or string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeErrorTests.java new file mode 100644 index 000000000000..9e6f81e649d8 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToGeoShapeErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToGeoShapeTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToGeoShape(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "geo_point or geo_shape or string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java index 38d871b17e8b..9ac823553491 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToGeoShapeTests.java @@ -66,7 +66,7 @@ public class ToGeoShapeTests extends AbstractScalarFunctionTestCase { List.of() ); } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "geo_point or geo_shape or string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPErrorTests.java new file mode 100644 index 000000000000..31dd36041a6f --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToIPErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToIPTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToIP(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "ip or string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java index 66358b7ba744..22517ae99596 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java @@ -61,9 +61,7 @@ public class ToIPTests extends AbstractScalarFunctionTestCase { bytesRef -> parseIP(((BytesRef) bytesRef).utf8ToString()), emptyList() ); - - // add null as parameter - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "ip or string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerErrorTests.java new file mode 100644 index 000000000000..6ab9c0be50a7 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerErrorTests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToIntegerErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToIntegerTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToInteger(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo( + typeErrorMessage(false, validPerPosition, signature, (v, p) -> "boolean or counter_integer or datetime or numeric or string") + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java index 6a3f7022c9d3..0bd2aa24e80d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIntegerTests.java @@ -271,11 +271,7 @@ public class ToIntegerTests extends AbstractScalarFunctionTestCase { List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks( - true, - suppliers, - (v, p) -> "boolean or counter_integer or datetime or numeric or string" - ); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongErrorTests.java new file mode 100644 index 000000000000..cc93e4f05079 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongErrorTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToLongErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToLongTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToLong(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo( + typeErrorMessage( + false, + validPerPosition, + signature, + (v, p) -> "boolean or counter_integer or counter_long or date_nanos or datetime or numeric or string" + ) + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java index 49daeba43f13..c2d7565c3e6f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLongTests.java @@ -236,11 +236,7 @@ public class ToLongTests extends AbstractScalarFunctionTestCase { l -> ((Integer) l).longValue(), List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks( - true, - suppliers, - (v, p) -> "boolean or counter_integer or counter_long or date_nanos or datetime or numeric or string" - ); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansErrorTests.java new file mode 100644 index 000000000000..a132448aa38e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToRadiansErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToRadiansTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToRadians(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "numeric")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java index 6e99e720ad5e..92a5bdac99db 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToRadiansTests.java @@ -70,7 +70,7 @@ public class ToRadiansTests extends AbstractScalarFunctionTestCase { List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java new file mode 100644 index 000000000000..31088139b891 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringErrorTests.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToStringErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToStringTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToString(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> { + /* + * In general ToString should support all signatures. While building a + * new type you may we to temporarily remove this. + */ + throw new UnsupportedOperationException("all signatures should be supported"); + })); + } + + @Override + protected void assertNumberOfCheckedSignatures(int checked) { + /* + * In general ToString should support all signatures. While building a + * new type you may we to temporarily relax this. + */ + assertThat("all signatures should be supported", checked, equalTo(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java index 3b30e4b353ae..c89f8c34b845 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java @@ -141,12 +141,9 @@ public class ToStringTests extends AbstractScalarFunctionTestCase { v -> new BytesRef(v.toString()), List.of() ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> typeErrorString); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } - private static String typeErrorString = - "boolean or cartesian_point or cartesian_shape or datetime or geo_point or geo_shape or ip or numeric or string or version"; - @Override protected Expression build(Source source, List args) { return new ToString(source, args.get(0)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationErrorTests.java new file mode 100644 index 000000000000..c44212ed74b7 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToTimeDurationErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToTimeDurationTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToTimeDuration(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "time_duration or string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java index 00555f5c668c..6486f6efcdd3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java @@ -70,9 +70,7 @@ public class ToTimeDurationTests extends AbstractScalarFunctionTestCase { })); } } - return parameterSuppliersFromTypedData( - errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), (v, p) -> "time_duration or string") - ); + return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongErrorTests.java new file mode 100644 index 000000000000..9fbc036feaa3 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToUnsignedLongErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToUnsignedLongTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToUnsignedLong(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "boolean or datetime or numeric or string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java index ca48bb029f22..adf104e8db5b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToUnsignedLongTests.java @@ -246,7 +246,7 @@ public class ToUnsignedLongTests extends AbstractScalarFunctionTestCase { ) ); - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "boolean or datetime or numeric or string"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionLongErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionLongErrorTests.java new file mode 100644 index 000000000000..9dde42cb1265 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionLongErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class ToVersionLongErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(ToVersionTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new ToVersion(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string or version")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java index 57f11331818d..3a3026d1db67 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToVersionTests.java @@ -60,7 +60,7 @@ public class ToVersionTests extends AbstractScalarFunctionTestCase { ); } - return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "string or version"); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override From 84d04a620f7b9e3befaf2e1705176c91571cc726 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 9 Jan 2025 14:56:31 +0000 Subject: [PATCH 25/33] Rename methods in ChunkedToXContentHelper to be clearer (#119342) --- .../ingest/geoip/IngestGeoIpMetadata.java | 2 +- .../ClusterAllocationExplanation.java | 4 +- .../allocation/DesiredBalanceResponse.java | 14 +-- .../admin/cluster/node/stats/NodeStats.java | 12 +-- .../reroute/ClusterRerouteResponse.java | 2 +- .../segments/IndicesSegmentResponse.java | 14 ++- .../stats/FieldUsageStatsResponse.java | 4 +- .../indices/stats/IndicesStatsResponse.java | 6 +- .../action/search/SearchResponse.java | 8 +- .../elasticsearch/cluster/ClusterInfo.java | 10 +- .../elasticsearch/cluster/ClusterState.java | 2 +- .../metadata/ComponentTemplateMetadata.java | 2 +- .../ComposableIndexTemplateMetadata.java | 2 +- .../cluster/metadata/DataStreamMetadata.java | 2 +- .../cluster/metadata/Metadata.java | 8 +- .../metadata/NodesShutdownMetadata.java | 2 +- .../ShutdownShardMigrationStatus.java | 4 +- .../xcontent/ChunkedToXContentHelper.java | 92 ++++++++++--------- .../upgrades/FeatureMigrationResults.java | 2 +- .../nodes/BaseNodesXContentResponseTests.java | 2 +- .../autoscaling/AutoscalingMetadata.java | 2 +- .../xpack/core/ccr/AutoFollowMetadata.java | 6 +- .../xpack/core/enrich/EnrichMetadata.java | 2 +- .../core/ilm/IndexLifecycleMetadata.java | 2 +- .../core/ml/inference/ModelAliasMetadata.java | 2 +- .../search/action/AsyncSearchResponse.java | 4 +- .../core/slm/SnapshotLifecycleMetadata.java | 2 +- .../xpack/esql/action/EsqlExecutionInfo.java | 2 +- .../xpack/esql/action/EsqlQueryResponse.java | 4 +- .../esql/action/ResponseXContentUtils.java | 4 +- .../ServerSentEventsRestActionListener.java | 2 +- .../action/GetFlamegraphResponse.java | 16 ++-- .../action/GetStackTracesResponse.java | 8 +- .../shutdown/SingleNodeShutdownStatus.java | 36 ++++---- ...positoryVerifyIntegrityResponseStream.java | 8 +- 35 files changed, 145 insertions(+), 149 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java index 5b681644cd99..58ff64c97b2e 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpMetadata.java @@ -92,7 +92,7 @@ public final class IngestGeoIpMetadata implements Metadata.Custom { @Override public Iterator toXContentChunked(ToXContent.Params ignored) { - return Iterators.concat(ChunkedToXContentHelper.xContentValuesMap(DATABASES_FIELD.getPreferredName(), databases)); + return Iterators.concat(ChunkedToXContentHelper.xContentObjectFields(DATABASES_FIELD.getPreferredName(), databases)); } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java index 23f88540c266..3b6a161c3db8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanation.java @@ -37,7 +37,7 @@ import java.util.Iterator; import java.util.Locale; import static org.elasticsearch.cluster.routing.allocation.AbstractAllocationDecision.discoveryNodeToXContent; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.singleChunk; +import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk; /** * A {@code ClusterAllocationExplanation} is an explanation of why a shard is unassigned, @@ -169,7 +169,7 @@ public final class ClusterAllocationExplanation implements ChunkedToXContentObje } public Iterator toXContentChunked(ToXContent.Params params) { - return Iterators.concat(singleChunk((builder, p) -> { + return Iterators.concat(chunk((builder, p) -> { builder.startObject(); if (isSpecificShard() == false) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java index 2f22b63f78bc..2a35851f0017 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/DesiredBalanceResponse.java @@ -33,8 +33,8 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk; import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endObject; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.singleChunk; import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startObject; public class DesiredBalanceResponse extends ActionResponse implements ChunkedToXContentObject { @@ -88,7 +88,7 @@ public class DesiredBalanceResponse extends ActionResponse implements ChunkedToX @Override public Iterator toXContentChunked(ToXContent.Params params) { return Iterators.concat( - singleChunk( + chunk( (builder, p) -> builder.startObject() .field("stats", stats) .field("cluster_balance_stats", clusterBalanceStats) @@ -101,16 +101,16 @@ public class DesiredBalanceResponse extends ActionResponse implements ChunkedToX Iterators.flatMap( indexEntry.getValue().entrySet().iterator(), shardEntry -> Iterators.concat( - singleChunk((builder, p) -> builder.field(String.valueOf(shardEntry.getKey()))), + chunk((builder, p) -> builder.field(String.valueOf(shardEntry.getKey()))), shardEntry.getValue().toXContentChunked(params) ) ), endObject() ) ), - singleChunk((builder, p) -> builder.endObject().startObject("cluster_info")), + chunk((builder, p) -> builder.endObject().startObject("cluster_info")), clusterInfo.toXContentChunked(params), - singleChunk((builder, p) -> builder.endObject().endObject()) + chunk((builder, p) -> builder.endObject().endObject()) ); } @@ -173,9 +173,9 @@ public class DesiredBalanceResponse extends ActionResponse implements ChunkedToX @Override public Iterator toXContentChunked(ToXContent.Params params) { return Iterators.concat( - singleChunk((builder, p) -> builder.startObject().startArray("current")), + chunk((builder, p) -> builder.startObject().startArray("current")), current().iterator(), - singleChunk((builder, p) -> builder.endArray().field("desired").value(desired, p).endObject()) + chunk((builder, p) -> builder.endArray().field("desired").value(desired, p).endObject()) ); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java index 4ead19fbf30e..ae4d6cb92c08 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java @@ -45,7 +45,7 @@ import java.util.Iterator; import java.util.Map; import java.util.Objects; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.singleChunk; +import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk; /** * Node statistics (dynamic, changes depending on when created). @@ -347,7 +347,7 @@ public class NodeStats extends BaseNodeResponse implements ChunkedToXContent { @Override public Iterator toXContentChunked(ToXContent.Params outerParams) { - return Iterators.concat(singleChunk((builder, params) -> { + return Iterators.concat(chunk((builder, params) -> { builder.field("name", getNode().getName()); builder.field("transport_address", getNode().getAddress().toString()); builder.field("host", getNode().getHostName()); @@ -369,9 +369,7 @@ public class NodeStats extends BaseNodeResponse implements ChunkedToXContent { return builder; }), ifPresent(getIndices()).toXContentChunked(outerParams), - singleChunk( - (builder, p) -> builder.value(ifPresent(getOs()), p).value(ifPresent(getProcess()), p).value(ifPresent(getJvm()), p) - ), + chunk((builder, p) -> builder.value(ifPresent(getOs()), p).value(ifPresent(getProcess()), p).value(ifPresent(getJvm()), p)), ifPresent(getThreadPool()).toXContentChunked(outerParams), singleChunkIfPresent(getFs()), ifPresent(getTransport()).toXContentChunked(outerParams), @@ -382,7 +380,7 @@ public class NodeStats extends BaseNodeResponse implements ChunkedToXContent { ifPresent(getIngestStats()).toXContentChunked(outerParams), singleChunkIfPresent(getAdaptiveSelectionStats()), singleChunkIfPresent(getScriptCacheStats()), - singleChunk( + chunk( (builder, p) -> builder.value(ifPresent(getIndexingPressureStats()), p) .value(ifPresent(getRepositoriesStats()), p) .value(ifPresent(getNodeAllocationStats()), p) @@ -399,6 +397,6 @@ public class NodeStats extends BaseNodeResponse implements ChunkedToXContent { } private static Iterator singleChunkIfPresent(ToXContent toXContent) { - return toXContent == null ? Collections.emptyIterator() : ChunkedToXContentHelper.singleChunk(toXContent); + return toXContent == null ? Collections.emptyIterator() : ChunkedToXContentHelper.chunk(toXContent); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java index 3f12c9e7a5bd..f04db5eec667 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponse.java @@ -97,7 +97,7 @@ public class ClusterRerouteResponse extends ActionResponse implements IsAcknowle return Iterators.concat( Iterators.single((builder, params) -> builder.startObject().field(ACKNOWLEDGED_KEY, isAcknowledged())), emitState(outerParams) - ? ChunkedToXContentHelper.wrapWithObject("state", state.toXContentChunked(outerParams)) + ? ChunkedToXContentHelper.object("state", state.toXContentChunked(outerParams)) : Collections.emptyIterator(), Iterators.single((builder, params) -> { if (params.paramAsBoolean("explain", false)) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java index 45d784d301bf..d34e71f715a5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java @@ -78,9 +78,7 @@ public class IndicesSegmentResponse extends ChunkedBroadcastResponse { getIndices().values().iterator(), indexSegments -> Iterators.concat( - ChunkedToXContentHelper.singleChunk( - (builder, p) -> builder.startObject(indexSegments.getIndex()).startObject(Fields.SHARDS) - ), + ChunkedToXContentHelper.chunk((builder, p) -> builder.startObject(indexSegments.getIndex()).startObject(Fields.SHARDS)), Iterators.flatMap( indexSegments.iterator(), indexSegment -> Iterators.concat( @@ -90,7 +88,7 @@ public class IndicesSegmentResponse extends ChunkedBroadcastResponse { indexSegment.iterator(), shardSegments -> Iterators.concat( - ChunkedToXContentHelper.singleChunk((builder, p) -> { + ChunkedToXContentHelper.chunk((builder, p) -> { builder.startObject(); builder.startObject(Fields.ROUTING); @@ -112,7 +110,7 @@ public class IndicesSegmentResponse extends ChunkedBroadcastResponse { shardSegments.iterator(), segment -> Iterators.concat( - ChunkedToXContentHelper.singleChunk((builder, p) -> { + ChunkedToXContentHelper.chunk((builder, p) -> { builder.startObject(segment.getName()); builder.field(Fields.GENERATION, segment.getGeneration()); builder.field(Fields.NUM_DOCS, segment.getNumDocs()); @@ -132,7 +130,7 @@ public class IndicesSegmentResponse extends ChunkedBroadcastResponse { return builder; }), getSegmentSortChunks(segment.getSegmentSort()), - ChunkedToXContentHelper.singleChunk((builder, p) -> { + ChunkedToXContentHelper.chunk((builder, p) -> { if (segment.attributes != null && segment.attributes.isEmpty() == false) { builder.field("attributes", segment.attributes); } @@ -141,13 +139,13 @@ public class IndicesSegmentResponse extends ChunkedBroadcastResponse { }) ) ), - ChunkedToXContentHelper.singleChunk((builder, p) -> builder.endObject().endObject()) + ChunkedToXContentHelper.chunk((builder, p) -> builder.endObject().endObject()) ) ), ChunkedToXContentHelper.endArray() ) ), - ChunkedToXContentHelper.singleChunk((builder, p) -> builder.endObject().endObject()) + ChunkedToXContentHelper.chunk((builder, p) -> builder.endObject().endObject()) ) ), ChunkedToXContentHelper.endObject() diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java index 50b46aa5e284..dfb22a4f0cba 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/FieldUsageStatsResponse.java @@ -50,9 +50,9 @@ public class FieldUsageStatsResponse extends ChunkedBroadcastResponse { return Iterators.flatMap( stats.entrySet().stream().sorted(Map.Entry.comparingByKey()).iterator(), entry -> Iterators.concat( - ChunkedToXContentHelper.singleChunk((builder, p) -> builder.startObject(entry.getKey()).startArray("shards")), + ChunkedToXContentHelper.chunk((builder, p) -> builder.startObject(entry.getKey()).startArray("shards")), entry.getValue().iterator(), - ChunkedToXContentHelper.singleChunk((builder, p) -> builder.endArray().endObject()) + ChunkedToXContentHelper.chunk((builder, p) -> builder.endArray().endObject()) ) ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java index 91e0e7cbc1df..d6c9a3e2e544 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java @@ -204,7 +204,7 @@ public class IndicesStatsResponse extends ChunkedBroadcastResponse { if (level == ClusterStatsLevel.INDICES || level == ClusterStatsLevel.SHARDS) { return Iterators.concat( - ChunkedToXContentHelper.singleChunk((builder, p) -> { + ChunkedToXContentHelper.chunk((builder, p) -> { commonStats(builder, p); return builder.startObject(Fields.INDICES); }), @@ -212,7 +212,7 @@ public class IndicesStatsResponse extends ChunkedBroadcastResponse { getIndices().values().iterator(), indexStats -> Iterators.concat( - ChunkedToXContentHelper.singleChunk((builder, p) -> { + ChunkedToXContentHelper.chunk((builder, p) -> { builder.startObject(indexStats.getIndex()); builder.field("uuid", indexStats.getUuid()); if (indexStats.getHealth() != null) { @@ -257,7 +257,7 @@ public class IndicesStatsResponse extends ChunkedBroadcastResponse { ChunkedToXContentHelper.endObject() ); } else { - return ChunkedToXContentHelper.singleChunk((builder, p) -> { + return ChunkedToXContentHelper.chunk((builder, p) -> { commonStats(builder, p); return builder; }); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java index 6074a591a976..787dc14f6cd9 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchResponse.java @@ -391,13 +391,13 @@ public class SearchResponse extends ActionResponse implements ChunkedToXContentO public Iterator innerToXContentChunked(ToXContent.Params params) { return Iterators.concat( - ChunkedToXContentHelper.singleChunk(SearchResponse.this::headerToXContent), + ChunkedToXContentHelper.chunk(SearchResponse.this::headerToXContent), Iterators.single(clusters), Iterators.concat( hits.toXContentChunked(params), - aggregations == null ? Collections.emptyIterator() : ChunkedToXContentHelper.singleChunk(aggregations), - suggest == null ? Collections.emptyIterator() : ChunkedToXContentHelper.singleChunk(suggest), - profileResults == null ? Collections.emptyIterator() : ChunkedToXContentHelper.singleChunk(profileResults) + aggregations == null ? Collections.emptyIterator() : ChunkedToXContentHelper.chunk(aggregations), + suggest == null ? Collections.emptyIterator() : ChunkedToXContentHelper.chunk(suggest), + profileResults == null ? Collections.emptyIterator() : ChunkedToXContentHelper.chunk(profileResults) ) ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java index fc7aa65e80f5..66fbe35fa52b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java @@ -36,8 +36,8 @@ import java.util.Set; import static org.elasticsearch.cluster.routing.ShardRouting.newUnassigned; import static org.elasticsearch.cluster.routing.UnassignedInfo.Reason.REINITIALIZED; +import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk; import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endArray; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.singleChunk; import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startObject; /** @@ -162,7 +162,7 @@ public class ClusterInfo implements ChunkedToXContent, Writeable { } return builder.endObject(); // end $nodename }), - singleChunk( + chunk( (builder, p) -> builder.endObject() // end "nodes" .startObject("shard_sizes") ), @@ -171,7 +171,7 @@ public class ClusterInfo implements ChunkedToXContent, Writeable { shardSizes.entrySet().iterator(), c -> (builder, p) -> builder.humanReadableField(c.getKey() + "_bytes", c.getKey(), ByteSizeValue.ofBytes(c.getValue())) ), - singleChunk( + chunk( (builder, p) -> builder.endObject() // end "shard_sizes" .startObject("shard_data_set_sizes") ), @@ -183,12 +183,12 @@ public class ClusterInfo implements ChunkedToXContent, Writeable { ByteSizeValue.ofBytes(c.getValue()) ) ), - singleChunk( + chunk( (builder, p) -> builder.endObject() // end "shard_data_set_sizes" .startObject("shard_paths") ), Iterators.map(dataPath.entrySet().iterator(), c -> (builder, p) -> builder.field(c.getKey().toString(), c.getValue())), - singleChunk( + chunk( (builder, p) -> builder.endObject() // end "shard_paths" .startArray("reserved_sizes") ), diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java index ddc74ec5ff24..62f2947d06a4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterState.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterState.java @@ -760,7 +760,7 @@ public class ClusterState implements ChunkedToXContent, Diffable { metrics.contains(Metric.CUSTOMS) ? Iterators.flatMap( customs.entrySet().iterator(), - e -> ChunkedToXContentHelper.wrapWithObject(e.getKey(), e.getValue().toXContentChunked(outerParams)) + e -> ChunkedToXContentHelper.object(e.getKey(), e.getValue().toXContentChunked(outerParams)) ) : Collections.emptyIterator() ); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java index 054ba1415f96..ed1ad1fc9ebb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComponentTemplateMetadata.java @@ -103,7 +103,7 @@ public class ComponentTemplateMetadata implements Metadata.Custom { @Override public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentValuesMap(COMPONENT_TEMPLATE.getPreferredName(), componentTemplates); + return ChunkedToXContentHelper.xContentObjectFields(COMPONENT_TEMPLATE.getPreferredName(), componentTemplates); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java index a2c3775d26d8..95f08963b62b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateMetadata.java @@ -104,7 +104,7 @@ public class ComposableIndexTemplateMetadata implements Metadata.Custom { @Override public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentValuesMap(INDEX_TEMPLATE.getPreferredName(), indexTemplates); + return ChunkedToXContentHelper.xContentObjectFields(INDEX_TEMPLATE.getPreferredName(), indexTemplates); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java index 8606c26ac946..67117df97441 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamMetadata.java @@ -234,7 +234,7 @@ public class DataStreamMetadata implements Metadata.Custom { @Override public Iterator toXContentChunked(ToXContent.Params ignored) { return Iterators.concat( - ChunkedToXContentHelper.xContentValuesMap(DATA_STREAM.getPreferredName(), dataStreams), + ChunkedToXContentHelper.xContentObjectFields(DATA_STREAM.getPreferredName(), dataStreams), ChunkedToXContentHelper.startObject(DATA_STREAM_ALIASES.getPreferredName()), dataStreamAliases.values().iterator(), ChunkedToXContentHelper.endObject() diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index a9617058acd6..6a4f24818003 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -1517,7 +1517,7 @@ public class Metadata implements Iterable, Diffable, Ch : Collections.emptyIterator(); final Iterator indices = context == XContentContext.API - ? ChunkedToXContentHelper.wrapWithObject("indices", indices().values().iterator()) + ? ChunkedToXContentHelper.object("indices", indices().values().iterator()) : Collections.emptyIterator(); return Iterators.concat(start, Iterators.single((builder, params) -> { @@ -1528,7 +1528,7 @@ public class Metadata implements Iterable, Diffable, Ch return builder.endObject(); }), persistentSettings, - ChunkedToXContentHelper.wrapWithObject( + ChunkedToXContentHelper.object( "templates", Iterators.map( templates().values().iterator(), @@ -1539,10 +1539,10 @@ public class Metadata implements Iterable, Diffable, Ch Iterators.flatMap( customs.entrySet().iterator(), entry -> entry.getValue().context().contains(context) - ? ChunkedToXContentHelper.wrapWithObject(entry.getKey(), entry.getValue().toXContentChunked(p)) + ? ChunkedToXContentHelper.object(entry.getKey(), entry.getValue().toXContentChunked(p)) : Collections.emptyIterator() ), - ChunkedToXContentHelper.wrapWithObject("reserved_state", reservedStateMetadata().values().iterator()), + ChunkedToXContentHelper.object("reserved_state", reservedStateMetadata().values().iterator()), ChunkedToXContentHelper.endObject() ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java index 25b99943d9ec..50e0de92f75b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadata.java @@ -191,7 +191,7 @@ public class NodesShutdownMetadata implements Metadata.Custom { @Override public Iterator toXContentChunked(ToXContent.Params params) { - return ChunkedToXContentHelper.xContentValuesMap(NODES_FIELD.getPreferredName(), nodes); + return ChunkedToXContentHelper.xContentObjectFields(NODES_FIELD.getPreferredName(), nodes); } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java index a224b3ae8231..72448b56e3a0 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java @@ -27,8 +27,8 @@ import java.util.Collections; import java.util.Iterator; import java.util.Objects; +import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk; import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endObject; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.singleChunk; import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startObject; public class ShutdownShardMigrationStatus implements Writeable, ChunkedToXContentObject { @@ -168,7 +168,7 @@ public class ShutdownShardMigrationStatus implements Writeable, ChunkedToXConten public Iterator toXContentChunked(ToXContent.Params params) { return Iterators.concat( startObject(), - singleChunk((builder, p) -> buildHeader(builder)), + chunk((builder, p) -> buildHeader(builder)), Objects.nonNull(allocationDecision) ? Iterators.concat(startObject(NODE_ALLOCATION_DECISION_KEY), allocationDecision.toXContentChunked(params), endObject()) : Collections.emptyIterator(), diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java index 52c16f88e61f..7c7a1ea4a189 100644 --- a/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java +++ b/server/src/main/java/org/elasticsearch/common/xcontent/ChunkedToXContentHelper.java @@ -21,69 +21,60 @@ public enum ChunkedToXContentHelper { ; public static Iterator startObject() { - return Iterators.single(((builder, params) -> builder.startObject())); + return Iterators.single((b, p) -> b.startObject()); } public static Iterator startObject(String name) { - return Iterators.single(((builder, params) -> builder.startObject(name))); + return Iterators.single((b, p) -> b.startObject(name)); } public static Iterator endObject() { - return Iterators.single(((builder, params) -> builder.endObject())); + return Iterators.single((b, p) -> b.endObject()); } public static Iterator startArray() { - return Iterators.single(((builder, params) -> builder.startArray())); + return Iterators.single((b, p) -> b.startArray()); } public static Iterator startArray(String name) { - return Iterators.single(((builder, params) -> builder.startArray(name))); + return Iterators.single((b, p) -> b.startArray(name)); } public static Iterator endArray() { - return Iterators.single(((builder, params) -> builder.endArray())); - } - - public static Iterator map(String name, Map map) { - return map(name, map, entry -> (ToXContent) (builder, params) -> builder.field(entry.getKey(), entry.getValue())); - } - - public static Iterator xContentFragmentValuesMap(String name, Map map) { - return map( - name, - map, - entry -> (ToXContent) (builder, params) -> entry.getValue().toXContent(builder.startObject(entry.getKey()), params).endObject() - ); - } - - public static Iterator xContentValuesMap(String name, Map map) { - return map( - name, - map, - entry -> (ToXContent) (builder, params) -> entry.getValue().toXContent(builder.field(entry.getKey()), params) - ); + return Iterators.single((b, p) -> b.endArray()); } /** - * Like xContentFragmentValuesMap, but allows the underlying XContent object to define its own "name" with startObject(string) - * and endObject, rather than assuming that the key in the map should be the name in the XContent output. - * @param name name to use in the XContent for the outer object wrapping the map being rendered to XContent - * @param map map being rendered to XContent + * Defines an object named {@code name}, with the contents of each field set by {@code map} */ - public static Iterator xContentFragmentValuesMapCreateOwnName(String name, Map map) { - return map(name, map, entry -> (ToXContent) (builder, params) -> entry.getValue().toXContent(builder, params)); + public static Iterator object(String name, Map map) { + return object(name, map, e -> (b, p) -> b.field(e.getKey(), e.getValue())); + } + + /** + * Defines an object named {@code name}, with the contents of each field created from each entry in {@code map} + */ + public static Iterator xContentObjectFields(String name, Map map) { + return object(name, map, e -> (b, p) -> e.getValue().toXContent(b.field(e.getKey()), p)); + } + + /** + * Defines an object named {@code name}, with the contents of each field each another object created from each entry in {@code map} + */ + public static Iterator xContentObjectFieldObjects(String name, Map map) { + return object(name, map, e -> (b, p) -> e.getValue().toXContent(b.startObject(e.getKey()), p).endObject()); } public static Iterator field(String name, boolean value) { - return Iterators.single(((builder, params) -> builder.field(name, value))); + return Iterators.single((b, p) -> b.field(name, value)); } public static Iterator field(String name, long value) { - return Iterators.single(((builder, params) -> builder.field(name, value))); + return Iterators.single((b, p) -> b.field(name, value)); } public static Iterator field(String name, String value) { - return Iterators.single(((builder, params) -> builder.field(name, value))); + return Iterators.single((b, p) -> b.field(name, value)); } public static Iterator optionalField(String name, String value) { @@ -107,7 +98,7 @@ public enum ChunkedToXContentHelper { } public static Iterator array(String name, Iterator contents) { - return Iterators.concat(ChunkedToXContentHelper.startArray(name), contents, ChunkedToXContentHelper.endArray()); + return Iterators.concat(startArray(name), contents, endArray()); } /** @@ -119,19 +110,32 @@ public enum ChunkedToXContentHelper { * @return Iterator composing field name and value serialization */ public static Iterator array(String name, Iterator contents, ToXContent.Params params) { - return Iterators.concat( - ChunkedToXContentHelper.startArray(name), - Iterators.flatMap(contents, c -> c.toXContentChunked(params)), - ChunkedToXContentHelper.endArray() - ); + return Iterators.concat(startArray(name), Iterators.flatMap(contents, c -> c.toXContentChunked(params)), endArray()); } - public static Iterator wrapWithObject(String name, Iterator iterator) { + /** + * Defines an object named {@code name}, with the contents set by {@code iterator} + */ + public static Iterator object(String name, Iterator iterator) { return Iterators.concat(startObject(name), iterator, endObject()); } - public static Iterator map(String name, Map map, Function, ToXContent> toXContent) { - return wrapWithObject(name, Iterators.map(map.entrySet().iterator(), toXContent)); + /** + * Defines an object named {@code name}, with the contents set by calling {@code toXContent} on each entry in {@code map} + */ + public static Iterator object(String name, Map map, Function, ToXContent> toXContent) { + return object(name, Iterators.map(map.entrySet().iterator(), toXContent)); + } + + /** + * Creates an Iterator of a single ToXContent object that serializes the given object as a single chunk. Just wraps {@link + * Iterators#single}, but still useful because it avoids any type ambiguity. + * + * @param item Item to wrap + * @return Singleton iterator for the given item. + */ + public static Iterator chunk(ToXContent item) { + return Iterators.single(item); } /** diff --git a/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java b/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java index d8badd6847c9..38d6f338073d 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java +++ b/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java @@ -58,7 +58,7 @@ public class FeatureMigrationResults implements Metadata.Custom { @Override public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentValuesMap(RESULTS_FIELD.getPreferredName(), featureStatuses); + return ChunkedToXContentHelper.xContentObjectFields(RESULTS_FIELD.getPreferredName(), featureStatuses); } /** diff --git a/server/src/test/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponseTests.java b/server/src/test/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponseTests.java index 03a555fac602..81db5bf8b955 100644 --- a/server/src/test/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponseTests.java @@ -36,7 +36,7 @@ public class BaseNodesXContentResponseTests extends ESTestCase { final var fullResponse = new BaseNodesXContentResponse<>(ClusterName.DEFAULT, List.of(new TestNodeResponse(node)), List.of()) { @Override protected Iterator xContentChunks(ToXContent.Params outerParams) { - return ChunkedToXContentHelper.singleChunk((b, p) -> b.startObject("content").endObject()); + return ChunkedToXContentHelper.chunk((b, p) -> b.startObject("content").endObject()); } @Override diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java index d4e605d1ba66..07fceffe11da 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java @@ -119,7 +119,7 @@ public class AutoscalingMetadata implements Metadata.Custom { @Override public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentValuesMap(POLICIES_FIELD.getPreferredName(), policies); + return ChunkedToXContentHelper.xContentObjectFields(POLICIES_FIELD.getPreferredName(), policies); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java index 82f3598f2be3..4c6c18db367d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/AutoFollowMetadata.java @@ -151,9 +151,9 @@ public class AutoFollowMetadata extends AbstractNamedDiffable i @Override public Iterator toXContentChunked(ToXContent.Params params) { return Iterators.concat( - ChunkedToXContentHelper.xContentFragmentValuesMap(PATTERNS_FIELD.getPreferredName(), patterns), - ChunkedToXContentHelper.map(FOLLOWED_LEADER_INDICES_FIELD.getPreferredName(), followedLeaderIndexUUIDs), - ChunkedToXContentHelper.map(HEADERS.getPreferredName(), headers) + ChunkedToXContentHelper.xContentObjectFieldObjects(PATTERNS_FIELD.getPreferredName(), patterns), + ChunkedToXContentHelper.object(FOLLOWED_LEADER_INDICES_FIELD.getPreferredName(), followedLeaderIndexUUIDs), + ChunkedToXContentHelper.object(HEADERS.getPreferredName(), headers) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java index ee57e54e1fde..e433093cdfb9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/EnrichMetadata.java @@ -99,7 +99,7 @@ public final class EnrichMetadata extends AbstractNamedDiffable @Override public Iterator toXContentChunked(ToXContent.Params ignored) { - return ChunkedToXContentHelper.xContentFragmentValuesMap(POLICIES.getPreferredName(), policies); + return ChunkedToXContentHelper.xContentObjectFieldObjects(POLICIES.getPreferredName(), policies); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java index af7d4455ecd0..a0e733107e0f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/IndexLifecycleMetadata.java @@ -118,7 +118,7 @@ public class IndexLifecycleMetadata implements Metadata.Custom { @Override public Iterator toXContentChunked(ToXContent.Params ignored) { return Iterators.concat( - ChunkedToXContentHelper.xContentValuesMap(POLICIES_FIELD.getPreferredName(), policyMetadatas), + ChunkedToXContentHelper.xContentObjectFields(POLICIES_FIELD.getPreferredName(), policyMetadatas), Iterators.single((builder, params) -> builder.field(OPERATION_MODE_FIELD.getPreferredName(), operationMode)) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/ModelAliasMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/ModelAliasMetadata.java index 0e30d501258c..bdf060bf6d7a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/ModelAliasMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/ModelAliasMetadata.java @@ -93,7 +93,7 @@ public class ModelAliasMetadata implements Metadata.Custom { @Override public Iterator toXContentChunked(ToXContent.Params params) { - return ChunkedToXContentHelper.xContentValuesMap(MODEL_ALIASES.getPreferredName(), modelAliases); + return ChunkedToXContentHelper.xContentObjectFields(MODEL_ALIASES.getPreferredName(), modelAliases); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java index 1c3e7bd08526..1d63d2710bd6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/search/action/AsyncSearchResponse.java @@ -234,7 +234,7 @@ public class AsyncSearchResponse extends ActionResponse implements ChunkedToXCon @Override public Iterator toXContentChunked(ToXContent.Params params) { - return Iterators.concat(ChunkedToXContentHelper.singleChunk((builder, p) -> { + return Iterators.concat(ChunkedToXContentHelper.chunk((builder, p) -> { builder.startObject(); if (id != null) { builder.field("id", id); @@ -257,7 +257,7 @@ public class AsyncSearchResponse extends ActionResponse implements ChunkedToXCon return builder; }), searchResponse == null ? Collections.emptyIterator() : searchResponse.toXContentChunked(params), - ChunkedToXContentHelper.singleChunk((builder, p) -> { + ChunkedToXContentHelper.chunk((builder, p) -> { if (error != null) { builder.startObject("error"); ElasticsearchException.generateThrowableXContent(builder, params, error); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java index e8f1ae855691..136070429c36 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/SnapshotLifecycleMetadata.java @@ -145,7 +145,7 @@ public class SnapshotLifecycleMetadata implements Metadata.Custom { @Override public Iterator toXContentChunked(ToXContent.Params params) { return Iterators.concat( - ChunkedToXContentHelper.xContentValuesMap(POLICIES_FIELD.getPreferredName(), this.snapshotConfigurations), + ChunkedToXContentHelper.xContentObjectFields(POLICIES_FIELD.getPreferredName(), this.snapshotConfigurations), Iterators.single( (builder, p) -> builder.field(OPERATION_MODE_FIELD.getPreferredName(), operationMode) .field(STATS_FIELD.getPreferredName(), this.slmStats) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java index c1afa728bc37..61c7135cef42 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlExecutionInfo.java @@ -247,7 +247,7 @@ public class EsqlExecutionInfo implements ChunkedToXContentObject, Writeable { ChunkedToXContentHelper.field(PARTIAL_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.PARTIAL)), ChunkedToXContentHelper.field(FAILED_FIELD.getPreferredName(), getClusterStateCount(Cluster.Status.FAILED)), // each Cluster object defines its own field object name - ChunkedToXContentHelper.xContentFragmentValuesMapCreateOwnName("details", clusterInfo), + ChunkedToXContentHelper.object("details", clusterInfo.values().iterator()), ChunkedToXContentHelper.endObject() ); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java index 01b1821c01b9..8530d9b48da0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java @@ -187,7 +187,7 @@ public class EsqlQueryResponse extends org.elasticsearch.xpack.core.esql.action. private Iterator asyncPropertiesOrEmpty() { if (isAsync) { - return ChunkedToXContentHelper.singleChunk((builder, params) -> { + return ChunkedToXContentHelper.chunk((builder, params) -> { if (asyncExecutionId != null) { builder.field("id", asyncExecutionId); } @@ -206,7 +206,7 @@ public class EsqlQueryResponse extends org.elasticsearch.xpack.core.esql.action. Iterator tookTime; if (executionInfo != null && executionInfo.overallTook() != null) { - tookTime = ChunkedToXContentHelper.singleChunk((builder, p) -> { + tookTime = ChunkedToXContentHelper.chunk((builder, p) -> { builder.field("took", executionInfo.overallTook().millis()); return builder; }); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java index d7d0d9033d3b..eb1f4f95db6f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseXContentUtils.java @@ -28,7 +28,7 @@ final class ResponseXContentUtils { * Returns the column headings for the given columns. */ static Iterator allColumns(List columns, String name) { - return ChunkedToXContentHelper.singleChunk((builder, params) -> { + return ChunkedToXContentHelper.chunk((builder, params) -> { builder.startArray(name); for (ColumnInfo col : columns) { col.toXContent(builder, params); @@ -42,7 +42,7 @@ final class ResponseXContentUtils { * for always-null columns to a {@code null_columns} section. */ static Iterator nonNullColumns(List columns, boolean[] nullColumns, String name) { - return ChunkedToXContentHelper.singleChunk((builder, params) -> { + return ChunkedToXContentHelper.chunk((builder, params) -> { builder.startArray(name); for (int c = 0; c < columns.size(); c++) { if (nullColumns[c] == false) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java index 042c8b8a8346..62cbcf902a9e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rest/ServerSentEventsRestActionListener.java @@ -158,7 +158,7 @@ public class ServerSentEventsRestActionListener implements ActionListener Iterators.concat(ChunkedToXContentHelper.startObject(), ChunkedToXContentHelper.singleChunk((b, p) -> { + return params -> Iterators.concat(ChunkedToXContentHelper.startObject(), ChunkedToXContentHelper.chunk((b, p) -> { // Render the exception with a simple message if (channel.detailedErrorsEnabled() == false) { String message = "No ElasticsearchException found"; diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetFlamegraphResponse.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetFlamegraphResponse.java index 24f2f287f4cd..2419e2e1dc0c 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetFlamegraphResponse.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetFlamegraphResponse.java @@ -195,7 +195,7 @@ public class GetFlamegraphResponse extends ActionResponse implements ChunkedToXC ChunkedToXContentHelper.array("ExeFilename", Iterators.map(fileNames.iterator(), e -> (b, p) -> b.value(e))), ChunkedToXContentHelper.array("AddressOrLine", Iterators.map(addressOrLines.iterator(), e -> (b, p) -> b.value(e))), ChunkedToXContentHelper.array("FunctionName", Iterators.map(functionNames.iterator(), e -> (b, p) -> b.value(e))), - ChunkedToXContentHelper.singleChunk((b, p) -> { + ChunkedToXContentHelper.chunk((b, p) -> { b.startArray("FunctionOffset"); for (int functionOffset : functionOffsets) { b.value(functionOffset); @@ -203,28 +203,28 @@ public class GetFlamegraphResponse extends ActionResponse implements ChunkedToXC return b.endArray(); }), ChunkedToXContentHelper.array("SourceFilename", Iterators.map(sourceFileNames.iterator(), e -> (b, p) -> b.value(e))), - ChunkedToXContentHelper.singleChunk((b, p) -> { + ChunkedToXContentHelper.chunk((b, p) -> { b.startArray("SourceLine"); for (int sourceLine : sourceLines) { b.value(sourceLine); } return b.endArray(); }), - ChunkedToXContentHelper.singleChunk((b, p) -> { + ChunkedToXContentHelper.chunk((b, p) -> { b.startArray("CountInclusive"); for (long countInclusive : countInclusive) { b.value(countInclusive); } return b.endArray(); }), - ChunkedToXContentHelper.singleChunk((b, p) -> { + ChunkedToXContentHelper.chunk((b, p) -> { b.startArray("CountExclusive"); for (long c : countExclusive) { b.value(c); } return b.endArray(); }), - ChunkedToXContentHelper.singleChunk((b, p) -> { + ChunkedToXContentHelper.chunk((b, p) -> { b.startArray("AnnualCO2TonsInclusive"); for (double co2Tons : annualCO2TonsInclusive) { // write as raw value - we need direct control over the output representation (here: limit to 4 decimal places) @@ -232,21 +232,21 @@ public class GetFlamegraphResponse extends ActionResponse implements ChunkedToXC } return b.endArray(); }), - ChunkedToXContentHelper.singleChunk((b, p) -> { + ChunkedToXContentHelper.chunk((b, p) -> { b.startArray("AnnualCO2TonsExclusive"); for (double co2Tons : annualCO2TonsExclusive) { b.rawValue(NumberUtils.doubleToString(co2Tons)); } return b.endArray(); }), - ChunkedToXContentHelper.singleChunk((b, p) -> { + ChunkedToXContentHelper.chunk((b, p) -> { b.startArray("AnnualCostsUSDInclusive"); for (double costs : annualCostsUSDInclusive) { b.rawValue(NumberUtils.doubleToString(costs)); } return b.endArray(); }), - ChunkedToXContentHelper.singleChunk((b, p) -> { + ChunkedToXContentHelper.chunk((b, p) -> { b.startArray("AnnualCostsUSDExclusive"); for (double costs : annualCostsUSDExclusive) { b.rawValue(NumberUtils.doubleToString(costs)); diff --git a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java index e9757d3806b6..c2dd51acfd76 100644 --- a/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java +++ b/x-pack/plugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/action/GetStackTracesResponse.java @@ -93,14 +93,14 @@ public class GetStackTracesResponse extends ActionResponse implements ChunkedToX public Iterator toXContentChunked(ToXContent.Params params) { return Iterators.concat( ChunkedToXContentHelper.startObject(), - optional("stack_traces", stackTraces, ChunkedToXContentHelper::xContentValuesMap), - optional("stack_frames", stackFrames, ChunkedToXContentHelper::xContentValuesMap), - optional("executables", executables, ChunkedToXContentHelper::map), + optional("stack_traces", stackTraces, ChunkedToXContentHelper::xContentObjectFields), + optional("stack_frames", stackFrames, ChunkedToXContentHelper::xContentObjectFields), + optional("executables", executables, ChunkedToXContentHelper::object), // render only count for backwards-compatibility optional( "stack_trace_events", stackTraceEvents, - (n, v) -> ChunkedToXContentHelper.map(n, v, entry -> (b, p) -> b.field(entry.getKey(), entry.getValue().count)) + (n, v) -> ChunkedToXContentHelper.object(n, v, entry -> (b, p) -> b.field(entry.getKey(), entry.getValue().count)) ), Iterators.single((b, p) -> b.field("total_frames", totalFrames)), Iterators.single((b, p) -> b.field("sampling_rate", samplingRate)), diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java index 6078f1cbfa96..6a9116daf2ca 100644 --- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java +++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/SingleNodeShutdownStatus.java @@ -25,8 +25,8 @@ import java.io.IOException; import java.util.Iterator; import java.util.Objects; +import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.chunk; import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.endObject; -import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.singleChunk; import static org.elasticsearch.common.xcontent.ChunkedToXContentHelper.startObject; public class SingleNodeShutdownStatus implements Writeable, ChunkedToXContentObject { @@ -116,7 +116,7 @@ public class SingleNodeShutdownStatus implements Writeable, ChunkedToXContentObj @Override public Iterator toXContentChunked(ToXContent.Params params) { - return Iterators.concat(startObject(), singleChunk((builder, p) -> { + return Iterators.concat(startObject(), chunk((builder, p) -> { builder.field(SingleNodeShutdownMetadata.NODE_ID_FIELD.getPreferredName(), metadata.getNodeId()); builder.field(SingleNodeShutdownMetadata.NODE_EPHEMERAL_ID_FIELD.getPreferredName(), metadata.getNodeEphemeralId()); builder.field(SingleNodeShutdownMetadata.TYPE_FIELD.getPreferredName(), metadata.getType()); @@ -134,23 +134,19 @@ public class SingleNodeShutdownStatus implements Writeable, ChunkedToXContentObj ); builder.field(STATUS.getPreferredName(), overallStatus()); return builder; - }), - ChunkedToXContentHelper.field(SHARD_MIGRATION_FIELD.getPreferredName(), shardMigrationStatus, params), - singleChunk((builder, p) -> { - builder.field(PERSISTENT_TASKS_FIELD.getPreferredName(), persistentTasksStatus); - builder.field(PLUGINS_STATUS.getPreferredName(), pluginsStatus); - if (metadata.getTargetNodeName() != null) { - builder.field(TARGET_NODE_NAME_FIELD.getPreferredName(), metadata.getTargetNodeName()); - } - if (metadata.getGracePeriod() != null) { - builder.timestampField( - SingleNodeShutdownMetadata.GRACE_PERIOD_FIELD.getPreferredName(), - metadata.getGracePeriod().getStringRep() - ); - } - return builder; - }), - endObject() - ); + }), ChunkedToXContentHelper.field(SHARD_MIGRATION_FIELD.getPreferredName(), shardMigrationStatus, params), chunk((builder, p) -> { + builder.field(PERSISTENT_TASKS_FIELD.getPreferredName(), persistentTasksStatus); + builder.field(PLUGINS_STATUS.getPreferredName(), pluginsStatus); + if (metadata.getTargetNodeName() != null) { + builder.field(TARGET_NODE_NAME_FIELD.getPreferredName(), metadata.getTargetNodeName()); + } + if (metadata.getGracePeriod() != null) { + builder.timestampField( + SingleNodeShutdownMetadata.GRACE_PERIOD_FIELD.getPreferredName(), + metadata.getGracePeriod().getStringRep() + ); + } + return builder; + }), endObject()); } } diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseStream.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseStream.java index 7ea9bfe6f2b2..0eb7182ef99e 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseStream.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/integrity/RepositoryVerifyIntegrityResponseStream.java @@ -61,7 +61,7 @@ class RepositoryVerifyIntegrityResponseStream extends AbstractRefCounted { assert streamingXContentResponse == null; streamingXContentResponse = new StreamingXContentResponse(restChannel, restChannel.request(), () -> {}); streamingXContentResponse.writeFragment( - p0 -> ChunkedToXContentHelper.singleChunk((b, p) -> b.startObject().startArray("log")), + p0 -> ChunkedToXContentHelper.chunk((b, p) -> b.startObject().startArray("log")), releasable ); } @@ -74,7 +74,7 @@ class RepositoryVerifyIntegrityResponseStream extends AbstractRefCounted { anomalyCount.incrementAndGet(); } streamingXContentResponse.writeFragment( - p0 -> ChunkedToXContentHelper.singleChunk((b, p) -> b.startObject().value(chunk, p).endObject()), + p0 -> ChunkedToXContentHelper.chunk((b, p) -> b.startObject().value(chunk, p).endObject()), releasable ); } @@ -89,7 +89,7 @@ class RepositoryVerifyIntegrityResponseStream extends AbstractRefCounted { // success - finish the response with the final results assert streamingXContentResponse != null; streamingXContentResponse.writeFragment( - p0 -> ChunkedToXContentHelper.singleChunk( + p0 -> ChunkedToXContentHelper.chunk( (b, p) -> b.endArray() .startObject("results") .field("status", repositoryVerifyIntegrityResponse.finalTaskStatus()) @@ -116,7 +116,7 @@ class RepositoryVerifyIntegrityResponseStream extends AbstractRefCounted { if (streamingXContentResponse != null) { // failure after starting the response - finish the response with a rendering of the final exception streamingXContentResponse.writeFragment( - p0 -> ChunkedToXContentHelper.singleChunk( + p0 -> ChunkedToXContentHelper.chunk( (b, p) -> b.endArray() .startObject("exception") .value((bb, pp) -> ElasticsearchException.generateFailureXContent(bb, pp, e, true)) From f383c868514cbc3f8c0bef178d62cd9eaa0e99e6 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 9 Jan 2025 16:04:25 +0100 Subject: [PATCH 26/33] [Inference API] Remove unused settings parameter in TransportUpdateInferenceModelAction (#119853) --- .../inference/action/TransportUpdateInferenceModelAction.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java index 03a88e5228fa..3c47de1ad64d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java @@ -21,7 +21,6 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.inference.InferenceService; @@ -80,8 +79,7 @@ public class TransportUpdateInferenceModelAction extends TransportMasterNodeActi IndexNameExpressionResolver indexNameExpressionResolver, ModelRegistry modelRegistry, InferenceServiceRegistry serviceRegistry, - Client client, - Settings settings + Client client ) { super( UpdateInferenceModelAction.NAME, From c41897e9c4730a7131b7cf1077c9e9d54237329c Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Thu, 9 Jan 2025 15:17:22 +0000 Subject: [PATCH 27/33] Remove the features supported feature (#119758) This removes the features_supported feature from the codebase, as all nodes communicable with 9.0 will support features --- .../upgrades/ClusterFeatureMigrationIT.java | 61 ------------------- .../features/ClusterFeaturesIT.java | 3 +- .../FeatureInfrastructureFeatures.java | 9 --- .../features/FeatureService.java | 4 -- .../features/FeatureSpecification.java | 2 +- 5 files changed, 2 insertions(+), 77 deletions(-) delete mode 100644 qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ClusterFeatureMigrationIT.java diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ClusterFeatureMigrationIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ClusterFeatureMigrationIT.java deleted file mode 100644 index 2b3e10acc15e..000000000000 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/ClusterFeatureMigrationIT.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.upgrades; - -import com.carrotsearch.randomizedtesting.annotations.Name; - -import org.elasticsearch.client.Request; -import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.features.FeatureService; -import org.junit.Before; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.hasSize; - -public class ClusterFeatureMigrationIT extends AbstractRollingUpgradeTestCase { - - @Before - public void checkMigrationVersion() { - assumeFalse( - "This checks migrations from before cluster features were introduced", - oldClusterHasFeature(FeatureService.FEATURES_SUPPORTED) - ); - } - - public ClusterFeatureMigrationIT(@Name("upgradedNodes") int upgradedNodes) { - super(upgradedNodes); - } - - public void testClusterFeatureMigration() throws IOException { - if (isUpgradedCluster()) { - // check the nodes all have a feature in their cluster state (there should always be features_supported) - var response = entityAsMap(adminClient().performRequest(new Request("GET", "/_cluster/state/nodes"))); - List nodeFeatures = (List) XContentMapValues.extractValue("nodes_features", response); - assertThat(nodeFeatures, hasSize(adminClient().getNodes().size())); - - Map> features = nodeFeatures.stream() - .map(o -> (Map) o) - .collect(Collectors.toMap(m -> (String) m.get("node_id"), m -> (List) m.get("features"))); - - Set missing = features.entrySet() - .stream() - .filter(e -> e.getValue().contains(FeatureService.FEATURES_SUPPORTED.id()) == false) - .map(Map.Entry::getKey) - .collect(Collectors.toSet()); - assertThat(missing + " out of " + features.keySet() + " does not have the required feature", missing, empty()); - } - } -} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/features/ClusterFeaturesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/features/ClusterFeaturesIT.java index 74fd945ed377..b756b4ca770c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/features/ClusterFeaturesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/features/ClusterFeaturesIT.java @@ -30,7 +30,6 @@ public class ClusterFeaturesIT extends ESIntegTestCase { FeatureService service = internalCluster().getCurrentMasterNodeInstance(FeatureService.class); - assertThat(service.getNodeFeatures(), hasKey(FeatureService.FEATURES_SUPPORTED.id())); assertThat(service.getNodeFeatures(), hasKey(FeatureService.TEST_FEATURES_ENABLED.id())); // check the nodes all have a feature in their cluster state (there should always be features_supported) @@ -38,7 +37,7 @@ public class ClusterFeaturesIT extends ESIntegTestCase { var features = response.getState().clusterFeatures().nodeFeatures(); Set missing = features.entrySet() .stream() - .filter(e -> e.getValue().contains(FeatureService.FEATURES_SUPPORTED.id()) == false) + .filter(e -> e.getValue().contains(FeatureService.TEST_FEATURES_ENABLED.id()) == false) .map(Map.Entry::getKey) .collect(Collectors.toSet()); assertThat(missing + " out of " + features.keySet() + " does not have the required feature", missing, empty()); diff --git a/server/src/main/java/org/elasticsearch/features/FeatureInfrastructureFeatures.java b/server/src/main/java/org/elasticsearch/features/FeatureInfrastructureFeatures.java index 76afb5eba8a4..b61d866b0f1b 100644 --- a/server/src/main/java/org/elasticsearch/features/FeatureInfrastructureFeatures.java +++ b/server/src/main/java/org/elasticsearch/features/FeatureInfrastructureFeatures.java @@ -13,18 +13,9 @@ import java.util.Set; /** * This class specifies features for the features functionality itself. - *

    - * This adds a feature {@code features_supported} indicating that a node supports node features. - * Nodes that do not support features won't have this feature in its feature set, - * so this can be checked without needing to look at the node version. */ public class FeatureInfrastructureFeatures implements FeatureSpecification { - @Override - public Set getFeatures() { - return Set.of(FeatureService.FEATURES_SUPPORTED); - } - @Override public Set getTestFeatures() { return Set.of(FeatureService.TEST_FEATURES_ENABLED); diff --git a/server/src/main/java/org/elasticsearch/features/FeatureService.java b/server/src/main/java/org/elasticsearch/features/FeatureService.java index da71b8f0ec2f..9f4ec6209c6f 100644 --- a/server/src/main/java/org/elasticsearch/features/FeatureService.java +++ b/server/src/main/java/org/elasticsearch/features/FeatureService.java @@ -26,10 +26,6 @@ import java.util.Map; */ public class FeatureService { - /** - * A feature indicating that node features are supported. - */ - public static final NodeFeature FEATURES_SUPPORTED = new NodeFeature("features_supported", true); public static final NodeFeature TEST_FEATURES_ENABLED = new NodeFeature("test_features_enabled"); private static final Logger logger = LogManager.getLogger(FeatureService.class); diff --git a/server/src/main/java/org/elasticsearch/features/FeatureSpecification.java b/server/src/main/java/org/elasticsearch/features/FeatureSpecification.java index c37bc4488f10..391dad48f734 100644 --- a/server/src/main/java/org/elasticsearch/features/FeatureSpecification.java +++ b/server/src/main/java/org/elasticsearch/features/FeatureSpecification.java @@ -31,7 +31,7 @@ import java.util.Set; */ public interface FeatureSpecification { /** - * Returns a set of regular features that this node supports. + * Returns a set of features that this node supports. */ default Set getFeatures() { return Set.of(); From f214dcaa3a7061641858cb981cbd3b499e4af539 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 9 Jan 2025 16:23:53 +0100 Subject: [PATCH 28/33] Remove unused client field in TransportPutInferenceModelAction (#119849) --- .../inference/action/TransportPutInferenceModelAction.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java index 7c2a139672e8..393055428e44 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportPutInferenceModelAction.java @@ -13,7 +13,6 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeAction; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -60,7 +59,6 @@ public class TransportPutInferenceModelAction extends TransportMasterNodeAction< private final ModelRegistry modelRegistry; private final InferenceServiceRegistry serviceRegistry; - private final Client client; private volatile boolean skipValidationAndStart; @Inject @@ -72,7 +70,6 @@ public class TransportPutInferenceModelAction extends TransportMasterNodeAction< IndexNameExpressionResolver indexNameExpressionResolver, ModelRegistry modelRegistry, InferenceServiceRegistry serviceRegistry, - Client client, Settings settings ) { super( @@ -88,7 +85,6 @@ public class TransportPutInferenceModelAction extends TransportMasterNodeAction< ); this.modelRegistry = modelRegistry; this.serviceRegistry = serviceRegistry; - this.client = client; this.skipValidationAndStart = InferencePlugin.SKIP_VALIDATE_AND_START.get(settings); clusterService.getClusterSettings() .addSettingsUpdateConsumer(InferencePlugin.SKIP_VALIDATE_AND_START, this::setSkipValidationAndStart); From 70e5a67904fffdd43862ead4644706df9446f7fc Mon Sep 17 00:00:00 2001 From: Arianna Laudazzi <46651782+alaudazzi@users.noreply.github.com> Date: Thu, 9 Jan 2025 16:24:20 +0100 Subject: [PATCH 29/33] [AutoOps] Reference AutoOps solution on troubleshooting pages (#119630) * Reference AutoOps on troubleshooting pages * Integrate reviewer's feedback --- docs/reference/monitoring/overview.asciidoc | 4 ++++ docs/reference/troubleshooting.asciidoc | 4 ++++ .../common-issues/circuit-breaker-errors.asciidoc | 4 ++++ .../common-issues/diagnose-unassigned-shards.asciidoc | 4 ++++ .../common-issues/disk-usage-exceeded.asciidoc | 4 ++++ .../troubleshooting/common-issues/high-cpu-usage.asciidoc | 4 ++++ .../common-issues/high-jvm-memory-pressure.asciidoc | 4 ++++ .../troubleshooting/common-issues/hotspotting.asciidoc | 4 ++++ .../common-issues/red-yellow-cluster-status.asciidoc | 4 ++++ .../troubleshooting/common-issues/rejected-requests.asciidoc | 4 ++++ .../data/increase-cluster-shard-limit.asciidoc | 3 +++ .../troubleshooting/data/increase-shard-limit.asciidoc | 4 ++++ .../troubleshooting/data/increase-tier-capacity.asciidoc | 3 +++ docs/reference/troubleshooting/diagnostic.asciidoc | 4 ++++ .../troubleshooting/fix-common-cluster-issues.asciidoc | 4 ++++ .../snapshot/repeated-snapshot-failures.asciidoc | 3 +++ .../troubleshooting/troubleshooting-shards-capacity.asciidoc | 4 ++++ .../troubleshooting-unbalanced-cluster.asciidoc | 4 ++++ .../troubleshooting/troubleshooting-unstable-cluster.asciidoc | 4 ++++ 19 files changed, 73 insertions(+) diff --git a/docs/reference/monitoring/overview.asciidoc b/docs/reference/monitoring/overview.asciidoc index d2a0130f71bb..5c5d016f45ea 100644 --- a/docs/reference/monitoring/overview.asciidoc +++ b/docs/reference/monitoring/overview.asciidoc @@ -13,6 +13,10 @@ All of the monitoring metrics are stored in {es}, which enables you to easily visualize the data in {kib}. By default, the monitoring metrics are stored in local indices. +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + TIP: In production, we strongly recommend using a separate monitoring cluster. Using a separate monitoring cluster prevents production cluster outages from impacting your ability to access your monitoring data. It also prevents diff --git a/docs/reference/troubleshooting.asciidoc b/docs/reference/troubleshooting.asciidoc index ceff8619062c..75cd0c1be49f 100644 --- a/docs/reference/troubleshooting.asciidoc +++ b/docs/reference/troubleshooting.asciidoc @@ -6,6 +6,10 @@ This section provides a series of troubleshooting solutions aimed at helping users fix problems that an {es} deployment might encounter. +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + [discrete] [[troubleshooting-general]] === General diff --git a/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc b/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc index 74793f8a625b..8ebe962eaed8 100644 --- a/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc +++ b/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc @@ -12,6 +12,10 @@ memory pressure if usage consistently exceeds 85%. See https://www.youtube.com/watch?v=k3wYlRVbMSw[this video] for a walkthrough of diagnosing circuit breaker errors. +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + [discrete] [[diagnose-circuit-breaker-errors]] ==== Diagnose circuit breaker errors diff --git a/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc b/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc index e1ceefb92bbe..c6bbcd69b055 100644 --- a/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc +++ b/docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc @@ -11,3 +11,7 @@ include::{es-ref-dir}/tab-widgets/troubleshooting/data/diagnose-unassigned-shard See https://www.youtube.com/watch?v=v2mbeSd1vTQ[this video] for a walkthrough of monitoring allocation health. +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + diff --git a/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc b/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc index 2222d09c26db..a2342c449c88 100644 --- a/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc +++ b/docs/reference/troubleshooting/common-issues/disk-usage-exceeded.asciidoc @@ -21,6 +21,10 @@ usage falls below the <>. To achieve this, {es} attempts to rebalance some of the affected node's shards to other nodes in the same data tier. +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + [[fix-watermark-errors-rebalance]] ==== Monitor rebalancing diff --git a/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc b/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc index 96a9a8f1e32b..7d226a891251 100644 --- a/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc +++ b/docs/reference/troubleshooting/common-issues/high-cpu-usage.asciidoc @@ -11,6 +11,10 @@ depleted, {es} will reject search requests until more threads are available. You might experience high CPU usage if a <>, and therefore the nodes assigned to that tier, is experiencing more traffic than other tiers. This imbalance in resource utilization is also known as <>. +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + [discrete] [[diagnose-high-cpu-usage]] ==== Diagnose high CPU usage diff --git a/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc b/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc index 3469a0ca5bf4..842b7edda73a 100644 --- a/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc +++ b/docs/reference/troubleshooting/common-issues/high-jvm-memory-pressure.asciidoc @@ -6,6 +6,10 @@ High JVM memory usage can degrade cluster performance and trigger taking steps to reduce memory pressure if a node's JVM memory usage consistently exceeds 85%. +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + [discrete] [[diagnose-high-jvm-memory-pressure]] ==== Diagnose high JVM memory pressure diff --git a/docs/reference/troubleshooting/common-issues/hotspotting.asciidoc b/docs/reference/troubleshooting/common-issues/hotspotting.asciidoc index 344e36a999a3..53226dbfedb5 100644 --- a/docs/reference/troubleshooting/common-issues/hotspotting.asciidoc +++ b/docs/reference/troubleshooting/common-issues/hotspotting.asciidoc @@ -11,6 +11,10 @@ may occur in {es} when resource utilizations are unevenly distributed across ongoing significantly unique utilization may lead to cluster bottlenecks and should be reviewed. +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + See link:https://www.youtube.com/watch?v=Q5ODJ5nIKAM[this video] for a walkthrough of troubleshooting a hot spotting issue. [discrete] diff --git a/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc b/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc index 4289242deb48..c07e92c05899 100644 --- a/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc +++ b/docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc @@ -22,6 +22,10 @@ the remaining problems so management and cleanup activities can proceed. See https://www.youtube.com/watch?v=v2mbeSd1vTQ[this video] for a walkthrough of monitoring allocation health. +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + [discrete] [[diagnose-cluster-status]] ==== Diagnose your cluster status diff --git a/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc b/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc index 83f6962a219e..7b0e3eb51680 100644 --- a/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc +++ b/docs/reference/troubleshooting/common-issues/rejected-requests.asciidoc @@ -12,6 +12,10 @@ thread pool returns a `TOO_MANY_REQUESTS` error message. * High <> that exceeds the <>. +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + [discrete] [[check-rejected-tasks]] ==== Check rejected tasks diff --git a/docs/reference/troubleshooting/data/increase-cluster-shard-limit.asciidoc b/docs/reference/troubleshooting/data/increase-cluster-shard-limit.asciidoc index 8b8703f9a9dc..ddd088138552 100644 --- a/docs/reference/troubleshooting/data/increase-cluster-shard-limit.asciidoc +++ b/docs/reference/troubleshooting/data/increase-cluster-shard-limit.asciidoc @@ -16,5 +16,8 @@ In order to fix this follow the next steps: include::{es-ref-dir}/tab-widgets/troubleshooting/data/increase-cluster-shard-limit-widget.asciidoc[] +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** diff --git a/docs/reference/troubleshooting/data/increase-shard-limit.asciidoc b/docs/reference/troubleshooting/data/increase-shard-limit.asciidoc index 121b5348ab36..51bb9fda09fe 100644 --- a/docs/reference/troubleshooting/data/increase-shard-limit.asciidoc +++ b/docs/reference/troubleshooting/data/increase-shard-limit.asciidoc @@ -14,5 +14,9 @@ In order to fix this follow the next steps: include::{es-ref-dir}/tab-widgets/troubleshooting/data/total-shards-per-node-widget.asciidoc[] +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + diff --git a/docs/reference/troubleshooting/data/increase-tier-capacity.asciidoc b/docs/reference/troubleshooting/data/increase-tier-capacity.asciidoc index 362a14c3874d..09d7fedb89e6 100644 --- a/docs/reference/troubleshooting/data/increase-tier-capacity.asciidoc +++ b/docs/reference/troubleshooting/data/increase-tier-capacity.asciidoc @@ -17,5 +17,8 @@ In order to fix this follow the next steps: include::{es-ref-dir}/tab-widgets/troubleshooting/data/increase-tier-capacity-widget.asciidoc[] +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** diff --git a/docs/reference/troubleshooting/diagnostic.asciidoc b/docs/reference/troubleshooting/diagnostic.asciidoc index c6d46b9e94fc..d806fa2b986c 100644 --- a/docs/reference/troubleshooting/diagnostic.asciidoc +++ b/docs/reference/troubleshooting/diagnostic.asciidoc @@ -15,6 +15,10 @@ https://discuss.elastic.co[Elastic Discuss] to minimize turnaround time. See this https://www.youtube.com/watch?v=Bb6SaqhqYHw[this video] for a walkthrough of capturing an {es} diagnostic. +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + [discrete] [[diagnostic-tool-requirements]] === Requirements diff --git a/docs/reference/troubleshooting/fix-common-cluster-issues.asciidoc b/docs/reference/troubleshooting/fix-common-cluster-issues.asciidoc index 47244ac17e99..5aa90215d7ae 100644 --- a/docs/reference/troubleshooting/fix-common-cluster-issues.asciidoc +++ b/docs/reference/troubleshooting/fix-common-cluster-issues.asciidoc @@ -3,6 +3,10 @@ This guide describes how to fix common errors and problems with {es} clusters. +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + <>:: Fix watermark errors that occur when a data node is critically low on disk space and has reached the flood-stage disk usage watermark. diff --git a/docs/reference/troubleshooting/snapshot/repeated-snapshot-failures.asciidoc b/docs/reference/troubleshooting/snapshot/repeated-snapshot-failures.asciidoc index 2496781c0c8f..7a7e606e2ca7 100644 --- a/docs/reference/troubleshooting/snapshot/repeated-snapshot-failures.asciidoc +++ b/docs/reference/troubleshooting/snapshot/repeated-snapshot-failures.asciidoc @@ -14,5 +14,8 @@ information about the problem: include::{es-ref-dir}/tab-widgets/troubleshooting/snapshot/repeated-snapshot-failures-widget.asciidoc[] +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** diff --git a/docs/reference/troubleshooting/troubleshooting-shards-capacity.asciidoc b/docs/reference/troubleshooting/troubleshooting-shards-capacity.asciidoc index bc8fb7290f1e..d9871855faa8 100644 --- a/docs/reference/troubleshooting/troubleshooting-shards-capacity.asciidoc +++ b/docs/reference/troubleshooting/troubleshooting-shards-capacity.asciidoc @@ -8,3 +8,7 @@ The current shards capacity of the cluster is available in the <>. include::{es-ref-dir}/tab-widgets/troubleshooting/troubleshooting-shards-capacity-widget.asciidoc[] + +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** diff --git a/docs/reference/troubleshooting/troubleshooting-unbalanced-cluster.asciidoc b/docs/reference/troubleshooting/troubleshooting-unbalanced-cluster.asciidoc index a1d4f5df9c4f..4ac3155c7fa8 100644 --- a/docs/reference/troubleshooting/troubleshooting-unbalanced-cluster.asciidoc +++ b/docs/reference/troubleshooting/troubleshooting-unbalanced-cluster.asciidoc @@ -7,6 +7,10 @@ Elasticsearch balances shards across data tiers to achieve a good compromise bet * disk usage * write load (for indices in data streams) +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + Elasticsearch does not take into account the amount or complexity of search queries when rebalancing shards. This is indirectly achieved by balancing shard count and disk usage. diff --git a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc index e47b85aa9954..ab0faa3b6c71 100644 --- a/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc +++ b/docs/reference/troubleshooting/troubleshooting-unstable-cluster.asciidoc @@ -17,6 +17,10 @@ logs. * The master may appear busy due to frequent cluster state updates. +**** +If you're using Elastic Cloud Hosted, then you can use AutoOps to monitor your cluster. AutoOps significantly simplifies cluster management with performance recommendations, resource utilization visibility, real-time issue detection and resolution paths. For more information, refer to https://www.elastic.co/guide/en/cloud/current/ec-autoops.html[Monitor with AutoOps]. +**** + To troubleshoot a cluster in this state, first ensure the cluster has a <>. Next, focus on the nodes unexpectedly leaving the cluster ahead of all other issues. It will not be From 26744829feba0d3ea583a7482ca6544074842420 Mon Sep 17 00:00:00 2001 From: Tim Grein Date: Thu, 9 Jan 2025 16:38:18 +0100 Subject: [PATCH 30/33] [Inference API] Inline two variables in TransportDeleteInferenceEndpointAction (#119848) --- .../action/TransportDeleteInferenceEndpointAction.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java index c1dbd8cfec9d..bdbd190a5120 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportDeleteInferenceEndpointAction.java @@ -205,16 +205,11 @@ public class TransportDeleteInferenceEndpointAction extends TransportMasterNodeA } private static Set endpointIsReferencedInIndex(final ClusterState state, final String inferenceEndpointId) { - Set indexes = extractIndexesReferencingInferenceEndpoints(state.getMetadata(), Set.of(inferenceEndpointId)); - return indexes; + return extractIndexesReferencingInferenceEndpoints(state.getMetadata(), Set.of(inferenceEndpointId)); } private static Set endpointIsReferencedInPipelines(final ClusterState state, final String inferenceEndpointId) { - Set modelIdsReferencedByPipelines = InferenceProcessorInfoExtractor.pipelineIdsForResource( - state, - Set.of(inferenceEndpointId) - ); - return modelIdsReferencedByPipelines; + return InferenceProcessorInfoExtractor.pipelineIdsForResource(state, Set.of(inferenceEndpointId)); } @Override From 547a567c13f89be74db51ba1be9e34182b21206a Mon Sep 17 00:00:00 2001 From: David Kyle Date: Thu, 9 Jan 2025 15:44:01 +0000 Subject: [PATCH 31/33] [DOCS][ML] Document the text_expansion task type (#119581) --- docs/reference/ml/ml-shared.asciidoc | 6 +++++ .../apis/put-trained-models.asciidoc | 23 +++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index 4948db48664e..af384c2c9001 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -1167,6 +1167,12 @@ tag::inference-config-text-embedding-size[] The number of dimensions in the embedding vector produced by the model. end::inference-config-text-embedding-size[] +tag::inference-config-text-expansion[] +The text expansion task works with sparse embedding models to transform an input sequence +into a vector of weighted tokens. These embeddings capture semantic meanings and +context and can be used in a <> field for powerful insights. +end::inference-config-text-expansion[] + tag::inference-config-text-similarity[] Text similarity takes an input sequence and compares it with another input sequence. This is commonly referred to as cross-encoding. This task is useful for ranking document text when comparing it to another provided text input. diff --git a/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc b/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc index 256c0d29f8d2..ccd76b709576 100644 --- a/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc +++ b/docs/reference/ml/trained-models/apis/put-trained-models.asciidoc @@ -395,10 +395,10 @@ the model definition is not supplied. (Required, object) The default configuration for inference. This can be: `regression`, `classification`, `fill_mask`, `ner`, `question_answering`, -`text_classification`, `text_embedding` or `zero_shot_classification`. +`text_classification`, `text_embedding`, `text_expansion` or `zero_shot_classification`. If `regression` or `classification`, it must match the `target_type` of the underlying `definition.trained_model`. If `fill_mask`, `ner`, -`question_answering`, `text_classification`, or `text_embedding`; the +`question_answering`, `text_classification`, `text_embedding` or `text_expansion`; the `model_type` must be `pytorch`. + .Properties of `inference_config` @@ -592,6 +592,25 @@ Refer to <> to review the properties of the `tokenization` object. ===== +`text_expansion`::: +(Object, optional) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-expansion] ++ +.Properties of text_expansion inference +[%collapsible%open] +===== +`results_field`:::: +(Optional, string) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-results-field] + +`tokenization`:::: +(Optional, object) +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-nlp-tokenization] ++ +Refer to <> to review the properties of the +`tokenization` object. +===== + `text_similarity`::: (Object, optional) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=inference-config-text-similarity] From 9c9670bbfd6cdb3679689627894b9659646f06a7 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 9 Jan 2025 16:44:25 +0100 Subject: [PATCH 32/33] Optimize CachedSupplier a little (#119563) This thing shows up a lot in profiling many-shards searches and fields caps. This is mainly due to expensive `initResult` but we can at least get a little better cache behavior here on some architectures by saving one volatile read. --- .../common/util/CachedSupplier.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/util/CachedSupplier.java b/server/src/main/java/org/elasticsearch/common/util/CachedSupplier.java index e5ab006bd3c1..f394d9d8367c 100644 --- a/server/src/main/java/org/elasticsearch/common/util/CachedSupplier.java +++ b/server/src/main/java/org/elasticsearch/common/util/CachedSupplier.java @@ -19,7 +19,9 @@ import java.util.function.Supplier; public final class CachedSupplier implements Supplier { private volatile Supplier supplier; - private volatile T result; + // result does not need to be volatile as we only read it after reading that the supplier got nulled out. Since we null out the + // supplier after setting the result, total store order from an observed volatile write is sufficient to make a plain read safe. + private T result; public static CachedSupplier wrap(Supplier supplier) { if (supplier instanceof CachedSupplier c) { @@ -38,14 +40,18 @@ public final class CachedSupplier implements Supplier { if (supplier == null) { return result; } - initResult(); - return result; + return initResult(); } - private synchronized void initResult() { - if (supplier != null) { - result = supplier.get(); + private synchronized T initResult() { + var s = supplier; + if (s != null) { + T res = s.get(); + result = res; supplier = null; + return res; + } else { + return result; } } From ae6ebbe22e3a493e65332692ab8c90c96523e05e Mon Sep 17 00:00:00 2001 From: Pawan Kartik Date: Thu, 9 Jan 2025 16:01:55 +0000 Subject: [PATCH 33/33] ESQL: `connect_transport_exception` should be thrown instead of `verification_exception` when ENRICH-ing if remote is disconnected (#119750) * fix: `verification_exception` is thrown instead of `connect_transport_exception` in ENRICH In the context of ENRICH, if a remote is disconnected and skip unavailable is set to `true`, then `verification_exception` is thrown instead of `connect_transport_exception`. This PR fixes this and adds the IT tests for ENRICH for RCS 1 and RCS 2. * Update docs/changelog/119750.yaml * Update 119750.yaml --------- Co-authored-by: Michael Peterson --- docs/changelog/119750.yaml | 6 + .../esql/session/EsqlSessionCCSUtils.java | 4 + ...terEsqlRCS1EnrichUnavailableRemotesIT.java | 353 ++++++++++++++++ ...terEsqlRCS2EnrichUnavailableRemotesIT.java | 380 ++++++++++++++++++ 4 files changed, 743 insertions(+) create mode 100644 docs/changelog/119750.yaml create mode 100644 x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java create mode 100644 x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java diff --git a/docs/changelog/119750.yaml b/docs/changelog/119750.yaml new file mode 100644 index 000000000000..2ec5c298d0eb --- /dev/null +++ b/docs/changelog/119750.yaml @@ -0,0 +1,6 @@ +pr: 119750 +summary: "ESQL: `connect_transport_exception` should be thrown instead of `verification_exception`\ + \ when ENRICH-ing if remote is disconnected" +area: Search +type: bug +issues: [] diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java index 95f7a37ce4d6..f8670a8e6d05 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java @@ -211,6 +211,10 @@ class EsqlSessionCCSUtils { * Mark it as SKIPPED with 0 shards searched and took=0. */ for (String c : clustersWithNoMatchingIndices) { + if (executionInfo.getCluster(c).getStatus() == EsqlExecutionInfo.Cluster.Status.SKIPPED) { + // if cluster was already marked SKIPPED during enrich policy resolution, do not overwrite + continue; + } final String indexExpression = executionInfo.getCluster(c).getIndexExpression(); if (missingIndicesIsFatal(c, executionInfo)) { String error = Strings.format( diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java new file mode 100644 index 000000000000..47c7ac8241fc --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java @@ -0,0 +1,353 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.remotecluster; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; + +public class CrossClusterEsqlRCS1EnrichUnavailableRemotesIT extends AbstractRemoteClusterSecurityTestCase { + private static final AtomicBoolean SSL_ENABLED_REF = new AtomicBoolean(); + + static { + fulfillingCluster = ElasticsearchCluster.local() + .name("fulfilling-cluster") + .nodes(1) + .module("x-pack-autoscaling") + .module("x-pack-esql") + .module("x-pack-enrich") + .module("x-pack-ml") + .module("ingest-common") + .apply(commonClusterConfig) + .setting("remote_cluster.port", "0") + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) + .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key") + .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt") + .setting("xpack.security.authc.token.enabled", "true") + .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password") + .node(0, spec -> spec.setting("remote_cluster_server.enabled", "true")) + .build(); + + queryCluster = ElasticsearchCluster.local() + .name("query-cluster") + .module("x-pack-autoscaling") + .module("x-pack-esql") + .module("x-pack-enrich") + .module("x-pack-ml") + .module("ingest-common") + .apply(commonClusterConfig) + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) + .build(); + } + + @ClassRule + public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster); + + private final String[] modes = { "_coordinator", "_remote" }; + + @Before + public void setupPreRequisites() throws IOException { + setupRolesAndPrivileges(); + setSourceData(); + + var policy = createPolicy("employees-policy", "employees", "email", new String[] { "id", "designation" }); + // Create the enrich policy on both clusters. + assertOK(client().performRequest(policy)); + assertOK(performRequestAgainstFulfillingCluster(policy)); + + // Execute the enrich policy on both clusters. + var exec = executePolicy("employees-policy"); + assertOK(client().performRequest(exec)); + assertOK(performRequestAgainstFulfillingCluster(exec)); + } + + public void testEsqlEnrichWithSkipUnavailable() throws Exception { + esqlEnrichWithRandomSkipUnavailable(); + esqlEnrichWithSkipUnavailableTrue(); + esqlEnrichWithSkipUnavailableFalse(); + } + + private void esqlEnrichWithRandomSkipUnavailable() throws Exception { + configureRemoteCluster("my_remote_cluster", fulfillingCluster, true, randomBoolean(), randomBoolean()); + + String query = "FROM to-be-enr*,my_remote_cluster:to-be-enr* | ENRICH " + randomFrom(modes) + ":employees-policy | LIMIT 10"; + Response response = client().performRequest(esqlRequest(query)); + + Map map = responseAsMap(response); + ArrayList values = (ArrayList) map.get("values"); + Map clusters = (Map) map.get("_clusters"); + Map clusterDetails = (Map) clusters.get("details"); + Map localClusterDetails = (Map) clusterDetails.get("(local)"); + Map remoteClusterDetails = (Map) clusterDetails.get("my_remote_cluster"); + + assertOK(response); + assertThat((int) map.get("took"), greaterThan(0)); + assertThat(values.size(), is(6)); + for (int i = 0; i < 6; i++) { + ArrayList value = (ArrayList) values.get(i); + // Size is 3: ID, Email, Designation. + assertThat(value.size(), is(3)); + // Email + assertThat((String) value.get(0), endsWith("@corp.co")); + // ID + assertThat(value.get(1), is(i + 1)); + } + + assertThat((int) clusters.get("total"), is(2)); + assertThat((int) clusters.get("successful"), is(2)); + assertThat((int) clusters.get("running"), is(0)); + assertThat((int) clusters.get("skipped"), is(0)); + assertThat((int) clusters.get("partial"), is(0)); + assertThat((int) clusters.get("failed"), is(0)); + + assertThat(clusterDetails.size(), is(2)); + assertThat((int) localClusterDetails.get("took"), greaterThan(0)); + assertThat(localClusterDetails.get("status"), is("successful")); + + assertThat((int) remoteClusterDetails.get("took"), greaterThan(0)); + assertThat(remoteClusterDetails.get("status"), is("successful")); + } + + @SuppressWarnings("unchecked") + private void esqlEnrichWithSkipUnavailableTrue() throws Exception { + configureRemoteCluster("my_remote_cluster", fulfillingCluster, true, randomBoolean(), true); + + try { + fulfillingCluster.stop(true); + + String query = "FROM to-be-enriched,my_remote_cluster:to-be-enriched | ENRICH employees-policy | LIMIT 10"; + Response response = client().performRequest(esqlRequest(query)); + + Map map = responseAsMap(response); + ArrayList values = (ArrayList) map.get("values"); + Map clusters = (Map) map.get("_clusters"); + Map clusterDetails = (Map) clusters.get("details"); + Map localClusterDetails = (Map) clusterDetails.get("(local)"); + Map remoteClusterDetails = (Map) clusterDetails.get("my_remote_cluster"); + + assertOK(response); + assertThat((int) map.get("took"), greaterThan(0)); + assertThat(values.size(), is(3)); + + // We only have 3 values since the remote cluster is turned off. + for (int i = 0; i < 3; i++) { + ArrayList value = (ArrayList) values.get(i); + // Size is 3: ID, Email, Designation. + assertThat(value.size(), is(3)); + // Email + assertThat((String) value.get(0), endsWith("@corp.co")); + // ID + assertThat(value.get(1), is(i + 1)); + } + + assertThat((int) clusters.get("total"), is(2)); + assertThat((int) clusters.get("successful"), is(1)); + assertThat((int) clusters.get("running"), is(0)); + assertThat((int) clusters.get("skipped"), is(1)); + assertThat((int) clusters.get("partial"), is(0)); + assertThat((int) clusters.get("failed"), is(0)); + + assertThat(clusterDetails.size(), is(2)); + assertThat((int) localClusterDetails.get("took"), greaterThan(0)); + assertThat(localClusterDetails.get("status"), is("successful")); + + assertThat((int) remoteClusterDetails.get("took"), greaterThan(0)); + assertThat(remoteClusterDetails.get("status"), is("skipped")); + + ArrayList remoteClusterFailures = (ArrayList) remoteClusterDetails.get("failures"); + assertThat(remoteClusterFailures.size(), equalTo(1)); + Map failuresMap = (Map) remoteClusterFailures.get(0); + + Map reason = (Map) failuresMap.get("reason"); + assertThat(reason.get("type").toString(), equalTo("connect_transport_exception")); + assertThat(reason.get("reason").toString(), containsString("Unable to connect to [my_remote_cluster]")); + } finally { + fulfillingCluster.start(); + closeFulfillingClusterClient(); + initFulfillingClusterClient(); + } + } + + private void esqlEnrichWithSkipUnavailableFalse() throws Exception { + configureRemoteCluster("my_remote_cluster", fulfillingCluster, true, randomBoolean(), false); + + try { + fulfillingCluster.stop(true); + + String query = "FROM to-be-enr*,my_remote_cluster:to-be-enr* | ENRICH " + randomFrom(modes) + ":employees-policy | LIMIT 10"; + ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(query))); + assertThat(ex.getMessage(), containsString("connect_transport_exception")); + } finally { + fulfillingCluster.start(); + closeFulfillingClusterClient(); + initFulfillingClusterClient(); + } + } + + private void setupRolesAndPrivileges() throws IOException { + var putUserRequest = new Request("PUT", "/_security/user/" + REMOTE_SEARCH_USER); + putUserRequest.setJsonEntity(""" + { + "password": "x-pack-test-password", + "roles" : ["remote_search"] + }"""); + assertOK(adminClient().performRequest(putUserRequest)); + + var putRoleOnRemoteClusterRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE); + putRoleOnRemoteClusterRequest.setJsonEntity(""" + { + "indices": [ + { + "names": ["*"], + "privileges": ["read"] + } + ], + "cluster": [ "monitor_enrich", "manage_own_api_key" ], + "remote_indices": [ + { + "names": ["*"], + "privileges": ["read"], + "clusters": ["my_remote_cluster"] + } + ], + "remote_cluster": [ + { + "privileges": ["monitor_enrich"], + "clusters": ["my_remote_cluster"] + } + ] + }"""); + assertOK(adminClient().performRequest(putRoleOnRemoteClusterRequest)); + } + + private void setSourceData() throws IOException { + Request createIndex = new Request("PUT", "employees"); + createIndex.setJsonEntity(""" + { + "mappings": { + "properties": { + "id": { "type": "integer" }, + "email": { "type": "text" }, + "designation": { "type": "text" } + } + } + } + """); + assertOK(client().performRequest(createIndex)); + assertOK(performRequestAgainstFulfillingCluster(createIndex)); + + Request bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity(""" + { "index": { "_index": "employees" } } + { "id": 1, "email": "a@corp.co", "designation": "SDE intern"} + { "index": { "_index": "employees" } } + { "id": 2, "email": "b@corp.co", "designation": "SDE 1"} + { "index": { "_index": "employees" } } + { "id": 3, "email": "c@corp.co", "designation": "SDE 2"} + { "index": { "_index": "employees" } } + { "id": 4, "email": "d@corp.co", "designation": "SSE"} + { "index": { "_index": "employees" } } + { "id": 5, "email": "e@corp.co", "designation": "PSE 1"} + { "index": { "_index": "employees" } } + { "id": 6, "email": "f@corp.co", "designation": "PSE 2"} + """); + assertOK(client().performRequest(bulkRequest)); + assertOK(performRequestAgainstFulfillingCluster(bulkRequest)); + + createIndex = new Request("PUT", "to-be-enriched"); + createIndex.setJsonEntity(""" + { + "mappings": { + "properties": { + "email": { "type": "text" } + } + } + } + """); + assertOK(client().performRequest(createIndex)); + assertOK(performRequestAgainstFulfillingCluster(createIndex)); + + bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity(""" + { "index": { "_index": "to-be-enriched" } } + { "email": "a@corp.co"} + { "index": { "_index": "to-be-enriched" } } + { "email": "b@corp.co"} + { "index": { "_index": "to-be-enriched" } } + { "email": "c@corp.co"} + """); + assertOK(client().performRequest(bulkRequest)); + + bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity(""" + { "index": { "_index": "to-be-enriched" } } + { "email": "d@corp.co"} + { "index": { "_index": "to-be-enriched" } } + { "email": "e@corp.co"} + { "index": { "_index": "to-be-enriched" } } + { "email": "f@corp.co"} + """); + assertOK(performRequestAgainstFulfillingCluster(bulkRequest)); + } + + private Request createPolicy(String policyName, String matchIndex, String matchField, String[] enrichFields) throws IOException { + XContentBuilder body = JsonXContent.contentBuilder(); + body.startObject(); + body.startObject("match"); + body.field("indices", matchIndex); + body.field("match_field", matchField); + body.field("enrich_fields", enrichFields); + + body.endObject(); + body.endObject(); + + return makeRequest("PUT", "_enrich/policy/" + policyName, body); + } + + private Request executePolicy(String policyName) { + return new Request("PUT", "_enrich/policy/employees-policy/_execute"); + } + + private Request esqlRequest(String query) throws IOException { + XContentBuilder body = JsonXContent.contentBuilder(); + + body.startObject(); + body.field("query", query); + body.field("include_ccs_metadata", true); + body.endObject(); + + return makeRequest("POST", "_query", body); + } + + private Request makeRequest(String method, String endpoint, XContentBuilder requestBody) { + Request request = new Request(method, endpoint); + request.setJsonEntity(Strings.toString(requestBody)); + return request; + } +} diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java new file mode 100644 index 000000000000..da59e3c77273 --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java @@ -0,0 +1,380 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.remotecluster; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.util.resource.Resource; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; + +public class CrossClusterEsqlRCS2EnrichUnavailableRemotesIT extends AbstractRemoteClusterSecurityTestCase { + private static final AtomicReference> API_KEY_MAP_REF = new AtomicReference<>(); + + static { + fulfillingCluster = ElasticsearchCluster.local() + .name("fulfilling-cluster") + .nodes(1) + .module("x-pack-autoscaling") + .module("x-pack-esql") + .module("x-pack-enrich") + .module("x-pack-ml") + .module("ingest-common") + .apply(commonClusterConfig) + .setting("remote_cluster.port", "0") + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.remote_cluster_server.ssl.enabled", "true") + .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key") + .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt") + .setting("xpack.security.authc.token.enabled", "true") + .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password") + .node(0, spec -> spec.setting("remote_cluster_server.enabled", "true")) + .build(); + + queryCluster = ElasticsearchCluster.local() + .name("query-cluster") + .module("x-pack-autoscaling") + .module("x-pack-esql") + .module("x-pack-enrich") + .module("x-pack-ml") + .module("ingest-common") + .apply(commonClusterConfig) + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.remote_cluster_client.ssl.enabled", "true") + .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt") + .setting("xpack.security.authc.token.enabled", "true") + .keystore("cluster.remote.my_remote_cluster.credentials", () -> { + if (API_KEY_MAP_REF.get() == null) { + final Map apiKeyMap = createCrossClusterAccessApiKey(""" + { + "search": [ + { + "names": ["*"] + } + ] + }"""); + API_KEY_MAP_REF.set(apiKeyMap); + } + return (String) API_KEY_MAP_REF.get().get("encoded"); + }) + .rolesFile(Resource.fromClasspath("roles.yml")) + .user(REMOTE_METRIC_USER, PASS.toString(), "read_remote_shared_metrics", false) + .build(); + } + + @ClassRule + public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster); + + private String[] modes = { "_coordinator", "_remote" }; + + @Before + public void setupPreRequisites() throws IOException { + setupRolesAndPrivileges(); + setSourceData(); + + var policy = createPolicy("employees-policy", "employees", "email", new String[] { "id", "designation" }); + // Create the enrich policy on both clusters. + assertOK(client().performRequest(policy)); + assertOK(performRequestAgainstFulfillingCluster(policy)); + + // Execute the enrich policy on both clusters. + var exec = executePolicy("employees-policy"); + assertOK(client().performRequest(exec)); + assertOK(performRequestAgainstFulfillingCluster(exec)); + } + + public void testEsqlEnrichWithSkipUnavailable() throws Exception { + esqlEnrichWithRandomSkipUnavailable(); + esqlEnrichWithSkipUnavailableTrue(); + esqlEnrichWithSkipUnavailableFalse(); + } + + private void esqlEnrichWithRandomSkipUnavailable() throws Exception { + configureRemoteCluster("my_remote_cluster", fulfillingCluster, false, randomBoolean(), randomBoolean()); + + String query = "FROM to-be-enr*,my_remote_cluster:to-be-enr* | ENRICH " + randomFrom(modes) + ":employees-policy | LIMIT 10"; + Response response = performRequestWithRemoteSearchUser(esqlRequest(query)); + + Map map = responseAsMap(response); + ArrayList values = (ArrayList) map.get("values"); + Map clusters = (Map) map.get("_clusters"); + Map clusterDetails = (Map) clusters.get("details"); + Map localClusterDetails = (Map) clusterDetails.get("(local)"); + Map remoteClusterDetails = (Map) clusterDetails.get("my_remote_cluster"); + + assertOK(response); + assertThat((int) map.get("took"), greaterThan(0)); + assertThat(values.size(), is(6)); + for (int i = 0; i < 6; i++) { + ArrayList value = (ArrayList) values.get(i); + // Size is 3: ID, Email, Designation. + assertThat(value.size(), is(3)); + // Email + assertThat((String) value.get(0), endsWith("@corp.co")); + // ID + assertThat(value.get(1), is(i + 1)); + } + + assertThat((int) clusters.get("total"), is(2)); + assertThat((int) clusters.get("successful"), is(2)); + assertThat((int) clusters.get("running"), is(0)); + assertThat((int) clusters.get("skipped"), is(0)); + assertThat((int) clusters.get("partial"), is(0)); + assertThat((int) clusters.get("failed"), is(0)); + + assertThat(clusterDetails.size(), is(2)); + assertThat((int) localClusterDetails.get("took"), greaterThan(0)); + assertThat(localClusterDetails.get("status"), is("successful")); + + assertThat((int) remoteClusterDetails.get("took"), greaterThan(0)); + assertThat(remoteClusterDetails.get("status"), is("successful")); + } + + @SuppressWarnings("unchecked") + private void esqlEnrichWithSkipUnavailableTrue() throws Exception { + configureRemoteCluster("my_remote_cluster", fulfillingCluster, false, randomBoolean(), true); + + try { + fulfillingCluster.stop(true); + + String query = "FROM to-be-enriched,my_remote_cluster:to-be-enriched | ENRICH employees-policy | LIMIT 10"; + Response response = performRequestWithRemoteSearchUser(esqlRequest(query)); + + Map map = responseAsMap(response); + ArrayList values = (ArrayList) map.get("values"); + Map clusters = (Map) map.get("_clusters"); + Map clusterDetails = (Map) clusters.get("details"); + Map localClusterDetails = (Map) clusterDetails.get("(local)"); + Map remoteClusterDetails = (Map) clusterDetails.get("my_remote_cluster"); + + assertOK(response); + assertThat((int) map.get("took"), greaterThan(0)); + assertThat(values.size(), is(3)); + + // We only have 3 values since the remote cluster is turned off. + for (int i = 0; i < 3; i++) { + ArrayList value = (ArrayList) values.get(i); + // Size is 3: ID, Email, Designation. + assertThat(value.size(), is(3)); + // Email + assertThat((String) value.get(0), endsWith("@corp.co")); + // ID + assertThat(value.get(1), is(i + 1)); + } + + assertThat((int) clusters.get("total"), is(2)); + assertThat((int) clusters.get("successful"), is(1)); + assertThat((int) clusters.get("running"), is(0)); + assertThat((int) clusters.get("skipped"), is(1)); + assertThat((int) clusters.get("partial"), is(0)); + assertThat((int) clusters.get("failed"), is(0)); + + assertThat(clusterDetails.size(), is(2)); + assertThat((int) localClusterDetails.get("took"), greaterThan(0)); + assertThat(localClusterDetails.get("status"), is("successful")); + + assertThat((int) remoteClusterDetails.get("took"), greaterThan(0)); + assertThat(remoteClusterDetails.get("status"), is("skipped")); + + ArrayList remoteClusterFailures = (ArrayList) remoteClusterDetails.get("failures"); + assertThat(remoteClusterFailures.size(), equalTo(1)); + Map failuresMap = (Map) remoteClusterFailures.get(0); + + Map reason = (Map) failuresMap.get("reason"); + assertThat(reason.get("type").toString(), equalTo("connect_transport_exception")); + assertThat(reason.get("reason").toString(), containsString("Unable to connect to [my_remote_cluster]")); + } finally { + fulfillingCluster.start(); + closeFulfillingClusterClient(); + initFulfillingClusterClient(); + } + } + + private void esqlEnrichWithSkipUnavailableFalse() throws Exception { + configureRemoteCluster("my_remote_cluster", fulfillingCluster, false, randomBoolean(), false); + + try { + fulfillingCluster.stop(true); + + String query = "FROM to-be-enr*,my_remote_cluster:to-be-enr* | ENRICH " + randomFrom(modes) + ":employees-policy | LIMIT 10"; + ResponseException ex = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(esqlRequest(query))); + assertThat(ex.getMessage(), containsString("connect_transport_exception")); + } finally { + fulfillingCluster.start(); + closeFulfillingClusterClient(); + initFulfillingClusterClient(); + } + } + + private void setupRolesAndPrivileges() throws IOException { + var putUserRequest = new Request("PUT", "/_security/user/" + REMOTE_SEARCH_USER); + putUserRequest.setJsonEntity(""" + { + "password": "x-pack-test-password", + "roles" : ["remote_search"] + }"""); + assertOK(adminClient().performRequest(putUserRequest)); + + var putRoleOnRemoteClusterRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE); + putRoleOnRemoteClusterRequest.setJsonEntity(""" + { + "indices": [ + { + "names": ["*"], + "privileges": ["read"] + } + ], + "cluster": [ "monitor_enrich", "manage_own_api_key" ], + "remote_indices": [ + { + "names": ["*"], + "privileges": ["read"], + "clusters": ["my_remote_cluster"] + } + ], + "remote_cluster": [ + { + "privileges": ["monitor_enrich"], + "clusters": ["my_remote_cluster"] + } + ] + }"""); + assertOK(adminClient().performRequest(putRoleOnRemoteClusterRequest)); + } + + private void setSourceData() throws IOException { + Request createIndex = new Request("PUT", "employees"); + createIndex.setJsonEntity(""" + { + "mappings": { + "properties": { + "id": { "type": "integer" }, + "email": { "type": "text" }, + "designation": { "type": "text" } + } + } + } + """); + assertOK(client().performRequest(createIndex)); + assertOK(performRequestAgainstFulfillingCluster(createIndex)); + + Request bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity(""" + { "index": { "_index": "employees" } } + { "id": 1, "email": "a@corp.co", "designation": "SDE intern"} + { "index": { "_index": "employees" } } + { "id": 2, "email": "b@corp.co", "designation": "SDE 1"} + { "index": { "_index": "employees" } } + { "id": 3, "email": "c@corp.co", "designation": "SDE 2"} + { "index": { "_index": "employees" } } + { "id": 4, "email": "d@corp.co", "designation": "SSE"} + { "index": { "_index": "employees" } } + { "id": 5, "email": "e@corp.co", "designation": "PSE 1"} + { "index": { "_index": "employees" } } + { "id": 6, "email": "f@corp.co", "designation": "PSE 2"} + """); + assertOK(client().performRequest(bulkRequest)); + assertOK(performRequestAgainstFulfillingCluster(bulkRequest)); + + createIndex = new Request("PUT", "to-be-enriched"); + createIndex.setJsonEntity(""" + { + "mappings": { + "properties": { + "email": { "type": "text" } + } + } + } + """); + assertOK(client().performRequest(createIndex)); + assertOK(performRequestAgainstFulfillingCluster(createIndex)); + + bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity(""" + { "index": { "_index": "to-be-enriched" } } + { "email": "a@corp.co"} + { "index": { "_index": "to-be-enriched" } } + { "email": "b@corp.co"} + { "index": { "_index": "to-be-enriched" } } + { "email": "c@corp.co"} + """); + assertOK(client().performRequest(bulkRequest)); + + bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity(""" + { "index": { "_index": "to-be-enriched" } } + { "email": "d@corp.co"} + { "index": { "_index": "to-be-enriched" } } + { "email": "e@corp.co"} + { "index": { "_index": "to-be-enriched" } } + { "email": "f@corp.co"} + """); + assertOK(performRequestAgainstFulfillingCluster(bulkRequest)); + } + + private Request createPolicy(String policyName, String matchIndex, String matchField, String[] enrichFields) throws IOException { + XContentBuilder body = JsonXContent.contentBuilder(); + body.startObject(); + body.startObject("match"); + body.field("indices", matchIndex); + body.field("match_field", matchField); + body.field("enrich_fields", enrichFields); + + body.endObject(); + body.endObject(); + + return makeRequest("PUT", "_enrich/policy/" + policyName, body); + } + + private Request executePolicy(String policyName) { + return new Request("PUT", "_enrich/policy/employees-policy/_execute"); + } + + private Request esqlRequest(String query) throws IOException { + XContentBuilder body = JsonXContent.contentBuilder(); + + body.startObject(); + body.field("query", query); + body.field("include_ccs_metadata", true); + body.endObject(); + + return makeRequest("POST", "_query", body); + } + + private Request makeRequest(String method, String endpoint, XContentBuilder requestBody) { + Request request = new Request(method, endpoint); + request.setJsonEntity(Strings.toString(requestBody)); + return request; + } + + private Response performRequestWithRemoteSearchUser(final Request request) throws IOException { + request.setOptions( + RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", headerFromRandomAuthMethod(REMOTE_SEARCH_USER, PASS)) + ); + return client().performRequest(request); + } +}