diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml index 5bf4c8923ef7..238edd7b7092 100644 --- a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml @@ -85,3 +85,27 @@ setup: - match: {fields.day_of_week.keyword.type: keyword} - match: {fields.day_of_week.keyword.searchable: true} - match: {fields.day_of_week.keyword.aggregatable: true} + +--- +"Field caps with errors in runtime mappings section throws": + + - skip: + version: " - 7.11.99" + reason: Runtime mappings support was added in 7.12 + + - do: + catch: bad_request + field_caps: + index: test-* + fields: "*" + body: + runtime_mappings: + day_of_week: + type: keyword + script: + source: "bad syntax" + + - match: { error.type: "script_exception" } + - match: { error.reason: "compile error" } + - match: { error.script : "bad syntax" } + - match: { error.lang : "painless" } diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml index 9a5ea9ee1bcc..1afc26e38603 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml @@ -61,10 +61,13 @@ - is_false: fields.geo.keyword.on_aggregatable_indices - do: + catch: missing field_caps: index: 'my_remote_cluster:some_index_that_doesnt_exist' fields: [number] - - match: { 'fields': {} } # empty response - this index doesn't exists + + - match: { error.type: "index_not_found_exception" } + - match: { error.reason: "no such index [some_index_that_doesnt_exist]" } - do: field_caps: @@ -86,6 +89,23 @@ - match: {fields.number.keyword.aggregatable: true} - match: {fields.number.keyword.type: keyword} + - do: + catch: bad_request + field_caps: + index: 'my_remote_cluster:field_caps_index_1' + fields: [number] + body: + runtime_mappings: + day_of_week: + type: keyword + script: + source: "bad syntax" + + - match: { error.type: "script_exception" } + - match: { error.reason: "compile error" } + - match: { error.script : "bad syntax" } + - match: { error.lang : "painless" } + --- "Get field caps from remote cluster with index filter": - skip: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java new file mode 100644 index 000000000000..c5f422a9b627 --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.search.fieldcaps; + +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.fieldcaps.FieldCapabilitiesIT.ExceptionOnRewriteQueryBuilder; +import org.elasticsearch.search.fieldcaps.FieldCapabilitiesIT.ExceptionOnRewriteQueryPlugin; +import org.elasticsearch.test.AbstractMultiClustersTestCase; +import org.elasticsearch.transport.RemoteTransportException; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; + +public class CCSFieldCapabilitiesIT extends AbstractMultiClustersTestCase { + + @Override + protected Collection remoteClusterAlias() { + return List.of("remote_cluster"); + } + + @Override + protected Collection> nodePlugins(String clusterAlias) { + final List> plugins = new ArrayList<>(super.nodePlugins(clusterAlias)); + plugins.add(ExceptionOnRewriteQueryPlugin.class); + return plugins; + } + + public void testFailuresFromRemote() { + final Client localClient = client(LOCAL_CLUSTER); + final Client remoteClient = client("remote_cluster"); + String localIndex = "local_test"; + assertAcked(localClient.admin().indices().prepareCreate(localIndex)); + localClient.prepareIndex(localIndex).setId("1").setSource("foo", "bar").get(); + localClient.admin().indices().prepareRefresh(localIndex).get(); + + String remoteErrorIndex = "remote_test_error"; + assertAcked(remoteClient.admin().indices().prepareCreate(remoteErrorIndex)); + remoteClient.prepareIndex(remoteErrorIndex).setId("2").setSource("foo", "bar").get(); + remoteClient.admin().indices().prepareRefresh(remoteErrorIndex).get(); + + // regular field_caps across clusters + FieldCapabilitiesResponse response = client().prepareFieldCaps("*", "remote_cluster:*").setFields("*").get(); + assertThat(Arrays.asList(response.getIndices()), containsInAnyOrder(localIndex, "remote_cluster:" + remoteErrorIndex)); + + // adding an index filter so remote call should fail + response = client().prepareFieldCaps("*", "remote_cluster:*") + .setFields("*") + .setIndexFilter(new ExceptionOnRewriteQueryBuilder()) + .get(); + assertThat(response.getIndices()[0], equalTo(localIndex)); + assertThat(response.getFailedIndices()[0], equalTo("remote_cluster:*")); + FieldCapabilitiesFailure failure = response.getFailures() + .stream() + .filter(f -> Arrays.asList(f.getIndices()).contains("remote_cluster:*")) + .findFirst().get(); + Exception ex = failure.getException(); + assertEquals(RemoteTransportException.class, ex.getClass()); + Throwable cause = ExceptionsHelper.unwrapCause(ex); + assertEquals(IllegalArgumentException.class, cause.getClass()); + assertEquals("I throw because I choose to.", cause.getMessage()); + + // if we only query the remote we should get back an exception only + ex = expectThrows( + IllegalArgumentException.class, + () -> client().prepareFieldCaps("remote_cluster:*") + .setFields("*") + .setIndexFilter(new ExceptionOnRewriteQueryBuilder()) + .get()); + assertEquals("I throw because I choose to.", ex.getMessage()); + + // add an index that doesn't fail to the remote + assertAcked(remoteClient.admin().indices().prepareCreate("okay_remote_index")); + remoteClient.prepareIndex("okay_remote_index").setId("2").setSource("foo", "bar").get(); + remoteClient.admin().indices().prepareRefresh("okay_remote_index").get(); + + response = client().prepareFieldCaps("*", "remote_cluster:*") + .setFields("*") + .setIndexFilter(new ExceptionOnRewriteQueryBuilder()) + .get(); + assertThat(Arrays.asList(response.getIndices()), containsInAnyOrder(localIndex, "remote_cluster:okay_remote_index")); + assertThat(response.getFailedIndices()[0], equalTo("remote_cluster:" + remoteErrorIndex)); + failure = response.getFailures() + .stream() + .filter(f -> Arrays.asList(f.getIndices()).contains("remote_cluster:" + remoteErrorIndex)) + .findFirst().get(); + ex = failure.getException(); + assertEquals(RemoteTransportException.class, ex.getClass()); + cause = ExceptionsHelper.unwrapCause(ex); + assertEquals(IllegalArgumentException.class, cause.getClass()); + assertEquals("I throw because I choose to.", cause.getMessage()); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index 1eb3639c872f..9022005b56c5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -8,18 +8,28 @@ package org.elasticsearch.search.fieldcaps; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; import org.elasticsearch.action.fieldcaps.FieldCapabilities; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.transport.RemoteTransportException; import org.junit.Before; import java.io.IOException; @@ -27,15 +37,20 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.function.Predicate; +import static java.util.Collections.singletonList; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.containsInAnyOrder; public class FieldCapabilitiesIT extends ESIntegTestCase { + @Override @Before public void setUp() throws Exception { super.setUp(); @@ -92,7 +107,7 @@ public class FieldCapabilitiesIT extends ESIntegTestCase { @Override protected Collection> nodePlugins() { - return List.of(TestMapperPlugin.class); + return List.of(TestMapperPlugin.class, ExceptionOnRewriteQueryPlugin.class); } public void testFieldAlias() { @@ -258,6 +273,48 @@ public class FieldCapabilitiesIT extends ESIntegTestCase { } } + public void testWithRunntimeMappings() throws InterruptedException { + Map runtimeFields = new HashMap<>(); + runtimeFields.put("day_of_week", Collections.singletonMap("type", "keyword")); + FieldCapabilitiesResponse response = client().prepareFieldCaps().setFields("*").setRuntimeFields(runtimeFields).get(); + Map runtimeField = response.getField("day_of_week"); + assertNotNull(runtimeField); + assertEquals("day_of_week", runtimeField.get("keyword").getName()); + assertEquals("keyword", runtimeField.get("keyword").getType()); + assertTrue(runtimeField.get("keyword").isSearchable()); + assertTrue(runtimeField.get("keyword").isAggregatable()); + } + + public void testFailures() throws InterruptedException { + // in addition to the existing "old_index" and "new_index", create two where the test query throws an error on rewrite + assertAcked(prepareCreate("index1-error")); + assertAcked(prepareCreate("index2-error")); + ensureGreen("index1-error", "index2-error"); + FieldCapabilitiesResponse response = client().prepareFieldCaps() + .setFields("*") + .setIndexFilter(new ExceptionOnRewriteQueryBuilder()) + .get(); + assertEquals(1, response.getFailures().size()); + assertEquals(2, response.getFailedIndices().length); + assertThat(response.getFailures().get(0).getIndices(), arrayContainingInAnyOrder("index1-error", "index2-error")); + Exception failure = response.getFailures().get(0).getException(); + assertEquals(RemoteTransportException.class, failure.getClass()); + assertEquals(IllegalArgumentException.class, failure.getCause().getClass()); + assertEquals("I throw because I choose to.", failure.getCause().getMessage()); + + // the "indices" section should not include failed ones + assertThat(Arrays.asList(response.getIndices()), containsInAnyOrder("old_index", "new_index")); + + // if all requested indices failed, we fail the request by throwing the exception + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> client().prepareFieldCaps("index1-error", "index2-error") + .setFields("*") + .setIndexFilter(new ExceptionOnRewriteQueryBuilder()) + .get()); + assertEquals("I throw because I choose to.", ex.getMessage()); + } + private void assertIndices(FieldCapabilitiesResponse response, String... indices) { assertNotNull(response.getIndices()); Arrays.sort(indices); @@ -265,6 +322,72 @@ public class FieldCapabilitiesIT extends ESIntegTestCase { assertArrayEquals(indices, response.getIndices()); } + /** + * Adds an "exception" query that throws on rewrite if the index name contains the string "error" + */ + public static class ExceptionOnRewriteQueryPlugin extends Plugin implements SearchPlugin { + + public ExceptionOnRewriteQueryPlugin() {} + + @Override + public List> getQueries() { + return singletonList( + new QuerySpec<>("exception", ExceptionOnRewriteQueryBuilder::new, p -> new ExceptionOnRewriteQueryBuilder()) + ); + } + } + + static class ExceptionOnRewriteQueryBuilder extends AbstractQueryBuilder { + + public static final String NAME = "exception"; + + ExceptionOnRewriteQueryBuilder() {} + + ExceptionOnRewriteQueryBuilder(StreamInput in) throws IOException { + super(in); + } + + @Override + protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { + SearchExecutionContext searchExecutionContext = queryRewriteContext.convertToSearchExecutionContext(); + if (searchExecutionContext != null) { + if (searchExecutionContext.indexMatches("*error*")) { + throw new IllegalArgumentException("I throw because I choose to."); + }; + } + return this; + } + + @Override + protected void doWriteTo(StreamOutput out) {} + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.endObject(); + } + + @Override + protected Query doToQuery(SearchExecutionContext context) { + return new MatchAllDocsQuery(); + } + + @Override + protected boolean doEquals(ExceptionOnRewriteQueryBuilder other) { + return false; + } + + @Override + protected int doHashCode() { + return 0; + } + + @Override + public String getWriteableName() { + return NAME; + } + } + public static final class TestMapperPlugin extends Plugin implements MapperPlugin { @Override public Map getMetadataMappers() { diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFailure.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFailure.java new file mode 100644 index 000000000000..5e2b78452a21 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesFailure.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.fieldcaps; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +public class FieldCapabilitiesFailure implements Writeable, ToXContentObject { + + private static final ParseField INDICES_FIELD = new ParseField("indices"); + private static final ParseField FAILURE_FIELD = new ParseField("failure"); + private final List indices; + private final Exception exception; + + public FieldCapabilitiesFailure(String[] indices, Exception exception) { + this.indices = new ArrayList<>(Arrays.asList(Objects.requireNonNull(indices))); + this.exception = Objects.requireNonNull(exception); + } + + public FieldCapabilitiesFailure(StreamInput in) throws IOException { + this.indices = in.readStringList(); + this.exception = in.readException(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field(INDICES_FIELD.getPreferredName(), indices); + builder.startObject(FAILURE_FIELD.getPreferredName()); + { + ElasticsearchException.generateFailureXContent(builder, params, exception, true); + } + builder.endObject(); + } + builder.endObject(); + return builder; + } + + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("field_capabilities_failure", true, a -> { + return new FieldCapabilitiesFailure(((List) a[0]).toArray(String[]::new), (Exception) a[1]); + }); + + static { + PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES_FIELD); + PARSER.declareObject( + ConstructingObjectParser.constructorArg(), + (p, c) -> { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, p.currentToken(), p); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, p.nextToken(), p); + Exception e = ElasticsearchException.failureFromXContent(p); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, p.nextToken(), p); + return e; + }, + FAILURE_FIELD + ); + } + + public static FieldCapabilitiesFailure fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeStringCollection(indices); + out.writeException(exception); + } + + public String[] getIndices() { + return indices.toArray(String[]::new); + } + + public Exception getException() { + return exception; + } + + FieldCapabilitiesFailure addIndex(String index) { + this.indices.add(index); + return this; + } + + FieldCapabilitiesFailure addIndices(List indices) { + this.indices.addAll(indices); + return this; + } +} diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java index e048644a3b9b..c16d31f5958e 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java @@ -12,6 +12,8 @@ import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.index.query.QueryBuilder; +import java.util.Map; + public class FieldCapabilitiesRequestBuilder extends ActionRequestBuilder { public FieldCapabilitiesRequestBuilder(ElasticsearchClient client, FieldCapabilitiesAction action, @@ -36,4 +38,9 @@ public class FieldCapabilitiesRequestBuilder extends ActionRequestBuilder runtimeFieldSection) { + request().runtimeFields(runtimeFieldSection); + return this; + } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java index 173036afd12d..650b2e6a6db6 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java @@ -36,24 +36,36 @@ import java.util.stream.Collectors; public class FieldCapabilitiesResponse extends ActionResponse implements ToXContentObject { private static final ParseField INDICES_FIELD = new ParseField("indices"); private static final ParseField FIELDS_FIELD = new ParseField("fields"); + private static final ParseField FAILED_INDICES_FIELD = new ParseField("failed_indices"); + private static final ParseField FAILURES_FIELD = new ParseField("failures"); private final String[] indices; private final Map> responseMap; + private final List failures; private final List indexResponses; - public FieldCapabilitiesResponse(String[] indices, Map> responseMap) { - this(indices, responseMap, Collections.emptyList()); + public FieldCapabilitiesResponse( + String[] indices, + Map> responseMap, + List failures + ) { + this(indices, responseMap, Collections.emptyList(), failures); } - FieldCapabilitiesResponse(List indexResponses) { - this(Strings.EMPTY_ARRAY, Collections.emptyMap(), indexResponses); + public FieldCapabilitiesResponse(String[] indices, Map> responseMap) { + this(indices, responseMap, Collections.emptyList(), Collections.emptyList()); + } + + FieldCapabilitiesResponse(List indexResponses, List failures) { + this(Strings.EMPTY_ARRAY, Collections.emptyMap(), indexResponses, failures); } private FieldCapabilitiesResponse(String[] indices, Map> responseMap, - List indexResponses) { + List indexResponses, List failures) { this.responseMap = Objects.requireNonNull(responseMap); this.indexResponses = Objects.requireNonNull(indexResponses); this.indices = indices; + this.failures = failures; } public FieldCapabilitiesResponse(StreamInput in) throws IOException { @@ -65,22 +77,26 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont } this.responseMap = in.readMap(StreamInput::readString, FieldCapabilitiesResponse::readField); indexResponses = in.readList(FieldCapabilitiesIndexResponse::new); + if (in.getVersion().onOrAfter(Version.CURRENT)) { + this.failures = in.readList(FieldCapabilitiesFailure::new); + } else { + this.failures = Collections.emptyList(); + } } /** - * Used for serialization - */ - FieldCapabilitiesResponse() { - this(Strings.EMPTY_ARRAY, Collections.emptyMap(), Collections.emptyList()); - } - - /** - * Get the concrete list of indices that were requested. + * Get the concrete list of indices that were requested and returned a response. */ public String[] getIndices() { return indices; } + /** + * Get the concrete list of indices that failed + */ + public String[] getFailedIndices() { + return this.failures.stream().map(FieldCapabilitiesFailure::getIndices).flatMap(s -> Arrays.stream(s)).toArray(String[]::new); + } /** * Get the field capabilities map. @@ -89,6 +105,12 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont return responseMap; } + /** + * Get possible request failures keyed by index name + */ + public List getFailures() { + return failures; + } /** * Returns the actual per-index field caps responses @@ -127,6 +149,9 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont } out.writeMap(responseMap, StreamOutput::writeString, FieldCapabilitiesResponse::writeField); out.writeList(indexResponses); + if (out.getVersion().onOrAfter(Version.CURRENT)) { + out.writeList(failures); + } } private static void writeField(StreamOutput out, Map map) throws IOException { @@ -141,6 +166,10 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont builder.startObject(); builder.field(INDICES_FIELD.getPreferredName(), indices); builder.field(FIELDS_FIELD.getPreferredName(), responseMap); + if (this.failures.size() > 0) { + builder.field(FAILED_INDICES_FIELD.getPreferredName(), getFailedIndices().length); + builder.field(FAILURES_FIELD.getPreferredName(), failures); + } builder.endObject(); return builder; } @@ -151,19 +180,27 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("field_capabilities_response", true, - a -> { - List indices = a[0] == null ? Collections.emptyList() : (List) a[0]; - return new FieldCapabilitiesResponse(indices.stream().toArray(String[]::new), - ((List>>) a[1]).stream().collect(Collectors.toMap(Tuple::v1, Tuple::v2))); - }); + new ConstructingObjectParser<>("field_capabilities_response", true, a -> { + Map> responseMap = ((List>>) a[0]).stream() + .collect(Collectors.toMap(Tuple::v1, Tuple::v2)); + List indices = a[1] == null ? Collections.emptyList() : (List) a[1]; + List failures = a[2] == null + ? Collections.emptyList() + : (List) a[2]; + return new FieldCapabilitiesResponse(indices.stream().toArray(String[]::new), responseMap, failures); + }); static { - PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD); PARSER.declareNamedObjects(ConstructingObjectParser.constructorArg(), (p, c, n) -> { Map typeToCapabilities = parseTypeToCapabilities(p, n); return new Tuple<>(n, typeToCapabilities); }, FIELDS_FIELD); + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), INDICES_FIELD); + PARSER.declareObjectArray( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> FieldCapabilitiesFailure.fromXContent(p), + FAILURES_FIELD + ); } private static Map parseTypeToCapabilities(XContentParser parser, String name) throws IOException { @@ -187,12 +224,13 @@ public class FieldCapabilitiesResponse extends ActionResponse implements ToXCont FieldCapabilitiesResponse that = (FieldCapabilitiesResponse) o; return Arrays.equals(indices, that.indices) && Objects.equals(responseMap, that.responseMap) && - Objects.equals(indexResponses, that.indexResponses); + Objects.equals(indexResponses, that.indexResponses) && + Objects.equals(failures, that.failures); } @Override public int hashCode() { - int result = Objects.hash(responseMap, indexResponses); + int result = Objects.hash(responseMap, indexResponses, failures); result = 31 * result + Arrays.hashCode(indices); return result; } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 9441bae1fed7..219ee6e426fc 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.OriginalIndices; @@ -19,6 +20,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.indices.IndicesService; @@ -87,47 +89,55 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction indexResponses = Collections.synchronizedList(new ArrayList<>()); - final ActionListener> countDownListener = new ActionListener<>() { - @Override - public void onResponse(List results) { - for (FieldCapabilitiesIndexResponse res : results) { - if (res.canMatch()) { - indexResponses.add(res); - } - } - countDown(); - } + final FailureCollector indexFailures = new FailureCollector(); - @Override - public void onFailure(Exception e) { - // TODO we should somehow inform the user that we failed - countDown(); - } - - private void countDown() { - if (completionCounter.countDown()) { + final Runnable countDown = () -> { + if (completionCounter.countDown()) { + List failures = indexFailures.values(); + if (indexResponses.size() > 0) { if (request.isMergeResults()) { - listener.onResponse(merge(indexResponses, request.includeUnmapped())); + listener.onResponse(merge(indexResponses, request.includeUnmapped(), new ArrayList<>(failures))); } else { - listener.onResponse(new FieldCapabilitiesResponse(indexResponses)); + listener.onResponse(new FieldCapabilitiesResponse(indexResponses, new ArrayList<>(failures))); + } + } else { + // we have no responses at all, maybe because of errors + if (indexFailures.size() > 0) { + // throw back the first exception + listener.onFailure(failures.iterator().next().getException()); + } else { + listener.onResponse(new FieldCapabilitiesResponse(Collections.emptyList(), Collections.emptyList())); } } } }; for (String index : concreteIndices) { - client.executeLocally(TransportFieldCapabilitiesIndexAction.TYPE, + client.executeLocally( + TransportFieldCapabilitiesIndexAction.TYPE, new FieldCapabilitiesIndexRequest( request.fields(), index, localIndices, request.indexFilter(), - nowInMillis, request.runtimeFields() + nowInMillis, + request.runtimeFields() ), - ActionListener.wrap( - response -> countDownListener.onResponse(Collections.singletonList(response)), - countDownListener::onFailure - ) + new ActionListener() { + @Override + public void onResponse(FieldCapabilitiesIndexResponse result) { + if (result.canMatch()) { + indexResponses.add(result); + } + countDown.run(); + } + + @Override + public void onFailure(Exception e) { + indexFailures.collect(e, index); + countDown.run(); + } + } ); } @@ -145,25 +155,36 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction { - List remotes = new ArrayList<>(); - for (FieldCapabilitiesIndexResponse resp : response.getIndexResponses()) { - remotes.add(new FieldCapabilitiesIndexResponse( + remoteClusterClient.fieldCaps(remoteRequest, ActionListener.wrap(response -> { + for (FieldCapabilitiesIndexResponse resp : response.getIndexResponses()) { + indexResponses.add( + new FieldCapabilitiesIndexResponse( RemoteClusterAware.buildRemoteIndexName(clusterAlias, resp.getIndexName()), - resp.get(), resp.canMatch())); - } - countDownListener.onResponse(remotes); - }, countDownListener::onFailure)); + resp.get(), + resp.canMatch() + ) + ); + } + for (FieldCapabilitiesFailure failure : response.getFailures()) { + Exception ex = failure.getException(); + indexFailures.collectRemoteException(ex, clusterAlias, failure.getIndices()); + } + countDown.run(); + }, ex -> { + indexFailures.collectRemoteException(ex, clusterAlias, originalIndices.indices()); + countDown.run(); + } + )); } } - private FieldCapabilitiesResponse merge(List indexResponses, boolean includeUnmapped) { - String[] indices = indexResponses.stream() - .map(FieldCapabilitiesIndexResponse::getIndexName) - .sorted() - .toArray(String[]::new); - final Map> responseMapBuilder = new HashMap<> (); + private FieldCapabilitiesResponse merge( + List indexResponses, + boolean includeUnmapped, + List failures + ) { + String[] indices = indexResponses.stream().map(FieldCapabilitiesIndexResponse::getIndexName).sorted().toArray(String[]::new); + final Map> responseMapBuilder = new HashMap<>(); for (FieldCapabilitiesIndexResponse response : indexResponses) { innerMerge(responseMapBuilder, response); } @@ -180,8 +201,8 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction typeMap) { @@ -211,4 +232,35 @@ public class TransportFieldCapabilitiesAction extends HandledTransportAction, FieldCapabilitiesFailure> indexFailures = Collections.synchronizedMap( + new HashMap<>() + ); + + List values() { + return new ArrayList<>(indexFailures.values()); + } + + void collect(Exception e, String index) { + // we deduplicate exceptions on the underlying causes message and classname + // we unwrap the cause to e.g. group RemoteTransportexceptions coming from different nodes if the cause is the same + Throwable cause = ExceptionsHelper.unwrapCause(e); + Tuple groupingKey = new Tuple(cause.getMessage(), cause.getClass().getName()); + indexFailures.compute( + groupingKey, + (k, v) -> v == null ? new FieldCapabilitiesFailure(new String[] {index}, e) : v.addIndex(index) + ); + } + + void collectRemoteException(Exception ex, String clusterAlias, String[] remoteIndices) { + for (String failedIndex : remoteIndices) { + collect(ex, RemoteClusterAware.buildRemoteIndexName(clusterAlias, failedIndex)); + } + } + + int size() { + return this.indexFailures.size(); + } + } } diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index 0a6b5fb47b7e..5a37caa4b742 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -240,7 +240,12 @@ public class TransportFieldCapabilitiesIndexAction ShardRouting shardRouting = nextRoutingOrNull(); if (shardRouting == null) { if (canMatchShard == false) { - listener.onResponse(new FieldCapabilitiesIndexResponse(request.index(), Collections.emptyMap(), false)); + if (lastFailure == null) { + listener.onResponse(new FieldCapabilitiesIndexResponse(request.index(), Collections.emptyMap(), false)); + } else { + logger.debug(() -> new ParameterizedMessage("{}: failed to execute [{}]", null, request), lastFailure); + listener.onFailure(lastFailure); + } } else { if (lastFailure == null || isShardNotAvailableException(lastFailure)) { listener.onFailure(new NoShardAvailableActionException(null, diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java index 02cdb7f22f87..7d7f92faec27 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponseTests.java @@ -8,9 +8,16 @@ package org.elasticsearch.action.fieldcaps; +import org.elasticsearch.ElasticsearchExceptionTests; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.hamcrest.Matchers; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -23,13 +30,15 @@ public class FieldCapabilitiesResponseTests extends AbstractWireSerializingTestC @Override protected FieldCapabilitiesResponse createTestInstance() { + FieldCapabilitiesResponse randomResponse; List responses = new ArrayList<>(); int numResponse = randomIntBetween(0, 10); for (int i = 0; i < numResponse; i++) { responses.add(createRandomIndexResponse()); } - return new FieldCapabilitiesResponse(responses); + randomResponse = new FieldCapabilitiesResponse(responses, Collections.emptyList()); + return randomResponse; } @Override @@ -91,6 +100,57 @@ public class FieldCapabilitiesResponseTests extends AbstractWireSerializingTestC FieldCapabilitiesTests.randomFieldCaps(toReplace))); break; } - return new FieldCapabilitiesResponse(null, mutatedResponses); + return new FieldCapabilitiesResponse(null, mutatedResponses, Collections.emptyList()); + } + + public void testFailureSerialization() throws IOException { + FieldCapabilitiesResponse randomResponse = createResponseWithFailures(); + FieldCapabilitiesResponse deserialized = copyInstance(randomResponse); + assertThat(deserialized.getIndices(), Matchers.equalTo(randomResponse.getIndices())); + // only match size of failure list and indices, most exceptions don't support 'equals' + List deserializedFailures = deserialized.getFailures(); + assertEquals(deserializedFailures.size(), randomResponse.getFailures().size()); + int i = 0; + for (FieldCapabilitiesFailure originalFailure : randomResponse.getFailures()) { + FieldCapabilitiesFailure deserializedFaliure = deserializedFailures.get(i); + assertThat(deserializedFaliure.getIndices(), Matchers.equalTo(originalFailure.getIndices())); + i++; + } + } + + public void testFailureParsing() throws IOException { + FieldCapabilitiesResponse randomResponse = createResponseWithFailures(); + boolean humanReadable = randomBoolean(); + XContentType xContentType = randomFrom(XContentType.values()); + BytesReference originalBytes = toShuffledXContent(randomResponse, xContentType, ToXContent.EMPTY_PARAMS, humanReadable); + FieldCapabilitiesResponse parsedResponse; + try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) { + parsedResponse = FieldCapabilitiesResponse.fromXContent(parser); + assertNull(parser.nextToken()); + } + assertNotSame(parsedResponse, randomResponse); + assertThat(parsedResponse.getIndices(), Matchers.equalTo(randomResponse.getIndices())); + // only match size of failure list and indices, most exceptions don't support 'equals' + List deserializedFailures = parsedResponse.getFailures(); + assertEquals(deserializedFailures.size(), randomResponse.getFailures().size()); + int i = 0; + for (FieldCapabilitiesFailure originalFailure : randomResponse.getFailures()) { + FieldCapabilitiesFailure deserializedFaliure = deserializedFailures.get(i); + assertThat(deserializedFaliure.getIndices(), Matchers.equalTo(originalFailure.getIndices())); + i++; + } + } + + private FieldCapabilitiesResponse createResponseWithFailures() { + String[] indices = randomArray(randomIntBetween(1, 5), String[]::new, () -> randomAlphaOfLength(5)); + List failures = new ArrayList<>(); + for (String index : indices) { + if (randomBoolean() || failures.size() == 0) { + failures.add(new FieldCapabilitiesFailure(new String[] {index}, ElasticsearchExceptionTests.randomExceptions().v2())); + } else { + failures.get(failures.size() - 1).addIndex(index); + } + } + return new FieldCapabilitiesResponse(indices, Collections.emptyMap(), failures); } } diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java index e614afcb9345..7442d84f05e9 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/MergedFieldCapabilitiesResponseTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.test.AbstractSerializingTestCase; import java.io.IOException; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Predicate; @@ -85,7 +86,8 @@ public class MergedFieldCapabilitiesResponseTests extends AbstractSerializingTes FieldCapabilitiesTests.randomFieldCaps(toReplace))); break; } - return new FieldCapabilitiesResponse(null, mutatedResponses); + // TODO pass real list + return new FieldCapabilitiesResponse(null, mutatedResponses, Collections.emptyList()); } @Override @@ -105,7 +107,7 @@ public class MergedFieldCapabilitiesResponseTests extends AbstractSerializingTes String generatedResponse = BytesReference.bytes(builder).utf8ToString(); assertEquals(( "{" + - " \"indices\": null," + + " \"indices\": [\"index1\",\"index2\",\"index3\",\"index4\"]," + " \"fields\": {" + " \"rating\": { " + " \"keyword\": {" + @@ -133,15 +135,16 @@ public class MergedFieldCapabilitiesResponseTests extends AbstractSerializingTes " \"aggregatable\": false" + " }" + " }" + - " }" + + " }," + + " \"failed_indices\":2," + + " \"failures\":[" + + " { \"indices\": [\"errorindex\", \"errorindex2\"]," + + " \"failure\" : {\"error\":{\"root_cause\":[{\"type\":\"illegal_argument_exception\"," + + " \"reason\":\"test\"}],\"type\":\"illegal_argument_exception\",\"reason\":\"test\"}}}" + + " ]" + "}").replaceAll("\\s+", ""), generatedResponse); } - public void testEmptyResponse() throws IOException { - FieldCapabilitiesResponse testInstance = new FieldCapabilitiesResponse(); - assertSerialization(testInstance); - } - private static FieldCapabilitiesResponse createSimpleResponse() { Map titleCapabilities = new HashMap<>(); titleCapabilities.put("text", new FieldCapabilities("title", "text", false, true, false, @@ -162,6 +165,10 @@ public class MergedFieldCapabilitiesResponseTests extends AbstractSerializingTes Map> responses = new HashMap<>(); responses.put("title", titleCapabilities); responses.put("rating", ratingCapabilities); - return new FieldCapabilitiesResponse(null, responses); + + List failureMap = List.of( + new FieldCapabilitiesFailure(new String[] { "errorindex", "errorindex2" }, new IllegalArgumentException("test")) + ); + return new FieldCapabilitiesResponse(new String[] {"index1", "index2", "index3", "index4"}, responses, failureMap); } }