From cca7c153debaca557e7ee86d43529e09fba937f3 Mon Sep 17 00:00:00 2001 From: Michael Peterson Date: Thu, 14 Nov 2024 10:43:49 -0500 Subject: [PATCH] ESQL: Honor skip_unavailable setting for nonmatching indices errors at planning time (#116348) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds support to ES|QL planning time (EsqlSession code) for dealing with non-matching indices and how that relates to the remote cluster skip_unavailable setting and also how to deal with missing indices on the local cluster (if included in the query). For clusters included in an ES|QL query: • `skip_unavailable=true` means: if no data is returned from that cluster (due to cluster-not-connected, no matching indices, a missing concrete index or shard failures during searches), it is not a "fatal" error that causes the entire query to fail. Instead it is just a failure on that particular cluster and partial data should be returned from other clusters. • `skip_unavailable=false` means: if no data is returned from that cluster (for the same reasons enumerated above), then the whole query should fail. This allows users to ensure that data is returned from a "required" cluster. • For the local cluster, ES|QL assumes `allow_no_indices=true` and the skip_unavailable setting does not apply (in part because there is no way for a user to set skip_unavailable for the local cluster) Based on discussions with ES|QL team members, we defined the following rules to be enforced with respect to non-matching index expressions: **Rules enforced at planning time** P1. fail the query if there are no matching indices on any cluster (VerificationException) P2. fail the query if a skip_unavailable:false cluster has no matching indices (the local cluster already has this rule enforced at planning time) P3. fail query if the local cluster has no matching indices and a concrete index was specified **Rules enforced at execution time** For missing concrete (no wildcards present) index expressions: E1. fail the query when it was specified for the local cluster or a skip_unavailable=false remote cluster E2: on skip_unavailable=true clusters: an error fails the query on that cluster, but not the entire query (data from other clusters still returned) **Notes on the rules** P1: this already happens, no new code needed in this PR P2: The reason we need to enforce rule 2 at planning time is that when there are no matching indices from field caps the EsIndex that is created (and passed into IndexResolution.valid) leaves that cluster out of the list, so at execution time it will not attempt to query that cluster at all, so execution time will not catch missing concrete indices. And even if it did get queried at execution time it wouldn't fail on wildcard only indices where none of them matched. P3: Right now `FROM remote:existent,nomatch` does NOT throw a failure (for same reason described in rule 2 above) so that needs to be enforced in this PR. This PR deals with enforcing and testing the planning time rules: P1, P2 and P3. A follow-on PR will address changes needed for handling the execution time rules. **Notes on PR scope** This PR covers nonsecured clusters (`xpack.security.enabled: false`) and security using certs ("RCS1). In my testing I've founding that api-key based security ("RCS2") is not behaving the same, so that work has been deferred to a follow-on PR. Partially addresses https://github.com/elastic/elasticsearch/issues/114531 --- docs/changelog/116348.yaml | 5 + .../esql/action/CrossClustersQueryIT.java | 1073 +++++++++++++---- .../xpack/esql/index/IndexResolution.java | 63 +- .../xpack/esql/plugin/ComputeService.java | 24 +- .../xpack/esql/session/EsqlSession.java | 2 +- .../esql/session/EsqlSessionCCSUtils.java | 99 +- .../xpack/esql/session/IndexResolver.java | 20 +- .../session/EsqlSessionCCSUtilsTests.java | 60 +- .../CrossClusterEsqlRCS1MissingIndicesIT.java | 574 +++++++++ .../RemoteClusterSecurityEsqlIT.java | 55 +- 10 files changed, 1670 insertions(+), 305 deletions(-) create mode 100644 docs/changelog/116348.yaml create mode 100644 x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1MissingIndicesIT.java diff --git a/docs/changelog/116348.yaml b/docs/changelog/116348.yaml new file mode 100644 index 000000000000..927ffc5a6121 --- /dev/null +++ b/docs/changelog/116348.yaml @@ -0,0 +1,5 @@ +pr: 116348 +summary: "ESQL: Honor skip_unavailable setting for nonmatching indices errors at planning time" +area: ES|QL +type: enhancement +issues: [ 114531 ] diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java index ba44adb5a85e..6801e1f4eb40 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java @@ -10,6 +10,7 @@ package org.elasticsearch.xpack.esql.action; import org.elasticsearch.Build; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Priority; @@ -21,12 +22,16 @@ import org.elasticsearch.compute.operator.DriverProfile; import org.elasticsearch.compute.operator.exchange.ExchangeService; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractMultiClustersTestCase; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.XContentTestUtils; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; @@ -35,30 +40,36 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { - private static final String REMOTE_CLUSTER = "cluster-a"; + private static final String REMOTE_CLUSTER_1 = "cluster-a"; + private static final String REMOTE_CLUSTER_2 = "remote-b"; @Override protected Collection remoteClusterAlias() { - return List.of(REMOTE_CLUSTER); + return List.of(REMOTE_CLUSTER_1, REMOTE_CLUSTER_2); } @Override protected Map skipUnavailableForRemoteClusters() { - return Map.of(REMOTE_CLUSTER, randomBoolean()); + return Map.of(REMOTE_CLUSTER_1, randomBoolean()); } @Override @@ -90,7 +101,7 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { Tuple includeCCSMetadata = randomIncludeCCSMetadata(); Boolean requestIncludeMeta = includeCCSMetadata.v1(); boolean responseExpectMeta = includeCCSMetadata.v2(); - try (EsqlQueryResponse resp = runQuery("from logs-*,*:logs-* | stats sum (v)", requestIncludeMeta)) { + try (EsqlQueryResponse resp = runQuery("from logs-*,c*:logs-* | stats sum (v)", requestIncludeMeta)) { List> values = getValuesList(resp); assertThat(values, hasSize(1)); assertThat(values.get(0), equalTo(List.of(330L))); @@ -102,9 +113,9 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, LOCAL_CLUSTER))); - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1); assertThat(remoteCluster.getIndexExpression(), equalTo("logs-*")); assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); @@ -128,7 +139,7 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { assertClusterMetadataInResponse(resp, responseExpectMeta); } - try (EsqlQueryResponse resp = runQuery("from logs-*,*:logs-* | stats count(*) by tag | sort tag | keep tag", requestIncludeMeta)) { + try (EsqlQueryResponse resp = runQuery("from logs-*,c*:logs-* | stats count(*) by tag | sort tag | keep tag", requestIncludeMeta)) { List> values = getValuesList(resp); assertThat(values, hasSize(2)); assertThat(values.get(0), equalTo(List.of("local"))); @@ -141,9 +152,9 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, LOCAL_CLUSTER))); - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1); assertThat(remoteCluster.getIndexExpression(), equalTo("logs-*")); assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); @@ -168,171 +179,695 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { } } - public void testSearchesWhereMissingIndicesAreSpecified() { - Map testClusterInfo = setupTwoClusters(); - int localNumShards = (Integer) testClusterInfo.get("local.num_shards"); - int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards"); + public void testSearchesAgainstNonMatchingIndicesWithLocalOnly() { + Map testClusterInfo = setupClusters(2); + String localIndex = (String) testClusterInfo.get("local.index"); + + { + String q = "FROM nomatch," + localIndex; + IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> runQuery(q, false)); + assertThat(e.getDetailedMessage(), containsString("no such index [nomatch]")); + + // MP TODO: am I able to fix this from the field-caps call? Yes, if we detect concrete vs. wildcard expressions in user query + // TODO bug - this does not throw; uncomment this test once https://github.com/elastic/elasticsearch/issues/114495 is fixed + // String limit0 = q + " | LIMIT 0"; + // VerificationException ve = expectThrows(VerificationException.class, () -> runQuery(limit0, false)); + // assertThat(ve.getDetailedMessage(), containsString("No matching indices for [nomatch]")); + } + + { + // no failure since concrete index matches, so wildcard matching is lenient + String q = "FROM nomatch*," + localIndex; + try (EsqlQueryResponse resp = runQuery(q, false)) { + // we are only testing that this does not throw an Exception, so the asserts below are minimal + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(false)); + } + + String limit0 = q + " | LIMIT 0"; + try (EsqlQueryResponse resp = runQuery(limit0, false)) { + // we are only testing that this does not throw an Exception, so the asserts below are minimal + assertThat(resp.columns().size(), greaterThanOrEqualTo(1)); + assertThat(getValuesList(resp).size(), equalTo(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(false)); + } + } + { + String q = "FROM nomatch"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, false)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, false)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch]")); + } + { + String q = "FROM nomatch*"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, false)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch*]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, false)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch*]")); + } + } + + public void testSearchesAgainstIndicesWithNoMappingsSkipUnavailableTrue() { + int numClusters = 2; + setupClusters(numClusters); + Map clusterToEmptyIndexMap = createEmptyIndicesWithNoMappings(numClusters); + setSkipUnavailable(REMOTE_CLUSTER_1, randomBoolean()); Tuple includeCCSMetadata = randomIncludeCCSMetadata(); Boolean requestIncludeMeta = includeCCSMetadata.v1(); boolean responseExpectMeta = includeCCSMetadata.v2(); - // since a valid local index was specified, the invalid index on cluster-a does not throw an exception, - // but instead is simply ignored - ensure this is captured in the EsqlExecutionInfo - try (EsqlQueryResponse resp = runQuery("from logs-*,cluster-a:no_such_index | stats sum (v)", requestIncludeMeta)) { - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - List> values = getValuesList(resp); - assertThat(values, hasSize(1)); - assertThat(values.get(0), equalTo(List.of(45L))); + try { + String emptyIndex = clusterToEmptyIndexMap.get(REMOTE_CLUSTER_1); + String q = Strings.format("FROM cluster-a:%s", emptyIndex); + // query without referring to fields should work + { + String limit1 = q + " | LIMIT 1"; + try (EsqlQueryResponse resp = runQuery(limit1, requestIncludeMeta)) { + assertThat(resp.columns().size(), equalTo(1)); + assertThat(resp.columns().get(0).name(), equalTo("")); + assertThat(getValuesList(resp).size(), equalTo(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of(new ExpectedCluster(REMOTE_CLUSTER_1, emptyIndex, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0)) + ); + } - assertNotNull(executionInfo); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - long overallTookMillis = executionInfo.overallTook().millis(); - assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + String limit0 = q + " | LIMIT 0"; + try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { + assertThat(resp.columns().size(), equalTo(1)); + assertThat(resp.columns().get(0).name(), equalTo("")); + assertThat(getValuesList(resp).size(), equalTo(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of(new ExpectedCluster(REMOTE_CLUSTER_1, emptyIndex, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0)) + ); + } + } - assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); + // query that refers to missing fields should throw: + // "type": "verification_exception", + // "reason": "Found 1 problem\nline 2:7: Unknown column [foo]", + { + String keepQuery = q + " | KEEP foo | LIMIT 100"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(keepQuery, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown column [foo]")); + } - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); - assertThat(remoteCluster.getIndexExpression(), equalTo("no_such_index")); - assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); - assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); - assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); - assertThat(remoteCluster.getTotalShards(), equalTo(0)); // 0 since no matching index, thus no shards to search - assertThat(remoteCluster.getSuccessfulShards(), equalTo(0)); - assertThat(remoteCluster.getSkippedShards(), equalTo(0)); - assertThat(remoteCluster.getFailedShards(), equalTo(0)); - - EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); - assertThat(localCluster.getIndexExpression(), equalTo("logs-*")); - assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); - assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); - assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); - assertThat(localCluster.getTotalShards(), equalTo(localNumShards)); - assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards)); - assertThat(localCluster.getSkippedShards(), equalTo(0)); - assertThat(localCluster.getFailedShards(), equalTo(0)); + } finally { + clearSkipUnavailable(); } + } - // since the remote cluster has a valid index expression, the missing local index is ignored - // make this is captured in the EsqlExecutionInfo - try ( - EsqlQueryResponse resp = runQuery( - "from no_such_index,*:logs-* | stats count(*) by tag | sort tag | keep tag", - requestIncludeMeta - ) - ) { - List> values = getValuesList(resp); - assertThat(values, hasSize(1)); - assertThat(values.get(0), equalTo(List.of("remote"))); + public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableTrue() { + int numClusters = 3; + Map testClusterInfo = setupClusters(numClusters); + int localNumShards = (Integer) testClusterInfo.get("local.num_shards"); + int remote1NumShards = (Integer) testClusterInfo.get("remote.num_shards"); + int remote2NumShards = (Integer) testClusterInfo.get("remote2.num_shards"); + String localIndex = (String) testClusterInfo.get("local.index"); + String remote1Index = (String) testClusterInfo.get("remote.index"); + String remote2Index = (String) testClusterInfo.get("remote2.index"); - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertNotNull(executionInfo); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - long overallTookMillis = executionInfo.overallTook().millis(); - assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + createIndexAliases(numClusters); + setSkipUnavailable(REMOTE_CLUSTER_1, true); + setSkipUnavailable(REMOTE_CLUSTER_2, true); - assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); - assertThat(remoteCluster.getIndexExpression(), equalTo("logs-*")); - assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); - assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); - assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); - assertThat(remoteCluster.getTotalShards(), equalTo(remoteNumShards)); - assertThat(remoteCluster.getSuccessfulShards(), equalTo(remoteNumShards)); - assertThat(remoteCluster.getSkippedShards(), equalTo(0)); - assertThat(remoteCluster.getFailedShards(), equalTo(0)); + try { + // missing concrete local index is fatal + { + String q = "FROM nomatch,cluster-a:" + randomFrom(remote1Index, IDX_ALIAS, FILTERED_IDX_ALIAS); + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch]")); - EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); - assertThat(localCluster.getIndexExpression(), equalTo("no_such_index")); - // TODO: a follow on PR will change this to throw an Exception when the local cluster requests a concrete index that is missing - assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); - assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); - assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); - assertThat(localCluster.getTotalShards(), equalTo(0)); - assertThat(localCluster.getSuccessfulShards(), equalTo(0)); - assertThat(localCluster.getSkippedShards(), equalTo(0)); - assertThat(localCluster.getFailedShards(), equalTo(0)); + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch]")); + } + + // missing concrete remote index is not fatal when skip_unavailable=true (as long as an index matches on another cluster) + { + String localIndexName = randomFrom(localIndex, IDX_ALIAS, FILTERED_IDX_ALIAS); + String q = Strings.format("FROM %s,cluster-a:nomatch", localIndexName); + try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + new ExpectedCluster(LOCAL_CLUSTER, localIndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, localNumShards), + new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0) + ) + ); + } + + String limit0 = q + " | LIMIT 0"; + try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { + assertThat(resp.columns().size(), greaterThan(0)); + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + new ExpectedCluster(LOCAL_CLUSTER, localIndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), + new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0) + ) + ); + } + } + + // since there is at least one matching index in the query, the missing wildcarded local index is not an error + { + String remoteIndexName = randomFrom(remote1Index, IDX_ALIAS, FILTERED_IDX_ALIAS); + String q = "FROM nomatch*,cluster-a:" + remoteIndexName; + try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched + new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), + new ExpectedCluster( + REMOTE_CLUSTER_1, + remoteIndexName, + EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, + remote1NumShards + ) + ) + ); + } + + String limit0 = q + " | LIMIT 0"; + try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), equalTo(0)); + assertThat(resp.columns().size(), greaterThan(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched + new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), + // LIMIT 0 searches always have total shards = 0 + new ExpectedCluster(REMOTE_CLUSTER_1, remoteIndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0) + ) + ); + } + } + + // since at least one index of the query matches on some cluster, a wildcarded index on skip_un=true is not an error + { + String localIndexName = randomFrom(localIndex, IDX_ALIAS, FILTERED_IDX_ALIAS); + String q = Strings.format("FROM %s,cluster-a:nomatch*", localIndexName); + try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + new ExpectedCluster(LOCAL_CLUSTER, localIndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, localNumShards), + new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch*", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0) + ) + ); + } + + String limit0 = q + " | LIMIT 0"; + try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { + assertThat(resp.columns().size(), greaterThan(0)); + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + new ExpectedCluster(LOCAL_CLUSTER, localIndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), + new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch*", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0) + ) + ); + } + } + + // an error is thrown if there are no matching indices at all, even when the cluster is skip_unavailable=true + { + // with non-matching concrete index + String q = "FROM cluster-a:nomatch"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); + } + + // an error is thrown if there are no matching indices at all, even when the cluster is skip_unavailable=true and the + // index was wildcarded + { + // with non-matching wildcard index + String q = "FROM cluster-a:nomatch*"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); + } + + // an error is thrown if there are no matching indices at all - local with wildcard, remote with concrete + { + String q = "FROM nomatch*,cluster-a:nomatch"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch*]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch*]")); + } + + // an error is thrown if there are no matching indices at all - local with wildcard, remote with wildcard + { + String q = "FROM nomatch*,cluster-a:nomatch*"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch*]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch*]")); + } + + // an error is thrown if there are no matching indices at all - local with concrete, remote with concrete + { + String q = "FROM nomatch,cluster-a:nomatch"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch]")); + } + + // an error is thrown if there are no matching indices at all - local with concrete, remote with wildcard + { + String q = "FROM nomatch,cluster-a:nomatch*"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch]")); + } + + // since cluster-a is skip_unavailable=true and at least one cluster has a matching indices, no error is thrown + { + // TODO solve in follow-on PR which does skip_unavailable handling at execution time + // String q = Strings.format("FROM %s,cluster-a:nomatch,cluster-a:%s*", localIndex, remote1Index); + // try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { + // assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); + // EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + // assertThat(executionInfo.isCrossClusterSearch(), is(true)); + // assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + // assertExpectedClustersForMissingIndicesTests(executionInfo, List.of( + // // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched + // new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0), + // new ExpectedCluster(REMOTE_CLUSTER_1, "*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, remote2NumShards) + // )); + // } + + // TODO: handle LIMIT 0 for this case in follow-on PR + // String limit0 = q + " | LIMIT 0"; + // try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { + // assertThat(resp.columns().size(), greaterThanOrEqualTo(1)); + // assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(0)); + // EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + // assertThat(executionInfo.isCrossClusterSearch(), is(true)); + // assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + // assertExpectedClustersForMissingIndicesTests(executionInfo, List.of( + // // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched + // new ExpectedCluster(LOCAL_CLUSTER, localIndex, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), + // new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch," + remote1Index + "*", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0) + // )); + // } + } + + // tests with three clusters --- + + // since cluster-a is skip_unavailable=true and at least one cluster has a matching indices, no error is thrown + // cluster-a should be marked as SKIPPED with VerificationException + { + String remote2IndexName = randomFrom(remote2Index, IDX_ALIAS, FILTERED_IDX_ALIAS); + String q = Strings.format("FROM nomatch*,cluster-a:nomatch,%s:%s", REMOTE_CLUSTER_2, remote2IndexName); + try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched + new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), + new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0), + new ExpectedCluster( + REMOTE_CLUSTER_2, + remote2IndexName, + EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, + remote2NumShards + ) + ) + ); + } + + String limit0 = q + " | LIMIT 0"; + try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { + assertThat(resp.columns().size(), greaterThanOrEqualTo(1)); + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched + new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), + new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0), + new ExpectedCluster(REMOTE_CLUSTER_2, remote2IndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0) + ) + ); + } + } + + // since cluster-a is skip_unavailable=true and at least one cluster has a matching indices, no error is thrown + // cluster-a should be marked as SKIPPED with a "NoMatchingIndicesException" since a wildcard index was requested + { + String remote2IndexName = randomFrom(remote2Index, IDX_ALIAS, FILTERED_IDX_ALIAS); + String q = Strings.format("FROM nomatch*,cluster-a:nomatch*,%s:%s", REMOTE_CLUSTER_2, remote2IndexName); + try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched + new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), + new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch*", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0), + new ExpectedCluster( + REMOTE_CLUSTER_2, + remote2IndexName, + EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, + remote2NumShards + ) + ) + ); + } + + String limit0 = q + " | LIMIT 0"; + try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { + assertThat(resp.columns().size(), greaterThanOrEqualTo(1)); + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched + new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), + new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch*", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0), + new ExpectedCluster(REMOTE_CLUSTER_2, remote2IndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0) + ) + ); + } + } + } finally { + clearSkipUnavailable(); } + } - // when multiple invalid indices are specified on the remote cluster, both should be ignored and present - // in the index expression of the EsqlExecutionInfo and with an indication that zero shards were searched - try ( - EsqlQueryResponse resp = runQuery( - "FROM no_such_index*,*:no_such_index1,*:no_such_index2,logs-1 | STATS COUNT(*) by tag | SORT tag | KEEP tag", - requestIncludeMeta - ) - ) { - List> values = getValuesList(resp); - assertThat(values, hasSize(1)); - assertThat(values.get(0), equalTo(List.of("local"))); + public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableFalse() { + int numClusters = 3; + Map testClusterInfo = setupClusters(numClusters); + int remote1NumShards = (Integer) testClusterInfo.get("remote.num_shards"); + String localIndex = (String) testClusterInfo.get("local.index"); + String remote1Index = (String) testClusterInfo.get("remote.index"); + String remote2Index = (String) testClusterInfo.get("remote2.index"); - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertNotNull(executionInfo); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - long overallTookMillis = executionInfo.overallTook().millis(); - assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + createIndexAliases(numClusters); + setSkipUnavailable(REMOTE_CLUSTER_1, false); + setSkipUnavailable(REMOTE_CLUSTER_2, false); - assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); + Tuple includeCCSMetadata = randomIncludeCCSMetadata(); + Boolean requestIncludeMeta = includeCCSMetadata.v1(); + boolean responseExpectMeta = includeCCSMetadata.v2(); - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); - assertThat(remoteCluster.getIndexExpression(), equalTo("no_such_index1,no_such_index2")); - assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); - assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); - assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); - assertThat(remoteCluster.getTotalShards(), equalTo(0)); - assertThat(remoteCluster.getSuccessfulShards(), equalTo(0)); - assertThat(remoteCluster.getSkippedShards(), equalTo(0)); - assertThat(remoteCluster.getFailedShards(), equalTo(0)); + try { + // missing concrete local index is an error + { + String q = "FROM nomatch,cluster-a:" + remote1Index; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch]")); - EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); - assertThat(localCluster.getIndexExpression(), equalTo("no_such_index*,logs-1")); - assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); - assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); - assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); - assertThat(localCluster.getTotalShards(), equalTo(localNumShards)); - assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards)); - assertThat(localCluster.getSkippedShards(), equalTo(0)); - assertThat(localCluster.getFailedShards(), equalTo(0)); + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch]")); + } + + // missing concrete remote index is fatal when skip_unavailable=false + { + String q = "FROM logs*,cluster-a:nomatch"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); + } + + // No error since local non-matching has wildcard and the remote cluster matches + { + String remote1IndexName = randomFrom(remote1Index, IDX_ALIAS, FILTERED_IDX_ALIAS); + String q = Strings.format("FROM nomatch*,%s:%s", REMOTE_CLUSTER_1, remote1IndexName); + try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + // local cluster is never marked as SKIPPED even when no matcing indices - just marked as 0 shards searched + new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), + new ExpectedCluster( + REMOTE_CLUSTER_1, + remote1IndexName, + EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, + remote1NumShards + ) + ) + ); + } + + String limit0 = q + " | LIMIT 0"; + try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), equalTo(0)); + assertThat(resp.columns().size(), greaterThan(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + // local cluster is never marked as SKIPPED even when no matcing indices - just marked as 0 shards searched + new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), + // LIMIT 0 searches always have total shards = 0 + new ExpectedCluster(REMOTE_CLUSTER_1, remote1IndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0) + ) + ); + } + } + + // query is fatal since cluster-a has skip_unavailable=false and has no matching indices + { + String q = Strings.format("FROM %s,cluster-a:nomatch*", randomFrom(localIndex, IDX_ALIAS, FILTERED_IDX_ALIAS)); + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); + } + + // an error is thrown if there are no matching indices at all - single remote cluster with concrete index expression + { + String q = "FROM cluster-a:nomatch"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); + } + + // an error is thrown if there are no matching indices at all - single remote cluster with wildcard index expression + { + String q = "FROM cluster-a:nomatch*"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); + } + + // an error is thrown if there are no matching indices at all - local with wildcard, remote with concrete + { + String q = "FROM nomatch*,cluster-a:nomatch"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch*]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch*]")); + } + + // an error is thrown if there are no matching indices at all - local with wildcard, remote with wildcard + { + String q = "FROM nomatch*,cluster-a:nomatch*"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch*]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch*]")); + } + + // an error is thrown if there are no matching indices at all - local with concrete, remote with concrete + { + String q = "FROM nomatch,cluster-a:nomatch"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch]")); + } + + // an error is thrown if there are no matching indices at all - local with concrete, remote with wildcard + { + String q = "FROM nomatch,cluster-a:nomatch*"; + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch]")); + } + + // Missing concrete index on skip_unavailable=false cluster is a fatal error, even when another index expression + // against that cluster matches + { + String remote2IndexName = randomFrom(remote2Index, IDX_ALIAS, FILTERED_IDX_ALIAS); + String q = Strings.format("FROM %s,cluster-a:nomatch,cluster-a:%s*", localIndex, remote2IndexName); + IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("no such index [nomatch]")); + + // TODO: in follow on PR, add support for throwing a VerificationException from this scenario + // String limit0 = q + " | LIMIT 0"; + // VerificationException e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + // assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch]")); + } + + // --- test against 3 clusters + + // skip_unavailable=false cluster having no matching indices is a fatal error. This error + // is fatal at plan time, so it throws VerificationException, not IndexNotFoundException (thrown at execution time) + { + String localIndexName = randomFrom(localIndex, IDX_ALIAS, FILTERED_IDX_ALIAS); + String remote2IndexName = randomFrom(remote2Index, IDX_ALIAS, FILTERED_IDX_ALIAS); + String q = Strings.format("FROM %s*,cluster-a:nomatch,%s:%s*", localIndexName, REMOTE_CLUSTER_2, remote2IndexName); + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); + } + + // skip_unavailable=false cluster having no matching indices is a fatal error (even if wildcarded) + { + String localIndexName = randomFrom(localIndex, IDX_ALIAS, FILTERED_IDX_ALIAS); + String remote2IndexName = randomFrom(remote2Index, IDX_ALIAS, FILTERED_IDX_ALIAS); + String q = Strings.format("FROM %s*,cluster-a:nomatch*,%s:%s*", localIndexName, REMOTE_CLUSTER_2, remote2IndexName); + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); + } + } finally { + clearSkipUnavailable(); } + } - // wildcard on remote cluster that matches nothing - should be present in EsqlExecutionInfo marked as SKIPPED, no shards searched - try (EsqlQueryResponse resp = runQuery("from cluster-a:no_such_index*,logs-* | stats sum (v)", requestIncludeMeta)) { - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - List> values = getValuesList(resp); - assertThat(values, hasSize(1)); - assertThat(values.get(0), equalTo(List.of(45L))); + record ExpectedCluster(String clusterAlias, String indexExpression, EsqlExecutionInfo.Cluster.Status status, Integer totalShards) {} - assertNotNull(executionInfo); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - long overallTookMillis = executionInfo.overallTook().millis(); - assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + public void assertExpectedClustersForMissingIndicesTests(EsqlExecutionInfo executionInfo, List expected) { + long overallTookMillis = executionInfo.overallTook().millis(); + assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); - assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); + Set expectedClusterAliases = expected.stream().map(c -> c.clusterAlias()).collect(Collectors.toSet()); + assertThat(executionInfo.clusterAliases(), equalTo(expectedClusterAliases)); - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); - assertThat(remoteCluster.getIndexExpression(), equalTo("no_such_index*")); - assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); - assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); - assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); - assertThat(remoteCluster.getTotalShards(), equalTo(0)); - assertThat(remoteCluster.getSuccessfulShards(), equalTo(0)); - assertThat(remoteCluster.getSkippedShards(), equalTo(0)); - assertThat(remoteCluster.getFailedShards(), equalTo(0)); - - EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); - assertThat(localCluster.getIndexExpression(), equalTo("logs-*")); - assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); - assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); - assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); - assertThat(localCluster.getTotalShards(), equalTo(localNumShards)); - assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards)); - assertThat(localCluster.getSkippedShards(), equalTo(0)); - assertThat(localCluster.getFailedShards(), equalTo(0)); + for (ExpectedCluster expectedCluster : expected) { + EsqlExecutionInfo.Cluster cluster = executionInfo.getCluster(expectedCluster.clusterAlias()); + String msg = cluster.getClusterAlias(); + assertThat(msg, cluster.getIndexExpression(), equalTo(expectedCluster.indexExpression())); + assertThat(msg, cluster.getStatus(), equalTo(expectedCluster.status())); + assertThat(msg, cluster.getTook().millis(), greaterThanOrEqualTo(0L)); + assertThat(msg, cluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); + assertThat(msg, cluster.getTotalShards(), equalTo(expectedCluster.totalShards())); + if (cluster.getStatus() == EsqlExecutionInfo.Cluster.Status.SUCCESSFUL) { + assertThat(msg, cluster.getSuccessfulShards(), equalTo(expectedCluster.totalShards())); + assertThat(msg, cluster.getSkippedShards(), equalTo(0)); + } else if (cluster.getStatus() == EsqlExecutionInfo.Cluster.Status.SKIPPED) { + assertThat(msg, cluster.getSuccessfulShards(), equalTo(0)); + assertThat(msg, cluster.getSkippedShards(), equalTo(expectedCluster.totalShards())); + assertThat(msg, cluster.getFailures().size(), equalTo(1)); + assertThat(msg, cluster.getFailures().get(0).getCause(), instanceOf(VerificationException.class)); + String expectedMsg = "Unknown index [" + expectedCluster.indexExpression() + "]"; + assertThat(msg, cluster.getFailures().get(0).getCause().getMessage(), containsString(expectedMsg)); + } + // currently failed shards is always zero - change this once we start allowing partial data for individual shard failures + assertThat(msg, cluster.getFailedShards(), equalTo(0)); } } @@ -359,6 +894,10 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); } + // skip_un must be true for the next test or it will fail on "cluster-a:no_such_index*" with a + // VerificationException because there are no matching indices for that skip_un=false cluster. + setSkipUnavailable(REMOTE_CLUSTER_1, true); + // cluster-foo* matches nothing and so should not be present in the EsqlExecutionInfo try ( EsqlQueryResponse resp = runQuery( @@ -376,9 +915,9 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, LOCAL_CLUSTER))); - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1); assertThat(remoteCluster.getIndexExpression(), equalTo("no_such_index*")); assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); @@ -395,6 +934,8 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { assertThat(localCluster.getSuccessfulShards(), equalTo(localNumShards)); assertThat(localCluster.getSkippedShards(), equalTo(0)); assertThat(localCluster.getFailedShards(), equalTo(0)); + } finally { + clearSkipUnavailable(); } } @@ -403,10 +944,12 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { * (which involves cross-cluster field-caps calls), it is a coordinator only operation at query time * which uses a different pathway compared to queries that require data node (and remote data node) operations * at query time. + * + * Note: the tests covering "nonmatching indices" also do LIMIT 0 tests. + * This one is mostly focuses on took time values. */ public void testCCSExecutionOnSearchesWithLimit0() { setupTwoClusters(); - Tuple includeCCSMetadata = randomIncludeCCSMetadata(); Boolean requestIncludeMeta = includeCCSMetadata.v1(); boolean responseExpectMeta = includeCCSMetadata.v2(); @@ -427,9 +970,9 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { long overallTookMillis = executionInfo.overallTook().millis(); assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); + assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER_1, LOCAL_CLUSTER))); - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1); assertThat(remoteCluster.getIndexExpression(), equalTo("*")); assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); @@ -449,66 +992,6 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { assertThat(remoteCluster.getSkippedShards(), equalTo(0)); assertThat(remoteCluster.getFailedShards(), equalTo(0)); } - - try (EsqlQueryResponse resp = runQuery("FROM logs*,cluster-a:nomatch* | LIMIT 0", requestIncludeMeta)) { - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertNotNull(executionInfo); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - long overallTookMillis = executionInfo.overallTook().millis(); - assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); - - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); - assertThat(remoteCluster.getIndexExpression(), equalTo("nomatch*")); - assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); - assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); - assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); - assertThat(remoteCluster.getTotalShards(), equalTo(0)); - assertThat(remoteCluster.getSuccessfulShards(), equalTo(0)); - assertThat(remoteCluster.getSkippedShards(), equalTo(0)); - assertThat(remoteCluster.getFailedShards(), equalTo(0)); - - EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); - assertThat(localCluster.getIndexExpression(), equalTo("logs*")); - assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); - assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); - assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); - assertThat(localCluster.getTotalShards(), equalTo(0)); - assertThat(localCluster.getSuccessfulShards(), equalTo(0)); - assertThat(localCluster.getSkippedShards(), equalTo(0)); - assertThat(localCluster.getFailedShards(), equalTo(0)); - } - - try (EsqlQueryResponse resp = runQuery("FROM nomatch*,cluster-a:* | LIMIT 0", requestIncludeMeta)) { - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertNotNull(executionInfo); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - long overallTookMillis = executionInfo.overallTook().millis(); - assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertThat(executionInfo.clusterAliases(), equalTo(Set.of(REMOTE_CLUSTER, LOCAL_CLUSTER))); - - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); - assertThat(remoteCluster.getIndexExpression(), equalTo("*")); - assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); - assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); - assertThat(remoteCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); - assertThat(remoteCluster.getTotalShards(), equalTo(0)); - assertThat(remoteCluster.getSuccessfulShards(), equalTo(0)); - assertThat(remoteCluster.getSkippedShards(), equalTo(0)); - assertThat(remoteCluster.getFailedShards(), equalTo(0)); - - EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(LOCAL_CLUSTER); - assertThat(localCluster.getIndexExpression(), equalTo("nomatch*")); - assertThat(localCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); - assertThat(localCluster.getTook().millis(), greaterThanOrEqualTo(0L)); - assertThat(localCluster.getTook().millis(), lessThanOrEqualTo(overallTookMillis)); - assertThat(remoteCluster.getTotalShards(), equalTo(0)); - assertThat(remoteCluster.getSuccessfulShards(), equalTo(0)); - assertThat(remoteCluster.getSkippedShards(), equalTo(0)); - assertThat(remoteCluster.getFailedShards(), equalTo(0)); - } } public void testMetadataIndex() { @@ -536,7 +1019,7 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1); assertThat(remoteCluster.getIndexExpression(), equalTo("logs*")); assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); @@ -571,12 +1054,12 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put("index.routing.rebalance.enable", "none")) .get(); waitForNoInitializingShards(client(LOCAL_CLUSTER), TimeValue.timeValueSeconds(30), "logs-1"); - client(REMOTE_CLUSTER).admin() + client(REMOTE_CLUSTER_1).admin() .indices() .prepareUpdateSettings("logs-2") .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put("index.routing.rebalance.enable", "none")) .get(); - waitForNoInitializingShards(client(REMOTE_CLUSTER), TimeValue.timeValueSeconds(30), "logs-2"); + waitForNoInitializingShards(client(REMOTE_CLUSTER_1), TimeValue.timeValueSeconds(30), "logs-2"); final int localOnlyProfiles; { EsqlQueryRequest request = EsqlQueryRequest.syncEsqlQueryRequest(); @@ -593,7 +1076,7 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); assertNotNull(executionInfo); - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1); assertNull(remoteCluster); assertThat(executionInfo.isCrossClusterSearch(), is(false)); assertThat(executionInfo.includeCCSMetadata(), is(false)); @@ -621,7 +1104,7 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { assertThat(executionInfo.includeCCSMetadata(), is(false)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1); assertThat(remoteCluster.getIndexExpression(), equalTo("logs*")); assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); @@ -654,7 +1137,7 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { assertThat(executionInfo.includeCCSMetadata(), is(false)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1); assertThat(remoteCluster.getIndexExpression(), equalTo("logs*")); assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); @@ -704,7 +1187,7 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { assertThat(executionInfo.includeCCSMetadata(), is(false)); assertThat(executionInfo.overallTook().millis(), greaterThanOrEqualTo(0L)); - EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER); + EsqlExecutionInfo.Cluster remoteCluster = executionInfo.getCluster(REMOTE_CLUSTER_1); assertThat(remoteCluster.getIndexExpression(), equalTo("logs*")); assertThat(remoteCluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL)); assertThat(remoteCluster.getTook().millis(), greaterThanOrEqualTo(0L)); @@ -792,22 +1275,37 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { } Map setupTwoClusters() { - String localIndex = "logs-1"; - int numShardsLocal = randomIntBetween(1, 5); - populateLocalIndices(localIndex, numShardsLocal); + return setupClusters(2); + } + + private static String LOCAL_INDEX = "logs-1"; + private static String IDX_ALIAS = "alias1"; + private static String FILTERED_IDX_ALIAS = "alias-filtered-1"; + private static String REMOTE_INDEX = "logs-2"; + + Map setupClusters(int numClusters) { + assert numClusters == 2 || numClusters == 3 : "2 or 3 clusters supported not: " + numClusters; + int numShardsLocal = randomIntBetween(1, 5); + populateLocalIndices(LOCAL_INDEX, numShardsLocal); - String remoteIndex = "logs-2"; int numShardsRemote = randomIntBetween(1, 5); - populateRemoteIndices(remoteIndex, numShardsRemote); + populateRemoteIndices(REMOTE_CLUSTER_1, REMOTE_INDEX, numShardsRemote); Map clusterInfo = new HashMap<>(); clusterInfo.put("local.num_shards", numShardsLocal); - clusterInfo.put("local.index", localIndex); + clusterInfo.put("local.index", LOCAL_INDEX); clusterInfo.put("remote.num_shards", numShardsRemote); - clusterInfo.put("remote.index", remoteIndex); + clusterInfo.put("remote.index", REMOTE_INDEX); - String skipUnavailableKey = Strings.format("cluster.remote.%s.skip_unavailable", REMOTE_CLUSTER); - Setting skipUnavailableSetting = cluster(REMOTE_CLUSTER).clusterService().getClusterSettings().get(skipUnavailableKey); + if (numClusters == 3) { + int numShardsRemote2 = randomIntBetween(1, 5); + populateRemoteIndices(REMOTE_CLUSTER_2, REMOTE_INDEX, numShardsRemote2); + clusterInfo.put("remote2.index", REMOTE_INDEX); + clusterInfo.put("remote2.num_shards", numShardsRemote2); + } + + String skipUnavailableKey = Strings.format("cluster.remote.%s.skip_unavailable", REMOTE_CLUSTER_1); + Setting skipUnavailableSetting = cluster(REMOTE_CLUSTER_1).clusterService().getClusterSettings().get(skipUnavailableKey); boolean skipUnavailable = (boolean) cluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY).clusterService() .getClusterSettings() .get(skipUnavailableSetting); @@ -816,6 +1314,98 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { return clusterInfo; } + /** + * For the local cluster and REMOTE_CLUSTER_1 it creates a standard alias to the index created in populateLocalIndices + * and populateRemoteIndices. It also creates a filtered alias against those indices that looks like: + * PUT /_aliases + * { + * "actions": [ + * { + * "add": { + * "index": "my_index", + * "alias": "my_alias", + * "filter": { + * "terms": { + * "v": [1, 2, 4] + * } + * } + * } + * } + * ] + * } + */ + void createIndexAliases(int numClusters) { + assert numClusters == 2 || numClusters == 3 : "Only 2 or 3 clusters allowed in createIndexAliases"; + + int[] allowed = new int[] { 1, 2, 4 }; + QueryBuilder filterBuilder = new TermsQueryBuilder("v", allowed); + + { + Client localClient = client(LOCAL_CLUSTER); + IndicesAliasesResponse indicesAliasesResponse = localClient.admin() + .indices() + .prepareAliases() + .addAlias(LOCAL_INDEX, IDX_ALIAS) + .addAlias(LOCAL_INDEX, FILTERED_IDX_ALIAS, filterBuilder) + .get(); + assertFalse(indicesAliasesResponse.hasErrors()); + } + { + Client remoteClient = client(REMOTE_CLUSTER_1); + IndicesAliasesResponse indicesAliasesResponse = remoteClient.admin() + .indices() + .prepareAliases() + .addAlias(REMOTE_INDEX, IDX_ALIAS) + .addAlias(REMOTE_INDEX, FILTERED_IDX_ALIAS, filterBuilder) + .get(); + assertFalse(indicesAliasesResponse.hasErrors()); + } + if (numClusters == 3) { + Client remoteClient = client(REMOTE_CLUSTER_2); + IndicesAliasesResponse indicesAliasesResponse = remoteClient.admin() + .indices() + .prepareAliases() + .addAlias(REMOTE_INDEX, IDX_ALIAS) + .addAlias(REMOTE_INDEX, FILTERED_IDX_ALIAS, filterBuilder) + .get(); + assertFalse(indicesAliasesResponse.hasErrors()); + } + } + + Map createEmptyIndicesWithNoMappings(int numClusters) { + assert numClusters == 2 || numClusters == 3 : "Only 2 or 3 clusters supported in createEmptyIndicesWithNoMappings"; + + Map clusterToEmptyIndexMap = new HashMap<>(); + + String localIndexName = randomAlphaOfLength(14).toLowerCase(Locale.ROOT) + "1"; + clusterToEmptyIndexMap.put(LOCAL_CLUSTER, localIndexName); + Client localClient = client(LOCAL_CLUSTER); + assertAcked( + localClient.admin().indices().prepareCreate(localIndexName).setSettings(Settings.builder().put("index.number_of_shards", 1)) + ); + + String remote1IndexName = randomAlphaOfLength(14).toLowerCase(Locale.ROOT) + "2"; + clusterToEmptyIndexMap.put(REMOTE_CLUSTER_1, remote1IndexName); + Client remote1Client = client(REMOTE_CLUSTER_1); + assertAcked( + remote1Client.admin().indices().prepareCreate(remote1IndexName).setSettings(Settings.builder().put("index.number_of_shards", 1)) + ); + + if (numClusters == 3) { + String remote2IndexName = randomAlphaOfLength(14).toLowerCase(Locale.ROOT) + "3"; + clusterToEmptyIndexMap.put(REMOTE_CLUSTER_2, remote2IndexName); + Client remote2Client = client(REMOTE_CLUSTER_2); + assertAcked( + remote2Client.admin() + .indices() + .prepareCreate(remote2IndexName) + .setSettings(Settings.builder().put("index.number_of_shards", 1)) + ); + } + + return clusterToEmptyIndexMap; + } + void populateLocalIndices(String indexName, int numShards) { Client localClient = client(LOCAL_CLUSTER); assertAcked( @@ -831,8 +1421,8 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { localClient.admin().indices().prepareRefresh(indexName).get(); } - void populateRemoteIndices(String indexName, int numShards) { - Client remoteClient = client(REMOTE_CLUSTER); + void populateRemoteIndices(String clusterAlias, String indexName, int numShards) { + Client remoteClient = client(clusterAlias); assertAcked( remoteClient.admin() .indices() @@ -845,4 +1435,23 @@ public class CrossClustersQueryIT extends AbstractMultiClustersTestCase { } remoteClient.admin().indices().prepareRefresh(indexName).get(); } + + private void setSkipUnavailable(String clusterAlias, boolean skip) { + client(LOCAL_CLUSTER).admin() + .cluster() + .prepareUpdateSettings(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT) + .setPersistentSettings(Settings.builder().put("cluster.remote." + clusterAlias + ".skip_unavailable", skip).build()) + .get(); + } + + private void clearSkipUnavailable() { + Settings.Builder settingsBuilder = Settings.builder() + .putNull("cluster.remote." + REMOTE_CLUSTER_1 + ".skip_unavailable") + .putNull("cluster.remote." + REMOTE_CLUSTER_2 + ".skip_unavailable"); + client(LOCAL_CLUSTER).admin() + .cluster() + .prepareUpdateSettings(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT) + .setPersistentSettings(settingsBuilder.build()) + .get(); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java index b2eaefcf09d6..88366bbf9a7c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java @@ -12,22 +12,37 @@ import org.elasticsearch.core.Nullable; import java.util.Collections; import java.util.Map; import java.util.Objects; +import java.util.Set; public final class IndexResolution { - public static IndexResolution valid(EsIndex index, Map unavailableClusters) { + /** + * @param index EsIndex encapsulating requested index expression, resolved mappings and index modes from field-caps. + * @param resolvedIndices Set of concrete indices resolved by field-caps. (This information is not always present in the EsIndex). + * @param unavailableClusters Remote clusters that could not be contacted during planning + * @return valid IndexResolution + */ + public static IndexResolution valid( + EsIndex index, + Set resolvedIndices, + Map unavailableClusters + ) { Objects.requireNonNull(index, "index must not be null if it was found"); + Objects.requireNonNull(resolvedIndices, "resolvedIndices must not be null"); Objects.requireNonNull(unavailableClusters, "unavailableClusters must not be null"); - return new IndexResolution(index, null, unavailableClusters); + return new IndexResolution(index, null, resolvedIndices, unavailableClusters); } + /** + * Use this method only if the set of concrete resolved indices is the same as EsIndex#concreteIndices(). + */ public static IndexResolution valid(EsIndex index) { - return valid(index, Collections.emptyMap()); + return valid(index, index.concreteIndices(), Collections.emptyMap()); } public static IndexResolution invalid(String invalid) { Objects.requireNonNull(invalid, "invalid must not be null to signal that the index is invalid"); - return new IndexResolution(null, invalid, Collections.emptyMap()); + return new IndexResolution(null, invalid, Collections.emptySet(), Collections.emptyMap()); } public static IndexResolution notFound(String name) { @@ -39,12 +54,20 @@ public final class IndexResolution { @Nullable private final String invalid; + // all indices found by field-caps + private final Set resolvedIndices; // remote clusters included in the user's index expression that could not be connected to private final Map unavailableClusters; - private IndexResolution(EsIndex index, @Nullable String invalid, Map unavailableClusters) { + private IndexResolution( + EsIndex index, + @Nullable String invalid, + Set resolvedIndices, + Map unavailableClusters + ) { this.index = index; this.invalid = invalid; + this.resolvedIndices = resolvedIndices; this.unavailableClusters = unavailableClusters; } @@ -64,8 +87,8 @@ public final class IndexResolution { } /** - * Is the index valid for use with ql? Returns {@code false} if the - * index wasn't found. + * Is the index valid for use with ql? + * @return {@code false} if the index wasn't found. */ public boolean isValid() { return invalid == null; @@ -75,10 +98,17 @@ public final class IndexResolution { * @return Map of unavailable clusters (could not be connected to during field-caps query). Key of map is cluster alias, * value is the {@link FieldCapabilitiesFailure} describing the issue. */ - public Map getUnavailableClusters() { + public Map unavailableClusters() { return unavailableClusters; } + /** + * @return all indices found by field-caps (regardless of whether they had any mappings) + */ + public Set resolvedIndices() { + return resolvedIndices; + } + @Override public boolean equals(Object obj) { if (obj == null || obj.getClass() != getClass()) { @@ -87,16 +117,29 @@ public final class IndexResolution { IndexResolution other = (IndexResolution) obj; return Objects.equals(index, other.index) && Objects.equals(invalid, other.invalid) + && Objects.equals(resolvedIndices, other.resolvedIndices) && Objects.equals(unavailableClusters, other.unavailableClusters); } @Override public int hashCode() { - return Objects.hash(index, invalid, unavailableClusters); + return Objects.hash(index, invalid, resolvedIndices, unavailableClusters); } @Override public String toString() { - return invalid != null ? invalid : index.name(); + return invalid != null + ? invalid + : "IndexResolution{" + + "index=" + + index + + ", invalid='" + + invalid + + '\'' + + ", resolvedIndices=" + + resolvedIndices + + ", unavailableClusters=" + + unavailableClusters + + '}'; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index ffad379001ed..76de337ded5c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -251,19 +251,17 @@ public class ComputeService { if (execInfo.isCrossClusterSearch()) { assert execInfo.planningTookTime() != null : "Planning took time should be set on EsqlExecutionInfo but is null"; for (String clusterAlias : execInfo.clusterAliases()) { - // took time and shard counts for SKIPPED clusters were added at end of planning, so only update other cases here - if (execInfo.getCluster(clusterAlias).getStatus() != EsqlExecutionInfo.Cluster.Status.SKIPPED) { - execInfo.swapCluster( - clusterAlias, - (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setTook(execInfo.overallTook()) - .setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL) - .setTotalShards(0) - .setSuccessfulShards(0) - .setSkippedShards(0) - .setFailedShards(0) - .build() - ); - } + execInfo.swapCluster(clusterAlias, (k, v) -> { + var builder = new EsqlExecutionInfo.Cluster.Builder(v).setTook(execInfo.overallTook()) + .setTotalShards(0) + .setSuccessfulShards(0) + .setSkippedShards(0) + .setFailedShards(0); + if (v.getStatus() == EsqlExecutionInfo.Cluster.Status.RUNNING) { + builder.setStatus(EsqlExecutionInfo.Cluster.Status.SUCCESSFUL); + } + return builder.build(); + }); } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 504689fdac39..c576d15f9260 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -309,7 +309,7 @@ public class EsqlSession { // resolution to updateExecutionInfo if (indexResolution.isValid()) { EsqlSessionCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); - EsqlSessionCCSUtils.updateExecutionInfoWithUnavailableClusters(executionInfo, indexResolution.getUnavailableClusters()); + EsqlSessionCCSUtils.updateExecutionInfoWithUnavailableClusters(executionInfo, indexResolution.unavailableClusters()); if (executionInfo.isCrossClusterSearch() && executionInfo.getClusterStateCount(EsqlExecutionInfo.Cluster.Status.RUNNING) == 0) { // for a CCS, if all clusters have been marked as SKIPPED, nothing to search so send a sentinel diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java index 80709d8f6c4f..4fe2fef7e3f4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java @@ -17,6 +17,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.transport.ConnectTransportException; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.transport.RemoteTransportException; +import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo; import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.index.IndexResolution; @@ -33,7 +34,6 @@ class EsqlSessionCCSUtils { private EsqlSessionCCSUtils() {} - // visible for testing static Map determineUnavailableRemoteClusters(List failures) { Map unavailableRemotes = new HashMap<>(); for (FieldCapabilitiesFailure failure : failures) { @@ -75,10 +75,10 @@ class EsqlSessionCCSUtils { /** * Whether to return an empty result (HTTP status 200) for a CCS rather than a top level 4xx/5xx error. - * + *

* For cases where field-caps had no indices to search and the remotes were unavailable, we * return an empty successful response (200) if all remotes are marked with skip_unavailable=true. - * + *

* Note: a follow-on PR will expand this logic to handle cases where no indices could be found to match * on any of the requested clusters. */ @@ -132,7 +132,6 @@ class EsqlSessionCCSUtils { } } - // visible for testing static String createIndexExpressionFromAvailableClusters(EsqlExecutionInfo executionInfo) { StringBuilder sb = new StringBuilder(); for (String clusterAlias : executionInfo.clusterAliases()) { @@ -181,39 +180,91 @@ class EsqlSessionCCSUtils { } } - // visible for testing static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionInfo executionInfo, IndexResolution indexResolution) { Set clustersWithResolvedIndices = new HashSet<>(); // determine missing clusters - for (String indexName : indexResolution.get().indexNameWithModes().keySet()) { + for (String indexName : indexResolution.resolvedIndices()) { clustersWithResolvedIndices.add(RemoteClusterAware.parseClusterAlias(indexName)); } Set clustersRequested = executionInfo.clusterAliases(); Set clustersWithNoMatchingIndices = Sets.difference(clustersRequested, clustersWithResolvedIndices); - clustersWithNoMatchingIndices.removeAll(indexResolution.getUnavailableClusters().keySet()); + clustersWithNoMatchingIndices.removeAll(indexResolution.unavailableClusters().keySet()); + + /** + * Rules enforced at planning time around non-matching indices + * P1. fail query if no matching indices on any cluster (VerificationException) - that is handled elsewhere (TODO: document where) + * P2. fail query if a skip_unavailable:false cluster has no matching indices (the local cluster already has this rule + * enforced at planning time) + * P3. fail query if the local cluster has no matching indices and a concrete index was specified + */ + String fatalErrorMessage = null; /* * These are clusters in the original request that are not present in the field-caps response. They were - * specified with an index or indices that do not exist, so the search on that cluster is done. + * specified with an index expression matched no indices, so the search on that cluster is done. * Mark it as SKIPPED with 0 shards searched and took=0. */ for (String c : clustersWithNoMatchingIndices) { - // TODO: in a follow-on PR, throw a Verification(400 status code) for local and remotes with skip_unavailable=false if - // they were requested with one or more concrete indices - // for now we never mark the local cluster as SKIPPED - final var status = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY.equals(c) - ? EsqlExecutionInfo.Cluster.Status.SUCCESSFUL - : EsqlExecutionInfo.Cluster.Status.SKIPPED; - executionInfo.swapCluster( - c, - (k, v) -> new EsqlExecutionInfo.Cluster.Builder(v).setStatus(status) - .setTook(new TimeValue(0)) - .setTotalShards(0) - .setSuccessfulShards(0) - .setSkippedShards(0) - .setFailedShards(0) - .build() - ); + final String indexExpression = executionInfo.getCluster(c).getIndexExpression(); + if (missingIndicesIsFatal(c, executionInfo)) { + String error = Strings.format( + "Unknown index [%s]", + (c.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) ? indexExpression : c + ":" + indexExpression) + ); + if (fatalErrorMessage == null) { + fatalErrorMessage = error; + } else { + fatalErrorMessage += "; " + error; + } + } else { + // handles local cluster (when no concrete indices requested) and skip_unavailable=true clusters + EsqlExecutionInfo.Cluster.Status status; + ShardSearchFailure failure; + if (c.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { + status = EsqlExecutionInfo.Cluster.Status.SUCCESSFUL; + failure = null; + } else { + status = EsqlExecutionInfo.Cluster.Status.SKIPPED; + failure = new ShardSearchFailure(new VerificationException("Unknown index [" + indexExpression + "]")); + } + executionInfo.swapCluster(c, (k, v) -> { + var builder = new EsqlExecutionInfo.Cluster.Builder(v).setStatus(status) + .setTook(new TimeValue(0)) + .setTotalShards(0) + .setSuccessfulShards(0) + .setSkippedShards(0) + .setFailedShards(0); + if (failure != null) { + builder.setFailures(List.of(failure)); + } + return builder.build(); + }); + } } + if (fatalErrorMessage != null) { + throw new VerificationException(fatalErrorMessage); + } + } + + // visible for testing + static boolean missingIndicesIsFatal(String clusterAlias, EsqlExecutionInfo executionInfo) { + // missing indices on local cluster is fatal only if a concrete index requested + if (clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { + return concreteIndexRequested(executionInfo.getCluster(clusterAlias).getIndexExpression()); + } + return executionInfo.getCluster(clusterAlias).isSkipUnavailable() == false; + } + + private static boolean concreteIndexRequested(String indexExpression) { + for (String expr : indexExpression.split(",")) { + if (expr.charAt(0) == '<' || expr.startsWith("-<")) { + // skip date math expressions + continue; + } + if (expr.indexOf('*') < 0) { + return true; + } + } + return false; } // visible for testing diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index 210f991306ba..0be8cf820d34 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.session; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; @@ -143,21 +144,24 @@ public class IndexResolver { fields.put(name, field); } + Map unavailableRemotes = EsqlSessionCCSUtils.determineUnavailableRemoteClusters( + fieldCapsResponse.getFailures() + ); + + Map concreteIndices = Maps.newMapWithExpectedSize(fieldCapsResponse.getIndexResponses().size()); + for (FieldCapabilitiesIndexResponse ir : fieldCapsResponse.getIndexResponses()) { + concreteIndices.put(ir.getIndexName(), ir.getIndexMode()); + } + boolean allEmpty = true; for (FieldCapabilitiesIndexResponse ir : fieldCapsResponse.getIndexResponses()) { allEmpty &= ir.get().isEmpty(); } if (allEmpty) { // If all the mappings are empty we return an empty set of resolved indices to line up with QL - return IndexResolution.valid(new EsIndex(indexPattern, rootFields, Map.of())); + return IndexResolution.valid(new EsIndex(indexPattern, rootFields, Map.of()), concreteIndices.keySet(), unavailableRemotes); } - - Map concreteIndices = Maps.newMapWithExpectedSize(fieldCapsResponse.getIndexResponses().size()); - for (FieldCapabilitiesIndexResponse ir : fieldCapsResponse.getIndexResponses()) { - concreteIndices.put(ir.getIndexName(), ir.getIndexMode()); - } - EsIndex esIndex = new EsIndex(indexPattern, rootFields, concreteIndices); - return IndexResolution.valid(esIndex, EsqlSessionCCSUtils.determineUnavailableRemoteClusters(fieldCapsResponse.getFailures())); + return IndexResolution.valid(new EsIndex(indexPattern, rootFields, concreteIndices), concreteIndices.keySet(), unavailableRemotes); } private boolean allNested(List caps) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java index e60024ecd5db..60b632c443f8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.transport.NoSeedNodeLeftException; import org.elasticsearch.transport.NoSuchRemoteClusterException; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.transport.RemoteTransportException; +import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlExecutionInfo; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.index.EsIndex; @@ -228,7 +229,8 @@ public class EsqlSessionCCSUtilsTests extends ESTestCase { IndexMode.STANDARD ) ); - IndexResolution indexResolution = IndexResolution.valid(esIndex, Map.of()); + + IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), Map.of()); EsqlSessionCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); @@ -266,7 +268,7 @@ public class EsqlSessionCCSUtilsTests extends ESTestCase { IndexMode.STANDARD ) ); - IndexResolution indexResolution = IndexResolution.valid(esIndex, Map.of()); + IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), Map.of()); EsqlSessionCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); @@ -293,7 +295,7 @@ public class EsqlSessionCCSUtilsTests extends ESTestCase { EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); - executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false)); + executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", true)); EsIndex esIndex = new EsIndex( "logs*,remote2:mylogs1,remote2:mylogs2,remote2:logs*", @@ -302,7 +304,7 @@ public class EsqlSessionCCSUtilsTests extends ESTestCase { ); // remote1 is unavailable var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect")); - IndexResolution indexResolution = IndexResolution.valid(esIndex, Map.of(remote1Alias, failure)); + IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), Map.of(remote1Alias, failure)); EsqlSessionCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); @@ -341,8 +343,12 @@ public class EsqlSessionCCSUtilsTests extends ESTestCase { ); var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect")); - IndexResolution indexResolution = IndexResolution.valid(esIndex, Map.of(remote1Alias, failure)); - EsqlSessionCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); + IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), Map.of(remote1Alias, failure)); + VerificationException ve = expectThrows( + VerificationException.class, + () -> EsqlSessionCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution) + ); + assertThat(ve.getDetailedMessage(), containsString("Unknown index [remote2:mylogs1,mylogs2,logs*]")); } } @@ -579,4 +585,46 @@ public class EsqlSessionCCSUtilsTests extends ESTestCase { assertThat(remoteFailures.get(0).reason(), containsString("unable to connect to remote cluster")); } } + + public void testMissingIndicesIsFatal() { + String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; + String remote1Alias = "remote1"; + String remote2Alias = "remote2"; + String remote3Alias = "remote3"; + + // scenario 1: cluster is skip_unavailable=true - not fatal + { + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); + executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); + executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "mylogs1,mylogs2,logs*", true)); + assertThat(EsqlSessionCCSUtils.missingIndicesIsFatal(remote1Alias, executionInfo), equalTo(false)); + } + + // scenario 2: cluster is local cluster and had no concrete indices - not fatal + { + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); + executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); + executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "mylogs1,mylogs2,logs*", true)); + assertThat(EsqlSessionCCSUtils.missingIndicesIsFatal(localClusterAlias, executionInfo), equalTo(false)); + } + + // scenario 3: cluster is local cluster and user specified a concrete index - fatal + { + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); + String localIndexExpr = randomFrom("foo*,logs", "logs", "logs,metrics", "bar*,x*,logs", "logs-1,*x*"); + executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, localIndexExpr, false)); + executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "mylogs1,mylogs2,logs*", true)); + assertThat(EsqlSessionCCSUtils.missingIndicesIsFatal(localClusterAlias, executionInfo), equalTo(true)); + } + + // scenario 4: cluster is skip_unavailable=false - always fatal + { + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); + executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "*", false)); + String indexExpr = randomFrom("foo*,logs", "logs", "bar*,x*,logs", "logs-1,*x*", "*"); + executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, indexExpr, false)); + assertThat(EsqlSessionCCSUtils.missingIndicesIsFatal(remote1Alias, executionInfo), equalTo(true)); + } + + } } diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1MissingIndicesIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1MissingIndicesIT.java new file mode 100644 index 000000000000..0f39104511be --- /dev/null +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1MissingIndicesIT.java @@ -0,0 +1,574 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.remotecluster; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.hamcrest.Matchers; +import org.junit.ClassRule; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +/** + * Tests cross-cluster ES|QL queries under RCS1.0 security model for cases where index expressions do not match + * to ensure handling of those matches the expected rules defined in EsqlSessionCrossClusterUtils. + */ +public class CrossClusterEsqlRCS1MissingIndicesIT extends AbstractRemoteClusterSecurityTestCase { + + private static final AtomicBoolean SSL_ENABLED_REF = new AtomicBoolean(); + + static { + // remote cluster + fulfillingCluster = ElasticsearchCluster.local() + .name("fulfilling-cluster") + .nodes(1) + .module("x-pack-esql") + .module("x-pack-enrich") + .apply(commonClusterConfig) + .setting("remote_cluster.port", "0") + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.remote_cluster_server.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) + .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key") + .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt") + .setting("xpack.security.authc.token.enabled", "true") + .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password") + .node(0, spec -> spec.setting("remote_cluster_server.enabled", "true")) + .build(); + + // "local" cluster + queryCluster = ElasticsearchCluster.local() + .name("query-cluster") + .module("x-pack-esql") + .module("x-pack-enrich") + .apply(commonClusterConfig) + .setting("xpack.ml.enabled", "false") + .setting("xpack.security.remote_cluster_client.ssl.enabled", () -> String.valueOf(SSL_ENABLED_REF.get())) + .build(); + } + + @ClassRule + public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster); + + private static final String INDEX1 = "points"; // on local cluster only + private static final String INDEX2 = "squares"; // on local and remote clusters + + record ExpectedCluster(String clusterAlias, String indexExpression, String status, Integer totalShards) {} + + @SuppressWarnings("unchecked") + public void assertExpectedClustersForMissingIndicesTests(Map responseMap, List expected) { + Map clusters = (Map) responseMap.get("_clusters"); + assertThat((int) responseMap.get("took"), greaterThan(0)); + + Map detailsMap = (Map) clusters.get("details"); + assertThat(detailsMap.size(), is(expected.size())); + + assertThat((int) clusters.get("total"), is(expected.size())); + assertThat((int) clusters.get("successful"), is((int) expected.stream().filter(ec -> ec.status().equals("successful")).count())); + assertThat((int) clusters.get("skipped"), is((int) expected.stream().filter(ec -> ec.status().equals("skipped")).count())); + assertThat((int) clusters.get("failed"), is((int) expected.stream().filter(ec -> ec.status().equals("failed")).count())); + + for (ExpectedCluster expectedCluster : expected) { + Map clusterDetails = (Map) detailsMap.get(expectedCluster.clusterAlias()); + String msg = expectedCluster.clusterAlias(); + + assertThat(msg, (int) clusterDetails.get("took"), greaterThan(0)); + assertThat(msg, clusterDetails.get("status"), is(expectedCluster.status())); + Map shards = (Map) clusterDetails.get("_shards"); + if (expectedCluster.totalShards() == null) { + assertThat(msg, (int) shards.get("total"), greaterThan(0)); + } else { + assertThat(msg, (int) shards.get("total"), is(expectedCluster.totalShards())); + } + + if (expectedCluster.status().equals("successful")) { + assertThat((int) shards.get("successful"), is((int) shards.get("total"))); + assertThat((int) shards.get("skipped"), is(0)); + + } else if (expectedCluster.status().equals("skipped")) { + assertThat((int) shards.get("successful"), is(0)); + assertThat((int) shards.get("skipped"), is((int) shards.get("total"))); + ArrayList failures = (ArrayList) clusterDetails.get("failures"); + assertThat(failures.size(), is(1)); + Map failure1 = (Map) failures.get(0); + Map innerReason = (Map) failure1.get("reason"); + String expectedMsg = "Unknown index [" + expectedCluster.indexExpression() + "]"; + assertThat(innerReason.get("reason").toString(), containsString(expectedMsg)); + assertThat(innerReason.get("type").toString(), containsString("verification_exception")); + + } else { + fail(msg + "; Unexpected status: " + expectedCluster.status()); + } + // currently failed shards is always zero - change this once we start allowing partial data for individual shard failures + assertThat((int) shards.get("failed"), is(0)); + } + } + + @SuppressWarnings("unchecked") + public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableTrue() throws Exception { + setupRolesAndPrivileges(); + setupIndex(); + + configureRemoteCluster(REMOTE_CLUSTER_ALIAS, fulfillingCluster, true, randomBoolean(), true); + + // missing concrete local index is an error + { + String q = Strings.format("FROM nomatch,%s:%s | STATS count(*)", REMOTE_CLUSTER_ALIAS, INDEX2); + + String limit1 = q + " | LIMIT 1"; + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); + assertThat(e.getMessage(), containsString("Unknown index [nomatch]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); + assertThat(e.getMessage(), Matchers.containsString("Unknown index [nomatch]")); + } + + // missing concrete remote index is not fatal when skip_unavailable=true (as long as an index matches on another cluster) + { + String q = Strings.format("FROM %s,%s:nomatch | STATS count(*)", INDEX1, REMOTE_CLUSTER_ALIAS); + + String limit1 = q + " | LIMIT 1"; + Response response = client().performRequest(esqlRequest(limit1)); + assertOK(response); + + Map map = responseAsMap(response); + assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); + assertThat(((ArrayList) map.get("values")).size(), greaterThanOrEqualTo(1)); + + assertExpectedClustersForMissingIndicesTests( + map, + List.of( + new ExpectedCluster("(local)", INDEX1, "successful", null), + new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "nomatch", "skipped", 0) + ) + ); + + String limit0 = q + " | LIMIT 0"; + response = client().performRequest(esqlRequest(limit0)); + assertOK(response); + + map = responseAsMap(response); + assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); + assertThat(((ArrayList) map.get("values")).size(), is(0)); + + assertExpectedClustersForMissingIndicesTests( + map, + List.of( + new ExpectedCluster("(local)", INDEX1, "successful", 0), + new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "nomatch", "skipped", 0) + ) + ); + } + + // since there is at least one matching index in the query, the missing wildcarded local index is not an error + { + String q = Strings.format("FROM nomatch*,%s:%s", REMOTE_CLUSTER_ALIAS, INDEX2); + + String limit1 = q + " | LIMIT 1"; + Response response = client().performRequest(esqlRequest(limit1)); + assertOK(response); + + Map map = responseAsMap(response); + assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); + assertThat(((ArrayList) map.get("values")).size(), greaterThanOrEqualTo(1)); + + assertExpectedClustersForMissingIndicesTests( + map, + List.of( + // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched + new ExpectedCluster("(local)", "nomatch*", "successful", 0), + new ExpectedCluster(REMOTE_CLUSTER_ALIAS, INDEX2, "successful", null) + ) + ); + + String limit0 = q + " | LIMIT 0"; + response = client().performRequest(esqlRequest(limit0)); + assertOK(response); + + map = responseAsMap(response); + assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); + assertThat(((ArrayList) map.get("values")).size(), is(0)); + + assertExpectedClustersForMissingIndicesTests( + map, + List.of( + // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched + new ExpectedCluster("(local)", "nomatch*", "successful", 0), + new ExpectedCluster(REMOTE_CLUSTER_ALIAS, INDEX2, "successful", 0) + ) + ); + } + + // since at least one index of the query matches on some cluster, a wildcarded index on skip_un=true is not an error + { + String q = Strings.format("FROM %s,%s:nomatch*", INDEX1, REMOTE_CLUSTER_ALIAS); + + String limit1 = q + " | LIMIT 1"; + Response response = client().performRequest(esqlRequest(limit1)); + assertOK(response); + + Map map = responseAsMap(response); + assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); + assertThat(((ArrayList) map.get("values")).size(), greaterThanOrEqualTo(1)); + + assertExpectedClustersForMissingIndicesTests( + map, + List.of( + new ExpectedCluster("(local)", INDEX1, "successful", null), + new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "nomatch*", "skipped", 0) + ) + ); + + String limit0 = q + " | LIMIT 0"; + response = client().performRequest(esqlRequest(limit0)); + assertOK(response); + + map = responseAsMap(response); + assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); + assertThat(((ArrayList) map.get("values")).size(), is(0)); + + assertExpectedClustersForMissingIndicesTests( + map, + List.of( + new ExpectedCluster("(local)", INDEX1, "successful", 0), + new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "nomatch*", "skipped", 0) + ) + ); + } + + // an error is thrown if there are no matching indices at all, even when the cluster is skip_unavailable=true + { + // with non-matching concrete index + String q = Strings.format("FROM %s:nomatch", REMOTE_CLUSTER_ALIAS); + + String limit1 = q + " | LIMIT 1"; + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); + assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch]", REMOTE_CLUSTER_ALIAS))); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); + assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch]", REMOTE_CLUSTER_ALIAS))); + } + + // an error is thrown if there are no matching indices at all, even when the cluster is skip_unavailable=true and the + // index was wildcarded + { + String q = Strings.format("FROM %s:nomatch*", REMOTE_CLUSTER_ALIAS); + + String limit1 = q + " | LIMIT 1"; + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); + assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch*]", REMOTE_CLUSTER_ALIAS))); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); + assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch*]", REMOTE_CLUSTER_ALIAS))); + } + + // an error is thrown if there are no matching indices at all + { + String localExpr = randomFrom("nomatch", "nomatch*"); + String remoteExpr = randomFrom("nomatch", "nomatch*"); + String q = Strings.format("FROM %s,%s:%s", localExpr, REMOTE_CLUSTER_ALIAS, remoteExpr); + + String limit1 = q + " | LIMIT 1"; + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); + assertThat(e.getMessage(), containsString("Unknown index")); + assertThat(e.getMessage(), containsString(Strings.format("%s:%s", REMOTE_CLUSTER_ALIAS, remoteExpr))); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); + assertThat(e.getMessage(), containsString("Unknown index")); + assertThat(e.getMessage(), containsString(Strings.format("%s:%s", REMOTE_CLUSTER_ALIAS, remoteExpr))); + } + + // TODO uncomment and test in follow-on PR which does skip_unavailable handling at execution time + // { + // String q = Strings.format("FROM %s,%s:nomatch,%s:%s*", INDEX1, REMOTE_CLUSTER_ALIAS, REMOTE_CLUSTER_ALIAS, INDEX2); + // + // String limit1 = q + " | LIMIT 1"; + // Response response = client().performRequest(esqlRequest(limit1)); + // assertOK(response); + // + // Map map = responseAsMap(response); + // assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); + // assertThat(((ArrayList) map.get("values")).size(), greaterThanOrEqualTo(1)); + // + // assertExpectedClustersForMissingIndicesTests(map, + // List.of( + // new ExpectedCluster("(local)", INDEX1, "successful", null), + // new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "nomatch," + INDEX2 + "*", "skipped", 0) + // ) + // ); + // + // String limit0 = q + " | LIMIT 0"; + // response = client().performRequest(esqlRequest(limit0)); + // assertOK(response); + // + // map = responseAsMap(response); + // assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); + // assertThat(((ArrayList) map.get("values")).size(), is(0)); + // + // assertExpectedClustersForMissingIndicesTests(map, + // List.of( + // new ExpectedCluster("(local)", INDEX1, "successful", 0), + // new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "nomatch," + INDEX2 + "*", "skipped", 0) + // ) + // ); + // } + } + + @SuppressWarnings("unchecked") + public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableFalse() throws Exception { + // Remote cluster is closed and skip_unavailable is set to false. + // Although the other cluster is open, we expect an Exception. + + setupRolesAndPrivileges(); + setupIndex(); + + configureRemoteCluster(REMOTE_CLUSTER_ALIAS, fulfillingCluster, true, randomBoolean(), false); + + // missing concrete local index is an error + { + String q = Strings.format("FROM nomatch,%s:%s | STATS count(*)", REMOTE_CLUSTER_ALIAS, INDEX2); + + String limit1 = q + " | LIMIT 1"; + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); + assertThat(e.getMessage(), containsString("Unknown index [nomatch]")); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); + assertThat(e.getMessage(), Matchers.containsString("Unknown index [nomatch]")); + } + + // missing concrete remote index is not fatal when skip_unavailable=true (as long as an index matches on another cluster) + { + String q = Strings.format("FROM %s,%s:nomatch | STATS count(*)", INDEX1, REMOTE_CLUSTER_ALIAS); + + String limit1 = q + " | LIMIT 1"; + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); + assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch]", REMOTE_CLUSTER_ALIAS))); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); + assertThat(e.getMessage(), Matchers.containsString(Strings.format("Unknown index [%s:nomatch]", REMOTE_CLUSTER_ALIAS))); + } + + // since there is at least one matching index in the query, the missing wildcarded local index is not an error + { + String q = Strings.format("FROM nomatch*,%s:%s", REMOTE_CLUSTER_ALIAS, INDEX2); + + String limit1 = q + " | LIMIT 1"; + Response response = client().performRequest(esqlRequest(limit1)); + assertOK(response); + + Map map = responseAsMap(response); + assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); + assertThat(((ArrayList) map.get("values")).size(), greaterThanOrEqualTo(1)); + + assertExpectedClustersForMissingIndicesTests( + map, + List.of( + // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched + new ExpectedCluster("(local)", "nomatch*", "successful", 0), + new ExpectedCluster(REMOTE_CLUSTER_ALIAS, INDEX2, "successful", null) + ) + ); + + String limit0 = q + " | LIMIT 0"; + response = client().performRequest(esqlRequest(limit0)); + assertOK(response); + + map = responseAsMap(response); + assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); + assertThat(((ArrayList) map.get("values")).size(), is(0)); + + assertExpectedClustersForMissingIndicesTests( + map, + List.of( + // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched + new ExpectedCluster("(local)", "nomatch*", "successful", 0), + new ExpectedCluster(REMOTE_CLUSTER_ALIAS, INDEX2, "successful", 0) + ) + ); + } + + // query is fatal since the remote cluster has skip_unavailable=false and has no matching indices + { + String q = Strings.format("FROM %s,%s:nomatch*", INDEX1, REMOTE_CLUSTER_ALIAS); + + String limit1 = q + " | LIMIT 1"; + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); + assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch*]", REMOTE_CLUSTER_ALIAS))); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); + assertThat(e.getMessage(), Matchers.containsString(Strings.format("Unknown index [%s:nomatch*]", REMOTE_CLUSTER_ALIAS))); + } + + // an error is thrown if there are no matching indices at all + { + // with non-matching concrete index + String q = Strings.format("FROM %s:nomatch", REMOTE_CLUSTER_ALIAS); + + String limit1 = q + " | LIMIT 1"; + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); + assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch]", REMOTE_CLUSTER_ALIAS))); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); + assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch]", REMOTE_CLUSTER_ALIAS))); + } + + // an error is thrown if there are no matching indices at all + { + String localExpr = randomFrom("nomatch", "nomatch*"); + String remoteExpr = randomFrom("nomatch", "nomatch*"); + String q = Strings.format("FROM %s,%s:%s", localExpr, REMOTE_CLUSTER_ALIAS, remoteExpr); + + String limit1 = q + " | LIMIT 1"; + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); + assertThat(e.getMessage(), containsString("Unknown index")); + assertThat(e.getMessage(), containsString(Strings.format("%s:%s", REMOTE_CLUSTER_ALIAS, remoteExpr))); + + String limit0 = q + " | LIMIT 0"; + e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); + assertThat(e.getMessage(), containsString("Unknown index")); + assertThat(e.getMessage(), containsString(Strings.format("%s:%s", REMOTE_CLUSTER_ALIAS, remoteExpr))); + } + + // error since the remote cluster with skip_unavailable=false specified a concrete index that is not found + { + String q = Strings.format("FROM %s,%s:nomatch,%s:%s*", INDEX1, REMOTE_CLUSTER_ALIAS, REMOTE_CLUSTER_ALIAS, INDEX2); + + String limit1 = q + " | LIMIT 1"; + ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); + assertThat(e.getMessage(), containsString(Strings.format("no such index [nomatch]", REMOTE_CLUSTER_ALIAS))); + assertThat(e.getMessage(), containsString(Strings.format("index_not_found_exception", REMOTE_CLUSTER_ALIAS))); + + // TODO: in follow on PR, add support for throwing a VerificationException from this scenario + // String limit0 = q + " | LIMIT 0"; + // e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); + // assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch]", REMOTE_CLUSTER_ALIAS))); + } + } + + private void setupRolesAndPrivileges() throws IOException { + var putUserRequest = new Request("PUT", "/_security/user/" + REMOTE_SEARCH_USER); + putUserRequest.setJsonEntity(""" + { + "password": "x-pack-test-password", + "roles" : ["remote_search"] + }"""); + assertOK(adminClient().performRequest(putUserRequest)); + + var putRoleOnRemoteClusterRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE); + putRoleOnRemoteClusterRequest.setJsonEntity(""" + { + "indices": [ + { + "names": ["points", "squares"], + "privileges": ["read", "read_cross_cluster", "create_index", "monitor"] + } + ], + "remote_indices": [ + { + "names": ["points", "squares"], + "privileges": ["read", "read_cross_cluster", "create_index", "monitor"], + "clusters": ["my_remote_cluster"] + } + ] + }"""); + assertOK(adminClient().performRequest(putRoleOnRemoteClusterRequest)); + } + + private void setupIndex() throws IOException { + Request createIndex = new Request("PUT", INDEX1); + createIndex.setJsonEntity(""" + { + "mappings": { + "properties": { + "id": { "type": "integer" }, + "score": { "type": "integer" } + } + } + } + """); + assertOK(client().performRequest(createIndex)); + + Request bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity(""" + { "index": { "_index": "points" } } + { "id": 1, "score": 75} + { "index": { "_index": "points" } } + { "id": 2, "score": 125} + { "index": { "_index": "points" } } + { "id": 3, "score": 100} + { "index": { "_index": "points" } } + { "id": 4, "score": 50} + { "index": { "_index": "points" } } + { "id": 5, "score": 150} + """); + assertOK(client().performRequest(bulkRequest)); + + createIndex = new Request("PUT", INDEX2); + createIndex.setJsonEntity(""" + { + "mappings": { + "properties": { + "num": { "type": "integer" }, + "square": { "type": "integer" } + } + } + } + """); + assertOK(client().performRequest(createIndex)); + + bulkRequest = new Request("POST", "/_bulk?refresh=true"); + bulkRequest.setJsonEntity(""" + { "index": {"_index": "squares"}} + { "num": 1, "square": 1 } + { "index": {"_index": "squares"}} + { "num": 4, "square": 4 } + { "index": {"_index": "squares"}} + { "num": 3, "square": 9 } + { "index": {"_index": "squares"}} + { "num": 4, "square": 16 } + """); + assertOK(performRequestAgainstFulfillingCluster(bulkRequest)); + } + + private Request esqlRequest(String query) throws IOException { + XContentBuilder body = JsonXContent.contentBuilder(); + + body.startObject(); + body.field("query", query); + body.field("include_ccs_metadata", true); + body.endObject(); + + Request request = new Request("POST", "_query"); + request.setJsonEntity(org.elasticsearch.common.Strings.toString(body)); + + return request; + } +} diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java index d5b3141b539e..74ef6f0dafe6 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java @@ -495,7 +495,7 @@ public class RemoteClusterSecurityEsqlIT extends AbstractRemoteClusterSecurityTe } /** - * Note: invalid_remote is "invalid" because it has a bogus API key and the cluster does not exist (cannot be connected to) + * Note: invalid_remote is "invalid" because it has a bogus API key */ @SuppressWarnings("unchecked") public void testCrossClusterQueryAgainstInvalidRemote() throws Exception { @@ -521,13 +521,19 @@ public class RemoteClusterSecurityEsqlIT extends AbstractRemoteClusterSecurityTe // invalid remote with local index should return local results { var q = "FROM invalid_remote:employees,employees | SORT emp_id DESC | LIMIT 10"; - Response response = performRequestWithRemoteSearchUser(esqlRequest(q)); - // TODO: when skip_unavailable=false for invalid_remote, a fatal exception should be thrown - // this does not yet happen because field-caps returns nothing for this cluster, rather - // than an error, so the current code cannot detect that error. Follow on PR will handle this. - assertLocalOnlyResults(response); + if (skipUnavailable) { + Response response = performRequestWithRemoteSearchUser(esqlRequest(q)); + // this does not yet happen because field-caps returns nothing for this cluster, rather + // than an error, so the current code cannot detect that error. Follow on PR will handle this. + assertLocalOnlyResultsAndSkippedRemote(response); + } else { + // errors from invalid remote should throw an exception if the cluster is marked with skip_unavailable=false + ResponseException error = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(esqlRequest(q))); + assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400)); + // TODO: in follow on PR, figure out why this is returning the wrong error - should be "cannot connect to invalid_remote" + assertThat(error.getMessage(), containsString("Unknown index [invalid_remote:employees]")); + } } - { var q = "FROM invalid_remote:employees | SORT emp_id DESC | LIMIT 10"; // errors from invalid remote should be ignored if the cluster is marked with skip_unavailable=true @@ -560,10 +566,9 @@ public class RemoteClusterSecurityEsqlIT extends AbstractRemoteClusterSecurityTe } else { // errors from invalid remote should throw an exception if the cluster is marked with skip_unavailable=false - ResponseException error = expectThrows(ResponseException.class, () -> { - final Response response1 = performRequestWithRemoteSearchUser(esqlRequest(q)); - }); + ResponseException error = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(esqlRequest(q))); assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(401)); + // TODO: in follow on PR, figure out why this is returning the wrong error - should be "cannot connect to invalid_remote" assertThat(error.getMessage(), containsString("unable to find apikey")); } } @@ -1049,7 +1054,7 @@ public class RemoteClusterSecurityEsqlIT extends AbstractRemoteClusterSecurityTe } @SuppressWarnings("unchecked") - private void assertLocalOnlyResults(Response response) throws IOException { + private void assertLocalOnlyResultsAndSkippedRemote(Response response) throws IOException { assertOK(response); Map responseAsMap = entityAsMap(response); List columns = (List) responseAsMap.get("columns"); @@ -1061,6 +1066,34 @@ public class RemoteClusterSecurityEsqlIT extends AbstractRemoteClusterSecurityTe .collect(Collectors.toList()); // local results assertThat(flatList, containsInAnyOrder("2", "4", "6", "8", "support", "management", "engineering", "marketing")); + Map clusters = (Map) responseAsMap.get("_clusters"); + + /* + clusters map: + {running=0, total=2, details={ + invalid_remote={_shards={total=0, failed=0, successful=0, skipped=0}, took=176, indices=employees, + failures=[{reason={reason=Unable to connect to [invalid_remote], type=connect_transport_exception}, + index=null, shard=-1}], status=skipped}, + (local)={_shards={total=1, failed=0, successful=1, skipped=0}, took=298, indices=employees, status=successful}}, + failed=0, partial=0, successful=1, skipped=1} + */ + + assertThat((int) clusters.get("total"), equalTo(2)); + assertThat((int) clusters.get("successful"), equalTo(1)); + assertThat((int) clusters.get("skipped"), equalTo(1)); + + Map details = (Map) clusters.get("details"); + Map invalidRemoteMap = (Map) details.get("invalid_remote"); + assertThat(invalidRemoteMap.get("status").toString(), equalTo("skipped")); + List failures = (List) invalidRemoteMap.get("failures"); + assertThat(failures.size(), equalTo(1)); + Map failureMap = (Map) failures.get(0); + Map reasonMap = (Map) failureMap.get("reason"); + assertThat(reasonMap.get("reason").toString(), containsString("Unable to connect to [invalid_remote]")); + assertThat(reasonMap.get("type").toString(), containsString("connect_transport_exception")); + + Map localCluster = (Map) details.get("(local)"); + assertThat(localCluster.get("status").toString(), equalTo("successful")); } @SuppressWarnings("unchecked")