github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-06-27 17:10:22 -04:00
Code Issues Projects Releases Packages Wiki Activity

Only aggregations require at least one shard request (#115314)

Browse source 
* unskipping shards only when aggs

* Update docs/changelog/115314.yaml

* fixed more tests

* null check for searchRequest.source()
This commit is contained in:
Matteo Piergiovanni 2024-10-25 08:50:05 +02:00 • committed by GitHub
parent bbd887a66a
commit 7f573c6c28
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 70 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

5
docs/changelog/115314.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 115314
summary: Only aggregations require at least one shard request
area: Search
type: enhancement
issues: []

2
modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java
View file

@ -412,7 +412,7 @@ public class TSDBIndexingIT extends ESSingleNodeTestCase {
assertResponse(client().search(searchRequest), searchResponse -> {
ElasticsearchAssertions.assertNoSearchHits(searchResponse);
assertThat(searchResponse.getTotalShards(), equalTo(2));
assertThat(searchResponse.getSkippedShards(), equalTo(1));
assertThat(searchResponse.getSkippedShards(), equalTo(2));
assertThat(searchResponse.getSuccessfulShards(), equalTo(2));
});
}

4
qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java
View file

@ -43,6 +43,7 @@ import org.elasticsearch.join.query.HasParentQueryBuilder;
import org.elasticsearch.rest.action.search.RestSearchAction;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
@ -580,13 +581,14 @@ public class CCSDuelIT extends ESRestTestCase {
public void testSortByFieldOneClusterHasNoResults() throws Exception {
assumeMultiClusterSetup();
// set to a value greater than the number of shards to avoid differences due to the skipping of shards
// setting aggs to avoid differences due to the skipping of shards when matching none
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
boolean onlyRemote = randomBoolean();
sourceBuilder.query(new TermQueryBuilder("_index", onlyRemote ? REMOTE_INDEX_NAME : INDEX_NAME));
sourceBuilder.sort("type.keyword", SortOrder.ASC);
sourceBuilder.sort("creationDate", SortOrder.DESC);
sourceBuilder.sort("user.keyword", SortOrder.ASC);
sourceBuilder.aggregation(AggregationBuilders.max("max").field("creationDate"));
CheckedConsumer<ObjectPath, IOException> responseChecker = response -> {
assertHits(response);
int size = response.evaluateArraySize("hits.hits");

12
qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/70_skip_shards.yml
View file

@ -166,8 +166,7 @@
- match: { hits.total.value: 0 }
- match: { _shards.total: 2 }
- match: { _shards.successful: 2 }
# When all shards are skipped current logic returns 1 to produce a valid search result
- match: { _shards.skipped : 1}
- match: { _shards.skipped : 2}
- match: { _shards.failed: 0 }
# check that skipped when we don't match the alias with a terms query
@ -183,8 +182,7 @@
- match: { hits.total.value: 0 }
- match: { _shards.total: 2 }
- match: { _shards.successful: 2 }
# When all shards are skipped current logic returns 1 to produce a valid search result
- match: { _shards.skipped : 1}
- match: { _shards.skipped : 2}
- match: { _shards.failed: 0 }
# check that skipped when we don't match the alias with a prefix query
@ -200,8 +198,7 @@
- match: { hits.total.value: 0 }
- match: { _shards.total: 2 }
- match: { _shards.successful: 2 }
# When all shards are skipped current logic returns 1 to produce a valid search result
- match: { _shards.skipped : 1}
- match: { _shards.skipped : 2}
- match: { _shards.failed: 0 }
# check that skipped when we don't match the alias with a wildcard query
@ -217,7 +214,6 @@
- match: { hits.total.value: 0 }
- match: { _shards.total: 2 }
- match: { _shards.successful: 2 }
# When all shards are skipped current logic returns 1 to produce a valid search result
- match: { _shards.skipped : 1}
- match: { _shards.skipped : 2}
- match: { _shards.failed: 0 }

4
qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/90_index_name_query.yml
View file

@ -81,7 +81,7 @@ teardown:
- match: { hits.total.value: 0 }
- match: { _shards.total: 2 }
- match: { _shards.successful: 2 }
- match: { _shards.skipped : 1}
- match: { _shards.skipped : 2}
- match: { _shards.failed: 0 }
- do:
@ -98,5 +98,5 @@ teardown:
- match: { hits.total.value: 0 }
- match: { _shards.total: 2 }
- match: { _shards.successful: 2 }
- match: { _shards.skipped : 1}
- match: { _shards.skipped : 2}
- match: { _shards.failed: 0 }

4
server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java
View file

@ -214,7 +214,7 @@ public class CrossClusterSearchIT extends AbstractMultiClustersTestCase {
// with DFS_QUERY_THEN_FETCH, the local shards are never skipped
assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(0));
} else {
assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(localNumShards - 1));
assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(localNumShards));
}
assertThat(localClusterSearchInfo.getFailedShards(), equalTo(0));
assertThat(localClusterSearchInfo.getFailures().size(), equalTo(0));
@ -224,7 +224,7 @@ public class CrossClusterSearchIT extends AbstractMultiClustersTestCase {
assertThat(remoteClusterSearchInfo.getTotalShards(), equalTo(remoteNumShards));
assertThat(remoteClusterSearchInfo.getSuccessfulShards(), equalTo(remoteNumShards));
if (clusters.isCcsMinimizeRoundtrips()) {
assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(remoteNumShards - 1));
assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(remoteNumShards));
} else {
assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(remoteNumShards));
}

6
server/src/internalClusterTest/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java
View file

@ -68,7 +68,11 @@ public class QueryProfilerIT extends ESIntegTestCase {
prepareSearch().setQuery(q).setTrackTotalHits(true).setProfile(true).setSearchType(SearchType.QUERY_THEN_FETCH),
response -> {
assertNotNull("Profile response element should not be null", response.getProfileResults());
assertThat("Profile response should not be an empty array", response.getProfileResults().size(), not(0));
if (response.getSkippedShards() == response.getSuccessfulShards()) {
assertEquals(0, response.getProfileResults().size());
} else {
assertThat("Profile response should not be an empty array", response.getProfileResults().size(), not(0));
}
for (Map.Entry<String, SearchProfileShardResult> shard : response.getProfileResults().entrySet()) {
for (QueryProfileShardResult searchProfiles : shard.getValue().getQueryProfileResults()) {
for (ProfileResult result : searchProfiles.getQueryResults()) {

12
server/src/internalClusterTest/java/org/elasticsearch/search/stats/FieldUsageStatsIT.java
View file

@ -158,11 +158,15 @@ public class FieldUsageStatsIT extends ESIntegTestCase {
assertTrue(stats.hasField("date_field"));
assertEquals(Set.of(UsageContext.POINTS), stats.get("date_field").keySet());
// can_match does not enter search stats
// there is a special case though where we have no hit but we need to get at least one search response in order
// to produce a valid search result with all the aggs etc., so we hit one of the two shards
long expectedShards = 2L * numShards;
if (numShards == 1) {
// with 1 shard and setPreFilterShardSize(1) we don't perform can_match phase but instead directly query the shard
expectedShards += 1;
}
assertEquals(
(2 * numShards) + 1,
expectedShards,
indicesAdmin().prepareStats("test")
.clear()
.setSearch(true)

4
server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
View file

@ -1458,6 +1458,8 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
SearchResponse.Clusters clusters
) {
if (preFilter) {
// only for aggs we need to contact shards even if there are no matches
boolean requireAtLeastOneMatch = searchRequest.source() != null && searchRequest.source().aggregations() != null;
return new CanMatchPreFilterSearchPhase(
logger,
searchTransportService,
@ -1469,7 +1471,7 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
shardIterators,
timeProvider,
task,
true,
requireAtLeastOneMatch,
searchService.getCoordinatorRewriteContextProvider(timeProvider::absoluteStartMillis),
listener.delegateFailureAndWrap(
(l, iters) -> newSearchPhase(

32
x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CrossClusterAsyncSearchIT.java
View file

@ -274,6 +274,8 @@ public class CrossClusterAsyncSearchIT extends AbstractMultiClustersTestCase {
boolean dfs = randomBoolean();
if (dfs) {
request.getSearchRequest().searchType(SearchType.DFS_QUERY_THEN_FETCH);
} else {
request.getSearchRequest().searchType(SearchType.QUERY_THEN_FETCH);
}
RangeQueryBuilder rangeQueryBuilder = new RangeQueryBuilder("@timestamp").from(100).to(2000);
request.getSearchRequest().source(new SearchSourceBuilder().query(rangeQueryBuilder).size(10));
@ -288,20 +290,30 @@ public class CrossClusterAsyncSearchIT extends AbstractMultiClustersTestCase {
assertTrue(response.isRunning());
SearchResponse.Clusters clusters = response.getSearchResponse().getClusters();
assertThat(clusters.getTotal(), equalTo(2));
assertTrue("search cluster results should be marked as partial", clusters.hasPartialResults());
if (dfs) {
assertTrue("search cluster results should be marked as partial", clusters.hasPartialResults());
} else {
assertFalse(
"search cluster results should not be marked as partial as all shards are skipped",
clusters.hasPartialResults()
);
}
SearchResponse.Cluster localClusterSearchInfo = clusters.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
assertNotNull(localClusterSearchInfo);
assertThat(localClusterSearchInfo.getStatus(), equalTo(SearchResponse.Cluster.Status.RUNNING));
if (dfs) {
assertThat(localClusterSearchInfo.getStatus(), equalTo(SearchResponse.Cluster.Status.RUNNING));
} else {
assertThat(localClusterSearchInfo.getStatus(), equalTo(SearchResponse.Cluster.Status.SUCCESSFUL));
}
SearchResponse.Cluster remoteClusterSearchInfo = clusters.getCluster(REMOTE_CLUSTER);
assertNotNull(remoteClusterSearchInfo);
assertThat(localClusterSearchInfo.getStatus(), equalTo(SearchResponse.Cluster.Status.RUNNING));
} finally {
response.decRef();
}
SearchListenerPlugin.waitSearchStarted();
if (dfs) {
SearchListenerPlugin.waitSearchStarted();
}
SearchListenerPlugin.allowQueryPhase();
waitForSearchTasksToFinish();
@ -331,7 +343,7 @@ public class CrossClusterAsyncSearchIT extends AbstractMultiClustersTestCase {
// no skipped shards locally when DFS_QUERY_THEN_FETCH is used
assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(0));
} else {
assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(localNumShards - 1));
assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(localNumShards));
}
assertThat(localClusterSearchInfo.getFailedShards(), equalTo(0));
assertThat(localClusterSearchInfo.getFailures().size(), equalTo(0));
@ -341,7 +353,7 @@ public class CrossClusterAsyncSearchIT extends AbstractMultiClustersTestCase {
assertThat(remoteClusterSearchInfo.getTotalShards(), equalTo(remoteNumShards));
assertThat(remoteClusterSearchInfo.getSuccessfulShards(), equalTo(remoteNumShards));
if (minimizeRoundtrips) {
assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(remoteNumShards - 1));
assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(remoteNumShards));
} else {
assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(remoteNumShards));
}
@ -377,7 +389,7 @@ public class CrossClusterAsyncSearchIT extends AbstractMultiClustersTestCase {
// no skipped shards locally when DFS_QUERY_THEN_FETCH is used
assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(0));
} else {
assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(localNumShards - 1));
assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(localNumShards));
}
assertThat(localClusterSearchInfo.getFailedShards(), equalTo(0));
assertThat(localClusterSearchInfo.getFailures().size(), equalTo(0));
@ -387,7 +399,7 @@ public class CrossClusterAsyncSearchIT extends AbstractMultiClustersTestCase {
assertThat(remoteClusterSearchInfo.getTotalShards(), equalTo(remoteNumShards));
assertThat(remoteClusterSearchInfo.getSuccessfulShards(), equalTo(remoteNumShards));
if (minimizeRoundtrips) {
assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(remoteNumShards - 1));
assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(remoteNumShards));
} else {
assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(remoteNumShards));
}

10
x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/SearchIdleTests.java
View file

@ -42,7 +42,6 @@ import java.util.concurrent.TimeUnit;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
public class SearchIdleTests extends ESSingleNodeTestCase {
@ -133,8 +132,7 @@ public class SearchIdleTests extends ESSingleNodeTestCase {
// WHEN
assertResponse(search("test*", "constant_keyword", randomAlphaOfLength(5), 5), searchResponse -> {
assertEquals(RestStatus.OK, searchResponse.status());
// NOTE: we need an empty result from at least one shard
assertEquals(idleIndexShardsCount + activeIndexShardsCount - 1, searchResponse.getSkippedShards());
assertEquals(idleIndexShardsCount + activeIndexShardsCount, searchResponse.getSkippedShards());
assertEquals(0, searchResponse.getFailedShards());
assertEquals(0, searchResponse.getHits().getHits().length);
});
@ -144,12 +142,8 @@ public class SearchIdleTests extends ESSingleNodeTestCase {
assertIdleShardsRefreshStats(beforeStatsResponse, afterStatsResponse);
// If no shards match the can match phase then at least one shard gets queries for an empty response.
// However, this affects the search idle stats.
List<ShardStats> active = Arrays.stream(afterStatsResponse.getShards()).filter(s -> s.isSearchIdle() == false).toList();
assertThat(active, hasSize(1));
assertThat(active.get(0).getShardRouting().getIndexName(), equalTo("test1"));
assertThat(active.get(0).getShardRouting().id(), equalTo(0));
assertThat(active, hasSize(0));
}
public void testSearchIdleConstantKeywordMatchOneIndex() throws InterruptedException {

5
x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankCoordinatorCanMatchIT.java
View file

@ -10,6 +10,7 @@ package org.elasticsearch.xpack.rank.rrf;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.search.TotalHits;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.common.Strings;
import org.elasticsearch.index.IndexSettings;
@ -206,10 +207,10 @@ public class RRFRankCoordinatorCanMatchIT extends ESIntegTestCase {
)
.setSize(5),
response -> {
assertNull(response.getHits().getTotalHits());
assertEquals(new TotalHits(0, TotalHits.Relation.EQUAL_TO), response.getHits().getTotalHits());
assertEquals(0, response.getHits().getHits().length);
assertEquals(5, response.getSuccessfulShards());
assertEquals(4, response.getSkippedShards());
assertEquals(5, response.getSkippedShards());
}
);

5
x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/rrf/RRFRankShardCanMatchIT.java
View file

@ -7,6 +7,7 @@
package org.elasticsearch.xpack.rank.rrf;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.search.SearchType;
@ -199,10 +200,10 @@ public class RRFRankShardCanMatchIT extends ESIntegTestCase {
)
.setSize(5),
response -> {
assertNull(response.getHits().getTotalHits());
assertEquals(new TotalHits(0, TotalHits.Relation.EQUAL_TO), response.getHits().getTotalHits());
assertEquals(0, response.getHits().getHits().length);
assertEquals(5, response.getSuccessfulShards());
assertEquals(4, response.getSkippedShards());
assertEquals(5, response.getSkippedShards());
}
);

12
x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsCanMatchOnCoordinatorIntegTests.java
View file

@ -384,11 +384,9 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
}
} else {
assertResponse(client().search(request), newSearchResponse -> {
// When all shards are skipped, at least one of them should be queried in order to
// provide a proper search response.
assertThat(newSearchResponse.getSkippedShards(), equalTo(indexOutsideSearchRangeShardCount - 1));
assertThat(newSearchResponse.getSuccessfulShards(), equalTo(indexOutsideSearchRangeShardCount - 1));
assertThat(newSearchResponse.getFailedShards(), equalTo(1));
assertThat(newSearchResponse.getSkippedShards(), equalTo(indexOutsideSearchRangeShardCount));
assertThat(newSearchResponse.getSuccessfulShards(), equalTo(indexOutsideSearchRangeShardCount));
assertThat(newSearchResponse.getFailedShards(), equalTo(0));
assertThat(newSearchResponse.getTotalShards(), equalTo(indexOutsideSearchRangeShardCount));
});
@ -748,9 +746,7 @@ public class SearchableSnapshotsCanMatchOnCoordinatorIntegTests extends BaseFroz
// All the regular index searches succeeded
assertThat(newSearchResponse.getSuccessfulShards(), equalTo(totalShards));
assertThat(newSearchResponse.getFailedShards(), equalTo(0));
// We have to query at least one node to construct a valid response, and we pick
// a shard that's available in order to construct the search response
assertThat(newSearchResponse.getSkippedShards(), equalTo(totalShards - 1));
assertThat(newSearchResponse.getSkippedShards(), equalTo(totalShards));
assertThat(newSearchResponse.getTotalShards(), equalTo(totalShards));
assertThat(newSearchResponse.getHits().getTotalHits().value(), equalTo(0L));
});

6
x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCCSCanMatchIT.java
View file

@ -197,15 +197,13 @@ public class TransformCCSCanMatchIT extends AbstractMultiClustersTestCase {
QueryBuilders.rangeQuery("@timestamp").from(100_000_000), // This query matches no documents
true,
0,
// All but 2 shards are skipped. TBH I don't know why this 2 shards are not skipped
oldLocalNumShards + newLocalNumShards + oldRemoteNumShards + newRemoteNumShards - 2
oldLocalNumShards + newLocalNumShards + oldRemoteNumShards + newRemoteNumShards
);
testSearchAction(
QueryBuilders.rangeQuery("@timestamp").from(100_000_000), // This query matches no documents
false,
0,
// All but 1 shards are skipped. TBH I don't know why this 1 shard is not skipped
oldLocalNumShards + newLocalNumShards + oldRemoteNumShards + newRemoteNumShards - 1
oldLocalNumShards + newLocalNumShards + oldRemoteNumShards + newRemoteNumShards
);
}

3
x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java
View file

@ -484,8 +484,7 @@ public class OldRepositoryAccessIT extends ESRestTestCase {
logger.info(searchResponse);
assertEquals(0, searchResponse.getHits().getTotalHits().value());
assertEquals(numberOfShards, searchResponse.getSuccessfulShards());
// When all shards are skipped, at least one of them is queried in order to provide a proper search response.
assertEquals(numberOfShards - 1, searchResponse.getSkippedShards());
assertEquals(numberOfShards, searchResponse.getSkippedShards());
} finally {
searchResponse.decRef();
}