Temporarily bypass competitive iteration for filters aggregation (#12… (#126962)

* Temporarily bypass competitive iteration for filters aggregation (#126956)

* Bump versions after 9.0.0 release

* fix merge conflict

* Remove 8.16 from branches.json

* Bring version-bump related changes from main

* [bwc] Add bugfix3 project (#126880)

* Sync version bump changes from main again

---------

Co-authored-by: Benjamin Trent <ben.w.trent@gmail.com>
Co-authored-by: elasticsearchmachine <infra-root+elasticsearchmachine@elastic.co>
Co-authored-by: elasticsearchmachine <58790826+elasticsearchmachine@users.noreply.github.com>
Co-authored-by: Brian Seeders <brian.seeders@elastic.co>
This commit is contained in:
Julio 2025-04-16 18:10:01 -06:00 committed by GitHub
parent 718315c5f8
commit d19b525eb1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 185 additions and 56 deletions

View file

@ -56,7 +56,7 @@ steps:
timeout_in_minutes: 300
matrix:
setup:
BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004

View file

@ -287,8 +287,8 @@ steps:
env:
BWC_VERSION: 8.15.5
- label: "{{matrix.image}} / 8.16.7 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.7
- label: "{{matrix.image}} / 8.16.6 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.6
timeout_in_minutes: 300
matrix:
setup:
@ -301,10 +301,10 @@ steps:
machineType: custom-16-32768
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 8.16.7
BWC_VERSION: 8.16.6
- label: "{{matrix.image}} / 8.17.5 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.5
- label: "{{matrix.image}} / 8.17.6 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.6
timeout_in_minutes: 300
matrix:
setup:
@ -317,10 +317,10 @@ steps:
machineType: custom-16-32768
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 8.17.5
BWC_VERSION: 8.17.6
- label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0
- label: "{{matrix.image}} / 8.18.1 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.1
timeout_in_minutes: 300
matrix:
setup:
@ -333,7 +333,7 @@ steps:
machineType: custom-16-32768
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 8.18.0
BWC_VERSION: 8.18.1
- label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0
@ -351,8 +351,8 @@ steps:
env:
BWC_VERSION: 8.19.0
- label: "{{matrix.image}} / 9.0.0 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.0
- label: "{{matrix.image}} / 9.0.1 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.1
timeout_in_minutes: 300
matrix:
setup:
@ -365,7 +365,7 @@ steps:
machineType: custom-16-32768
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 9.0.0
BWC_VERSION: 9.0.1
- label: "{{matrix.image}} / 9.1.0 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.0

View file

@ -306,8 +306,8 @@ steps:
- signal_reason: agent_stop
limit: 3
- label: 8.16.7 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.7#bwcTest
- label: 8.16.6 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.6#bwcTest
timeout_in_minutes: 300
agents:
provider: gcp
@ -316,7 +316,7 @@ steps:
buildDirectory: /dev/shm/bk
preemptible: true
env:
BWC_VERSION: 8.16.7
BWC_VERSION: 8.16.6
retry:
automatic:
- exit_status: "-1"
@ -325,8 +325,8 @@ steps:
- signal_reason: agent_stop
limit: 3
- label: 8.17.5 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.5#bwcTest
- label: 8.17.6 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.6#bwcTest
timeout_in_minutes: 300
agents:
provider: gcp
@ -335,7 +335,7 @@ steps:
buildDirectory: /dev/shm/bk
preemptible: true
env:
BWC_VERSION: 8.17.5
BWC_VERSION: 8.17.6
retry:
automatic:
- exit_status: "-1"
@ -344,8 +344,8 @@ steps:
- signal_reason: agent_stop
limit: 3
- label: 8.18.0 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.0#bwcTest
- label: 8.18.1 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.1#bwcTest
timeout_in_minutes: 300
agents:
provider: gcp
@ -354,7 +354,7 @@ steps:
buildDirectory: /dev/shm/bk
preemptible: true
env:
BWC_VERSION: 8.18.0
BWC_VERSION: 8.18.1
retry:
automatic:
- exit_status: "-1"
@ -382,8 +382,8 @@ steps:
- signal_reason: agent_stop
limit: 3
- label: 9.0.0 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.0#bwcTest
- label: 9.0.1 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.1#bwcTest
timeout_in_minutes: 300
agents:
provider: gcp
@ -392,7 +392,7 @@ steps:
buildDirectory: /dev/shm/bk
preemptible: true
env:
BWC_VERSION: 9.0.0
BWC_VERSION: 9.0.1
retry:
automatic:
- exit_status: "-1"
@ -486,7 +486,7 @@ steps:
setup:
ES_RUNTIME_JAVA:
- openjdk21
BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
@ -533,7 +533,7 @@ steps:
ES_RUNTIME_JAVA:
- openjdk21
- openjdk23
BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004

View file

@ -7,10 +7,6 @@ echo "steps:"
source .buildkite/scripts/branches.sh
for BRANCH in "${BRANCHES[@]}"; do
if [[ "$BRANCH" == "9.0" ]]; then
continue
fi
INTAKE_PIPELINE_SLUG="elasticsearch-intake"
BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')
LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit')

View file

@ -15,9 +15,9 @@ BWC_VERSION:
- "8.13.4"
- "8.14.3"
- "8.15.5"
- "8.16.7"
- "8.17.5"
- "8.18.0"
- "8.16.6"
- "8.17.6"
- "8.18.1"
- "8.19.0"
- "9.0.0"
- "9.0.1"
- "9.1.0"

View file

@ -1,7 +1,6 @@
BWC_VERSION:
- "8.16.7"
- "8.17.5"
- "8.18.0"
- "8.17.6"
- "8.18.1"
- "8.19.0"
- "9.0.0"
- "9.0.1"
- "9.1.0"

View file

@ -4,9 +4,6 @@
{
"branch": "main"
},
{
"branch": "8.16"
},
{
"branch": "9.0"
},

View file

@ -11,6 +11,7 @@ rootProject.name = "root"
include ":distribution:bwc:bugfix"
include ":distribution:bwc:bugfix2"
include ":distribution:bwc:bugfix3"
include ":distribution:bwc:minor"
include ":distribution:bwc:major"
include ":distribution:bwc:staged"

View file

@ -165,7 +165,7 @@ public class BwcVersions implements Serializable {
.sorted(reverseOrder(comparing(s -> Version.fromString(s, Version.Mode.RELAXED))))
.toList();
boolean existingBugfix = false;
int bugfixCount = 0;
boolean existingStaged = false;
for (int i = 0; i < featureFreezeBranches.size(); i++) {
String branch = featureFreezeBranches.get(i);
@ -198,9 +198,9 @@ public class BwcVersions implements Serializable {
result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:" + project));
existingStaged = true;
} else { // This is a bugfix
String project = existingBugfix ? "bugfix2" : "bugfix";
bugfixCount++;
String project = "bugfix" + (bugfixCount > 1 ? bugfixCount : "");
result.put(version, new UnreleasedVersionInfo(version, branch, ":distribution:bwc:" + project));
existingBugfix = true;
}
}

View file

@ -301,6 +301,7 @@ allprojects {
// ensure we have best possible caching of bwc builds
dependsOn ":distribution:bwc:bugfix:buildBwcLinuxTar"
dependsOn ":distribution:bwc:bugfix2:buildBwcLinuxTar"
dependsOn ":distribution:bwc:bugfix3:buildBwcLinuxTar"
dependsOn ":distribution:bwc:minor:buildBwcLinuxTar"
dependsOn ":distribution:bwc:staged:buildBwcLinuxTar"
dependsOn ":distribution:bwc:staged2:buildBwcLinuxTar"

View file

View file

@ -0,0 +1,5 @@
pr: 126956
summary: Temporarily bypass competitive iteration for filters aggregation
area: Aggregations
type: bug
issues: []

View file

@ -29,8 +29,10 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.index.query.QueryBuilders.termQuery;
import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
import static org.elasticsearch.search.aggregations.AggregationBuilders.avg;
import static org.elasticsearch.search.aggregations.AggregationBuilders.filters;
import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram;
@ -95,6 +97,118 @@ public class FiltersIT extends ESIntegTestCase {
ensureSearchable();
}
// This test replicates a strange filter query & filters aggregation behavior
// we apparently utilize competitive iterators strangely.
// See: https://github.com/elastic/elasticsearch/issues/126955
public void testSimpleWithFilterQuery() throws Exception {
createIndex("filters_idx");
String groupFieldName = "group";
String subGroupFieldName = "subGroup";
int numTotalGroup0 = 500;
String group0Name = "group0";
int numTotalGroup1 = 1000;
String group1Name = "group1";
int subGroup0 = 100;
String subGroup0Name = "subGroup0";
int subGroup1 = 50;
String subGroup1Name = "subGroup1";
int subGroup2 = 25;
String subGroup2Name = "subGroup2";
int others = 10;
String otherName = "others";
List<IndexRequestBuilder> builders = new ArrayList<>();
for (int i = 0; i < numTotalGroup0; i++) {
for (int j = 0; j < subGroup0; j++) {
XContentBuilder source = jsonBuilder().startObject()
.field(groupFieldName, group0Name)
.field(subGroupFieldName, subGroup0Name)
.endObject();
builders.add(prepareIndex("filters_idx").setSource(source));
}
for (int j = 0; j < subGroup1; j++) {
XContentBuilder source = jsonBuilder().startObject()
.field(groupFieldName, group0Name)
.field(subGroupFieldName, subGroup1Name)
.endObject();
builders.add(prepareIndex("filters_idx").setSource(source));
}
for (int j = 0; j < subGroup2; j++) {
XContentBuilder source = jsonBuilder().startObject()
.field(groupFieldName, group0Name)
.field(subGroupFieldName, subGroup2Name)
.endObject();
builders.add(prepareIndex("filters_idx").setSource(source));
}
for (int j = 0; j < others; j++) {
XContentBuilder source = jsonBuilder().startObject()
.field(groupFieldName, group0Name)
.field(subGroupFieldName, otherName)
.endObject();
builders.add(prepareIndex("filters_idx").setSource(source));
}
}
for (int i = 0; i < numTotalGroup1; i++) {
for (int j = 0; j < subGroup0; j++) {
XContentBuilder source = jsonBuilder().startObject()
.field(groupFieldName, group1Name)
.field(subGroupFieldName, subGroup0Name)
.endObject();
builders.add(prepareIndex("filters_idx").setSource(source));
}
for (int j = 0; j < subGroup1; j++) {
XContentBuilder source = jsonBuilder().startObject()
.field(groupFieldName, group1Name)
.field(subGroupFieldName, subGroup1Name)
.endObject();
builders.add(prepareIndex("filters_idx").setSource(source));
}
for (int j = 0; j < subGroup2; j++) {
XContentBuilder source = jsonBuilder().startObject()
.field(groupFieldName, group1Name)
.field(subGroupFieldName, subGroup2Name)
.endObject();
builders.add(prepareIndex("filters_idx").setSource(source));
}
for (int j = 0; j < others; j++) {
XContentBuilder source = jsonBuilder().startObject()
.field(groupFieldName, group1Name)
.field(subGroupFieldName, otherName)
.endObject();
builders.add(prepareIndex("filters_idx").setSource(source));
}
}
indexRandom(true, false, true, builders);
ensureSearchable();
assertNoFailuresAndResponse(
prepareSearch("filters_idx").setSize(0)
.setRequestCache(false)
.setTrackTotalHits(true)
.setQuery(boolQuery().filter(termQuery(groupFieldName + ".keyword", group0Name)))
.addAggregation(
filters(
"results",
new KeyedFilter(subGroup0Name, termsQuery(subGroupFieldName + ".keyword", subGroup0Name)),
new KeyedFilter(subGroup1Name, termsQuery(subGroupFieldName + ".keyword", subGroup1Name)),
new KeyedFilter(subGroup2Name, termsQuery(subGroupFieldName + ".keyword", subGroup2Name))
// This is key
).otherBucket(false)
),
searchResponse -> {
Filters filters = searchResponse.getAggregations().get("results");
assertThat(filters, notNullValue());
assertThat(filters.getName(), equalTo("results"));
Filters.Bucket bucket = filters.getBucketByKey(subGroup0Name);
assertThat(bucket, Matchers.notNullValue());
assertThat(bucket.getDocCount(), equalTo((long) subGroup0 * numTotalGroup0));
}
);
}
public void testSimple() throws Exception {
assertNoFailuresAndResponse(
prepareSearch("idx").addAggregation(

View file

@ -95,6 +95,7 @@ public class TransportVersions {
public static final TransportVersion INITIAL_ELASTICSEARCH_8_17_3 = def(8_797_0_03);
public static final TransportVersion INITIAL_ELASTICSEARCH_8_17_4 = def(8_797_0_04);
public static final TransportVersion INITIAL_ELASTICSEARCH_8_17_5 = def(8_797_0_05);
public static final TransportVersion INITIAL_ELASTICSEARCH_8_17_6 = def(8_797_0_06);
public static final TransportVersion INDEXING_PRESSURE_THROTTLING_STATS = def(8_798_0_00);
public static final TransportVersion REINDEX_DATA_STREAMS = def(8_799_0_00);
public static final TransportVersion ESQL_REMOVE_NODE_LEVEL_PLAN = def(8_800_0_00);
@ -140,6 +141,7 @@ public class TransportVersions {
public static final TransportVersion ML_INFERENCE_IBM_WATSONX_RERANK_ADDED = def(8_840_0_00);
public static final TransportVersion REMOVE_ALL_APPLICABLE_SELECTOR_BACKPORT_8_18 = def(8_840_0_01);
public static final TransportVersion RETRY_ILM_ASYNC_ACTION_REQUIRE_ERROR_8_18 = def(8_840_0_02);
public static final TransportVersion INITIAL_ELASTICSEARCH_8_18_1 = def(8_840_0_03);
public static final TransportVersion INITIAL_ELASTICSEARCH_8_19 = def(8_841_0_00);
public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED_BACKPORT_8_X = def(8_841_0_01);
public static final TransportVersion REMOVE_ALL_APPLICABLE_SELECTOR_BACKPORT_8_19 = def(8_841_0_02);
@ -169,6 +171,7 @@ public class TransportVersions {
public static final TransportVersion RETRY_ILM_ASYNC_ACTION_REQUIRE_ERROR_90 = def(9_000_0_07);
public static final TransportVersion RE_REMOVE_MIN_COMPATIBLE_SHARD_NODE_90 = def(9_000_0_08);
public static final TransportVersion STORED_SCRIPT_CONTENT_LENGTH_90 = def(9_000_0_09);
public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10);
public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00);
public static final TransportVersion REMOVE_SNAPSHOT_FAILURES = def(9_002_0_00);
public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED = def(9_003_0_00);
@ -215,10 +218,12 @@ public class TransportVersions {
public static final TransportVersion REMOTE_EXCEPTION = def(9_044_0_00);
public static final TransportVersion ESQL_REMOVE_AGGREGATE_TYPE = def(9_045_0_00);
public static final TransportVersion ADD_PROJECT_ID_TO_DSL_ERROR_INFO = def(9_046_0_00);
public static final TransportVersion SEMANTIC_TEXT_CHUNKING_CONFIG = def(9_047_00_0);
public static final TransportVersion REPO_ANALYSIS_COPY_BLOB = def(9_048_00_0);
public static final TransportVersion AMAZON_BEDROCK_TASK_SETTINGS = def(9_049_00_0);
public static final TransportVersion ESQL_REPORT_SHARD_PARTITIONING = def(9_050_00_0);
public static final TransportVersion SEMANTIC_TEXT_CHUNKING_CONFIG = def(9_047_0_00);
public static final TransportVersion REPO_ANALYSIS_COPY_BLOB = def(9_048_0_00);
public static final TransportVersion AMAZON_BEDROCK_TASK_SETTINGS = def(9_049_0_00);
public static final TransportVersion ESQL_REPORT_SHARD_PARTITIONING = def(9_050_0_00);
public static final TransportVersion ESQL_QUERY_PLANNING_DURATION = def(9_051_0_00);
public static final TransportVersion ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED = def(9_052_0_00);
/*
* STOP! READ THIS FIRST! No, really,
@ -285,7 +290,7 @@ public class TransportVersions {
* Reference to the minimum transport version that can be used with CCS.
* This should be the transport version used by the previous minor release.
*/
public static final TransportVersion MINIMUM_CCS_VERSION = BYTE_SIZE_VALUE_ALWAYS_USES_BYTES_1;
public static final TransportVersion MINIMUM_CCS_VERSION = STORED_SCRIPT_CONTENT_LENGTH_90;
/**
* Sorted list of all versions defined in this class

View file

@ -195,16 +195,18 @@ public class Version implements VersionId<Version>, ToXContentFragment {
public static final Version V_8_16_4 = new Version(8_16_04_99);
public static final Version V_8_16_5 = new Version(8_16_05_99);
public static final Version V_8_16_6 = new Version(8_16_06_99);
public static final Version V_8_16_7 = new Version(8_16_07_99);
public static final Version V_8_17_0 = new Version(8_17_00_99);
public static final Version V_8_17_1 = new Version(8_17_01_99);
public static final Version V_8_17_2 = new Version(8_17_02_99);
public static final Version V_8_17_3 = new Version(8_17_03_99);
public static final Version V_8_17_4 = new Version(8_17_04_99);
public static final Version V_8_17_5 = new Version(8_17_05_99);
public static final Version V_8_17_6 = new Version(8_17_06_99);
public static final Version V_8_18_0 = new Version(8_18_00_99);
public static final Version V_8_18_1 = new Version(8_18_01_99);
public static final Version V_8_19_0 = new Version(8_19_00_99);
public static final Version V_9_0_0 = new Version(9_00_00_99);
public static final Version V_9_0_1 = new Version(9_00_01_99);
public static final Version V_9_1_0 = new Version(9_01_00_99);
public static final Version CURRENT = V_9_1_0;

View file

@ -328,11 +328,13 @@ public abstract class FiltersAggregator extends BucketsAggregator {
hasOtherBucket
);
}
if (usesCompetitiveIterator) {
return new MultiFilterCompetitiveLeafCollector(sub, filterWrappers, numFilters, totalNumKeys, hasOtherBucket);
} else {
return new MultiFilterLeafCollector(sub, filterWrappers, numFilters, totalNumKeys, hasOtherBucket);
}
// TODO: https://github.com/elastic/elasticsearch/issues/126955
// competitive iterator is currently broken, we would rather be slow than broken
return new MultiFilterLeafCollector(sub, filterWrappers, numFilters, totalNumKeys, hasOtherBucket);
// if (usesCompetitiveIterator) {
// return new MultiFilterCompetitiveLeafCollector(sub, filterWrappers, numFilters, totalNumKeys, hasOtherBucket);
// } else {
// }
}
}

View file

@ -145,3 +145,6 @@
8.17.2,8797002
8.17.3,8797003
8.17.4,8797004
8.17.5,8797005
8.18.0,8840002
9.0.0,9000009

1 7.0.0 7000099
145 8.17.2 8797002
146 8.17.3 8797003
147 8.17.4 8797004
148 8.17.5 8797005
149 8.18.0 8840002
150 9.0.0 9000009

View file

@ -145,3 +145,6 @@
8.17.2,8521000
8.17.3,8521000
8.17.4,8521000
8.17.5,8521000
8.18.0,8525000
9.0.0,9009000

1 7.0.0 7000099
145 8.17.2 8521000
146 8.17.3 8521000
147 8.17.4 8521000
148 8.17.5 8521000
149 8.18.0 8525000
150 9.0.0 9009000

View file

@ -78,6 +78,7 @@ List projects = [
'distribution:packages:rpm',
'distribution:bwc:bugfix',
'distribution:bwc:bugfix2',
'distribution:bwc:bugfix3',
'distribution:bwc:maintenance',
'distribution:bwc:minor',
'distribution:bwc:staged',