github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-06-28 09:28:55 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'upstream/main' into lucene_snapshot

Browse source
This commit is contained in:
Simon Cooper 2025-06-27 11:47:57 +01:00
commit 301ebab068
2616 changed files with 49760 additions and 16836 deletions
Download patch file Download diff file Expand all files Collapse all files

4
.backportrc.json
View file

@ -1,9 +1,9 @@
{
"upstream" : "elastic/elasticsearch",
"targetBranchChoices" : [ "main", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
"targetBranchChoices" : [ "main", "9.1", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
"targetPRLabels" : [ "backport" ],
"branchLabelMapping" : {
"^v9.1.0$" : "main",
"^v9.2.0$" : "main",
"^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
}
}

2
.buildkite/pipelines/ecs-dynamic-template-tests.yml
View file

@ -4,7 +4,7 @@ steps:
timeout_in_minutes: 420
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
diskSizeGb: 350
machineType: custom-32-98304
notify:

27
.buildkite/pipelines/intake.template.yml
View file

@ -4,7 +4,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- wait
@ -13,7 +13,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: part2
@ -21,7 +21,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: part3
@ -29,7 +29,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: part4
@ -37,17 +37,26 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: part5
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart5
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: part6
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart6
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- group: bwc-snapshots
steps:
- label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
@ -58,11 +67,13 @@ steps:
BWC_VERSION: $BWC_LIST
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: "{{matrix.BWC_VERSION}}"
- label: bc-upgrade
command: ".buildkite/scripts/run-bc-upgrade-tests.sh"
- group: lucene-compat
steps:
- label: "{{matrix.LUCENE_VERSION}} / lucene-compat"
@ -78,7 +89,7 @@ steps:
- "10352e57d85505984582616e1e38530d3ec6ca59" # update to match last commit before lucene bump maintained from combat-lucene-10-0-0 branch
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
env:
@ -89,7 +100,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- wait

29
.buildkite/pipelines/intake.yml
View file

@ -5,7 +5,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- wait
@ -14,7 +14,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: part2
@ -22,7 +22,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: part3
@ -30,7 +30,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: part4
@ -38,17 +38,26 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: part5
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart5
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: part6
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart6
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- group: bwc-snapshots
steps:
- label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
@ -56,14 +65,16 @@ steps:
timeout_in_minutes: 300
matrix:
setup:
BWC_VERSION: ["8.17.8", "8.18.3", "8.19.0", "9.0.3", "9.1.0"]
BWC_VERSION: ["8.17.9", "8.18.4", "8.19.0", "9.0.4", "9.1.0", "9.2.0"]
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: "{{matrix.BWC_VERSION}}"
- label: bc-upgrade
command: ".buildkite/scripts/run-bc-upgrade-tests.sh"
- group: lucene-compat
steps:
- label: "{{matrix.LUCENE_VERSION}} / lucene-compat"
@ -79,7 +90,7 @@ steps:
- "10352e57d85505984582616e1e38530d3ec6ca59" # update to match last commit before lucene bump maintained from combat-lucene-10-0-0 branch
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
env:
@ -90,7 +101,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- wait

2
.buildkite/pipelines/lucene-snapshot/build-snapshot.yml
View file

@ -14,7 +14,7 @@ steps:
UPDATE_ES_LUCENE_SNAPSHOT: "true"
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- wait

22
.buildkite/pipelines/lucene-snapshot/run-tests.yml
View file

@ -4,7 +4,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- wait: null
@ -13,7 +13,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- label: part2
@ -21,7 +21,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- label: part3
@ -29,7 +29,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- label: part4
@ -37,12 +37,20 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- label: part5
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart5
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- label: part6
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkPart6
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
@ -60,7 +68,7 @@ steps:
- 8.10.0
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
env:
@ -70,6 +78,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/periodic-fwc.template.yml
View file

@ -4,7 +4,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
matrix:

2
.buildkite/pipelines/periodic-fwc.yml
View file

@ -5,7 +5,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
matrix:

2
.buildkite/pipelines/periodic-micro-benchmarks.yml
View file

@ -8,6 +8,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/periodic-packaging.bwc.template.yml
View file

@ -5,7 +5,7 @@
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}

1
.buildkite/pipelines/periodic-packaging.template.yml
View file

@ -12,7 +12,6 @@ steps:
- oraclelinux-8
- oraclelinux-9
- sles-15
- ubuntu-2004
- ubuntu-2204
- ubuntu-2404
- rocky-8

79
.buildkite/pipelines/periodic-packaging.yml
View file

@ -13,7 +13,6 @@ steps:
- oraclelinux-8
- oraclelinux-9
- sles-15
- ubuntu-2004
- ubuntu-2204
- ubuntu-2404
- rocky-8
@ -38,7 +37,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -54,7 +53,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -70,7 +69,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -86,7 +85,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -102,7 +101,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -118,7 +117,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -134,7 +133,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -150,7 +149,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -166,7 +165,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -182,7 +181,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -198,7 +197,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -214,7 +213,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -230,7 +229,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -246,7 +245,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -262,7 +261,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -278,7 +277,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -294,7 +293,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -303,37 +302,37 @@ steps:
env:
BWC_VERSION: 8.16.6
- label: "{{matrix.image}} / 8.17.8 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.8
- label: "{{matrix.image}} / 8.17.9 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.9
timeout_in_minutes: 300
matrix:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
machineType: custom-16-32768
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 8.17.8
BWC_VERSION: 8.17.9
- label: "{{matrix.image}} / 8.18.3 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.3
- label: "{{matrix.image}} / 8.18.4 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.4
timeout_in_minutes: 300
matrix:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
machineType: custom-16-32768
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 8.18.3
BWC_VERSION: 8.18.4
- label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0
@ -342,7 +341,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -351,21 +350,21 @@ steps:
env:
BWC_VERSION: 8.19.0
- label: "{{matrix.image}} / 9.0.3 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.3
- label: "{{matrix.image}} / 9.0.4 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.4
timeout_in_minutes: 300
matrix:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
machineType: custom-16-32768
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 9.0.3
BWC_VERSION: 9.0.4
- label: "{{matrix.image}} / 9.1.0 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.0
@ -374,7 +373,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
@ -383,6 +382,22 @@ steps:
env:
BWC_VERSION: 9.1.0
- label: "{{matrix.image}} / 9.2.0 / packaging-tests-upgrade"
command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.2.0
timeout_in_minutes: 300
matrix:
setup:
image:
- rocky-8
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}
machineType: custom-16-32768
buildDirectory: /dev/shm/bk
env:
BWC_VERSION: 9.2.0
- group: packaging-tests-windows
steps:
- label: "{{matrix.image}} / packaging-tests-windows"

4
.buildkite/pipelines/periodic-platform-support.yml
View file

@ -12,7 +12,6 @@ steps:
- oraclelinux-8
- oraclelinux-9
- sles-15
- ubuntu-2004
- ubuntu-2204
- ubuntu-2404
- rocky-8
@ -45,6 +44,7 @@ steps:
- checkPart3
- checkPart4
- checkPart5
- checkPart6
- checkRestCompat
agents:
provider: gcp
@ -63,7 +63,6 @@ steps:
setup:
image:
- almalinux-8-aarch64
- ubuntu-2004-aarch64
- ubuntu-2404-aarch64
GRADLE_TASK:
- checkPart1
@ -71,6 +70,7 @@ steps:
- checkPart3
- checkPart4
- checkPart5
- checkPart6
- checkRestCompat
agents:
provider: aws

2
.buildkite/pipelines/periodic.bwc.template.yml
View file

@ -3,7 +3,7 @@
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true

38
.buildkite/pipelines/periodic.template.yml
View file

@ -6,7 +6,7 @@ steps:
timeout_in_minutes: 420
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
diskSizeGb: 350
machineType: custom-32-98304
- label: encryption-at-rest
@ -14,7 +14,7 @@ steps:
timeout_in_minutes: 420
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
diskSizeGb: 350
machineType: custom-32-98304
- label: eql-correctness
@ -22,7 +22,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- label: example-plugins
@ -33,7 +33,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- group: java-fips-matrix
@ -51,10 +51,11 @@ steps:
- checkPart3
- checkPart4
- checkPart5
- checkPart6
- checkRestCompat
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
env:
@ -70,7 +71,7 @@ steps:
BWC_VERSION: $BWC_LIST
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
env:
@ -93,10 +94,11 @@ steps:
- checkPart3
- checkPart4
- checkPart5
- checkPart6
- checkRestCompat
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
env:
@ -113,7 +115,7 @@ steps:
BWC_VERSION: $BWC_LIST
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
env:
@ -124,7 +126,7 @@ steps:
timeout_in_minutes: 360
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
diskSizeGb: 350
machineType: custom-32-98304
- label: single-processor-node-tests
@ -132,7 +134,7 @@ steps:
timeout_in_minutes: 420
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
diskSizeGb: 350
machineType: custom-32-98304
- group: third-party tests
@ -148,7 +150,7 @@ steps:
timeout_in_minutes: 30
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n2-standard-8
buildDirectory: /dev/shm/bk
- label: third-party / azure
@ -162,7 +164,7 @@ steps:
timeout_in_minutes: 30
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n2-standard-8
buildDirectory: /dev/shm/bk
- label: third-party / gcs
@ -176,7 +178,7 @@ steps:
timeout_in_minutes: 30
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n2-standard-8
buildDirectory: /dev/shm/bk
- label: third-party / geoip
@ -185,7 +187,7 @@ steps:
timeout_in_minutes: 30
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n2-standard-8
buildDirectory: /dev/shm/bk
- label: third-party / s3
@ -199,7 +201,7 @@ steps:
timeout_in_minutes: 30
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n2-standard-8
buildDirectory: /dev/shm/bk
- group: lucene-compat
@ -217,7 +219,7 @@ steps:
- "10352e57d85505984582616e1e38530d3ec6ca59" # update to match last commit before lucene bump maintained from combat-lucene-10-0-0 branch
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
env:
@ -230,7 +232,7 @@ steps:
timeout_in_minutes: 20
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n2-standard-8
buildDirectory: /dev/shm/bk
if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
@ -239,7 +241,7 @@ steps:
timeout_in_minutes: 15
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n2-standard-2
- label: check-branch-protection-rules
command: .buildkite/scripts/branch-protection.sh

123
.buildkite/pipelines/periodic.yml
View file

@ -7,7 +7,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -26,7 +26,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -45,7 +45,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -64,7 +64,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -83,7 +83,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -102,7 +102,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -121,7 +121,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -140,7 +140,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -159,7 +159,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -178,7 +178,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -197,7 +197,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -216,7 +216,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -235,7 +235,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -254,7 +254,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -273,7 +273,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -292,7 +292,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -311,7 +311,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -325,17 +325,17 @@ steps:
- signal_reason: agent_stop
limit: 3
- label: 8.17.8 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.8#bwcTest
- label: 8.17.9 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.9#bwcTest
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
env:
BWC_VERSION: 8.17.8
BWC_VERSION: 8.17.9
retry:
automatic:
- exit_status: "-1"
@ -344,17 +344,17 @@ steps:
- signal_reason: agent_stop
limit: 3
- label: 8.18.3 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.3#bwcTest
- label: 8.18.4 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.4#bwcTest
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
env:
BWC_VERSION: 8.18.3
BWC_VERSION: 8.18.4
retry:
automatic:
- exit_status: "-1"
@ -368,7 +368,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -382,17 +382,17 @@ steps:
- signal_reason: agent_stop
limit: 3
- label: 9.0.3 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.3#bwcTest
- label: 9.0.4 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.4#bwcTest
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
env:
BWC_VERSION: 9.0.3
BWC_VERSION: 9.0.4
retry:
automatic:
- exit_status: "-1"
@ -406,7 +406,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
@ -420,12 +420,31 @@ steps:
- signal_reason: agent_stop
limit: 3
- label: 9.2.0 / bwc
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.2.0#bwcTest
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
preemptible: true
env:
BWC_VERSION: 9.2.0
retry:
automatic:
- exit_status: "-1"
limit: 3
signal_reason: none
- signal_reason: agent_stop
limit: 3
- label: concurrent-search-tests
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
timeout_in_minutes: 420
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
diskSizeGb: 350
machineType: custom-32-98304
- label: encryption-at-rest
@ -433,7 +452,7 @@ steps:
timeout_in_minutes: 420
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
diskSizeGb: 350
machineType: custom-32-98304
- label: eql-correctness
@ -441,7 +460,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- label: example-plugins
@ -452,7 +471,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
- group: java-fips-matrix
@ -470,10 +489,11 @@ steps:
- checkPart3
- checkPart4
- checkPart5
- checkPart6
- checkRestCompat
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
env:
@ -486,10 +506,10 @@ steps:
setup:
ES_RUNTIME_JAVA:
- openjdk21
BWC_VERSION: ["8.17.8", "8.18.3", "8.19.0", "9.0.3", "9.1.0"]
BWC_VERSION: ["8.17.9", "8.18.4", "8.19.0", "9.0.4", "9.1.0", "9.2.0"]
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
env:
@ -512,10 +532,11 @@ steps:
- checkPart3
- checkPart4
- checkPart5
- checkPart6
- checkRestCompat
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
env:
@ -529,10 +550,10 @@ steps:
ES_RUNTIME_JAVA:
- openjdk21
- openjdk23
BWC_VERSION: ["8.17.8", "8.18.3", "8.19.0", "9.0.3", "9.1.0"]
BWC_VERSION: ["8.17.9", "8.18.4", "8.19.0", "9.0.4", "9.1.0", "9.2.0"]
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
env:
@ -543,7 +564,7 @@ steps:
timeout_in_minutes: 360
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
diskSizeGb: 350
machineType: custom-32-98304
- label: single-processor-node-tests
@ -551,7 +572,7 @@ steps:
timeout_in_minutes: 420
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
diskSizeGb: 350
machineType: custom-32-98304
- group: third-party tests
@ -567,7 +588,7 @@ steps:
timeout_in_minutes: 30
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n2-standard-8
buildDirectory: /dev/shm/bk
- label: third-party / azure
@ -581,7 +602,7 @@ steps:
timeout_in_minutes: 30
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n2-standard-8
buildDirectory: /dev/shm/bk
- label: third-party / gcs
@ -595,7 +616,7 @@ steps:
timeout_in_minutes: 30
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n2-standard-8
buildDirectory: /dev/shm/bk
- label: third-party / geoip
@ -604,7 +625,7 @@ steps:
timeout_in_minutes: 30
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n2-standard-8
buildDirectory: /dev/shm/bk
- label: third-party / s3
@ -618,7 +639,7 @@ steps:
timeout_in_minutes: 30
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n2-standard-8
buildDirectory: /dev/shm/bk
- group: lucene-compat
@ -636,7 +657,7 @@ steps:
- "10352e57d85505984582616e1e38530d3ec6ca59" # update to match last commit before lucene bump maintained from combat-lucene-10-0-0 branch
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
env:
@ -649,7 +670,7 @@ steps:
timeout_in_minutes: 20
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n2-standard-8
buildDirectory: /dev/shm/bk
if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
@ -658,7 +679,7 @@ steps:
timeout_in_minutes: 15
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n2-standard-2
- label: check-branch-protection-rules
command: .buildkite/scripts/branch-protection.sh

2
.buildkite/pipelines/pull-request/build-benchmark.yml
View file

@ -22,6 +22,6 @@ steps:
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

67
.buildkite/pipelines/pull-request/bwc-snapshots.yml
View file

@ -6,9 +6,21 @@ config:
steps:
- group: bwc-snapshots
steps:
- label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
key: "bwc-snapshots"
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest
- label: "{{matrix.BWC_VERSION}} / Part 1 / bwc-snapshots"
key: "bwc-snapshots-part1"
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTestPart1
timeout_in_minutes: 300
matrix:
setup:
BWC_VERSION: $SNAPSHOT_BWC_VERSIONS
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: "{{matrix.BWC_VERSION}} / Part 2 / bwc-snapshots"
key: "bwc-snapshots-part2"
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTestPart2
timeout_in_minutes: 300
matrix:
setup:
@ -18,3 +30,52 @@ steps:
image: family/elasticsearch-ubuntu-2004
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: "{{matrix.BWC_VERSION}} / Part 3 / bwc-snapshots"
key: "bwc-snapshots-part3"
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTestPart3
timeout_in_minutes: 300
matrix:
setup:
BWC_VERSION: $SNAPSHOT_BWC_VERSIONS
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: "{{matrix.BWC_VERSION}} / Part 4 / bwc-snapshots"
key: "bwc-snapshots-part4"
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTestPart5
timeout_in_minutes: 300
matrix:
setup:
BWC_VERSION: $SNAPSHOT_BWC_VERSIONS
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: "{{matrix.BWC_VERSION}} / Part 5 / bwc-snapshots"
key: "bwc-snapshots-part5"
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTestPart5
timeout_in_minutes: 300
matrix:
setup:
BWC_VERSION: $SNAPSHOT_BWC_VERSIONS
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
- label: "{{matrix.BWC_VERSION}} / Part 6 / bwc-snapshots"
key: "bwc-snapshots-part6"
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTestPart6
timeout_in_minutes: 300
matrix:
setup:
BWC_VERSION: $SNAPSHOT_BWC_VERSIONS
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: n1-standard-32
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/cloud-deploy.yml
View file

@ -8,6 +8,6 @@ steps:
timeout_in_minutes: 20
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/docs-check.yml
View file

@ -9,6 +9,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/eql-correctness.yml
View file

@ -4,6 +4,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/example-plugins.yml
View file

@ -13,6 +13,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/full-bwc.yml
View file

@ -10,6 +10,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/packaging-tests-unix-sample.yml
View file

@ -13,7 +13,7 @@ steps:
setup:
image:
- rhel-8
- ubuntu-2004
- ubuntu-2404
PACKAGING_TASK:
- destructiveDistroTest.docker
- destructiveDistroTest.packages

1
.buildkite/pipelines/pull-request/packaging-tests-unix.yml
View file

@ -15,7 +15,6 @@ steps:
- oraclelinux-8
- oraclelinux-9
- sles-15
- ubuntu-2004
- ubuntu-2204
- ubuntu-2404
- rocky-8

2
.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml
View file

@ -12,7 +12,7 @@ steps:
setup:
image:
- rocky-8
- ubuntu-2004
- ubuntu-2404
agents:
provider: gcp
image: family/elasticsearch-{{matrix.image}}

2
.buildkite/pipelines/pull-request/part-1-arm.yml
View file

@ -6,7 +6,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: aws
imagePrefix: elasticsearch-ubuntu-2004-aarch64
imagePrefix: elasticsearch-ubuntu-2404-aarch64
instanceType: m6g.8xlarge
diskSizeGb: 350
diskType: gp3

2
.buildkite/pipelines/pull-request/part-1-fips.yml
View file

@ -8,6 +8,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/part-1.yml
View file

@ -6,6 +6,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/part-2-arm.yml
View file

@ -6,7 +6,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: aws
imagePrefix: elasticsearch-ubuntu-2004-aarch64
imagePrefix: elasticsearch-ubuntu-2404-aarch64
instanceType: m6g.8xlarge
diskSizeGb: 350
diskType: gp3

2
.buildkite/pipelines/pull-request/part-2-fips.yml
View file

@ -8,6 +8,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/part-2.yml
View file

@ -4,6 +4,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/part-3-arm.yml
View file

@ -6,7 +6,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: aws
imagePrefix: elasticsearch-ubuntu-2004-aarch64
imagePrefix: elasticsearch-ubuntu-2404-aarch64
instanceType: m6g.8xlarge
diskSizeGb: 350
diskType: gp3

2
.buildkite/pipelines/pull-request/part-3-fips.yml
View file

@ -8,6 +8,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/part-3.yml
View file

@ -6,6 +6,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/part-4-arm.yml
View file

@ -6,7 +6,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: aws
imagePrefix: elasticsearch-ubuntu-2004-aarch64
imagePrefix: elasticsearch-ubuntu-2404-aarch64
instanceType: m6g.8xlarge
diskSizeGb: 350
diskType: gp3

2
.buildkite/pipelines/pull-request/part-4-fips.yml
View file

@ -8,6 +8,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/part-4.yml
View file

@ -6,6 +6,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: n1-standard-32
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/part-5-arm.yml
View file

@ -6,7 +6,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: aws
imagePrefix: elasticsearch-ubuntu-2004-aarch64
imagePrefix: elasticsearch-ubuntu-2404-aarch64
instanceType: m6g.8xlarge
diskSizeGb: 350
diskType: gp3

2
.buildkite/pipelines/pull-request/part-5-fips.yml
View file

@ -8,6 +8,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/part-5.yml
View file

@ -6,6 +6,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

13
.buildkite/pipelines/pull-request/part-6-arm.yml Normal file
View file

@ -0,0 +1,13 @@
config:
allow-labels: "test-arm"
steps:
- label: part-6-arm
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart6
timeout_in_minutes: 300
agents:
provider: aws
imagePrefix: elasticsearch-ubuntu-2004-aarch64
instanceType: m6g.8xlarge
diskSizeGb: 350
diskType: gp3
diskName: /dev/sda1

13
.buildkite/pipelines/pull-request/part-6-fips.yml Normal file
View file

@ -0,0 +1,13 @@
config:
allow-labels:
- Team:Security
- test-fips
steps:
- label: part-6-fips
command: .ci/scripts/run-gradle.sh -Dignore.tests.seed -Dtests.fips.enabled=true checkPart6
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

14
.buildkite/pipelines/pull-request/part-6-windows.yml Normal file
View file

@ -0,0 +1,14 @@
config:
allow-labels: "test-windows"
steps:
- label: part-6-windows
command: .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-windows-2022
machineType: custom-32-98304
diskType: pd-ssd
diskSizeGb: 350
env:
GRADLE_TASK: checkPart6

10
.buildkite/pipelines/pull-request/part-6.yml Normal file
View file

@ -0,0 +1,10 @@
steps:
- label: part-6
command: |
.ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart6
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

6
.buildkite/pipelines/pull-request/pr-upgrade.yml Normal file
View file

@ -0,0 +1,6 @@
steps:
- label: pr-upgrade
command: ".buildkite/scripts/run-pr-upgrade-tests.sh"
agents:
image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
memory: "4G"

2
.buildkite/pipelines/pull-request/precommit.yml
View file

@ -9,6 +9,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

3
.buildkite/pipelines/pull-request/release-tests.yml
View file

@ -15,8 +15,9 @@ steps:
- checkPart3
- checkPart4
- checkPart5
- checkPart6
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
diskSizeGb: 350
machineType: custom-32-98304

2
.buildkite/pipelines/pull-request/rest-compatibility.yml
View file

@ -6,6 +6,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/pipelines/pull-request/validate-changelogs.yml
View file

@ -4,6 +4,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

18
.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap
View file

@ -12,7 +12,7 @@ exports[`generatePipelines should generate correct pipelines with a non-docs cha
{
"agents": {
"buildDirectory": "/dev/shm/bk",
"image": "family/elasticsearch-ubuntu-2004",
"image": "family/elasticsearch-ubuntu-2404",
"machineType": "custom-32-98304",
"provider": "gcp",
},
@ -63,7 +63,7 @@ exports[`generatePipelines should generate correct pipelines with only docs chan
{
"agents": {
"buildDirectory": "/dev/shm/bk",
"image": "family/elasticsearch-ubuntu-2004",
"image": "family/elasticsearch-ubuntu-2404",
"machineType": "custom-32-98304",
"provider": "gcp",
},
@ -89,7 +89,7 @@ exports[`generatePipelines should generate correct pipelines with full BWC expan
{
"agents": {
"buildDirectory": "/dev/shm/bk",
"image": "family/elasticsearch-ubuntu-2004",
"image": "family/elasticsearch-ubuntu-2404",
"machineType": "custom-32-98304",
"provider": "gcp",
},
@ -104,7 +104,7 @@ exports[`generatePipelines should generate correct pipelines with full BWC expan
{
"agents": {
"buildDirectory": "/dev/shm/bk",
"image": "family/elasticsearch-ubuntu-2004",
"image": "family/elasticsearch-ubuntu-2404",
"machineType": "custom-32-98304",
"provider": "gcp",
},
@ -119,7 +119,7 @@ exports[`generatePipelines should generate correct pipelines with full BWC expan
{
"agents": {
"buildDirectory": "/dev/shm/bk",
"image": "family/elasticsearch-ubuntu-2004",
"image": "family/elasticsearch-ubuntu-2404",
"machineType": "custom-32-98304",
"provider": "gcp",
},
@ -134,7 +134,7 @@ exports[`generatePipelines should generate correct pipelines with full BWC expan
{
"agents": {
"buildDirectory": "/dev/shm/bk",
"image": "family/elasticsearch-ubuntu-2004",
"image": "family/elasticsearch-ubuntu-2404",
"machineType": "custom-32-98304",
"provider": "gcp",
},
@ -149,7 +149,7 @@ exports[`generatePipelines should generate correct pipelines with full BWC expan
{
"agents": {
"buildDirectory": "/dev/shm/bk",
"image": "family/elasticsearch-ubuntu-2004",
"image": "family/elasticsearch-ubuntu-2404",
"machineType": "custom-32-98304",
"provider": "gcp",
},
@ -214,7 +214,7 @@ exports[`generatePipelines should generate correct pipelines with a non-docs cha
{
"agents": {
"buildDirectory": "/dev/shm/bk",
"image": "family/elasticsearch-ubuntu-2004",
"image": "family/elasticsearch-ubuntu-2404",
"machineType": "custom-32-98304",
"provider": "gcp",
},
@ -268,7 +268,7 @@ exports[`generatePipelines should generate correct pipelines with a non-docs cha
{
"agents": {
"buildDirectory": "/dev/shm/bk",
"image": "family/elasticsearch-ubuntu-2004",
"image": "family/elasticsearch-ubuntu-2404",
"machineType": "custom-32-98304",
"provider": "gcp",
},

2
.buildkite/scripts/pull-request/mocks/pipelines/bwc-snapshots.yml
View file

@ -14,7 +14,7 @@ steps:
BWC_VERSION: $SNAPSHOT_BWC_VERSIONS
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
env:

2
.buildkite/scripts/pull-request/mocks/pipelines/docs-check.yml
View file

@ -9,6 +9,6 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk

2
.buildkite/scripts/pull-request/mocks/pipelines/full-bwc.yml
View file

@ -10,7 +10,7 @@ steps:
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
image: family/elasticsearch-ubuntu-2404
machineType: custom-32-98304
buildDirectory: /dev/shm/bk
env:

7
.buildkite/scripts/release-tests.sh
View file

@ -9,8 +9,13 @@ export BEATS_DIR=$(pwd)/distribution/docker/build/artifacts/beats
mkdir -p ${BEATS_DIR}
curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
curl --fail -o "${BEATS_DIR}/metricbeat-fips-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-fips-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
curl --fail -o "${BEATS_DIR}/metricbeat-fips-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-fips-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
curl --fail -o "${BEATS_DIR}/filebeat-fips-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-fips-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
curl --fail -o "${BEATS_DIR}/filebeat-fips-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-fips-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
# Fetch ML artifacts
export ML_IVY_REPO=$(mktemp -d)

72
.buildkite/scripts/run-bc-upgrade-tests.sh Executable file
View file

@ -0,0 +1,72 @@
#!/bin/bash
#
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the "Elastic License
# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
# Public License v 1"; you may not use this file except in compliance with, at
# your election, the "Elastic License 2.0", the "GNU Affero General Public
# License v3.0 only", or the "Server Side Public License, v 1".
#
set -euo pipefail
echo "Selecting the most recent build from branch [$BUILDKITE_BRANCH]."
# Select the most recent build from the current branch.
# We collect snapshots, order by date, then collect BCs, order by date, and concat them; then we select the last.
# So if we have one (or more) BC, we will always prefer to use that. Otherwise we will use the latest snapshot.
MANIFEST_URL="$(curl -s https://artifacts.elastic.co/releases/TfEVhiaBGqR64ie0g0r0uUwNAbEQMu1Z/future-releases/stack.json |
jq ".releases[] |
select(.branch == \"$BUILDKITE_BRANCH\") |
select(.active_release == true) |
((.snapshots | to_entries | sort_by(.value.completed_at)) +
(.build_candidates | to_entries | sort_by(.value.completed_at))) |
last | .value.manifest_url")"
if [[ -z "$MANIFEST_URL" ]] || [[ "$MANIFEST_URL" == "null" ]]; then
echo "No snapshots or build candidates for branch [$BUILDKITE_BRANCH]."
echo "Skipping BC upgrade tests."
exit 0
fi
echo "Getting build manifest from [$MANIFEST_URL]"
# Note: we use eval to perform variable substitution for the curl arguments, and command substitution to
# set the output variable. Double quotes are not enough in this case.
MANIFEST="$(eval "curl -s $MANIFEST_URL")"
if [[ -z "$MANIFEST" ]]; then
echo "Cannot get the build manifest from [$MANIFEST_URL]"
exit 1
fi
CURRENT_VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-tools-internal/version.properties)
BC_VERSION=$(echo "$MANIFEST" | jq -r .version)
BC_BUILD_ID=$(echo "$MANIFEST" | jq -r .build_id)
BC_COMMIT_HASH=$(echo "$MANIFEST" | jq -r .projects.elasticsearch.commit_hash)
if [ "$CURRENT_VERSION-SNAPSHOT" != "$BC_VERSION" ]; then
echo "Version [$BC_VERSION] of BC (or snapshot) does not match current version [$CURRENT_VERSION] of branch [$BUILDKITE_BRANCH]."
echo "Skipping BC upgrade tests."
exit 0
fi
echo "Running BC upgrade tests on $BUILDKITE_BRANCH [$BC_VERSION] using BC (or snapshot) build of commit [$BC_COMMIT_HASH] with build id [$BC_BUILD_ID]."
cat <<EOF | buildkite-agent pipeline upload
steps:
- group: "bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH"
steps:
- label: "bc-upgrade-tests-part{{matrix.PART}}"
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTestPart{{matrix.PART}}
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
matrix:
setup:
PART: ["1", "2", "3", "4", "5", "6"]
EOF

42
.buildkite/scripts/run-pr-upgrade-tests.sh Normal file
View file

@ -0,0 +1,42 @@
#!/bin/bash
#
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the "Elastic License
# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
# Public License v 1"; you may not use this file except in compliance with, at
# your election, the "Elastic License 2.0", the "GNU Affero General Public
# License v3.0 only", or the "Server Side Public License, v 1".
#
set -euo pipefail
if [[ -z "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" ]]; then
echo "Not a pull request, skipping PR upgrade tests."
exit 0
fi
# Identify the merge base of the current commit (branch) and the base branch of the pull request.
# PR upgrade tests are run from the merge base to the current commit.
BASE_COMMIT=$(git merge-base $BUILDKITE_PULL_REQUEST_BASE_BRANCH $BUILDKITE_COMMIT)
VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-tools-internal/version.properties)
echo "Running PR upgrade tests from $BUILDKITE_PULL_REQUEST_BASE_BRANCH [$BASE_COMMIT] to $BUILDKITE_BRANCH [$BUILDKITE_COMMIT]."
cat << EOF | buildkite-agent pipeline upload
steps:
- group: "pr-upgrade $BUILDKITE_PULL_REQUEST_BASE_BRANCH -> $BUILDKITE_BRANCH"
steps:
- label: "pr-upgrade-part-{{matrix.PART}}"
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTestPart{{matrix.PART}}
timeout_in_minutes: 300
agents:
provider: gcp
image: family/elasticsearch-ubuntu-2004
machineType: n1-standard-32
buildDirectory: /dev/shm/bk
matrix:
setup:
PART: ["1", "2", "3", "4", "5", "6"]
EOF

7
.ci/bwcVersions
View file

@ -16,8 +16,9 @@ BWC_VERSION:
- "8.14.3"
- "8.15.5"
- "8.16.6"
- "8.17.8"
- "8.18.3"
- "8.17.9"
- "8.18.4"
- "8.19.0"
- "9.0.3"
- "9.0.4"
- "9.1.0"
- "9.2.0"

13
.ci/scripts/resolve-dra-manifest.sh
View file

@ -6,7 +6,8 @@ strip_version() {
}
fetch_build() {
curl -sS https://artifacts-$1.elastic.co/$2/latest/$3.json \
>&2 echo "Checking for build id: https://artifacts-$1.elastic.co/$2/latest/$3.json"
curl -sSf https://artifacts-$1.elastic.co/$2/latest/$3.json \
| jq -r '.build_id'
}
@ -15,7 +16,15 @@ BRANCH="${BRANCH:-$2}"
ES_VERSION="${ES_VERSION:-$3}"
WORKFLOW=${WORKFLOW:-$4}
LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $BRANCH)
if [[ "$WORKFLOW" == "staging" ]]; then
LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $ES_VERSION)
elif [[ "$WORKFLOW" == "snapshot" ]]; then
LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $BRANCH)
else
echo "Unknown workflow: $WORKFLOW"
exit 1
fi
LATEST_VERSION=$(strip_version $LATEST_BUILD)
# If the latest artifact version doesn't match what we expect, try the corresponding version branch.

7
.ci/snapshotBwcVersions
View file

@ -1,6 +1,7 @@
BWC_VERSION:
- "8.17.8"
- "8.18.3"
- "8.17.9"
- "8.18.4"
- "8.19.0"
- "9.0.3"
- "9.0.4"
- "9.1.0"
- "9.2.0"

2
.github/workflows/docs-build.yml vendored
View file

@ -16,4 +16,4 @@ jobs:
deployments: write
id-token: write
contents: read
pull-requests: read
pull-requests: write

71
.github/workflows/docs-preview-comment.yml vendored Normal file
View file

@ -0,0 +1,71 @@
name: "Docs preview comment"
on:
pull_request_target:
types: [opened, reopened, synchronize]
permissions:
contents: read
issues: write
pull-requests: write
jobs:
preview-links:
runs-on: ubuntu-latest
steps:
- name: Comment preview links for changed docs
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const pr = context.payload.pull_request;
const prNum = pr.number;
const owner = context.repo.owner;
const repo = context.repo.repo;
const base = `https://docs-v3-preview.elastic.dev/${owner}/${repo}/pull/${prNum}`;
// 1) List all files in this PR
const { data: files } = await github.rest.pulls.listFiles({
owner, repo, pull_number: prNum
});
// 2) Filter to only added/modified .md files (skip removed and _snippets/)
const links = files
.filter(f =>
f.status !== 'removed' &&
/\.md$/i.test(f.filename) &&
!/(^|\/)_snippets\//i.test(f.filename)
)
.map(f => {
let p = f.filename.replace(/^docs\//, '').replace(/\/index\.md$/i, '/');
if (p === f.filename.replace(/^docs\//, '')) p = p.replace(/\.md$/i, '');
return `- [\`${f.filename}\`](${base}/${p})`;
});
if (!links.length) return; // nothing to do
// 3) Build the comment body
const body = [
"🔍 **Preview links for changed docs:**",
"",
...links,
"",
"🔔 *The preview site may take up to **3 minutes** to finish building. These links will become live once it completes.*"
].join("\n");
// 4) Post or update a single bot comment
const { data: comments } = await github.rest.issues.listComments({
owner, repo, issue_number: prNum
});
const existing = comments.find(c =>
c.user.type === 'Bot' &&
c.body.startsWith("🔍 **Preview links for changed docs:**")
);
if (existing) {
await github.rest.issues.updateComment({
owner, repo,
comment_id: existing.id,
body
});
} else {
await github.rest.issues.createComment({
owner, repo,
issue_number: prNum,
body
});
}

12
CONTRIBUTING.md
View file

@ -203,6 +203,18 @@ Alternative manual steps for IntelliJ.
3. Navigate to the file `build-conventions/formatterConfig.xml`
4. Click "OK"
#### Options
When importing to IntelliJ, we offer a few options that can be used to
configure the behaviour of the import:
| Property | Description | Values (* = default) |
|--------------------------------------------|------------------------------------------------------------------------------------------------------|----------------------|
| `org.elasticsearch.idea-configuration-cache` | Should IntelliJ enable the Gradle Configuration cache to speed up builds when generating run configs | *`true`, `false` |
| `org.elasticsearch.idea-delegate-to-gradle` | Should IntelliJ use Gradle for all generated run / test configs or prompt each time | `true`, *`false` |
These options can be set anywhere on the Gradle config path including in `~/.gradle/gradle.properties`
### REST endpoint conventions
Elasticsearch typically uses singular nouns rather than plurals in URLs.

1
benchmarks/build.gradle
View file

@ -41,6 +41,7 @@ dependencies {
}
api(project(':libs:h3'))
api(project(':modules:aggregations'))
implementation project(':modules:mapper-extras');
api(project(':x-pack:plugin:esql-core'))
api(project(':x-pack:plugin:core'))
api(project(':x-pack:plugin:esql'))

15
benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java
View file

@ -40,6 +40,7 @@ import org.elasticsearch.compute.data.LongBlock;
import org.elasticsearch.compute.data.LongVector;
import org.elasticsearch.compute.data.Page;
import org.elasticsearch.compute.lucene.LuceneSourceOperator;
import org.elasticsearch.compute.lucene.ShardRefCounted;
import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator;
import org.elasticsearch.compute.operator.topn.TopNOperator;
import org.elasticsearch.core.IOUtils;
@ -477,6 +478,7 @@ public class ValuesSourceReaderBenchmark {
pages.add(
new Page(
new DocVector(
ShardRefCounted.ALWAYS_REFERENCED,
blockFactory.newConstantIntBlockWith(0, end - begin).asVector(),
blockFactory.newConstantIntBlockWith(ctx.ord, end - begin).asVector(),
docs.build(),
@ -512,7 +514,14 @@ public class ValuesSourceReaderBenchmark {
if (size >= BLOCK_LENGTH) {
pages.add(
new Page(
new DocVector(blockFactory.newConstantIntVector(0, size), leafs.build(), docs.build(), null).asBlock()
new DocVector(
ShardRefCounted.ALWAYS_REFERENCED,
blockFactory.newConstantIntVector(0, size),
leafs.build(),
docs.build(),
null
).asBlock()
)
);
docs = blockFactory.newIntVectorBuilder(BLOCK_LENGTH);
@ -525,6 +534,8 @@ public class ValuesSourceReaderBenchmark {
pages.add(
new Page(
new DocVector(
ShardRefCounted.ALWAYS_REFERENCED,
blockFactory.newConstantIntBlockWith(0, size).asVector(),
leafs.build().asBlock().asVector(),
docs.build(),
@ -551,6 +562,8 @@ public class ValuesSourceReaderBenchmark {
pages.add(
new Page(
new DocVector(
ShardRefCounted.ALWAYS_REFERENCED,
blockFactory.newConstantIntVector(0, 1),
blockFactory.newConstantIntVector(next.ord, 1),
blockFactory.newConstantIntVector(next.itr.nextInt(), 1),

8
benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java
View file

@ -29,6 +29,7 @@ import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
import org.elasticsearch.index.similarity.SimilarityService;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.plugins.MapperPlugin;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptCompiler;
import org.elasticsearch.script.ScriptContext;
@ -38,11 +39,16 @@ import org.elasticsearch.xcontent.XContentParserConfiguration;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
public class MapperServiceFactory {
public static MapperService create(String mappings) {
return create(mappings, Collections.emptyList());
}
public static MapperService create(String mappings, List<MapperPlugin> mapperPlugins) {
Settings settings = Settings.builder()
.put("index.number_of_replicas", 0)
.put("index.number_of_shards", 1)
@ -51,7 +57,7 @@ public class MapperServiceFactory {
.build();
IndexMetadata meta = IndexMetadata.builder("index").settings(settings).build();
IndexSettings indexSettings = new IndexSettings(meta, settings);
MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
MapperRegistry mapperRegistry = new IndicesModule(mapperPlugins).getMapperRegistry();
SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);

123
benchmarks/src/main/java/org/elasticsearch/benchmark/vector/Int4ScorerBenchmark.java Normal file
View file

@ -0,0 +1,123 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.benchmark.vector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.util.VectorUtil;
import org.elasticsearch.common.logging.LogConfigurator;
import org.elasticsearch.core.IOUtils;
import org.elasticsearch.simdvec.ES91Int4VectorsScorer;
import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import java.io.IOException;
import java.nio.file.Files;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@State(Scope.Benchmark)
// first iteration is complete garbage, so make sure we really warmup
@Warmup(iterations = 4, time = 1)
// real iterations. not useful to spend tons of time here, better to fork more
@Measurement(iterations = 5, time = 1)
// engage some noise reduction
@Fork(value = 1)
public class Int4ScorerBenchmark {
static {
LogConfigurator.configureESLogging(); // native access requires logging to be initialized
}
@Param({ "384", "702", "1024" })
int dims;
int numVectors = 200;
int numQueries = 10;
byte[] scratch;
byte[][] binaryVectors;
byte[][] binaryQueries;
ES91Int4VectorsScorer scorer;
Directory dir;
IndexInput in;
@Setup
public void setup() throws IOException {
binaryVectors = new byte[numVectors][dims];
dir = new MMapDirectory(Files.createTempDirectory("vectorData"));
try (IndexOutput out = dir.createOutput("vectors", IOContext.DEFAULT)) {
for (byte[] binaryVector : binaryVectors) {
for (int i = 0; i < dims; i++) {
// 4-bit quantization
binaryVector[i] = (byte) ThreadLocalRandom.current().nextInt(16);
}
out.writeBytes(binaryVector, 0, binaryVector.length);
}
}
in = dir.openInput("vectors", IOContext.DEFAULT);
binaryQueries = new byte[numVectors][dims];
for (byte[] binaryVector : binaryVectors) {
for (int i = 0; i < dims; i++) {
// 4-bit quantization
binaryVector[i] = (byte) ThreadLocalRandom.current().nextInt(16);
}
}
scratch = new byte[dims];
scorer = ESVectorizationProvider.getInstance().newES91Int4VectorsScorer(in, dims);
}
@TearDown
public void teardown() throws IOException {
IOUtils.close(dir, in);
}
@Benchmark
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
public void scoreFromArray(Blackhole bh) throws IOException {
for (int j = 0; j < numQueries; j++) {
in.seek(0);
for (int i = 0; i < numVectors; i++) {
in.readBytes(scratch, 0, dims);
bh.consume(VectorUtil.int4DotProduct(binaryQueries[j], scratch));
}
}
}
@Benchmark
@Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
for (int j = 0; j < numQueries; j++) {
in.seek(0);
for (int i = 0; i < numVectors; i++) {
bh.consume(scorer.int4DotProduct(binaryQueries[j]));
}
}
}
}

26
benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java
View file

@ -126,7 +126,10 @@ public class OSQScorerBenchmark {
in.readFloats(corrections, 0, corrections.length);
int addition = Short.toUnsignedInt(in.readShort());
float score = scorer.score(
result,
result.lowerInterval(),
result.upperInterval(),
result.quantizedComponentSum(),
result.additionalCorrection(),
VectorSimilarityFunction.EUCLIDEAN,
centroidDp,
corrections[0],
@ -150,7 +153,10 @@ public class OSQScorerBenchmark {
in.readFloats(corrections, 0, corrections.length);
int addition = Short.toUnsignedInt(in.readShort());
float score = scorer.score(
result,
result.lowerInterval(),
result.upperInterval(),
result.quantizedComponentSum(),
result.additionalCorrection(),
VectorSimilarityFunction.EUCLIDEAN,
centroidDp,
corrections[0],
@ -175,7 +181,10 @@ public class OSQScorerBenchmark {
in.readFloats(corrections, 0, corrections.length);
int addition = Short.toUnsignedInt(in.readShort());
float score = scorer.score(
result,
result.lowerInterval(),
result.upperInterval(),
result.quantizedComponentSum(),
result.additionalCorrection(),
VectorSimilarityFunction.EUCLIDEAN,
centroidDp,
corrections[0],
@ -196,7 +205,16 @@ public class OSQScorerBenchmark {
for (int j = 0; j < numQueries; j++) {
in.seek(0);
for (int i = 0; i < numVectors; i += 16) {
scorer.scoreBulk(binaryQueries[j], result, VectorSimilarityFunction.EUCLIDEAN, centroidDp, scratchScores);
scorer.scoreBulk(
binaryQueries[j],
result.lowerInterval(),
result.upperInterval(),
result.quantizedComponentSum(),
result.additionalCorrection(),
VectorSimilarityFunction.EUCLIDEAN,
centroidDp,
scratchScores
);
bh.consume(scratchScores);
}
}

10
benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/OptimizedTextBenchmark.java
View file

@ -15,6 +15,7 @@ import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.logging.LogConfigurator;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentFactory;
import org.elasticsearch.xcontent.XContentType;
@ -34,6 +35,7 @@ import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import java.io.IOException;
import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
@ -66,7 +68,7 @@ public class OptimizedTextBenchmark {
private SourceToParse[] sources;
private String randomValue(int length) {
final String CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
final String CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 ";
Random random = new Random();
StringBuilder builder = new StringBuilder(length);
for (int i = 0; i < length; i++) {
@ -83,17 +85,17 @@ public class OptimizedTextBenchmark {
"dynamic": false,
"properties": {
"field": {
"type": "keyword"
"type": "match_only_text"
}
}
}
}
""");
""", List.of(new MapperExtrasPlugin()));
sources = new SourceToParse[nDocs];
for (int i = 0; i < nDocs; i++) {
XContentBuilder b = XContentFactory.jsonBuilder();
b.startObject().field("field", randomValue(8)).endObject();
b.startObject().field("field", randomValue(512)).endObject();
sources[i] = new SourceToParse(UUIDs.randomBase64UUID(), BytesReference.bytes(b), XContentType.JSON);
}
}

3
branches.json
View file

@ -4,6 +4,9 @@
{
"branch": "main"
},
{
"branch": "9.1"
},
{
"branch": "9.0"
},

3
build-tools-internal/src/main/groovy/elasticsearch.ide.gradle
View file

@ -180,6 +180,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
// this path is produced by the extractLibs task above
String testLibraryPath = TestUtil.getTestLibraryPath("${elasticsearchProject.left()}/libs/native/libraries/build/platform")
def enableIdeaCC = providers.gradleProperty("org.elasticsearch.idea-configuration-cache").getOrElse("true").toBoolean()
def delegateToGradle = providers.gradleProperty("org.elasticsearch.idea-delegate-to-gradle").getOrElse("false").toBoolean()
idea {
project {
vcs = 'Git'
@ -188,7 +189,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
settings {
delegateActions {
delegateBuildRunToGradle = false
testRunner = 'choose_per_test'
testRunner = delegateToGradle ? 'gradle' : 'choose_per_test'
}
taskTriggers {
afterSync tasks.named('configureIdeCheckstyle'),

17
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java
View file

@ -10,7 +10,6 @@
package org.elasticsearch.gradle.internal;
import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin;
import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.artifacts.Configuration;
@ -28,14 +27,14 @@ public class DependenciesInfoPlugin implements Plugin<Project> {
var depsInfo = project.getTasks().register("dependenciesInfo", DependenciesInfoTask.class);
depsInfo.configure(t -> {
t.setRuntimeConfiguration(project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME));
t.setCompileOnlyConfiguration(
project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)
);
t.getConventionMapping().map("mappings", () -> {
var depLic = project.getTasks().named("dependencyLicenses", DependencyLicensesTask.class);
return depLic.get().getMappings();
});
var runtimeConfiguration = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
t.getRuntimeArtifacts().set(project.getProviders().provider(() -> runtimeConfiguration.getIncoming().getArtifacts()));
t.getClasspath().from(runtimeConfiguration);
var compileOnlyConfiguration = project.getConfigurations()
.getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
t.getCompileOnlyArtifacts().set(project.getProviders().provider(() -> compileOnlyConfiguration.getIncoming().getArtifacts()));
t.getClasspath().from(compileOnlyConfiguration);
});
Configuration dependenciesInfoFilesConfiguration = project.getConfigurations().create("dependenciesInfoFiles");
dependenciesInfoFilesConfiguration.setCanBeResolved(false);

123
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java
View file

@ -11,19 +11,22 @@ package org.elasticsearch.gradle.internal;
import org.elasticsearch.gradle.internal.precommit.DependencyLicensesTask;
import org.elasticsearch.gradle.internal.precommit.LicenseAnalyzer;
import org.gradle.api.artifacts.Configuration;
import org.gradle.api.artifacts.Dependency;
import org.gradle.api.artifacts.DependencySet;
import org.gradle.api.artifacts.ModuleVersionIdentifier;
import org.gradle.api.artifacts.ArtifactCollection;
import org.gradle.api.artifacts.ProjectDependency;
import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
import org.gradle.api.file.ConfigurableFileCollection;
import org.gradle.api.file.DirectoryProperty;
import org.gradle.api.file.ProjectLayout;
import org.gradle.api.internal.ConventionTask;
import org.gradle.api.model.ObjectFactory;
import org.gradle.api.provider.MapProperty;
import org.gradle.api.provider.Property;
import org.gradle.api.provider.Provider;
import org.gradle.api.provider.ProviderFactory;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.InputDirectory;
import org.gradle.api.tasks.InputFiles;
import org.gradle.api.tasks.Internal;
import org.gradle.api.tasks.Optional;
import org.gradle.api.tasks.OutputFile;
import org.gradle.api.tasks.TaskAction;
@ -34,6 +37,7 @@ import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@ -51,7 +55,50 @@ import javax.inject.Inject;
* <li>license: <a href="https://spdx.org/licenses/">SPDX license</a> identifier, custom license or UNKNOWN.</li>
* </ul>
*/
public class DependenciesInfoTask extends ConventionTask {
public abstract class DependenciesInfoTask extends ConventionTask {
@Inject
public abstract ProviderFactory getProviderFactory();
/**
* We have to use ArtifactCollection instead of ResolvedArtifactResult here as we're running
* into a an issue in Gradle: https://github.com/gradle/gradle/issues/27582
*/
@Internal
abstract Property<ArtifactCollection> getRuntimeArtifacts();
@Input
public Provider<Set<ModuleComponentIdentifier>> getRuntimeModules() {
return mapToModuleComponentIdentifiers(getRuntimeArtifacts().get());
}
@Internal
abstract Property<ArtifactCollection> getCompileOnlyArtifacts();
@Input
public Provider<Set<ModuleComponentIdentifier>> getCompileOnlyModules() {
return mapToModuleComponentIdentifiers(getCompileOnlyArtifacts().get());
}
/**
* We need to track file inputs here from the configurations we inspect to ensure we dont miss any
* artifact transforms that might be applied and fail due to missing task dependency to jar
* generating tasks.
* */
@InputFiles
abstract ConfigurableFileCollection getClasspath();
private Provider<Set<ModuleComponentIdentifier>> mapToModuleComponentIdentifiers(ArtifactCollection artifacts) {
return getProviderFactory().provider(
() -> artifacts.getArtifacts()
.stream()
.map(r -> r.getId())
.filter(id -> id instanceof ModuleComponentIdentifier)
.map(id -> (ModuleComponentIdentifier) id)
.collect(Collectors.toSet())
);
}
private final DirectoryProperty licensesDir;
@ -60,22 +107,6 @@ public class DependenciesInfoTask extends ConventionTask {
private LinkedHashMap<String, String> mappings;
public Configuration getRuntimeConfiguration() {
return runtimeConfiguration;
}
public void setRuntimeConfiguration(Configuration runtimeConfiguration) {
this.runtimeConfiguration = runtimeConfiguration;
}
public Configuration getCompileOnlyConfiguration() {
return compileOnlyConfiguration;
}
public void setCompileOnlyConfiguration(Configuration compileOnlyConfiguration) {
this.compileOnlyConfiguration = compileOnlyConfiguration;
}
/**
* Directory to read license files
*/
@ -102,17 +133,6 @@ public class DependenciesInfoTask extends ConventionTask {
this.outputFile = outputFile;
}
/**
* Dependencies to gather information from.
*/
@InputFiles
private Configuration runtimeConfiguration;
/**
* We subtract compile-only dependencies.
*/
@InputFiles
private Configuration compileOnlyConfiguration;
@Inject
public DependenciesInfoTask(ProjectLayout projectLayout, ObjectFactory objectFactory, ProviderFactory providerFactory) {
this.licensesDir = objectFactory.directoryProperty();
@ -123,22 +143,18 @@ public class DependenciesInfoTask extends ConventionTask {
@TaskAction
public void generateDependenciesInfo() throws IOException {
final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies();
// we have to resolve the transitive dependencies and create a group:artifactId:version map
final Set<String> compileOnlyArtifacts = compileOnlyConfiguration.getResolvedConfiguration()
.getResolvedArtifacts()
.stream()
.map(r -> {
ModuleVersionIdentifier id = r.getModuleVersion().getId();
return id.getGroup() + ":" + id.getName() + ":" + id.getVersion();
})
.collect(Collectors.toSet());
final Set<String> compileOnlyIds = getCompileOnlyModules().map(
set -> set.stream()
.map(id -> id.getModuleIdentifier().getGroup() + ":" + id.getModuleIdentifier().getName() + ":" + id.getVersion())
.collect(Collectors.toSet())
).get();
final StringBuilder output = new StringBuilder();
for (final Dependency dep : runtimeDependencies) {
Map<String, String> mappings = getMappings().get();
for (final ModuleComponentIdentifier dep : getRuntimeModules().get()) {
// we do not need compile-only dependencies here
if (compileOnlyArtifacts.contains(dep.getGroup() + ":" + dep.getName() + ":" + dep.getVersion())) {
String moduleName = dep.getModuleIdentifier().getName();
if (compileOnlyIds.contains(dep.getGroup() + ":" + moduleName + ":" + dep.getVersion())) {
continue;
}
@ -147,25 +163,20 @@ public class DependenciesInfoTask extends ConventionTask {
continue;
}
final String url = createURL(dep.getGroup(), dep.getName(), dep.getVersion());
final String dependencyName = DependencyLicensesTask.getDependencyName(getMappings(), dep.getName());
getLogger().info("mapped dependency " + dep.getGroup() + ":" + dep.getName() + " to " + dependencyName + " for license info");
final String url = createURL(dep.getGroup(), moduleName, dep.getVersion());
final String dependencyName = DependencyLicensesTask.getDependencyName(mappings, moduleName);
getLogger().info("mapped dependency " + dep.getGroup() + ":" + moduleName + " to " + dependencyName + " for license info");
final String licenseType = getLicenseType(dep.getGroup(), dependencyName);
output.append(dep.getGroup() + ":" + dep.getName() + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
output.append(dep.getGroup() + ":" + moduleName + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
}
Files.writeString(outputFile.toPath(), output.toString(), StandardOpenOption.CREATE);
}
@Input
public LinkedHashMap<String, String> getMappings() {
return mappings;
}
public void setMappings(LinkedHashMap<String, String> mappings) {
this.mappings = mappings;
}
@Optional
public abstract MapProperty<String, String> getMappings();
/**
* Create an URL on <a href="https://repo1.maven.org/maven2/">Maven Central</a>

1
build-tools-internal/src/main/resources/checkstyle_suppressions.xml
View file

@ -37,6 +37,7 @@
<suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
<suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIndexingRollingUpgradeIT.java" checks="LineLength" />
<suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]LogsdbIndexingRollingUpgradeIT.java" checks="LineLength" />
<suppress files="plugin[/\\]logsdb[/\\]qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]MatchOnlyTextRollingUpgradeIT.java" checks="LineLength" />
<!-- Gradle requires inputs to be seriablizable -->
<suppress files="build-tools-internal[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gradle[/\\]internal[/\\]precommit[/\\]TestingConventionRule.java" checks="RegexpSinglelineJava" />

4
build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt
View file

@ -61,3 +61,7 @@ org.apache.logging.log4j.message.ParameterizedMessage#<init>(java.lang.String, j
@defaultMessage Use WriteLoadForecaster#getForecastedWriteLoad instead
org.elasticsearch.cluster.metadata.IndexMetadata#getForecastedWriteLoad()
@defaultMessage Use org.elasticsearch.index.codec.vectors.OptimizedScalarQuantizer instead
org.apache.lucene.util.quantization.OptimizedScalarQuantizer#<init>(org.apache.lucene.index.VectorSimilarityFunction, float, int)
org.apache.lucene.util.quantization.OptimizedScalarQuantizer#<init>(org.apache.lucene.index.VectorSimilarityFunction)

2
build-tools-internal/src/main/resources/forbidden/es-server-signatures.txt
View file

@ -160,7 +160,7 @@ org.elasticsearch.cluster.ClusterFeatures#clusterHasFeature(org.elasticsearch.cl
@defaultMessage Do not construct this records outside the source files they are declared in
org.elasticsearch.cluster.SnapshotsInProgress$ShardSnapshotStatus#<init>(java.lang.String, org.elasticsearch.cluster.SnapshotsInProgress$ShardState, org.elasticsearch.repositories.ShardGeneration, java.lang.String, org.elasticsearch.repositories.ShardSnapshotResult)
org.elasticsearch.cluster.SnapshotDeletionsInProgress$Entry#<init>(java.lang.String, java.util.List, long, long, org.elasticsearch.cluster.SnapshotDeletionsInProgress$State, java.lang.String)
org.elasticsearch.cluster.SnapshotDeletionsInProgress$Entry#<init>(org.elasticsearch.cluster.metadata.ProjectId, java.lang.String, java.util.List, long, long, org.elasticsearch.cluster.SnapshotDeletionsInProgress$State, java.lang.String)
@defaultMessage Use a Thread constructor with a name, anonymous threads are more difficult to debug
java.lang.Thread#<init>(java.lang.Runnable)

2
build-tools-internal/version.properties
View file

@ -1,4 +1,4 @@
elasticsearch = 9.1.0
elasticsearch = 9.2.0
lucene = 10.3.0-snapshot-360eeb68ee0
bundled_jdk_vendor = openjdk

61
build.gradle
View file

@ -351,44 +351,45 @@ allprojects {
}
}
def splitForCI = { proj, partString ->
proj.tasks.register("check$partString") {
dependsOn 'check'
withReleaseBuild {
dependsOn 'assemble'
}
}
proj.tasks.addRule("Pattern: v<BWC_VERSION>#bwcTest$partString") { name ->
if(name.endsWith("#bwcTest$partString")) {
proj.project.getTasks().register(name) {
task -> task.dependsOn(proj.tasks.named { tskName -> tskName == (name - partString) })
}
}
}
proj.tasks.register("bcUpgradeTest$partString") {
dependsOn tasks.matching { it.name == 'bcUpgradeTest' }
withReleaseBuild {
dependsOn 'assemble'
}
}
}
plugins.withId('lifecycle-base') {
if (project.path.startsWith(":x-pack:")) {
if (project.path.contains("security") || project.path.contains(":ml")) {
tasks.register('checkPart4') {
dependsOn 'check'
withReleaseBuild {
dependsOn 'assemble'
}
}
splitForCI(project, "Part4")
} else if (project.path == ":x-pack:plugin" || project.path.contains("ql") || project.path.contains("smoke-test")) {
tasks.register('checkPart3') {
dependsOn 'check'
withReleaseBuild {
dependsOn 'assemble'
}
}
splitForCI(project, "Part3")
} else if (project.path.contains("multi-node")) {
tasks.register('checkPart5') {
dependsOn 'check'
withReleaseBuild {
dependsOn 'assemble'
}
}
splitForCI(project, "Part5")
} else {
tasks.register('checkPart2') {
dependsOn 'check'
withReleaseBuild {
dependsOn 'assemble'
}
}
splitForCI(project, "Part2")
}
} else if(project.path.startsWith(":qa:")) {
splitForCI(project, "Part6")
} else {
tasks.register('checkPart1') {
dependsOn 'check'
withReleaseBuild {
dependsOn 'assemble'
}
}
splitForCI(project, "Part1")
}
tasks.register('functionalTests') {
dependsOn 'check'

18
distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/MachineDependentHeapTests.java
View file

@ -10,6 +10,7 @@
package org.elasticsearch.server.cli;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.FeatureFlag;
import org.elasticsearch.test.ESTestCase;
import org.hamcrest.Matcher;
@ -55,7 +56,8 @@ public class MachineDependentHeapTests extends ESTestCase {
assertHeapOptions(64, containsInAnyOrder("-Xmx31744m", "-Xms31744m"), "master");
}
public void testMlOnlyOptions() throws Exception {
public void testMlOnlyOptions_new() throws Exception {
assumeTrue("feature flag must be enabled for new memory computation", new FeatureFlag("new_ml_memory_computation").isEnabled());
assertHeapOptions(1, containsInAnyOrder("-Xmx272m", "-Xms272m"), "ml");
assertHeapOptions(4, containsInAnyOrder("-Xmx1092m", "-Xms1092m"), "ml");
assertHeapOptions(32, containsInAnyOrder("-Xmx5460m", "-Xms5460m"), "ml");
@ -65,6 +67,20 @@ public class MachineDependentHeapTests extends ESTestCase {
assertHeapOptions(263, containsInAnyOrder("-Xmx21228m", "-Xms21228m"), "ml");
}
public void testMlOnlyOptions_old() throws Exception {
assumeTrue(
"feature flag must be disabled for old memory computation",
new FeatureFlag("new_ml_memory_computation").isEnabled() == false
);
assertHeapOptions(1, containsInAnyOrder("-Xmx408m", "-Xms408m"), "ml");
assertHeapOptions(4, containsInAnyOrder("-Xmx1636m", "-Xms1636m"), "ml");
assertHeapOptions(32, containsInAnyOrder("-Xmx8192m", "-Xms8192m"), "ml");
assertHeapOptions(64, containsInAnyOrder("-Xmx11468m", "-Xms11468m"), "ml");
// We'd never see a node this big in Cloud, but this assertion proves that the 31GB absolute maximum
// eventually kicks in (because 0.4 * 16 + 0.1 * (263 - 16) > 31)
assertHeapOptions(263, containsInAnyOrder("-Xmx31744m", "-Xms31744m"), "ml");
}
public void testDataNodeOptions() throws Exception {
assertHeapOptions(1, containsInAnyOrder("-Xmx512m", "-Xms512m"), "data");
assertHeapOptions(8, containsInAnyOrder("-Xmx4096m", "-Xms4096m"), "data");

5
docs/changelog/119967.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 119967
summary: Add `index_options` to `semantic_text` field mappings
area: Mapping
type: enhancement
issues: [ ]

5
docs/changelog/120869.yaml
View file

@ -1,5 +0,0 @@
pr: 120869
summary: Threadpool merge scheduler
area: Engine
type: feature
issues: []

5
docs/changelog/122497.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 122497
summary: Check if index patterns conform to valid format before validation
area: CCS
type: enhancement
issues: []

6
docs/changelog/125143.yaml
View file

@ -1,6 +0,0 @@
pr: 125143
summary: "ES|QL Support for ST_GEOHASH, ST_GEOTILE and ST_GEOHEX"
area: "ES|QL"
type: enhancement
issues:
- 123903

6
docs/changelog/125408.yaml
View file

@ -1,6 +0,0 @@
pr: 125408
summary: Prevent ML data retention logic from failing when deleting documents in read-only
indices
area: Machine Learning
type: bug
issues: []

18
docs/changelog/125921.yaml Normal file
View file

@ -0,0 +1,18 @@
pr: 125921
summary: Allow direct IO for BBQ rescoring
area: Vector Search
type: feature
highlight:
title: Allow direct IO for BBQ rescoring
body: |-
BBQ rescoring performance can be drastically affected by the amount of available
off-heap RAM for use by the system page cache. When there is not enough off-heap RAM
to fit all the vector data in memory, BBQ search latencies can be affected by as much as 5000x.
Specifying the `vector.rescoring.directio=true` Java option on all vector search
nodes modifies rescoring to use direct IO, which eliminates these very high latencies
from searches in low-memory scenarios, at a cost of a reduction
in vector search performance for BBQ indices when the vectors do all fit in memory.
This option is released in 9.1 as a tech preview whilst we analyse its effect
for a variety of use cases.
issues: []

5
docs/changelog/126866.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 126866
summary: Add recursive chunker
area: Machine Learning
type: enhancement
issues: []

6
docs/changelog/127532.yaml
View file

@ -1,6 +0,0 @@
pr: 127532
summary: Fix case insensitive comparisons to ""
area: ES|QL
type: bug
issues:
- 127431

5
docs/changelog/127613.yaml
View file

@ -1,5 +0,0 @@
pr: 127613
summary: Threadpool merge executor is aware of available disk space
area: Engine
type: feature
issues: []

5
docs/changelog/127621.yaml
View file

@ -1,5 +0,0 @@
pr: 127621
summary: Fix error message when changing the password for a user in the file realm
area: Security
type: bug
issues: []

17
docs/changelog/127636.yaml Normal file
View file

@ -0,0 +1,17 @@
pr: 127636
summary: Disallow mixed quoted/unquoted patterns in FROM
area: ES|QL
type: breaking
issues:
- 122651
breaking:
title: Disallow mixed quoted/unquoted patterns in FROM
area: ES|QL
details: "Previously, the ES|QL grammar allowed users to individually quote constituent strings in index patterns\
\ such as \"remote_cluster\":\"index_name\". This would allow users to write complex malformed index patterns\
\ that often slip through grammar and the subsequent validation. This could result in runtime errors\
\ that can be misleading. This change simplifies the grammar to early reject such malformed index patterns\
\ at the parsing stage, allowing users to write simpler queries and see more relevant and meaningful\
\ errors."
impact: "Users can write queries with simpler index patterns and see more meaningful and relevant errors."
notable: false

5
docs/changelog/127651.yaml
View file

@ -1,5 +0,0 @@
pr: 127651
summary: Add complete attribute to .fleet-agents docs
area: Infra/Plugins
type: bug
issues: []

6
docs/changelog/127962.yaml
View file

@ -1,6 +0,0 @@
pr: 127962
summary: Support DATE_NANOS in LOOKUP JOIN
area: ES|QL
type: bug
issues:
- 127249

6
docs/changelog/128036.yaml
View file

@ -1,6 +0,0 @@
pr: 128036
summary: Fix inner hits + aggregations concurrency bug
area: Search
type: bug
issues:
- 122419

5
docs/changelog/128291.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 128291
summary: Make `dense_vector` fields updatable to bbq_flat/bbq_hnsw
area: Vector Search
type: enhancement
issues: []

6
docs/changelog/128361.yaml Normal file
View file

@ -0,0 +1,6 @@
pr: 128361
summary: The follower index should wait until the time series end time passes before unfollowing the leader index.
area: ILM+SLM
type: bug
issues:
- 128129

5
docs/changelog/128362.yaml
View file

@ -1,5 +0,0 @@
pr: 128362
summary: Avoid unnecessary determinization in index pattern conflict checks
area: Indices APIs
type: bug
issues: []

7
docs/changelog/128472.yaml
View file

@ -1,7 +0,0 @@
pr: 128472
summary: "Aggs: Fix significant terms not finding background docuemnts for nested\
\ fields"
area: Aggregations
type: bug
issues:
- 101163

5
docs/changelog/128532.yaml
View file

@ -1,5 +0,0 @@
pr: 128532
summary: "Prevent invalid privileges in manage roles privilege"
area: "Authorization"
type: bug
issues: [127496]

6
docs/changelog/128610.yaml
View file

@ -1,6 +0,0 @@
pr: 128610
summary: "Handle the indices pattern `[\"*\", \"-*\"]` when grouping indices by cluster\
\ name"
area: CCS
type: bug
issues: []

6
docs/changelog/128635.yaml Normal file
View file

@ -0,0 +1,6 @@
pr: 128635
summary: Add `state` query param to Get snapshots API
area: Snapshot/Restore
type: enhancement
issues:
- 97446

Some files were not shown because too many files have changed in this diff Show more