github-mirrors/elasticsearch
1
0
Fork 0
mirror of https://github.com/elastic/elasticsearch.git synced 2025-04-24 23:27:25 -04:00
Code Issues Projects Releases Packages Wiki Activity

Merge main into multi-project

Browse source
This commit is contained in:
Tim Vernum 2024-10-25 13:36:22 +11:00
commit fd55e00fa7
212 changed files with 4120 additions and 2405 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.github/CODEOWNERS vendored
View file

@ -39,7 +39,6 @@ gradle @elastic/es-delivery
build-conventions @elastic/es-delivery
build-tools @elastic/es-delivery
build-tools-internal @elastic/es-delivery
*.gradle @elastic/es-delivery
.buildkite @elastic/es-delivery
.ci @elastic/es-delivery
.idea @elastic/es-delivery

2
README.asciidoc
View file

@ -56,8 +56,8 @@ Quickly set up Elasticsearch and Kibana in Docker for local development or testi
- If you're using Microsoft Windows, then install https://learn.microsoft.com/en-us/windows/wsl/install[Windows Subsystem for Linux (WSL)].
==== Trial license
This setup comes with a one-month trial license that includes all Elastic features.
This setup comes with a one-month trial of the Elastic *Platinum* license.
After the trial period, the license reverts to *Free and open - Basic*.
Refer to https://www.elastic.co/subscriptions[Elastic subscriptions] for more information.

3
build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java
View file

@ -21,9 +21,6 @@ public enum DockerBase {
// The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}", "-ironbank", "yum"),
// Base image with extras for Cloud
CLOUD("ubuntu:20.04", "-cloud", "apt-get"),
// Chainguard based wolfi image with latest jdk
// This is usually updated via renovatebot
// spotless:off

25
distribution/docker/build.gradle
View file

@ -288,20 +288,6 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
}
}
if (base == DockerBase.CLOUD) {
// If we're performing a release build, but `build.id` hasn't been set, we can
// infer that we're not at the Docker building stage of the build, and therefore
// we should skip the beats part of the build.
String buildId = providers.systemProperty('build.id').getOrNull()
boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
if (includeBeats) {
from configurations.getByName("filebeat_${architecture.classifier}")
from configurations.getByName("metricbeat_${architecture.classifier}")
}
// For some reason, the artifact name can differ depending on what repository we used.
rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
}
Provider<DockerSupportService> serviceProvider = GradleUtils.getBuildService(
project.gradle.sharedServices,
DockerSupportPlugin.DOCKER_SUPPORT_SERVICE_NAME
@ -381,7 +367,7 @@ private static List<String> generateTags(DockerBase base, Architecture architect
String image = "elasticsearch${base.suffix}"
String namespace = 'elasticsearch'
if (base == DockerBase.CLOUD || base == DockerBase.CLOUD_ESS) {
if (base == base == DockerBase.CLOUD_ESS) {
namespace += '-ci'
}
@ -439,7 +425,7 @@ void addBuildDockerImageTask(Architecture architecture, DockerBase base) {
}
if (base != DockerBase.IRON_BANK && base != DockerBase.CLOUD && base != DockerBase.CLOUD_ESS) {
if (base != DockerBase.IRON_BANK && base != DockerBase.CLOUD_ESS) {
tasks.named("assemble").configure {
dependsOn(buildDockerImageTask)
}
@ -548,10 +534,6 @@ subprojects { Project subProject ->
base = DockerBase.IRON_BANK
} else if (subProject.name.contains('cloud-ess-')) {
base = DockerBase.CLOUD_ESS
} else if (subProject.name.contains('cloud-')) {
base = DockerBase.CLOUD
} else if (subProject.name.contains('wolfi-ess')) {
base = DockerBase.WOLFI_ESS
} else if (subProject.name.contains('wolfi-')) {
base = DockerBase.WOLFI
}
@ -559,10 +541,9 @@ subprojects { Project subProject ->
final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
final String extension = base == DockerBase.UBI ? 'ubi.tar' :
(base == DockerBase.IRON_BANK ? 'ironbank.tar' :
(base == DockerBase.CLOUD ? 'cloud.tar' :
(base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' :
(base == DockerBase.WOLFI ? 'wolfi.tar' :
'docker.tar'))))
'docker.tar')))
final String artifactName = "elasticsearch${arch}${base.suffix}_test"
final String exportTaskName = taskName("export", architecture, base, 'DockerImage')

2
distribution/docker/cloud-docker-aarch64-export/build.gradle
View file

@ -1,2 +0,0 @@
// This file is intentionally blank. All configuration of the
// export is done in the parent project.

2
distribution/docker/cloud-docker-export/build.gradle
View file

@ -1,2 +0,0 @@
// This file is intentionally blank. All configuration of the
// export is done in the parent project.

2
distribution/docker/wolfi-ess-docker-aarch64-export/build.gradle
View file

@ -1,2 +0,0 @@
// This file is intentionally blank. All configuration of the
// export is done in the parent project.

2
distribution/docker/wolfi-ess-docker-export/build.gradle
View file

@ -1,2 +0,0 @@
// This file is intentionally blank. All configuration of the
// export is done in the parent project.

19
docs/changelog/113975.yaml Normal file
View file

@ -0,0 +1,19 @@
pr: 113975
summary: JDK locale database change
area: Mapping
type: breaking
issues: []
breaking:
title: JDK locale database change
area: Mapping
details: |
{es} 8.16 changes the version of the JDK that is included from version 22 to version 23. This changes the locale database that is used by Elasticsearch from the COMPAT database to the CLDR database. This change can cause significant differences to the textual date formats accepted by Elasticsearch, and to calculated week-dates.
If you run {es} 8.16 on JDK version 22 or below, it will use the COMPAT locale database to match the behavior of 8.15. However, starting with {es} 9.0, {es} will use the CLDR database regardless of JDK version it is run on.
impact: |
This affects you if you use custom date formats using textual or week-date field specifiers. If you use date fields or calculated week-dates that change between the COMPAT and CLDR databases, then this change will cause Elasticsearch to reject previously valid date fields as invalid data. You might need to modify your ingest or output integration code to account for the differences between these two JDK versions.
Starting in version 8.15.2, Elasticsearch will log deprecation warnings if you are using date format specifiers that might change on upgrading to JDK 23. These warnings are visible in Kibana.
For detailed guidance, refer to <<custom-date-format-locales,Differences in locale information between JDK versions>> and the https://ela.st/jdk-23-locales[Elastic blog].
notable: true

5
docs/changelog/114566.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 114566
summary: Use Azure blob batch API to delete blobs in batches
area: Distributed
type: enhancement
issues: []

6
docs/changelog/114665.yaml Normal file
View file

@ -0,0 +1,6 @@
pr: 114665
summary: Fixing remote ENRICH by pushing the Enrich inside `FragmentExec`
area: ES|QL
type: bug
issues:
- 105095

6
docs/changelog/114990.yaml Normal file
View file

@ -0,0 +1,6 @@
pr: 114990
summary: Allow for querries on `_tier` to skip shards in the `can_match` phase
area: Search
type: bug
issues:
- 114910

5
docs/changelog/115061.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 115061
summary: "[ES|QL] Simplify syntax of named parameter for identifier and pattern"
area: ES|QL
type: bug
issues: []

6
docs/changelog/115117.yaml Normal file
View file

@ -0,0 +1,6 @@
pr: 115117
summary: Report JVM stats for all memory pools (97046)
area: Infra/Core
type: bug
issues:
- 97046

5
docs/changelog/115383.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 115383
summary: Only publish desired balance gauges on master
area: Allocation
type: enhancement
issues: []

18
docs/changelog/115393.yaml Normal file
View file

@ -0,0 +1,18 @@
pr: 115393
summary: Remove deprecated local attribute from alias APIs
area: Indices APIs
type: breaking
issues: []
breaking:
title: Remove deprecated local attribute from alias APIs
area: REST API
details: >-
The following APIs no longer accept the `?local` query parameter:
`GET /_alias`, `GET /_aliases`, `GET /_alias/{name}`,
`HEAD /_alias/{name}`, `GET /{index}/_alias`, `HEAD /{index}/_alias`,
`GET /{index}/_alias/{name}`, `HEAD /{index}/_alias/{name}`,
`GET /_cat/aliases`, and `GET /_cat/aliases/{alias}`. This parameter
has been deprecated and ignored since version 8.12.
impact: >-
Cease usage of the `?local` query parameter when calling the listed APIs.
notable: false

29
docs/changelog/115399.yaml Normal file
View file

@ -0,0 +1,29 @@
pr: 115399
summary: Adding breaking change entry for retrievers
area: Search
type: breaking
issues: []
breaking:
title: Reworking RRF retriever to be evaluated during rewrite phase
area: REST API
details: |-
In this release (8.16), we have introduced major changes to the retrievers framework
and how they can be evaluated, focusing mainly on compound retrievers
like `rrf` and `text_similarity_reranker`, which allowed us to support full
composability (i.e. any retriever can be nested under any compound retriever),
as well as supporting additional search features like collapsing, explaining,
aggregations, and highlighting.
To ensure consistency, and given that this rework is not available until 8.16,
`rrf` and `text_similarity_reranker` retriever queries would now
throw an exception in a mixed cluster scenario, where there are nodes
both in current or later (i.e. >= 8.16) and previous ( <= 8.15) versions.
As part of the rework, we have also removed the `_rank` property from
the responses of an `rrf` retriever.
impact: |-
- Users will not be able to use the `rrf` and `text_similarity_reranker` retrievers in a mixed cluster scenario
with previous releases (i.e. prior to 8.16), and the request will throw an `IllegalArgumentException`.
- `_rank` has now been removed from the output of the `rrf` retrievers so trying to directly parse the field
will throw an exception
notable: false

5
docs/changelog/115429.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 115429
summary: "[otel-data] Add more kubernetes aliases"
area: Data streams
type: bug
issues: []

5
docs/changelog/115430.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 115430
summary: Prevent NPE if model assignment is removed while waiting to start
area: Machine Learning
type: bug
issues: []

5
docs/changelog/115459.yaml Normal file
View file

@ -0,0 +1,5 @@
pr: 115459
summary: Guard blob store local directory creation with `doPrivileged`
area: Infra/Core
type: bug
issues: []

6
docs/changelog/115594.yaml Normal file
View file

@ -0,0 +1,6 @@
pr: 115594
summary: Update `BlobCacheBufferedIndexInput::readVLong` to correctly handle negative
long values
area: Search
type: bug
issues: []

10
docs/reference/cat/alias.asciidoc
View file

@ -6,8 +6,8 @@
[IMPORTANT]
====
cat APIs are only intended for human consumption using the command line or the
{kib} console. They are _not_ intended for use by applications. For application
cat APIs are only intended for human consumption using the command line or the
{kib} console. They are _not_ intended for use by applications. For application
consumption, use the <<aliases,aliases API>>.
====
@ -45,8 +45,6 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h]
include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help]
include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local]
include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-s]
include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v]
@ -104,6 +102,6 @@ alias4 test1 - 2 1,2 -
This response shows that `alias2` has configured a filter, and specific routing
configurations in `alias3` and `alias4`.
If you only want to get information about specific aliases, you can specify
the aliases in comma-delimited format as a URL parameter, e.g.,
If you only want to get information about specific aliases, you can specify
the aliases in comma-delimited format as a URL parameter, e.g.,
/_cat/aliases/alias1,alias2.

2
docs/reference/cat/trainedmodel.asciidoc
View file

@ -116,7 +116,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v]
[source,console]
--------------------------------------------------
GET _cat/ml/trained_models?h=c,o,l,ct,v&v=ture
GET _cat/ml/trained_models?h=c,o,l,ct,v&v=true
--------------------------------------------------
// TEST[skip:kibana sample data]

180
docs/reference/data-streams/logs.asciidoc
View file

@ -8,14 +8,6 @@ A logs data stream is a data stream type that stores log data more efficiently.
In benchmarks, log data stored in a logs data stream used ~2.5 times less disk space than a regular data
stream. The exact impact will vary depending on your data set.
The following features are enabled in a logs data stream:
* <<synthetic-source,Synthetic source>>, which omits storing the `_source` field. When the document source is requested, it is synthesized from document fields upon retrieval.
* Index sorting. This yields a lower storage footprint. By default indices are sorted by `host.name` and `@timestamp` fields at index time.
* More space efficient compression for fields with <<doc-values,`doc_values`>> enabled.
[discrete]
[[how-to-use-logsds]]
=== Create a logs data stream
@ -50,3 +42,175 @@ DELETE _index_template/my-index-template
----
// TEST[continued]
////
[[logsdb-default-settings]]
[discrete]
[[logsdb-synthtic-source]]
=== Synthetic source
By default, `logsdb` mode uses <<synthetic-source,synthetic source>>, which omits storing the original `_source`
field and synthesizes it from doc values or stored fields upon document retrieval. Synthetic source comes with a few
restrictions which you can read more about in the <<synthetic-source,documentation>> section dedicated to it.
NOTE: When dealing with multi-value fields, the `index.mapping.synthetic_source_keep` setting controls how field values
are preserved for <<synthetic-source,synthetic source>> reconstruction. In `logsdb`, the default value is `arrays`,
which retains both duplicate values and the order of entries but not necessarily the exact structure when it comes to
array elements or objects. Preserving duplicates and ordering could be critical for some log fields. This could be the
case, for instance, for DNS A records, HTTP headers, or log entries that represent sequential or repeated events.
For more details on this setting and ways to refine or bypass it, check out <<synthetic-source-keep, this section>>.
[discrete]
[[logsdb-sort-settings]]
=== Index sort settings
The following settings are applied by default when using the `logsdb` mode for index sorting:
* `index.sort.field`: `["host.name", "@timestamp"]`
In `logsdb` mode, indices are sorted by `host.name` and `@timestamp` fields by default. For data streams, the
`@timestamp` field is automatically injected if it is not present.
* `index.sort.order`: `["desc", "desc"]`
The default sort order for both fields is descending (`desc`), prioritizing the latest data.
* `index.sort.mode`: `["min", "min"]`
The default sort mode is `min`, ensuring that indices are sorted by the minimum value of multi-value fields.
* `index.sort.missing`: `["_first", "_first"]`
Missing values are sorted to appear first (`_first`) in `logsdb` index mode.
`logsdb` index mode allows users to override the default sort settings. For instance, users can specify their own fields
and order for sorting by modifying the `index.sort.field` and `index.sort.order`.
When using default sort settings, the `host.name` field is automatically injected into the mappings of the
index as a `keyword` field to ensure that sorting can be applied. This guarantees that logs are efficiently sorted and
retrieved based on the `host.name` and `@timestamp` fields.
NOTE: If `subobjects` is set to `true` (which is the default), the `host.name` field will be mapped as an object field
named `host`, containing a `name` child field of type `keyword`. On the other hand, if `subobjects` is set to `false`,
a single `host.name` field will be mapped as a `keyword` field.
Once an index is created, the sort settings are immutable and cannot be modified. To apply different sort settings,
a new index must be created with the desired configuration. For data streams, this can be achieved by means of an index
rollover after updating relevant (component) templates.
If the default sort settings are not suitable for your use case, consider modifying them. Keep in mind that sort
settings can influence indexing throughput, query latency, and may affect compression efficiency due to the way data
is organized after sorting. For more details, refer to our documentation on
<<index-modules-index-sorting,index sorting>>.
NOTE: For <<data-streams, data streams>>, the `@timestamp` field is automatically injected if not already present.
However, if custom sort settings are applied, the `@timestamp` field is injected into the mappings, but it is not
automatically added to the list of sort fields.
[discrete]
[[logsdb-specialized-codecs]]
=== Specialized codecs
`logsdb` index mode uses the `best_compression` <<index-codec,codec>> by default, which applies {wikipedia}/Zstd[ZSTD]
compression to stored fields. Users are allowed to override it and switch to the `default` codec for faster compression
at the expense of slightly larger storage footprint.
`logsdb` index mode also adopts specialized codecs for numeric doc values that are crafted to optimize storage usage.
Users can rely on these specialized codecs being applied by default when using `logsdb` index mode.
Doc values encoding for numeric fields in `logsdb` follows a static sequence of codecs, applying each one in the
following order: delta encoding, offset encoding, Greatest Common Divisor GCD encoding, and finally Frame Of Reference
(FOR) encoding. The decision to apply each encoding is based on heuristics determined by the data distribution.
For example, before applying delta encoding, the algorithm checks if the data is monotonically non-decreasing or
non-increasing. If the data fits this pattern, delta encoding is applied; otherwise, the next encoding is considered.
The encoding is specific to each Lucene segment and is also re-applied at segment merging time. The merged Lucene segment
may use a different encoding compared to the original Lucene segments, based on the characteristics of the merged data.
The following methods are applied sequentially:
* **Delta encoding**:
a compression method that stores the difference between consecutive values instead of the actual values.
* **Offset encoding**:
a compression method that stores the difference from a base value rather than between consecutive values.
* **Greatest Common Divisor (GCD) encoding**:
a compression method that finds the greatest common divisor of a set of values and stores the differences
as multiples of the GCD.
* **Frame Of Reference (FOR) encoding**:
a compression method that determines the smallest number of bits required to encode a block of values and uses
bit-packing to fit such values into larger 64-bit blocks.
For keyword fields, **Run Length Encoding (RLE)** is applied to the ordinals, which represent positions in the Lucene
segment-level keyword dictionary. This compression is used when multiple consecutive documents share the same keyword.
[discrete]
[[logsdb-ignored-settings]]
=== `ignore_malformed`, `ignore_above`, `ignore_dynamic_beyond_limit`
By default, `logsdb` index mode sets `ignore_malformed` to `true`. This setting allows documents with malformed fields
to be indexed without causing indexing failures, ensuring that log data ingestion continues smoothly even when some
fields contain invalid or improperly formatted data.
Users can override this setting by setting `index.mapping.ignore_malformed` to `false`. However, this is not recommended
as it might result in documents with malformed fields being rejected and not indexed at all.
In `logsdb` index mode, the `index.mapping.ignore_above` setting is applied by default at the index level to ensure
efficient storage and indexing of large keyword fields.The index-level default for `ignore_above` is set to 8191
**characters**. If using UTF-8 encoding, this results in a limit of 32764 bytes, depending on character encoding.
The mapping-level `ignore_above` setting still takes precedence. If a specific field has an `ignore_above` value
defined in its mapping, that value will override the index-level `index.mapping.ignore_above` value. This default
behavior helps to optimize indexing performance by preventing excessively large string values from being indexed, while
still allowing users to customize the limit, overriding it at the mapping level or changing the index level default
setting.
In `logsdb` index mode, the setting `index.mapping.total_fields.ignore_dynamic_beyond_limit` is set to `true` by
default. This allows dynamically mapped fields to be added on top of statically defined fields without causing document
rejection, even after the total number of fields exceeds the limit defined by `index.mapping.total_fields.limit`. The
`index.mapping.total_fields.limit` setting specifies the maximum number of fields an index can have (static, dynamic
and runtime). When the limit is reached, new dynamically mapped fields will be ignored instead of failing the document
indexing, ensuring continued log ingestion without errors.
NOTE: When automatically injected, `host.name` and `@timestamp` contribute to the limit of mapped fields. When
`host.name` is mapped with `subobjects: true` it consists of two fields. When `host.name` is mapped with
`subobjects: false` it only consists of one field.
[discrete]
[[logsdb-nodocvalue-fields]]
=== Fields without doc values
When `logsdb` index mode uses synthetic `_source`, and `doc_values` are disabled for a field in the mapping,
Elasticsearch may set the `store` setting to `true` for that field as a last resort option to ensure that the field's
data is still available for reconstructing the document’s source when retrieving it via
<<synthetic-source,synthetic source>>.
For example, this happens with text fields when `store` is `false` and there is no suitable multi-field available to
reconstruct the original value in <<synthetic-source,synthetic source>>.
This automatic adjustment allows synthetic source to work correctly, even when doc values are not enabled for certain
fields.
[discrete]
[[logsdb-settings-summary]]
=== LogsDB settings summary
The following is a summary of key settings that apply when using `logsdb` index mode in Elasticsearch:
* **`index.mode`**: `"logsdb"`
* **`index.mapping.synthetic_source_keep`**: `"arrays"`
* **`index.sort.field`**: `["host.name", "@timestamp"]`
* **`index.sort.order`**: `["desc", "desc"]`
* **`index.sort.mode`**: `["min", "min"]`
* **`index.sort.missing`**: `["_first", "_first"]`
* **`index.codec`**: `"best_compression"`
* **`index.mapping.ignore_malformed`**: `true`
* **`index.mapping.ignore_above`**: `8191`
* **`index.mapping.total_fields.ignore_dynamic_beyond_limit`**: `true`

31
docs/reference/esql/esql-process-data-with-dissect-grok.asciidoc
View file

@ -40,7 +40,7 @@ delimiter-based pattern, and extracts the specified keys as columns.
For example, the following pattern:
[source,txt]
----
%{clientip} [%{@timestamp}] %{status}
%{clientip} [%{@timestamp}] %{status}
----
matches a log line of this format:
@ -76,8 +76,8 @@ ignore certain fields, append fields, skip over padding, etc.
===== Terminology
dissect pattern::
the set of fields and delimiters describing the textual
format. Also known as a dissection.
the set of fields and delimiters describing the textual
format. Also known as a dissection.
The dissection is described using a set of `%{}` sections:
`%{a} - %{b} - %{c}`
@ -91,14 +91,14 @@ Any set of characters other than `%{`, `'not }'`, or `}` is a delimiter.
key::
+
--
the text between the `%{` and `}`, exclusive of the `?`, `+`, `&` prefixes
and the ordinal suffix.
the text between the `%{` and `}`, exclusive of the `?`, `+`, `&` prefixes
and the ordinal suffix.
Examples:
* `%{?aaa}` - the key is `aaa`
* `%{+bbb/3}` - the key is `bbb`
* `%{&ccc}` - the key is `ccc`
* `%{?aaa}` - the key is `aaa`
* `%{+bbb/3}` - the key is `bbb`
* `%{&ccc}` - the key is `ccc`
--
[[esql-dissect-examples]]
@ -218,7 +218,7 @@ Putting it together as an {esql} query:
[source.merge.styled,esql]
----
include::{esql-specs}/docs.csv-spec[tag=grokWithEscape]
include::{esql-specs}/docs.csv-spec[tag=grokWithEscapeTripleQuotes]
----
`GROK` adds the following columns to the input table:
@ -239,15 +239,24 @@ with a `\`. For example, in the earlier pattern:
%{IP:ip} \[%{TIMESTAMP_ISO8601:@timestamp}\] %{GREEDYDATA:status}
----
In {esql} queries, the backslash character itself is a special character that
In {esql} queries, when using single quotes for strings, the backslash character itself is a special character that
needs to be escaped with another `\`. For this example, the corresponding {esql}
query becomes:
[source.merge.styled,esql]
----
include::{esql-specs}/docs.csv-spec[tag=grokWithEscape]
----
For this reason, in general it is more convenient to use triple quotes `"""` for GROK patterns,
that do not require escaping for backslash.
[source.merge.styled,esql]
----
include::{esql-specs}/docs.csv-spec[tag=grokWithEscapeTripleQuotes]
----
====
[[esql-grok-patterns]]
===== Grok patterns
@ -318,4 +327,4 @@ as the `GROK` command.
The `GROK` command does not support configuring <<custom-patterns,custom
patterns>>, or <<trace-match,multiple patterns>>. The `GROK` command is not
subject to <<grok-watchdog,Grok watchdog settings>>.
// end::grok-limitations[]
// end::grok-limitations[]

2
docs/reference/esql/functions/kibana/definition/like.json generated
View file

@ -42,7 +42,7 @@
}
],
"examples" : [
"FROM employees\n| WHERE first_name LIKE \"?b*\"\n| KEEP first_name, last_name"
"FROM employees\n| WHERE first_name LIKE \"\"\"?b*\"\"\"\n| KEEP first_name, last_name"
],
"preview" : false,
"snapshot_only" : false

2
docs/reference/esql/functions/kibana/definition/rlike.json generated
View file

@ -42,7 +42,7 @@
}
],
"examples" : [
"FROM employees\n| WHERE first_name RLIKE \".leja.*\"\n| KEEP first_name, last_name"
"FROM employees\n| WHERE first_name RLIKE \"\"\".leja.*\"\"\"\n| KEEP first_name, last_name"
],
"preview" : false,
"snapshot_only" : false

2
docs/reference/esql/functions/kibana/docs/like.md generated
View file

@ -15,6 +15,6 @@ The following wildcard characters are supported:
```
FROM employees
| WHERE first_name LIKE "?b*"
| WHERE first_name LIKE """?b*"""
| KEEP first_name, last_name
```

2
docs/reference/esql/functions/kibana/docs/rlike.md generated
View file

@ -10,6 +10,6 @@ expression. The right-hand side of the operator represents the pattern.
```
FROM employees
| WHERE first_name RLIKE ".leja.*"
| WHERE first_name RLIKE """.leja.*"""
| KEEP first_name, last_name
```

16
docs/reference/esql/functions/like.asciidoc
View file

@ -23,4 +23,20 @@ include::{esql-specs}/docs.csv-spec[tag=like]
|===
include::{esql-specs}/docs.csv-spec[tag=like-result]
|===
Matching the exact characters `*` and `.` will require escaping.
The escape character is backslash `\`. Since also backslash is a special character in string literals,
it will require further escaping.
[source.merge.styled,esql]
----
include::{esql-specs}/string.csv-spec[tag=likeEscapingSingleQuotes]
----
To reduce the overhead of escaping, we suggest using triple quotes strings `"""`
[source.merge.styled,esql]
----
include::{esql-specs}/string.csv-spec[tag=likeEscapingTripleQuotes]
----
// end::body[]

16
docs/reference/esql/functions/rlike.asciidoc
View file

@ -18,4 +18,20 @@ include::{esql-specs}/docs.csv-spec[tag=rlike]
|===
include::{esql-specs}/docs.csv-spec[tag=rlike-result]
|===
Matching special characters (eg. `.`, `*`, `(`...) will require escaping.
The escape character is backslash `\`. Since also backslash is a special character in string literals,
it will require further escaping.
[source.merge.styled,esql]
----
include::{esql-specs}/string.csv-spec[tag=rlikeEscapingSingleQuotes]
----
To reduce the overhead of escaping, we suggest using triple quotes strings `"""`
[source.merge.styled,esql]
----
include::{esql-specs}/string.csv-spec[tag=rlikeEscapingTripleQuotes]
----
// end::body[]

2
docs/reference/how-to/size-your-shards.asciidoc
View file

@ -572,7 +572,7 @@ PUT _cluster/settings
}
----
For more information, see <<troubleshooting-shards-capacity-issues,Troubleshooting shards capacity>>.
See this https://www.youtube.com/watch?v=tZKbDegt4-M[fixing "max shards open" video] for an example troubleshooting walkthrough. For more information, see <<troubleshooting-shards-capacity-issues,Troubleshooting shards capacity>>.
[discrete]
[[troubleshooting-max-docs-limit]]

62
docs/reference/images/semantic-options.svg Normal file
View file

@ -0,0 +1,62 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg viewBox="0 0 800 500" xmlns="http://www.w3.org/2000/svg">
<!-- Title -->
<text x="400" y="40" text-anchor="middle" font-size="24" fill="#000" font-family="Inter, Arial, sans-serif">Elasticsearch semantic search workflows</text>
<!-- Workflow boxes -->
<g transform="translate(0, 80)">
<!-- semantic_text workflow (recommended) -->
<rect x="50" y="50" width="200" height="300" rx="10" fill="#c2e0ff" stroke="#3f8dd6" stroke-width="2"/>
<text x="150" y="80" text-anchor="middle" font-size="16" font-weight="bold" fill="#2c5282" font-family="Inter, Arial, sans-serif">semantic_text</text>
<text x="150" y="100" text-anchor="middle" font-size="12" fill="#2c5282" font-family="Inter, Arial, sans-serif">(Recommended)</text>
<!-- Inference API workflow -->
<rect x="300" y="50" width="200" height="300" rx="10" fill="#d9f1e3" stroke="#38a169" stroke-width="2"/>
<text x="400" y="80" text-anchor="middle" font-size="16" font-weight="bold" fill="#276749" font-family="Inter, Arial, sans-serif">Inference API</text>
<!-- Model deployment workflow -->
<rect x="550" y="50" width="200" height="300" rx="10" fill="#feebc8" stroke="#dd6b20" stroke-width="2"/>
<text x="650" y="80" text-anchor="middle" font-size="16" font-weight="bold" fill="#9c4221" font-family="Inter, Arial, sans-serif">Model Deployment</text>
<!-- Complexity indicators -->
<text x="150" y="130" text-anchor="middle" font-size="12" fill="#2c5282" font-family="Inter, Arial, sans-serif">Complexity: Low</text>
<text x="400" y="130" text-anchor="middle" font-size="12" fill="#276749" font-family="Inter, Arial, sans-serif">Complexity: Medium</text>
<text x="650" y="130" text-anchor="middle" font-size="12" fill="#9c4221" font-family="Inter, Arial, sans-serif">Complexity: High</text>
<!-- Components in each workflow -->
<g transform="translate(60, 150)">
<!-- semantic_text components -->
<rect x="10" y="0" width="170" height="30" rx="5" fill="#fff" stroke="#3f8dd6"/>
<text x="95" y="20" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Create Inference Endpoint</text>
<rect x="10" y="40" width="170" height="30" rx="5" fill="#fff" stroke="#3f8dd6"/>
<text x="95" y="60" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Define Index Mapping</text>
<!-- Inference API components -->
<rect x="260" y="0" width="170" height="30" rx="5" fill="#fff" stroke="#38a169"/>
<text x="345" y="20" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Create Inference Endpoint</text>
<rect x="260" y="40" width="170" height="30" rx="5" fill="#fff" stroke="#38a169"/>
<text x="345" y="60" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Configure Model Settings</text>
<rect x="260" y="80" width="170" height="30" rx="5" fill="#fff" stroke="#38a169"/>
<text x="345" y="100" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Define Index Mapping</text>
<rect x="260" y="120" width="170" height="30" rx="5" fill="#fff" stroke="#38a169"/>
<text x="345" y="140" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Setup Ingest Pipeline</text>
<!-- Model deployment components -->
<rect x="510" y="0" width="170" height="30" rx="5" fill="#fff" stroke="#dd6b20"/>
<text x="595" y="20" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Select NLP Model</text>
<rect x="510" y="40" width="170" height="30" rx="5" fill="#fff" stroke="#dd6b20"/>
<text x="595" y="60" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Deploy with Eland Client</text>
<rect x="510" y="80" width="170" height="30" rx="5" fill="#fff" stroke="#dd6b20"/>
<text x="595" y="100" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Define Index Mapping</text>
<rect x="510" y="120" width="170" height="30" rx="5" fill="#fff" stroke="#dd6b20"/>
<text x="595" y="140" text-anchor="middle" font-size="12" font-family="Inter, Arial, sans-serif">Setup Ingest Pipeline</text>
</g>
</g>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 4.3 KiB

2
docs/reference/indices/alias-exists.asciidoc
View file

@ -52,8 +52,6 @@ Defaults to `all`.
(Optional, Boolean) If `false`, requests that include a missing data stream or
index in the `<target>` return an error. Defaults to `false`.
include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local]
[[alias-exists-api-response-codes]]
==== {api-response-codes-title}

2
docs/reference/indices/get-alias.asciidoc
View file

@ -58,5 +58,3 @@ Defaults to `all`.
`ignore_unavailable`::
(Optional, Boolean) If `false`, requests that include a missing data stream or
index in the `<target>` return an error. Defaults to `false`.
include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=local]

18
docs/reference/inference/inference-apis.asciidoc
View file

@ -34,6 +34,24 @@ Elastic –, then create an {infer} endpoint by the <<put-inference-api>>.
Now use <<semantic-search-semantic-text, semantic text>> to perform
<<semantic-search, semantic search>> on your data.
[discrete]
[[default-enpoints]]
=== Default {infer} endpoints
Your {es} deployment contains some preconfigured {infer} endpoints that makes it easier for you to use them when defining `semantic_text` fields or {infer} processors.
The following list contains the default {infer} endpoints listed by `inference_id`:
* `.elser-2-elasticsearch`: uses the {ml-docs}/ml-nlp-elser.html[ELSER] built-in trained model for `sparse_embedding` tasks (recommended for English language texts)
* `.multilingual-e5-small-elasticsearch`: uses the {ml-docs}/ml-nlp-e5.html[E5] built-in trained model for `text_embedding` tasks (recommended for non-English language texts)
Use the `inference_id` of the endpoint in a <<semantic-text,`semantic_text`>> field definition or when creating an <<inference-processor,{infer} processor>>.
The API call will automatically download and deploy the model which might take a couple of minutes.
Default {infer} enpoints have {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[adaptive allocations] enabled.
For these models, the minimum number of allocations is `0`.
If there is no {infer} activity that uses the endpoint, the number of allocations will scale down to `0` automatically after 15 minutes.
include::delete-inference.asciidoc[]
include::get-inference.asciidoc[]
include::post-inference.asciidoc[]

94
docs/reference/inference/service-elasticsearch.asciidoc
View file

@ -1,12 +1,9 @@
[[infer-service-elasticsearch]]
=== Elasticsearch {infer} service
Creates an {infer} endpoint to perform an {infer} task with the `elasticsearch`
service.
Creates an {infer} endpoint to perform an {infer} task with the `elasticsearch` service.
NOTE: If you use the E5 model through the `elasticsearch` service, the API
request will automatically download and deploy the model if it isn't downloaded
yet.
NOTE: If you use the ELSER or the E5 model through the `elasticsearch` service, the API request will automatically download and deploy the model if it isn't downloaded yet.
[discrete]
@ -56,6 +53,11 @@ These settings are specific to the `elasticsearch` service.
(Optional, object)
include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation]
`deployment_id`:::
(Optional, string)
The `deployment_id` of an existing trained model deployment.
When `deployment_id` is used the `model_id` is optional.
`enabled`::::
(Optional, Boolean)
include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-enabled]
@ -71,7 +73,7 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=adaptive-allocation-min-number]
`model_id`:::
(Required, string)
The name of the model to use for the {infer} task.
It can be the ID of either a built-in model (for example, `.multilingual-e5-small` for E5) or a text embedding model already
It can be the ID of either a built-in model (for example, `.multilingual-e5-small` for E5), a text embedding model already
{ml-docs}/ml-nlp-import-model.html#ml-nlp-import-script[uploaded through Eland].
`num_allocations`:::
@ -98,15 +100,44 @@ Returns the document instead of only the index. Defaults to `true`.
=====
[discrete]
[[inference-example-elasticsearch-elser]]
==== ELSER via the `elasticsearch` service
The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type.
The API request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model.
[source,console]
------------------------------------------------------------
PUT _inference/sparse_embedding/my-elser-model
{
"service": "elasticsearch",
"service_settings": {
"adaptive_allocations": { <1>
"enabled": true,
"min_number_of_allocations": 1,
"max_number_of_allocations": 10
},
"num_threads": 1,
"model_id": ".elser_model_2" <2>
}
}
------------------------------------------------------------
// TEST[skip:TBD]
<1> Adaptive allocations will be enabled with the minimum of 1 and the maximum of 10 allocations.
<2> The `model_id` must be the ID of one of the built-in ELSER models.
Valid values are `.elser_model_2` and `.elser_model_2_linux-x86_64`.
For further details, refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation].
[discrete]
[[inference-example-elasticsearch]]
==== E5 via the `elasticsearch` service
The following example shows how to create an {infer} endpoint called
`my-e5-model` to perform a `text_embedding` task type.
The following example shows how to create an {infer} endpoint called `my-e5-model` to perform a `text_embedding` task type.
The API request below will automatically download the E5 model if it isn't
already downloaded and then deploy the model.
The API request below will automatically download the E5 model if it isn't already downloaded and then deploy the model.
[source,console]
------------------------------------------------------------
@ -185,3 +216,46 @@ PUT _inference/text_embedding/my-e5-model
}
------------------------------------------------------------
// TEST[skip:TBD]
[discrete]
[[inference-example-existing-deployment]]
==== Using an existing model deployment with the `elasticsearch` service
The following example shows how to use an already existing model deployment when creating an {infer} endpoint.
[source,console]
------------------------------------------------------------
PUT _inference/sparse_embedding/use_existing_deployment
{
"service": "elasticsearch",
"service_settings": {
"deployment_id": ".elser_model_2" <1>
}
}
------------------------------------------------------------
// TEST[skip:TBD]
<1> The `deployment_id` of the already existing model deployment.
The API response contains the `model_id`, and the threads and allocations settings from the model deployment:
[source,console-result]
------------------------------------------------------------
{
"inference_id": "use_existing_deployment",
"task_type": "sparse_embedding",
"service": "elasticsearch",
"service_settings": {
"num_allocations": 2,
"num_threads": 1,
"model_id": ".elser_model_2",
"deployment_id": ".elser_model_2"
},
"chunking_settings": {
"strategy": "sentence",
"max_chunk_size": 250,
"sentence_overlap": 1
}
}
------------------------------------------------------------
// NOTCONSOLE

3
docs/reference/inference/service-elser.asciidoc
View file

@ -2,6 +2,7 @@
=== ELSER {infer} service
Creates an {infer} endpoint to perform an {infer} task with the `elser` service.
You can also deploy ELSER by using the <<infer-service-elasticsearch>>.
NOTE: The API request will automatically download and deploy the ELSER model if
it isn't already downloaded.
@ -128,7 +129,7 @@ If using the Python client, you can set the `timeout` parameter to a higher valu
[discrete]
[[inference-example-elser-adaptive-allocation]]
==== Setting adaptive allocation for the ELSER service
==== Setting adaptive allocations for the ELSER service
NOTE: For more information on how to optimize your ELSER endpoints, refer to {ml-docs}/ml-nlp-elser.html#elser-recommendations[the ELSER recommendations] section in the model documentation.
To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page.

4
docs/reference/mapping/params/format.asciidoc
View file

@ -34,13 +34,13 @@ down to the nearest day.
Completely customizable date formats are supported. The syntax for these is explained in
https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/time/format/DateTimeFormatter.html[DateTimeFormatter docs].
Note that whilst the built-in formats for week dates use the ISO definition of weekyears,
Note that while the built-in formats for week dates use the ISO definition of weekyears,
custom formatters using the `Y`, `W`, or `w` field specifiers use the JDK locale definition
of weekyears. This can result in different values between the built-in formats and custom formats
for week dates.
[[built-in-date-formats]]
==== Built In Formats
==== Built-in formats
Most of the below formats have a `strict` companion format, which means that
year, month and day parts of the month must use respectively 4, 2 and 2 digits

26
docs/reference/mapping/types/semantic-text.asciidoc
View file

@ -13,25 +13,47 @@ Long passages are <<auto-text-chunking, automatically chunked>> to smaller secti
The `semantic_text` field type specifies an inference endpoint identifier that will be used to generate embeddings.
You can create the inference endpoint by using the <<put-inference-api>>.
This field type and the <<query-dsl-semantic-query,`semantic` query>> type make it simpler to perform semantic search on your data.
If you don't specify an inference endpoint, the <<infer-service-elser,ELSER service>> is used by default.
Using `semantic_text`, you won't need to specify how to generate embeddings for your data, or how to index it.
The {infer} endpoint automatically determines the embedding generation, indexing, and query to use.
If you use the ELSER service, you can set up `semantic_text` with the following API request:
[source,console]
------------------------------------------------------------
PUT my-index-000001
{
"mappings": {
"properties": {
"inference_field": {
"type": "semantic_text"
}
}
}
}
------------------------------------------------------------
NOTE: In Serverless, you must create an {infer} endpoint using the <<put-inference-api>> and reference it when setting up `semantic_text` even if you use the ELSER service.
If you use a service other than ELSER, you must create an {infer} endpoint using the <<put-inference-api>> and reference it when setting up `semantic_text` as the following example demonstrates:
[source,console]
------------------------------------------------------------
PUT my-index-000002
{
"mappings": {
"properties": {
"inference_field": {
"type": "semantic_text",
"inference_id": "my-elser-endpoint"
"inference_id": "my-openai-endpoint" <1>
}
}
}
}
------------------------------------------------------------
// TEST[skip:Requires inference endpoint]
<1> The `inference_id` of the {infer} endpoint to use to generate embeddings.
The recommended way to use semantic_text is by having dedicated {infer} endpoints for ingestion and search.
@ -40,7 +62,7 @@ After creating dedicated {infer} endpoints for both, you can reference them usin
[source,console]
------------------------------------------------------------
PUT my-index-000002
PUT my-index-000003
{
"mappings": {
"properties": {

16
docs/reference/query-dsl/terms-set-query.asciidoc
View file

@ -159,12 +159,22 @@ GET /job-candidates/_search
`terms`::
+
--
(Required, array of strings) Array of terms you wish to find in the provided
(Required, array) Array of terms you wish to find in the provided
`<field>`. To return a document, a required number of terms must exactly match
the field values, including whitespace and capitalization.
The required number of matching terms is defined in the
`minimum_should_match_field` or `minimum_should_match_script` parameter.
The required number of matching terms is defined in the `minimum_should_match`,
`minimum_should_match_field` or `minimum_should_match_script` parameters. Exactly
one of these parameters must be provided.
--
`minimum_should_match`::
+
--
(Optional) Specification for the number of matching terms required to return
a document.
For valid values, see <<query-dsl-minimum-should-match, `minimum_should_match` parameter>>.
--
`minimum_should_match_field`::

7
docs/reference/query-dsl/text-expansion-query.asciidoc
View file

@ -7,6 +7,13 @@
deprecated[8.15.0, This query has been replaced by <<query-dsl-sparse-vector-query>>.]
.Deprecation usage note
****
You can continue using `rank_features` fields with `text_expansion` queries in the current version.
However, if you plan to upgrade, we recommend updating mappings to use the `sparse_vector` field type and <<docs-reindex,reindexing your data>>.
This will allow you to take advantage of the new capabilities and improvements available in newer versions.
****
The text expansion query uses a {nlp} model to convert the query text into a list of token-weight pairs which are then used in a query against a
<<sparse-vector,sparse vector>> or <<rank-features,rank features>> field.

2
docs/reference/release-notes/8.12.0.asciidoc
View file

@ -11,7 +11,7 @@ Also see <<breaking-changes-8.12,Breaking changes in 8.12>>.
+
When using `int8_hnsw` and the default `confidence_interval` (or any `confidence_interval` less than `1.0`) and when
there are deleted documents in the segments, quantiles may fail to build and prevent merging.
+
This issue is fixed in 8.12.1.
* When upgrading clusters from version 8.11.4 or earlier, if your cluster contains non-master-eligible nodes,

4
docs/reference/run-elasticsearch-locally.asciidoc
View file

@ -20,7 +20,7 @@ Refer to <<elasticsearch-intro-deploy, deployment options>> for a list of produc
Quickly set up {es} and {kib} in Docker for local development or testing, using the https://github.com/elastic/start-local?tab=readme-ov-file#-try-elasticsearch-and-kibana-locally[`start-local` script].
This setup comes with a one-month trial of the Elastic *Platinum* license.
This setup comes with a one-month trial license that includes all Elastic features.
After the trial period, the license reverts to *Free and open - Basic*.
Refer to https://www.elastic.co/subscriptions[Elastic subscriptions] for more information.
@ -84,4 +84,4 @@ Learn about customizing the setup, logging, and more.
[[local-dev-next-steps]]
=== Next steps
Use our <<quickstart,quick start guides>> to learn the basics of {es}.
Use our <<quickstart,quick start guides>> to learn the basics of {es}.

141
docs/reference/search/search-your-data/ingest-vectors.asciidoc Normal file
View file

@ -0,0 +1,141 @@
[[bring-your-own-vectors]]
=== Bring your own dense vector embeddings to {es}
++++
<titleabbrev>Bring your own dense vectors</titleabbrev>
++++
This tutorial demonstrates how to index documents that already have dense vector embeddings into {es}.
You'll also learn the syntax for searching these documents using a `knn` query.
You'll find links at the end of this tutorial for more information about deploying a text embedding model in {es}, so you can generate embeddings for queries on the fly.
[TIP]
====
This is an advanced use case.
Refer to <<semantic-search,Semantic search>> for an overview of your options for semantic search with {es}.
====
[discrete]
[[bring-your-own-vectors-create-index]]
=== Step 1: Create an index with `dense_vector` mapping
Each document in our simple dataset will have:
* A review: stored in a `review_text` field
* An embedding of that review: stored in a `review_vector` field
** The `review_vector` field is defined as a <<dense-vector,`dense_vector`>> data type.
[TIP]
====
The `dense_vector` type automatically uses `int8_hnsw` quantization by default to reduce the memory footprint required when searching float vectors.
Learn more about balancing performance and accuracy in <<dense-vector-quantization,Dense vector quantization>>.
====
[source,console]
----
PUT /amazon-reviews
{
"mappings": {
"properties": {
"review_vector": {
"type": "dense_vector",
"dims": 8, <1>
"index": true, <2>
"similarity": "cosine" <3>
},
"review_text": {
"type": "text"
}
}
}
}
----
// TEST SETUP
<1> The `dims` parameter must match the length of the embedding vector. Here we're using a simple 8-dimensional embedding for readability. If not specified, `dims` will be dynamically calculated based on the first indexed document.
<2> The `index` parameter is set to `true` to enable the use of the `knn` query.
<3> The `similarity` parameter defines the similarity function used to compare the query vector to the document vectors. `cosine` is the default similarity function for `dense_vector` fields in {es}.
[discrete]
[[bring-your-own-vectors-index-documents]]
=== Step 2: Index documents with embeddings
[discrete]
==== Index a single document
First, index a single document to understand the document structure.
[source,console]
----
PUT /amazon-reviews/_doc/1
{
"review_text": "This product is lifechanging! I'm telling all my friends about it.",
"review_vector": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] <1>
}
----
// TEST
<1> The size of the `review_vector` array is 8, matching the `dims` count specified in the mapping.
[discrete]
==== Bulk index multiple documents
In a production scenario, you'll want to index many documents at once using the <<docs-bulk,`_bulk` endpoint>>.
Here's an example of indexing multiple documents in a single `_bulk` request.
[source,console]
----
POST /_bulk
{ "index": { "_index": "amazon-reviews", "_id": "2" } }
{ "review_text": "This product is amazing! I love it.", "review_vector": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] }
{ "index": { "_index": "amazon-reviews", "_id": "3" } }
{ "review_text": "This product is terrible. I hate it.", "review_vector": [0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1] }
{ "index": { "_index": "amazon-reviews", "_id": "4" } }
{ "review_text": "This product is great. I can do anything with it.", "review_vector": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] }
{ "index": { "_index": "amazon-reviews", "_id": "5" } }
{ "review_text": "This product has ruined my life and the lives of my family and friends.", "review_vector": [0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1] }
----
// TEST[continued]
[discrete]
[[bring-your-own-vectors-search-documents]]
=== Step 3: Search documents with embeddings
Now you can query these document vectors using a <<knn-retriever,`knn` retriever>>.
`knn` is a type of vector search, which finds the `k` most similar documents to a query vector.
Here we're simply using a raw vector for the query text, for demonstration purposes.
[source,console]
----
POST /amazon-reviews/_search
{
"retriever": {
"knn": {
"field": "review_vector",
"query_vector": [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8], <1>
"k": 2, <2>
"num_candidates": 5 <3>
}
}
}
----
// TEST[skip:flakeyknnerror]
<1> In this simple example, we're sending a raw vector as the query text. In a real-world scenario, you'll need to generate vectors for queries using an embedding model.
<2> The `k` parameter specifies the number of results to return.
<3> The `num_candidates` parameter is optional. It limits the number of candidates returned by the search node. This can improve performance and reduce costs.
[discrete]
[[bring-your-own-vectors-learn-more]]
=== Learn more
In this simple example, we're sending a raw vector for the query text.
In a real-world scenario you won't know the query text ahead of time.
You'll need to generate query vectors, on the fly, using the same embedding model that generated the document vectors.
For this you'll need to deploy a text embedding model in {es} and use the <<knn-query-top-level-parameters,`query_vector_builder` parameter>>. Alternatively, you can generate vectors client-side and send them directly with the search request.
Learn how to <<semantic-search-deployed-nlp-model,use a deployed text embedding model>> for semantic search.
[TIP]
====
If you're just getting started with vector search in {es}, refer to <<semantic-search,Semantic search>>.
====

57
docs/reference/search/search-your-data/semantic-search-semantic-text.asciidoc
View file

@ -21,45 +21,11 @@ This tutorial uses the <<inference-example-elser,`elser` service>> for demonstra
[[semantic-text-requirements]]
==== Requirements
To use the `semantic_text` field type, you must have an {infer} endpoint deployed in
your cluster using the <<put-inference-api>>.
This tutorial uses the <<infer-service-elser,ELSER service>> for demonstration, which is created automatically as needed.
To use the `semantic_text` field type with an {infer} service other than ELSER, you must create an inference endpoint using the <<put-inference-api>>.
[discrete]
[[semantic-text-infer-endpoint]]
==== Create the {infer} endpoint
NOTE: In Serverless, you must create an {infer} endpoint using the <<put-inference-api>> and reference it when setting up `semantic_text` even if you use the ELSER service.
Create an inference endpoint by using the <<put-inference-api>>:
[source,console]
------------------------------------------------------------
PUT _inference/sparse_embedding/my-elser-endpoint <1>
{
"service": "elser", <2>
"service_settings": {
"adaptive_allocations": { <3>
"enabled": true,
"min_number_of_allocations": 3,
"max_number_of_allocations": 10
},
"num_threads": 1
}
}
------------------------------------------------------------
// TEST[skip:TBD]
<1> The task type is `sparse_embedding` in the path as the `elser` service will
be used and ELSER creates sparse vectors. The `inference_id` is
`my-elser-endpoint`.
<2> The `elser` service is used in this example.
<3> This setting enables and configures {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[adaptive allocations].
Adaptive allocations make it possible for ELSER to automatically scale up or down resources based on the current load on the process.
[NOTE]
====
You might see a 502 bad gateway error in the response when using the {kib} Console.
This error usually just reflects a timeout, while the model downloads in the background.
You can check the download progress in the {ml-app} UI.
If using the Python client, you can set the `timeout` parameter to a higher value.
====
[discrete]
[[semantic-text-index-mapping]]
@ -75,8 +41,7 @@ PUT semantic-embeddings
"mappings": {
"properties": {
"content": { <1>
"type": "semantic_text", <2>
"inference_id": "my-elser-endpoint" <3>
"type": "semantic_text" <2>
}
}
}
@ -85,18 +50,14 @@ PUT semantic-embeddings
// TEST[skip:TBD]
<1> The name of the field to contain the generated embeddings.
<2> The field to contain the embeddings is a `semantic_text` field.
<3> The `inference_id` is the inference endpoint you created in the previous step.
It will be used to generate the embeddings based on the input text.
Every time you ingest data into the related `semantic_text` field, this endpoint will be used for creating the vector representation of the text.
Since no `inference_id` is provided, the <<infer-service-elser,ELSER service>> is used by default.
To use a different {infer} service, you must create an {infer} endpoint first using the <<put-inference-api>> and then specify it in the `semantic_text` field mapping using the `inference_id` parameter.
[NOTE]
====
If you're using web crawlers or connectors to generate indices, you have to
<<indices-put-mapping,update the index mappings>> for these indices to
include the `semantic_text` field. Once the mapping is updated, you'll need to run
a full web crawl or a full connector sync. This ensures that all existing
documents are reprocessed and updated with the new semantic embeddings,
enabling semantic search on the updated data.
If you're using web crawlers or connectors to generate indices, you have to <<indices-put-mapping,update the index mappings>> for these indices to include the `semantic_text` field.
Once the mapping is updated, you'll need to run a full web crawl or a full connector sync.
This ensures that all existing documents are reprocessed and updated with the new semantic embeddings, enabling semantic search on the updated data.
====

3
docs/reference/search/search-your-data/semantic-search.asciidoc
View file

@ -8,6 +8,8 @@ Using an NLP model enables you to extract text embeddings out of text.
Embeddings are vectors that provide a numeric representation of a text.
Pieces of content with similar meaning have similar representations.
image::images/semantic-options.svg[Overview of semantic search workflows in {es}]
You have several options for using NLP models in the {stack}:
* use the `semantic_text` workflow (recommended)
@ -109,3 +111,4 @@ include::semantic-search-inference.asciidoc[]
include::semantic-search-elser.asciidoc[]
include::cohere-es.asciidoc[]
include::semantic-search-deploy-model.asciidoc[]
include::ingest-vectors.asciidoc[]

9
docs/reference/snapshot-restore/repository-azure.asciidoc
View file

@ -259,6 +259,15 @@ include::repository-shared-settings.asciidoc[]
`primary_only` or `secondary_only`. Defaults to `primary_only`. Note that if you set it
to `secondary_only`, it will force `readonly` to true.
`delete_objects_max_size`::
(integer) Sets the maxmimum batch size, betewen 1 and 256, used for `BlobBatch` requests. Defaults to 256 which is the maximum
number supported by the https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch#remarks[Azure blob batch API].
`max_concurrent_batch_deletes`::
(integer) Sets the maximum number of concurrent batch delete requests that will be submitted for any individual bulk delete with `BlobBatch`. Note that the effective number of concurrent deletes is further limited by the Azure client connection and event loop thread limits. Defaults to 10, minimum is 1, maximum is 100.
[[repository-azure-validation]]
==== Repository validation rules

5
gradle/verification-metadata.xml
View file

@ -144,6 +144,11 @@
<sha256 value="31915426834400cac854f48441c168d55aa6fc054527f28f1d242a7067affd14" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.azure" name="azure-storage-blob-batch" version="12.23.1">
<artifact name="azure-storage-blob-batch-12.23.1.jar">
<sha256 value="8c11749c783222873f63f22575aa5ae7ee8f285388183b82d1a18db21f4d2eba" origin="Generated by Gradle"/>
</artifact>
</component>
<component group="com.azure" name="azure-storage-common" version="12.26.1">
<artifact name="azure-storage-common-12.26.1.jar">
<sha256 value="b0297ac1a9017ccd8a1e5cf41fb8d00ff0adbdd06849f6c5aafb3208708264dd" origin="Generated by Gradle"/>

2
modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java
View file

@ -138,7 +138,7 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
// Initialize the failure store.
RolloverRequest rolloverRequest = new RolloverRequest("with-fs", null);
rolloverRequest.setIndicesOptions(
IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES).build()
IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
);
response = client.execute(RolloverAction.INSTANCE, rolloverRequest).get();
assertTrue(response.isAcknowledged());

2
modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/IngestFailureStoreMetricsIT.java
View file

@ -195,7 +195,7 @@ public class IngestFailureStoreMetricsIT extends ESIntegTestCase {
// Initialize failure store.
var rolloverRequest = new RolloverRequest(dataStream, null);
rolloverRequest.setIndicesOptions(
IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES).build()
IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
);
var rolloverResponse = client().execute(RolloverAction.INSTANCE, rolloverRequest).actionGet();
var failureStoreIndex = rolloverResponse.getNewIndex();

6
modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java
View file

@ -950,7 +950,7 @@ public class DataStreamLifecycleService implements ClusterStateListener, Closeab
UpdateSettingsRequest updateMergePolicySettingsRequest = new UpdateSettingsRequest();
updateMergePolicySettingsRequest.indicesOptions(
IndicesOptions.builder(updateMergePolicySettingsRequest.indicesOptions())
.selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE)
.selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
.build()
);
updateMergePolicySettingsRequest.indices(indexName);
@ -1412,9 +1412,7 @@ public class DataStreamLifecycleService implements ClusterStateListener, Closeab
RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null).masterNodeTimeout(TimeValue.MAX_VALUE);
if (rolloverFailureStore) {
rolloverRequest.setIndicesOptions(
IndicesOptions.builder(rolloverRequest.indicesOptions())
.selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES)
.build()
IndicesOptions.builder(rolloverRequest.indicesOptions()).selectorOptions(IndicesOptions.SelectorOptions.FAILURES).build()
);
}
rolloverRequest.setConditions(rolloverConfiguration.resolveRolloverConditions(dataRetention));

8
modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java
View file

@ -225,11 +225,11 @@ public class DataStreamLifecycleServiceTests extends ESTestCase {
assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class));
RolloverRequest rolloverBackingIndexRequest = (RolloverRequest) clientSeenRequests.get(0);
assertThat(rolloverBackingIndexRequest.getRolloverTarget(), is(dataStreamName));
assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.ONLY_DATA));
assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.DATA));
assertThat(clientSeenRequests.get(1), instanceOf(RolloverRequest.class));
RolloverRequest rolloverFailureIndexRequest = (RolloverRequest) clientSeenRequests.get(1);
assertThat(rolloverFailureIndexRequest.getRolloverTarget(), is(dataStreamName));
assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.ONLY_FAILURES));
assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.FAILURES));
List<DeleteIndexRequest> deleteRequests = clientSeenRequests.subList(2, 5)
.stream()
.map(transportRequest -> (DeleteIndexRequest) transportRequest)
@ -1573,11 +1573,11 @@ public class DataStreamLifecycleServiceTests extends ESTestCase {
assertThat(clientSeenRequests.get(0), instanceOf(RolloverRequest.class));
RolloverRequest rolloverBackingIndexRequest = (RolloverRequest) clientSeenRequests.get(0);
assertThat(rolloverBackingIndexRequest.getRolloverTarget(), is(dataStreamName));
assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.ONLY_DATA));
assertThat(rolloverBackingIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.DATA));
assertThat(clientSeenRequests.get(1), instanceOf(RolloverRequest.class));
RolloverRequest rolloverFailureIndexRequest = (RolloverRequest) clientSeenRequests.get(1);
assertThat(rolloverFailureIndexRequest.getRolloverTarget(), is(dataStreamName));
assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.ONLY_FAILURES));
assertThat(rolloverFailureIndexRequest.indicesOptions().selectorOptions(), equalTo(IndicesOptions.SelectorOptions.FAILURES));
assertThat(
((DeleteIndexRequest) clientSeenRequests.get(2)).indices()[0],
is(dataStream.getFailureIndices().getIndices().get(0).getName())

21
modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java
View file

@ -12,6 +12,8 @@ package org.elasticsearch.kibana;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.client.internal.Client;
import org.elasticsearch.common.settings.Settings;
@ -37,6 +39,7 @@ import java.util.stream.Stream;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.startsWith;
/**
@ -150,15 +153,15 @@ public class KibanaThreadPoolIT extends ESIntegTestCase {
new Thread(() -> expectThrows(EsRejectedExecutionException.class, () -> getFuture.actionGet(SAFE_AWAIT_TIMEOUT))).start();
// intentionally commented out this test until https://github.com/elastic/elasticsearch/issues/97916 is fixed
// var e3 = expectThrows(
// SearchPhaseExecutionException.class,
// () -> client().prepareSearch(USER_INDEX)
// .setQuery(QueryBuilders.matchAllQuery())
// // Request times out if max concurrent shard requests is set to 1
// .setMaxConcurrentShardRequests(usually() ? SearchRequest.DEFAULT_MAX_CONCURRENT_SHARD_REQUESTS : randomIntBetween(2, 10))
// .get()
// );
// assertThat(e3.getMessage(), containsString("all shards failed"));
var e3 = expectThrows(
SearchPhaseExecutionException.class,
() -> client().prepareSearch(USER_INDEX)
.setQuery(QueryBuilders.matchAllQuery())
// Request times out if max concurrent shard requests is set to 1
.setMaxConcurrentShardRequests(usually() ? SearchRequest.DEFAULT_MAX_CONCURRENT_SHARD_REQUESTS : randomIntBetween(2, 10))
.get()
);
assertThat(e3.getMessage(), containsString("all shards failed"));
}
protected void runWithBlockedThreadPools(Runnable runnable) throws Exception {

1
modules/repository-azure/build.gradle
View file

@ -30,6 +30,7 @@ dependencies {
api "com.azure:azure-identity:1.13.2"
api "com.azure:azure-json:1.2.0"
api "com.azure:azure-storage-blob:12.27.1"
api "com.azure:azure-storage-blob-batch:12.23.1"
api "com.azure:azure-storage-common:12.26.1"
api "com.azure:azure-storage-internal-avro:12.12.1"
api "com.azure:azure-xml:1.1.0"

91
modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryMetricsTests.java
View file

@ -9,14 +9,18 @@
package org.elasticsearch.repositories.azure;
import com.sun.net.httpserver.Headers;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.Randomness;
import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.OperationPurpose;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.SuppressForbidden;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.repositories.RepositoriesMetrics;
@ -31,6 +35,7 @@ import org.junit.After;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
@ -43,6 +48,7 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static org.elasticsearch.repositories.azure.AbstractAzureServerTestCase.randomBlobContent;
import static org.elasticsearch.repositories.blobstore.BlobStoreTestUtil.randomPurpose;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
@ -225,6 +231,91 @@ public class AzureBlobStoreRepositoryMetricsTests extends AzureBlobStoreReposito
assertThat(recordedRequestTime, lessThanOrEqualTo(elapsedTimeMillis));
}
public void testBatchDeleteFailure() throws IOException {
final int deleteBatchSize = randomIntBetween(1, 30);
final String repositoryName = randomRepositoryName();
final String repository = createRepository(
repositoryName,
Settings.builder()
.put(repositorySettings(repositoryName))
.put(AzureRepository.Repository.DELETION_BATCH_SIZE_SETTING.getKey(), deleteBatchSize)
.build(),
true
);
final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData);
final BlobContainer container = getBlobContainer(dataNodeName, repository);
final List<String> blobsToDelete = new ArrayList<>();
final int numberOfBatches = randomIntBetween(3, 20);
final int numberOfBlobs = numberOfBatches * deleteBatchSize;
final int failedBatches = randomIntBetween(1, numberOfBatches);
for (int i = 0; i < numberOfBlobs; i++) {
byte[] bytes = randomBytes(randomInt(100));
String blobName = "index-" + randomAlphaOfLength(10);
container.writeBlob(randomPurpose(), blobName, new BytesArray(bytes), false);
blobsToDelete.add(blobName);
}
Randomness.shuffle(blobsToDelete);
clearMetrics(dataNodeName);
// Handler will fail one or more of the batch requests
final RequestHandler failNRequestRequestHandler = createFailNRequestsHandler(failedBatches);
// Exhaust the retries
IntStream.range(0, (numberOfBatches - failedBatches) + (failedBatches * (MAX_RETRIES + 1)))
.forEach(i -> requestHandlers.offer(failNRequestRequestHandler));
logger.info("--> Failing {} of {} batches", failedBatches, numberOfBatches);
final IOException exception = assertThrows(
IOException.class,
() -> container.deleteBlobsIgnoringIfNotExists(randomPurpose(), blobsToDelete.iterator())
);
assertEquals(Math.min(failedBatches, 10), exception.getSuppressed().length);
assertEquals(
(numberOfBatches - failedBatches) + (failedBatches * (MAX_RETRIES + 1L)),
getLongCounterTotal(dataNodeName, RepositoriesMetrics.METRIC_REQUESTS_TOTAL)
);
assertEquals((failedBatches * (MAX_RETRIES + 1L)), getLongCounterTotal(dataNodeName, RepositoriesMetrics.METRIC_EXCEPTIONS_TOTAL));
assertEquals(failedBatches * deleteBatchSize, container.listBlobs(randomPurpose()).size());
}
private long getLongCounterTotal(String dataNodeName, String metricKey) {
return getTelemetryPlugin(dataNodeName).getLongCounterMeasurement(metricKey)
.stream()
.mapToLong(Measurement::getLong)
.reduce(0L, Long::sum);
}
/**
* Creates a {@link RequestHandler} that will persistently fail the first <code>numberToFail</code> distinct requests
* it sees. Any other requests are passed through to the delegate.
*
* @param numberToFail The number of requests to fail
* @return the handler
*/
private static RequestHandler createFailNRequestsHandler(int numberToFail) {
final List<String> requestsToFail = new ArrayList<>(numberToFail);
return (exchange, delegate) -> {
final Headers requestHeaders = exchange.getRequestHeaders();
final String requestId = requestHeaders.get("X-ms-client-request-id").get(0);
boolean failRequest = false;
synchronized (requestsToFail) {
if (requestsToFail.contains(requestId)) {
failRequest = true;
} else if (requestsToFail.size() < numberToFail) {
requestsToFail.add(requestId);
failRequest = true;
}
}
if (failRequest) {
exchange.sendResponseHeaders(500, -1);
} else {
delegate.handle(exchange);
}
};
}
private void clearMetrics(String discoveryNode) {
internalCluster().getInstance(PluginsService.class, discoveryNode)
.filterPlugins(TestTelemetryPlugin.class)

24
modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java
View file

@ -89,7 +89,9 @@ public class AzureBlobStoreRepositoryTests extends ESMockAPIBasedRepositoryInteg
.put(super.repositorySettings(repoName))
.put(AzureRepository.Repository.MAX_SINGLE_PART_UPLOAD_SIZE_SETTING.getKey(), new ByteSizeValue(1, ByteSizeUnit.MB))
.put(AzureRepository.Repository.CONTAINER_SETTING.getKey(), "container")
.put(AzureStorageSettings.ACCOUNT_SETTING.getKey(), "test");
.put(AzureStorageSettings.ACCOUNT_SETTING.getKey(), "test")
.put(AzureRepository.Repository.DELETION_BATCH_SIZE_SETTING.getKey(), randomIntBetween(5, 256))
.put(AzureRepository.Repository.MAX_CONCURRENT_BATCH_DELETES_SETTING.getKey(), randomIntBetween(1, 10));
if (randomBoolean()) {
settingsBuilder.put(AzureRepository.Repository.BASE_PATH_SETTING.getKey(), randomFrom("test", "test/1"));
}
@ -249,6 +251,8 @@ public class AzureBlobStoreRepositoryTests extends ESMockAPIBasedRepositoryInteg
trackRequest("PutBlockList");
} else if (Regex.simpleMatch("PUT /*/*", request)) {
trackRequest("PutBlob");
} else if (Regex.simpleMatch("POST /*/*?*comp=batch*", request)) {
trackRequest("BlobBatch");
}
}
@ -279,10 +283,22 @@ public class AzureBlobStoreRepositoryTests extends ESMockAPIBasedRepositoryInteg
}
public void testDeleteBlobsIgnoringIfNotExists() throws Exception {
try (BlobStore store = newBlobStore()) {
// Test with a smaller batch size here
final int deleteBatchSize = randomIntBetween(1, 30);
final String repositoryName = randomRepositoryName();
createRepository(
repositoryName,
Settings.builder()
.put(repositorySettings(repositoryName))
.put(AzureRepository.Repository.DELETION_BATCH_SIZE_SETTING.getKey(), deleteBatchSize)
.build(),
true
);
try (BlobStore store = newBlobStore(repositoryName)) {
final BlobContainer container = store.blobContainer(BlobPath.EMPTY);
List<String> blobsToDelete = new ArrayList<>();
for (int i = 0; i < 10; i++) {
final int toDeleteCount = randomIntBetween(deleteBatchSize, 3 * deleteBatchSize);
final List<String> blobsToDelete = new ArrayList<>();
for (int i = 0; i < toDeleteCount; i++) {
byte[] bytes = randomBytes(randomInt(100));
String blobName = randomAlphaOfLength(10);
container.writeBlob(randomPurpose(), blobName, new BytesArray(bytes), false);

5
modules/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java
View file

@ -30,6 +30,8 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.core.Booleans;
import org.elasticsearch.logging.LogManager;
import org.elasticsearch.logging.Logger;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.repositories.AbstractThirdPartyRepositoryTestCase;
import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
@ -46,6 +48,7 @@ import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
private static final Logger logger = LogManager.getLogger(AzureStorageCleanupThirdPartyTests.class);
private static final boolean USE_FIXTURE = Booleans.parseBoolean(System.getProperty("test.azure.fixture", "true"));
private static final String AZURE_ACCOUNT = System.getProperty("test.azure.account");
@ -89,8 +92,10 @@ public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyReposi
MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString("azure.client.default.account", System.getProperty("test.azure.account"));
if (hasSasToken) {
logger.info("--> Using SAS token authentication");
secureSettings.setString("azure.client.default.sas_token", System.getProperty("test.azure.sas_token"));
} else {
logger.info("--> Using key authentication");
secureSettings.setString("azure.client.default.key", System.getProperty("test.azure.key"));
}
return secureSettings;

5
modules/repository-azure/src/main/java/module-info.java
View file

@ -18,10 +18,7 @@ module org.elasticsearch.repository.azure {
requires org.apache.logging.log4j;
requires org.apache.logging.log4j.core;
requires com.azure.core;
requires com.azure.http.netty;
requires com.azure.storage.blob;
requires com.azure.storage.common;
requires com.azure.identity;
requires io.netty.buffer;
@ -29,7 +26,7 @@ module org.elasticsearch.repository.azure {
requires io.netty.resolver;
requires io.netty.common;
requires reactor.core;
requires reactor.netty.core;
requires reactor.netty.http;
requires com.azure.storage.blob.batch;
}

2
modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobContainer.java
View file

@ -138,7 +138,7 @@ public class AzureBlobContainer extends AbstractBlobContainer {
}
@Override
public DeleteResult delete(OperationPurpose purpose) {
public DeleteResult delete(OperationPurpose purpose) throws IOException {
return blobStore.deleteBlobDirectory(purpose, keyPath);
}

175
modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureBlobStore.java
View file

@ -25,6 +25,10 @@ import com.azure.storage.blob.BlobContainerAsyncClient;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceAsyncClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.batch.BlobBatch;
import com.azure.storage.blob.batch.BlobBatchAsyncClient;
import com.azure.storage.blob.batch.BlobBatchClientBuilder;
import com.azure.storage.blob.batch.BlobBatchStorageException;
import com.azure.storage.blob.models.BlobErrorCode;
import com.azure.storage.blob.models.BlobItem;
import com.azure.storage.blob.models.BlobItemProperties;
@ -99,6 +103,8 @@ import static org.elasticsearch.core.Strings.format;
public class AzureBlobStore implements BlobStore {
private static final Logger logger = LogManager.getLogger(AzureBlobStore.class);
// See https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch#request-body
public static final int MAX_ELEMENTS_PER_BATCH = 256;
private static final long DEFAULT_READ_CHUNK_SIZE = new ByteSizeValue(32, ByteSizeUnit.MB).getBytes();
private static final int DEFAULT_UPLOAD_BUFFERS_SIZE = (int) new ByteSizeValue(64, ByteSizeUnit.KB).getBytes();
@ -110,6 +116,8 @@ public class AzureBlobStore implements BlobStore {
private final String container;
private final LocationMode locationMode;
private final ByteSizeValue maxSinglePartUploadSize;
private final int deletionBatchSize;
private final int maxConcurrentBatchDeletes;
private final RequestMetricsRecorder requestMetricsRecorder;
private final AzureClientProvider.RequestMetricsHandler requestMetricsHandler;
@ -129,6 +137,8 @@ public class AzureBlobStore implements BlobStore {
// locationMode is set per repository, not per client
this.locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings());
this.maxSinglePartUploadSize = Repository.MAX_SINGLE_PART_UPLOAD_SIZE_SETTING.get(metadata.settings());
this.deletionBatchSize = Repository.DELETION_BATCH_SIZE_SETTING.get(metadata.settings());
this.maxConcurrentBatchDeletes = Repository.MAX_CONCURRENT_BATCH_DELETES_SETTING.get(metadata.settings());
List<RequestMatcher> requestMatchers = List.of(
new RequestMatcher((httpMethod, url) -> httpMethod == HttpMethod.HEAD, Operation.GET_BLOB_PROPERTIES),
@ -147,17 +157,14 @@ public class AzureBlobStore implements BlobStore {
&& isPutBlockRequest(httpMethod, url) == false
&& isPutBlockListRequest(httpMethod, url) == false,
Operation.PUT_BLOB
)
),
new RequestMatcher(AzureBlobStore::isBlobBatch, Operation.BLOB_BATCH)
);
this.requestMetricsHandler = (purpose, method, url, metrics) -> {
try {
URI uri = url.toURI();
String path = uri.getPath() == null ? "" : uri.getPath();
// Batch delete requests
if (path.contains(container) == false) {
return;
}
assert path.contains(container) : uri.toString();
} catch (URISyntaxException ignored) {
return;
@ -172,6 +179,10 @@ public class AzureBlobStore implements BlobStore {
};
}
private static boolean isBlobBatch(HttpMethod method, URL url) {
return method == HttpMethod.POST && url.getQuery() != null && url.getQuery().contains("comp=batch");
}
private static boolean isListRequest(HttpMethod httpMethod, URL url) {
return httpMethod == HttpMethod.GET && url.getQuery() != null && url.getQuery().contains("comp=list");
}
@ -231,95 +242,101 @@ public class AzureBlobStore implements BlobStore {
}
}
// number of concurrent blob delete requests to use while bulk deleting
private static final int CONCURRENT_DELETES = 100;
public DeleteResult deleteBlobDirectory(OperationPurpose purpose, String path) {
public DeleteResult deleteBlobDirectory(OperationPurpose purpose, String path) throws IOException {
final AtomicInteger blobsDeleted = new AtomicInteger(0);
final AtomicLong bytesDeleted = new AtomicLong(0);
SocketAccess.doPrivilegedVoidException(() -> {
final BlobContainerAsyncClient blobContainerAsyncClient = asyncClient(purpose).getBlobContainerAsyncClient(container);
final AzureBlobServiceClient client = getAzureBlobServiceClientClient(purpose);
final BlobContainerAsyncClient blobContainerAsyncClient = client.getAsyncClient().getBlobContainerAsyncClient(container);
final ListBlobsOptions options = new ListBlobsOptions().setPrefix(path)
.setDetails(new BlobListDetails().setRetrieveMetadata(true));
try {
blobContainerAsyncClient.listBlobs(options, null).flatMap(blobItem -> {
if (blobItem.isPrefix() != null && blobItem.isPrefix()) {
return Mono.empty();
} else {
final String blobName = blobItem.getName();
BlobAsyncClient blobAsyncClient = blobContainerAsyncClient.getBlobAsyncClient(blobName);
final Mono<Void> deleteTask = getDeleteTask(blobName, blobAsyncClient);
bytesDeleted.addAndGet(blobItem.getProperties().getContentLength());
blobsDeleted.incrementAndGet();
return deleteTask;
}
}, CONCURRENT_DELETES).then().block();
} catch (Exception e) {
filterDeleteExceptionsAndRethrow(e, new IOException("Deleting directory [" + path + "] failed"));
}
final Flux<String> blobsFlux = blobContainerAsyncClient.listBlobs(options).filter(bi -> bi.isPrefix() == false).map(bi -> {
bytesDeleted.addAndGet(bi.getProperties().getContentLength());
blobsDeleted.incrementAndGet();
return bi.getName();
});
deleteListOfBlobs(client, blobsFlux);
});
return new DeleteResult(blobsDeleted.get(), bytesDeleted.get());
}
private static void filterDeleteExceptionsAndRethrow(Exception e, IOException exception) throws IOException {
int suppressedCount = 0;
for (Throwable suppressed : e.getSuppressed()) {
// We're only interested about the blob deletion exceptions and not in the reactor internals exceptions
if (suppressed instanceof IOException) {
exception.addSuppressed(suppressed);
suppressedCount++;
if (suppressedCount > 10) {
break;
}
}
@Override
public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator<String> blobNames) throws IOException {
if (blobNames.hasNext() == false) {
return;
}
SocketAccess.doPrivilegedVoidException(
() -> deleteListOfBlobs(
getAzureBlobServiceClientClient(purpose),
Flux.fromStream(StreamSupport.stream(Spliterators.spliteratorUnknownSize(blobNames, Spliterator.ORDERED), false))
)
);
}
private void deleteListOfBlobs(AzureBlobServiceClient azureBlobServiceClient, Flux<String> blobNames) throws IOException {
// We need to use a container-scoped BlobBatchClient, so the restype=container parameter
// is sent, and we can support all SAS token types
// See https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=shared-access-signatures#authorization
final BlobBatchAsyncClient batchAsyncClient = new BlobBatchClientBuilder(
azureBlobServiceClient.getAsyncClient().getBlobContainerAsyncClient(container)
).buildAsyncClient();
final List<Throwable> errors;
final AtomicInteger errorsCollected = new AtomicInteger(0);
try {
errors = blobNames.buffer(deletionBatchSize).flatMap(blobs -> {
final BlobBatch blobBatch = batchAsyncClient.getBlobBatch();
blobs.forEach(blob -> blobBatch.deleteBlob(container, blob));
return batchAsyncClient.submitBatch(blobBatch).then(Mono.<Throwable>empty()).onErrorResume(t -> {
// Ignore errors that are just 404s, send other errors downstream as values
if (AzureBlobStore.isIgnorableBatchDeleteException(t)) {
return Mono.empty();
} else {
// Propagate the first 10 errors only
if (errorsCollected.getAndIncrement() < 10) {
return Mono.just(t);
} else {
return Mono.empty();
}
}
});
}, maxConcurrentBatchDeletes).collectList().block();
} catch (Exception e) {
throw new IOException("Error deleting batches", e);
}
if (errors.isEmpty() == false) {
final int totalErrorCount = errorsCollected.get();
final String errorMessage = totalErrorCount > errors.size()
? "Some errors occurred deleting batches, the first "
+ errors.size()
+ " are included as suppressed, but the total count was "
+ totalErrorCount
: "Some errors occurred deleting batches, all errors included as suppressed";
final IOException ex = new IOException(errorMessage);
errors.forEach(ex::addSuppressed);
throw ex;
}
throw exception;
}
/**
* {@inheritDoc}
* <p>
* Note that in this Azure implementation we issue a series of individual
* <a href="https://learn.microsoft.com/en-us/rest/api/storageservices/delete-blob">delete blob</a> calls rather than aggregating
* deletions into <a href="https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch">blob batch</a> calls.
* The reason for this is that the blob batch endpoint has limited support for SAS token authentication.
* We can ignore {@link BlobBatchStorageException}s when they are just telling us some of the files were not found
*
* @see <a href="https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=shared-access-signatures#authorization">
* API docs around SAS auth limitations</a>
* @see <a href="https://github.com/Azure/azure-storage-java/issues/538">Java SDK issue</a>
* @see <a href="https://github.com/elastic/elasticsearch/pull/65140#discussion_r528752070">Discussion on implementing PR</a>
* @param exception An exception throw by batch delete
* @return true if it is safe to ignore, false otherwise
*/
@Override
public void deleteBlobsIgnoringIfNotExists(OperationPurpose purpose, Iterator<String> blobs) {
if (blobs.hasNext() == false) {
return;
}
BlobServiceAsyncClient asyncClient = asyncClient(purpose);
SocketAccess.doPrivilegedVoidException(() -> {
final BlobContainerAsyncClient blobContainerClient = asyncClient.getBlobContainerAsyncClient(container);
try {
Flux.fromStream(StreamSupport.stream(Spliterators.spliteratorUnknownSize(blobs, Spliterator.ORDERED), false))
.flatMap(blob -> getDeleteTask(blob, blobContainerClient.getBlobAsyncClient(blob)), CONCURRENT_DELETES)
.then()
.block();
} catch (Exception e) {
filterDeleteExceptionsAndRethrow(e, new IOException("Unable to delete blobs"));
private static boolean isIgnorableBatchDeleteException(Throwable exception) {
if (exception instanceof BlobBatchStorageException bbse) {
final Iterable<BlobStorageException> batchExceptions = bbse.getBatchExceptions();
for (BlobStorageException bse : batchExceptions) {
// If any requests failed with something other than a BLOB_NOT_FOUND, it is not ignorable
if (BlobErrorCode.BLOB_NOT_FOUND.equals(bse.getErrorCode()) == false) {
return false;
}
}
});
}
private static Mono<Void> getDeleteTask(String blobName, BlobAsyncClient blobAsyncClient) {
return blobAsyncClient.delete()
// Ignore not found blobs, as it's possible that due to network errors a request
// for an already deleted blob is retried, causing an error.
.onErrorResume(
e -> e instanceof BlobStorageException blobStorageException && blobStorageException.getStatusCode() == 404,
throwable -> Mono.empty()
)
.onErrorMap(throwable -> new IOException("Error deleting blob " + blobName, throwable));
return true;
}
return false;
}
public InputStream getInputStream(OperationPurpose purpose, String blob, long position, final @Nullable Long length) {
@ -363,8 +380,7 @@ public class AzureBlobStore implements BlobStore {
for (final BlobItem blobItem : containerClient.listBlobsByHierarchy("/", listBlobsOptions, null)) {
BlobItemProperties properties = blobItem.getProperties();
Boolean isPrefix = blobItem.isPrefix();
if (isPrefix != null && isPrefix) {
if (blobItem.isPrefix()) {
continue;
}
String blobName = blobItem.getName().substring(keyPath.length());
@ -689,7 +705,8 @@ public class AzureBlobStore implements BlobStore {
GET_BLOB_PROPERTIES("GetBlobProperties"),
PUT_BLOB("PutBlob"),
PUT_BLOCK("PutBlock"),
PUT_BLOCK_LIST("PutBlockList");
PUT_BLOCK_LIST("PutBlockList"),
BLOB_BATCH("BlobBatch");
private final String key;

14
modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureClientProvider.java
View file

@ -317,6 +317,11 @@ class AzureClientProvider extends AbstractLifecycleComponent {
@Override
public Mono<HttpResponse> process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) {
if (requestIsPartOfABatch(context)) {
// Batch deletes fire once for each of the constituent requests, and they have a null response. Ignore those, we'll track
// metrics at the bulk level.
return next.process();
}
Optional<Object> metricsData = context.getData(RequestMetricsTracker.ES_REQUEST_METRICS_CONTEXT_KEY);
if (metricsData.isPresent() == false) {
assert false : "No metrics object associated with request " + context.getHttpRequest();
@ -361,6 +366,11 @@ class AzureClientProvider extends AbstractLifecycleComponent {
@Override
public Mono<HttpResponse> process(HttpPipelineCallContext context, HttpPipelineNextPolicy next) {
if (requestIsPartOfABatch(context)) {
// Batch deletes fire once for each of the constituent requests, and they have a null response. Ignore those, we'll track
// metrics at the bulk level.
return next.process();
}
final RequestMetrics requestMetrics = new RequestMetrics();
context.setData(ES_REQUEST_METRICS_CONTEXT_KEY, requestMetrics);
return next.process().doOnSuccess((httpResponse) -> {
@ -389,6 +399,10 @@ class AzureClientProvider extends AbstractLifecycleComponent {
}
}
private static boolean requestIsPartOfABatch(HttpPipelineCallContext context) {
return context.getData("Batch-Operation-Info").isPresent();
}
/**
* The {@link RequestMetricsTracker} calls this when a request completes
*/

15
modules/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java
View file

@ -87,6 +87,21 @@ public class AzureRepository extends MeteredBlobStoreRepository {
DEFAULT_MAX_SINGLE_UPLOAD_SIZE,
Property.NodeScope
);
/**
* The batch size for batched delete requests
*/
static final Setting<Integer> DELETION_BATCH_SIZE_SETTING = Setting.intSetting(
"delete_objects_max_size",
AzureBlobStore.MAX_ELEMENTS_PER_BATCH,
1,
AzureBlobStore.MAX_ELEMENTS_PER_BATCH
);
/**
* The maximum number of concurrent batch deletes
*/
static final Setting<Integer> MAX_CONCURRENT_BATCH_DELETES_SETTING = Setting.intSetting("max_concurrent_batch_deletes", 10, 1, 100);
}
private final ByteSizeValue chunkSize;

10
modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerStatsTests.java
View file

@ -18,6 +18,7 @@ import org.junit.Before;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
public class AzureBlobContainerStatsTests extends AbstractAzureServerTestCase {
@ -47,6 +48,8 @@ public class AzureBlobContainerStatsTests extends AbstractAzureServerTestCase {
os.write(blobContent);
os.flush();
});
// BLOB_BATCH
blobStore.deleteBlobsIgnoringIfNotExists(purpose, List.of(randomIdentifier(), randomIdentifier(), randomIdentifier()).iterator());
Map<String, Long> stats = blobStore.stats();
String statsMapString = stats.toString();
@ -55,6 +58,7 @@ public class AzureBlobContainerStatsTests extends AbstractAzureServerTestCase {
assertEquals(statsMapString, Long.valueOf(1L), stats.get(statsKey(purpose, AzureBlobStore.Operation.GET_BLOB_PROPERTIES)));
assertEquals(statsMapString, Long.valueOf(1L), stats.get(statsKey(purpose, AzureBlobStore.Operation.PUT_BLOCK)));
assertEquals(statsMapString, Long.valueOf(1L), stats.get(statsKey(purpose, AzureBlobStore.Operation.PUT_BLOCK_LIST)));
assertEquals(statsMapString, Long.valueOf(1L), stats.get(statsKey(purpose, AzureBlobStore.Operation.BLOB_BATCH)));
}
public void testOperationPurposeIsNotReflectedInBlobStoreStatsWhenNotServerless() throws IOException {
@ -79,6 +83,11 @@ public class AzureBlobContainerStatsTests extends AbstractAzureServerTestCase {
os.write(blobContent);
os.flush();
});
// BLOB_BATCH
blobStore.deleteBlobsIgnoringIfNotExists(
purpose,
List.of(randomIdentifier(), randomIdentifier(), randomIdentifier()).iterator()
);
}
Map<String, Long> stats = blobStore.stats();
@ -88,6 +97,7 @@ public class AzureBlobContainerStatsTests extends AbstractAzureServerTestCase {
assertEquals(statsMapString, Long.valueOf(repeatTimes), stats.get(AzureBlobStore.Operation.GET_BLOB_PROPERTIES.getKey()));
assertEquals(statsMapString, Long.valueOf(repeatTimes), stats.get(AzureBlobStore.Operation.PUT_BLOCK.getKey()));
assertEquals(statsMapString, Long.valueOf(repeatTimes), stats.get(AzureBlobStore.Operation.PUT_BLOCK_LIST.getKey()));
assertEquals(statsMapString, Long.valueOf(repeatTimes), stats.get(AzureBlobStore.Operation.BLOB_BATCH.getKey()));
}
private static String statsKey(OperationPurpose purpose, AzureBlobStore.Operation operation) {

33
muted-tests.yml
View file

@ -23,9 +23,6 @@ tests:
- class: org.elasticsearch.xpack.security.authz.store.NativePrivilegeStoreCacheTests
method: testPopulationOfCacheWhenLoadingPrivilegesForAllApplications
issue: https://github.com/elastic/elasticsearch/issues/110789
- class: org.elasticsearch.xpack.searchablesnapshots.cache.common.CacheFileTests
method: testCacheFileCreatedAsSparseFile
issue: https://github.com/elastic/elasticsearch/issues/110801
- class: org.elasticsearch.nativeaccess.VectorSystemPropertyTests
method: testSystemPropertyDisabled
issue: https://github.com/elastic/elasticsearch/issues/110949
@ -258,30 +255,30 @@ tests:
- class: org.elasticsearch.xpack.test.rest.XPackRestIT
method: test {p0=esql/60_usage/Basic ESQL usage output (telemetry)}
issue: https://github.com/elastic/elasticsearch/issues/115231
- class: org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleEngineTests
method: testAddWithNoLastCheckedTimeButHasActivationTimeExecutesBeforeInitialInterval
issue: https://github.com/elastic/elasticsearch/issues/115339
- class: org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleEngineTests
method: testWatchWithLastCheckedTimeExecutesBeforeInitialInterval
issue: https://github.com/elastic/elasticsearch/issues/115354
- class: org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleEngineTests
method: testAddWithLastCheckedTimeExecutesBeforeInitialInterval
issue: https://github.com/elastic/elasticsearch/issues/115356
- class: org.elasticsearch.xpack.inference.DefaultEndPointsIT
method: testInferDeploysDefaultE5
issue: https://github.com/elastic/elasticsearch/issues/115361
- class: org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleEngineTests
method: testWatchWithNoLastCheckedTimeButHasActivationTimeExecutesBeforeInitialInterval
issue: https://github.com/elastic/elasticsearch/issues/115368
- class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
method: testProcessFileChanges
issue: https://github.com/elastic/elasticsearch/issues/115280
- class: org.elasticsearch.smoketest.SmokeTestIngestWithAllDepsClientYamlTestSuiteIT
method: test {yaml=ingest/80_ingest_simulate/Test mapping addition works with legacy templates}
issue: https://github.com/elastic/elasticsearch/issues/115412
- class: org.elasticsearch.xpack.security.FileSettingsRoleMappingsRestartIT
method: testFileSettingsReprocessedOnRestartWithoutVersionChange
issue: https://github.com/elastic/elasticsearch/issues/115450
- class: org.elasticsearch.xpack.restart.MLModelDeploymentFullClusterRestartIT
method: testDeploymentSurvivesRestart {cluster=UPGRADED}
issue: https://github.com/elastic/elasticsearch/issues/115528
- class: org.elasticsearch.test.apmintegration.MetricsApmIT
method: testApmIntegration
issue: https://github.com/elastic/elasticsearch/issues/115415
- class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
method: test {yaml=reference/esql/esql-across-clusters/line_197}
issue: https://github.com/elastic/elasticsearch/issues/115575
- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
method: test {yaml=cluster.stats/30_ccs_stats/cross-cluster search stats search}
issue: https://github.com/elastic/elasticsearch/issues/115600
- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
method: test {yaml=indices.create/10_basic/Create lookup index}
issue: https://github.com/elastic/elasticsearch/issues/115605
# Examples:
#

11
qa/packaging/src/test/java/org/elasticsearch/packaging/test/DockerTests.java
View file

@ -169,10 +169,7 @@ public class DockerTests extends PackagingTestCase {
* Checks that no plugins are initially active.
*/
public void test020PluginsListWithNoPlugins() {
assumeTrue(
"Only applies to non-Cloud images",
distribution.packaging != Packaging.DOCKER_CLOUD && distribution().packaging != Packaging.DOCKER_CLOUD_ESS
);
assumeTrue("Only applies to non-Cloud images", distribution().packaging != Packaging.DOCKER_CLOUD_ESS);
final Installation.Executables bin = installation.executables();
final Result r = sh.run(bin.pluginTool + " list");
@ -1116,8 +1113,8 @@ public class DockerTests extends PackagingTestCase {
*/
public void test171AdditionalCliOptionsAreForwarded() throws Exception {
assumeTrue(
"Does not apply to Cloud and Cloud ESS images, because they don't use the default entrypoint",
distribution.packaging != Packaging.DOCKER_CLOUD && distribution().packaging != Packaging.DOCKER_CLOUD_ESS
"Does not apply to Cloud ESS images, because they don't use the default entrypoint",
distribution().packaging != Packaging.DOCKER_CLOUD_ESS
);
runContainer(distribution(), builder().runArgs("bin/elasticsearch", "-Ecluster.name=kimchy").envVar("ELASTIC_PASSWORD", PASSWORD));
@ -1204,7 +1201,7 @@ public class DockerTests extends PackagingTestCase {
* Check that the Cloud image contains the required Beats
*/
public void test400CloudImageBundlesBeats() {
assumeTrue(distribution.packaging == Packaging.DOCKER_CLOUD || distribution.packaging == Packaging.DOCKER_CLOUD_ESS);
assumeTrue(distribution.packaging == Packaging.DOCKER_CLOUD_ESS);
final List<String> contents = listContents("/opt");
assertThat("Expected beats in /opt", contents, hasItems("filebeat", "metricbeat"));

5
qa/packaging/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java
View file

@ -436,10 +436,7 @@ public class KeystoreManagementTests extends PackagingTestCase {
switch (distribution.packaging) {
case TAR, ZIP -> assertThat(keystore, file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660));
case DEB, RPM -> assertThat(keystore, file(File, "root", "elasticsearch", p660));
case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> assertThat(
keystore,
DockerFileMatcher.file(p660)
);
case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> assertThat(keystore, DockerFileMatcher.file(p660));
default -> throw new IllegalStateException("Unknown Elasticsearch packaging type.");
}
}

6
qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java
View file

@ -245,7 +245,7 @@ public abstract class PackagingTestCase extends Assert {
installation = Packages.installPackage(sh, distribution);
Packages.verifyPackageInstallation(installation, distribution, sh);
}
case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> {
case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> {
installation = Docker.runContainer(distribution);
Docker.verifyContainerInstallation(installation);
}
@ -335,7 +335,6 @@ public abstract class PackagingTestCase extends Assert {
case DOCKER:
case DOCKER_UBI:
case DOCKER_IRON_BANK:
case DOCKER_CLOUD:
case DOCKER_CLOUD_ESS:
case DOCKER_WOLFI:
// nothing, "installing" docker image is running it
@ -358,7 +357,6 @@ public abstract class PackagingTestCase extends Assert {
case DOCKER:
case DOCKER_UBI:
case DOCKER_IRON_BANK:
case DOCKER_CLOUD:
case DOCKER_CLOUD_ESS:
case DOCKER_WOLFI:
// nothing, "installing" docker image is running it
@ -373,7 +371,7 @@ public abstract class PackagingTestCase extends Assert {
switch (distribution.packaging) {
case TAR, ZIP -> Archives.assertElasticsearchStarted(installation);
case DEB, RPM -> Packages.assertElasticsearchStarted(sh, installation);
case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> Docker.waitForElasticsearchToStart();
case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> Docker.waitForElasticsearchToStart();
default -> throw new IllegalStateException("Unknown Elasticsearch packaging type.");
}
}

5
qa/packaging/src/test/java/org/elasticsearch/packaging/util/Distribution.java
View file

@ -33,8 +33,6 @@ public class Distribution {
this.packaging = Packaging.DOCKER_UBI;
} else if (filename.endsWith(".ironbank.tar")) {
this.packaging = Packaging.DOCKER_IRON_BANK;
} else if (filename.endsWith(".cloud.tar")) {
this.packaging = Packaging.DOCKER_CLOUD;
} else if (filename.endsWith(".cloud-ess.tar")) {
this.packaging = Packaging.DOCKER_CLOUD_ESS;
} else if (filename.endsWith(".wolfi.tar")) {
@ -63,7 +61,7 @@ public class Distribution {
*/
public boolean isDocker() {
return switch (packaging) {
case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> true;
case DOCKER, DOCKER_UBI, DOCKER_IRON_BANK, DOCKER_CLOUD_ESS, DOCKER_WOLFI -> true;
default -> false;
};
}
@ -77,7 +75,6 @@ public class Distribution {
DOCKER(".docker.tar", Platforms.isDocker()),
DOCKER_UBI(".ubi.tar", Platforms.isDocker()),
DOCKER_IRON_BANK(".ironbank.tar", Platforms.isDocker()),
DOCKER_CLOUD(".cloud.tar", Platforms.isDocker()),
DOCKER_CLOUD_ESS(".cloud-ess.tar", Platforms.isDocker()),
DOCKER_WOLFI(".wolfi.tar", Platforms.isDocker());

2
qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java
View file

@ -532,7 +532,7 @@ public class Docker {
)
);
if (es.distribution.packaging == Packaging.DOCKER_CLOUD || es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS) {
if (es.distribution.packaging == Packaging.DOCKER_CLOUD_ESS) {
verifyCloudContainerInstallation(es);
}
}

1
qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/DockerRun.java
View file

@ -165,7 +165,6 @@ public class DockerRun {
case DOCKER -> "";
case DOCKER_UBI -> "-ubi";
case DOCKER_IRON_BANK -> "-ironbank";
case DOCKER_CLOUD -> "-cloud";
case DOCKER_CLOUD_ESS -> "-cloud-ess";
case DOCKER_WOLFI -> "-wolfi";
default -> throw new IllegalStateException("Unexpected distribution packaging type: " + distribution.packaging);

13
qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java
View file

@ -11,7 +11,6 @@ package org.elasticsearch.upgrades;
import com.carrotsearch.randomizedtesting.annotations.Name;
import org.elasticsearch.Build;
import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesRequest;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.ResponseException;
@ -82,8 +81,7 @@ public class DesiredNodesUpgradeIT extends AbstractRollingUpgradeTestCase {
Settings.builder().put(NODE_NAME_SETTING.getKey(), nodeName).build(),
1238.49922909,
ByteSizeValue.ofGb(32),
ByteSizeValue.ofGb(128),
clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version()
ByteSizeValue.ofGb(128)
)
)
.toList();
@ -153,8 +151,7 @@ public class DesiredNodesUpgradeIT extends AbstractRollingUpgradeTestCase {
Settings.builder().put(NODE_NAME_SETTING.getKey(), nodeName).build(),
processorsPrecision == ProcessorsPrecision.DOUBLE ? randomDoubleProcessorCount() : 0.5f,
ByteSizeValue.ofGb(randomIntBetween(10, 24)),
ByteSizeValue.ofGb(randomIntBetween(128, 256)),
clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version()
ByteSizeValue.ofGb(randomIntBetween(128, 256))
)
)
.toList();
@ -167,8 +164,7 @@ public class DesiredNodesUpgradeIT extends AbstractRollingUpgradeTestCase {
Settings.builder().put(NODE_NAME_SETTING.getKey(), nodeName).build(),
new DesiredNode.ProcessorsRange(minProcessors, minProcessors + randomIntBetween(10, 20)),
ByteSizeValue.ofGb(randomIntBetween(10, 24)),
ByteSizeValue.ofGb(randomIntBetween(128, 256)),
clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version()
ByteSizeValue.ofGb(randomIntBetween(128, 256))
);
}).toList();
}
@ -182,8 +178,7 @@ public class DesiredNodesUpgradeIT extends AbstractRollingUpgradeTestCase {
Settings.builder().put(NODE_NAME_SETTING.getKey(), nodeName).build(),
randomIntBetween(1, 24),
ByteSizeValue.ofGb(randomIntBetween(10, 24)),
ByteSizeValue.ofGb(randomIntBetween(128, 256)),
clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version()
ByteSizeValue.ofGb(randomIntBetween(128, 256))
)
)
.toList();

9
qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml
View file

@ -1537,6 +1537,8 @@ setup:
- not_exists: docs.0.doc.error
- do:
allowed_warnings:
- "index [foo-1] matches multiple legacy templates [global, my-legacy-template], composable templates will only match a single template"
indices.create:
index: foo-1
- match: { acknowledged: true }
@ -1586,6 +1588,13 @@ setup:
cluster_features: ["simulate.support.non.template.mapping"]
reason: "ingest simulate support for indices with mappings that didn't come from templates added in 8.17"
# A global match-everything legacy template is added to the cluster sometimes (rarely). We have to get rid of this template if it exists
# because this test is making sure we get correct behavior when an index matches *no* template:
- do:
indices.delete_template:
name: '*'
ignore: 404
# First, make sure that validation fails before we create the index (since we are only defining to bar field but trying to index a value
# for foo.
- do:

3
rest-api-spec/build.gradle
View file

@ -60,4 +60,7 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility")
task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility")
task.skipTest("indices.create/21_synthetic_source_stored/object param - nested object with stored array", "temporary until backported")
task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy")
task.skipTest("cluster.desired_nodes/10_basic/Test delete desired nodes with node_version generates a warning", "node_version warning is removed in 9.0")
task.skipTest("cluster.desired_nodes/10_basic/Test update desired nodes with node_version generates a warning", "node_version warning is removed in 9.0")
})

4
rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
View file

@ -36,10 +36,6 @@
"type":"string",
"description":"a short version of the Accept header, e.g. json, yaml"
},
"local":{
"type":"boolean",
"description":"Return local information, do not retrieve the state from master node (default: false)"
},
"h":{
"type":"list",
"description":"Comma-separated list of column names to display"

4
rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
View file

@ -61,10 +61,6 @@
],
"default":"all",
"description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
},
"local":{
"type":"boolean",
"description":"Return local information, do not retrieve the state from master node (default: false)"
}
}
}

4
rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_alias.json
View file

@ -79,10 +79,6 @@
],
"default": "all",
"description":"Whether to expand wildcard expression to concrete indices that are open, closed or both."
},
"local":{
"type":"boolean",
"description":"Return local information, do not retrieve the state from master node (default: false)"
}
}
}

13
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.aliases/10_basic.yml
View file

@ -484,16 +484,3 @@
test_alias \s+ test_index\n
my_alias \s+ test_index\n
$/
---
"Deprecated local parameter":
- requires:
cluster_features: ["gte_v8.12.0"]
test_runner_features: ["warnings"]
reason: verifying deprecation warnings from 8.12.0 onwards
- do:
cat.aliases:
local: true
warnings:
- "the [?local=true] query parameter to cat-aliases requests has no effect and will be removed in a future version"

95
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.desired_nodes/10_basic.yml
View file

@ -59,61 +59,6 @@ teardown:
- contains: { nodes: { settings: { node: { name: "instance-000187" } }, processors: 8.5, memory: "64gb", storage: "128gb" } }
- contains: { nodes: { settings: { node: { name: "instance-000188" } }, processors: 16.0, memory: "128gb", storage: "1tb" } }
---
"Test update desired nodes with node_version generates a warning":
- skip:
reason: "contains is a newly added assertion"
features: ["contains", "allowed_warnings"]
- do:
cluster.state: {}
# Get master node id
- set: { master_node: master }
- do:
nodes.info: {}
- set: { nodes.$master.version: es_version }
- do:
_internal.update_desired_nodes:
history_id: "test"
version: 1
body:
nodes:
- { settings: { "node.name": "instance-000187" }, processors: 8.5, memory: "64gb", storage: "128gb", node_version: $es_version }
allowed_warnings:
- "[version removal] Specifying node_version in desired nodes requests is deprecated."
- match: { replaced_existing_history_id: false }
- do:
_internal.get_desired_nodes: {}
- match:
$body:
history_id: "test"
version: 1
nodes:
- { settings: { node: { name: "instance-000187" } }, processors: 8.5, memory: "64gb", storage: "128gb", node_version: $es_version }
- do:
_internal.update_desired_nodes:
history_id: "test"
version: 2
body:
nodes:
- { settings: { "node.name": "instance-000187" }, processors: 8.5, memory: "64gb", storage: "128gb", node_version: $es_version }
- { settings: { "node.name": "instance-000188" }, processors: 16.0, memory: "128gb", storage: "1tb", node_version: $es_version }
allowed_warnings:
- "[version removal] Specifying node_version in desired nodes requests is deprecated."
- match: { replaced_existing_history_id: false }
- do:
_internal.get_desired_nodes: {}
- match: { history_id: "test" }
- match: { version: 2 }
- length: { nodes: 2 }
- contains: { nodes: { settings: { node: { name: "instance-000187" } }, processors: 8.5, memory: "64gb", storage: "128gb", node_version: $es_version } }
- contains: { nodes: { settings: { node: { name: "instance-000188" } }, processors: 16.0, memory: "128gb", storage: "1tb", node_version: $es_version } }
---
"Test update move to a new history id":
- skip:
reason: "contains is a newly added assertion"
@ -199,46 +144,6 @@ teardown:
_internal.get_desired_nodes: {}
- match: { status: 404 }
---
"Test delete desired nodes with node_version generates a warning":
- skip:
features: allowed_warnings
- do:
cluster.state: {}
- set: { master_node: master }
- do:
nodes.info: {}
- set: { nodes.$master.version: es_version }
- do:
_internal.update_desired_nodes:
history_id: "test"
version: 1
body:
nodes:
- { settings: { "node.external_id": "instance-000187" }, processors: 8.0, memory: "64gb", storage: "128gb", node_version: $es_version }
allowed_warnings:
- "[version removal] Specifying node_version in desired nodes requests is deprecated."
- match: { replaced_existing_history_id: false }
- do:
_internal.get_desired_nodes: {}
- match:
$body:
history_id: "test"
version: 1
nodes:
- { settings: { node: { external_id: "instance-000187" } }, processors: 8.0, memory: "64gb", storage: "128gb", node_version: $es_version }
- do:
_internal.delete_desired_nodes: {}
- do:
catch: missing
_internal.get_desired_nodes: {}
- match: { status: 404 }
---
"Test update desired nodes is idempotent":
- skip:
reason: "contains is a newly added assertion"

67
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml
View file

@ -149,3 +149,70 @@
indices.exists_alias:
name: logs_2022-12-31
- is_true: ''
---
"Create lookup index":
- requires:
test_runner_features: [ capabilities, default_shards ]
capabilities:
- method: PUT
path: /{index}
capabilities: [ lookup_index_mode ]
reason: "Support for 'lookup' index mode capability required"
- do:
indices.create:
index: "test_lookup"
body:
settings:
index.mode: lookup
- do:
indices.get_settings:
index: test_lookup
- match: { test_lookup.settings.index.number_of_shards: "1"}
- match: { test_lookup.settings.index.auto_expand_replicas: "0-all"}
---
"Create lookup index with one shard":
- requires:
test_runner_features: [ capabilities, default_shards ]
capabilities:
- method: PUT
path: /{index}
capabilities: [ lookup_index_mode ]
reason: "Support for 'lookup' index mode capability required"
- do:
indices.create:
index: "test_lookup"
body:
settings:
index:
mode: lookup
number_of_shards: 1
- do:
indices.get_settings:
index: test_lookup
- match: { test_lookup.settings.index.number_of_shards: "1"}
- match: { test_lookup.settings.index.auto_expand_replicas: "0-all"}
---
"Create lookup index with two shards":
- requires:
test_runner_features: [ capabilities ]
capabilities:
- method: PUT
path: /{index}
capabilities: [ lookup_index_mode ]
reason: "Support for 'lookup' index mode capability required"
- do:
catch: /illegal_argument_exception/
indices.create:
index: test_lookup
body:
settings:
index.mode: lookup
index.number_of_shards: 2

14
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.exists_alias/10_basic.yml
View file

@ -34,17 +34,3 @@
name: test_alias
- is_false: ''
---
"Test indices.exists_alias with local flag":
- skip:
features: ["allowed_warnings"]
- do:
indices.exists_alias:
name: test_alias
local: true
allowed_warnings:
- "the [?local=true] query parameter to get-aliases requests has no effect and will be removed in a future version"
- is_false: ''

29
rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.get_alias/10_basic.yml
View file

@ -289,21 +289,6 @@ setup:
index: non-existent
name: foo
---
"Get alias with local flag":
- skip:
features: ["allowed_warnings"]
- do:
indices.get_alias:
local: true
allowed_warnings:
- "the [?local=true] query parameter to get-aliases requests has no effect and will be removed in a future version"
- is_true: test_index
- is_true: test_index_2
---
"Get alias against closed indices":
- skip:
@ -329,17 +314,3 @@ setup:
- is_true: test_index
- is_false: test_index_2
---
"Deprecated local parameter":
- requires:
cluster_features: "gte_v8.12.0"
test_runner_features: ["warnings"]
reason: verifying deprecation warnings from 8.12.0 onwards
- do:
indices.get_alias:
local: true
warnings:
- "the [?local=true] query parameter to get-aliases requests has no effect and will be removed in a future version"

69
server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerMetricsIT.java Normal file
View file

@ -0,0 +1,69 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.cluster.routing.allocation.allocator;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.PluginsService;
import org.elasticsearch.telemetry.TestTelemetryPlugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.hamcrest.Matcher;
import java.util.Collection;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.not;
public class DesiredBalanceReconcilerMetricsIT extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return CollectionUtils.appendToCopy(super.nodePlugins(), TestTelemetryPlugin.class);
}
public void testDesiredBalanceGaugeMetricsAreOnlyPublishedByCurrentMaster() throws Exception {
internalCluster().ensureAtLeastNumDataNodes(2);
prepareCreate("test").setSettings(indexSettings(2, 1)).get();
ensureGreen();
assertOnlyMasterIsPublishingMetrics();
// fail over and check again
int numFailOvers = randomIntBetween(1, 3);
for (int i = 0; i < numFailOvers; i++) {
internalCluster().restartNode(internalCluster().getMasterName());
ensureGreen();
assertOnlyMasterIsPublishingMetrics();
}
}
private static void assertOnlyMasterIsPublishingMetrics() {
String masterNodeName = internalCluster().getMasterName();
String[] nodeNames = internalCluster().getNodeNames();
for (String nodeName : nodeNames) {
assertMetricsAreBeingPublished(nodeName, nodeName.equals(masterNodeName));
}
}
private static void assertMetricsAreBeingPublished(String nodeName, boolean shouldBePublishing) {
final TestTelemetryPlugin testTelemetryPlugin = internalCluster().getInstance(PluginsService.class, nodeName)
.filterPlugins(TestTelemetryPlugin.class)
.findFirst()
.orElseThrow();
testTelemetryPlugin.resetMeter();
testTelemetryPlugin.collect();
Matcher<Collection<?>> matcher = shouldBePublishing ? not(empty()) : empty();
assertThat(testTelemetryPlugin.getLongGaugeMeasurement(DesiredBalanceMetrics.UNASSIGNED_SHARDS_METRIC_NAME), matcher);
assertThat(testTelemetryPlugin.getLongGaugeMeasurement(DesiredBalanceMetrics.TOTAL_SHARDS_METRIC_NAME), matcher);
assertThat(testTelemetryPlugin.getLongGaugeMeasurement(DesiredBalanceMetrics.UNDESIRED_ALLOCATION_COUNT_METRIC_NAME), matcher);
assertThat(testTelemetryPlugin.getDoubleGaugeMeasurement(DesiredBalanceMetrics.UNDESIRED_ALLOCATION_RATIO_METRIC_NAME), matcher);
}
}

66
server/src/internalClusterTest/java/org/elasticsearch/gateway/GatewayIndexStateIT.java
View file

@ -17,7 +17,6 @@ import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.support.ActiveShardCount;
import org.elasticsearch.client.internal.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.coordination.CoordinationMetadata;
import org.elasticsearch.cluster.metadata.IndexGraveyard;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.MappingMetadata;
@ -27,14 +26,9 @@ import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
import org.elasticsearch.cluster.routing.RoutingTable;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.CheckedConsumer;
import org.elasticsearch.core.IOUtils;
import org.elasticsearch.env.BuildVersion;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.env.NodeMetadata;
import org.elasticsearch.index.IndexVersions;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.indices.IndexClosedException;
@ -46,13 +40,8 @@ import org.elasticsearch.test.InternalTestCluster.RestartCallback;
import org.elasticsearch.xcontent.XContentFactory;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
@ -60,7 +49,6 @@ import static org.elasticsearch.test.NodeRoles.nonDataNode;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.notNullValue;
@ -554,58 +542,4 @@ public class GatewayIndexStateIT extends ESIntegTestCase {
assertHitCount(prepareSearch().setQuery(matchAllQuery()), 1L);
}
public void testHalfDeletedIndexImport() throws Exception {
// It's possible for a 6.x node to add a tombstone for an index but not actually delete the index metadata from disk since that
// deletion is slightly deferred and may race against the node being shut down; if you upgrade to 7.x when in this state then the
// node won't start.
final String nodeName = internalCluster().startNode();
createIndex("test", 1, 0);
ensureGreen("test");
final Metadata metadata = internalCluster().getInstance(ClusterService.class).state().metadata();
final Path[] paths = internalCluster().getInstance(NodeEnvironment.class).nodeDataPaths();
final String nodeId = clusterAdmin().prepareNodesInfo(nodeName).clear().get().getNodes().get(0).getNode().getId();
writeBrokenMeta(nodeEnvironment -> {
for (final Path path : paths) {
IOUtils.rm(path.resolve(PersistedClusterStateService.METADATA_DIRECTORY_NAME));
}
MetaStateWriterUtils.writeGlobalState(
nodeEnvironment,
"test",
Metadata.builder(metadata)
// we remove the manifest file, resetting the term and making this look like an upgrade from 6.x, so must also reset the
// term in the coordination metadata
.coordinationMetadata(CoordinationMetadata.builder(metadata.coordinationMetadata()).term(0L).build())
// add a tombstone but do not delete the index metadata from disk
.putCustom(
IndexGraveyard.TYPE,
IndexGraveyard.builder().addTombstone(metadata.getProject().index("test").getIndex()).build()
)
.build()
);
NodeMetadata.FORMAT.writeAndCleanup(
new NodeMetadata(nodeId, BuildVersion.current(), metadata.getProject().oldestIndexVersion()),
paths
);
});
ensureGreen();
assertBusy(() -> assertThat(internalCluster().getInstance(NodeEnvironment.class).availableIndexFolders(), empty()));
}
private void writeBrokenMeta(CheckedConsumer<NodeEnvironment, IOException> writer) throws Exception {
Map<String, NodeEnvironment> nodeEnvironments = Stream.of(internalCluster().getNodeNames())
.collect(Collectors.toMap(Function.identity(), nodeName -> internalCluster().getInstance(NodeEnvironment.class, nodeName)));
internalCluster().fullRestart(new RestartCallback() {
@Override
public Settings onNodeStopped(String nodeName) throws Exception {
final NodeEnvironment nodeEnvironment = nodeEnvironments.get(nodeName);
writer.accept(nodeEnvironment);
return super.onNodeStopped(nodeName);
}
});
}
}

219
server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java Normal file
View file

@ -0,0 +1,219 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
package org.elasticsearch.index;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
import org.elasticsearch.action.admin.indices.shrink.ResizeAction;
import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
import org.elasticsearch.action.admin.indices.shrink.ResizeType;
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse;
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase;
import java.util.Map;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
public class LookupIndexModeIT extends ESIntegTestCase {
@Override
protected int numberOfShards() {
return 1;
}
public void testBasic() {
internalCluster().ensureAtLeastNumDataNodes(1);
Settings.Builder lookupSettings = Settings.builder().put("index.mode", "lookup");
if (randomBoolean()) {
lookupSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
}
CreateIndexRequest createRequest = new CreateIndexRequest("hosts");
createRequest.settings(lookupSettings);
createRequest.simpleMapping("ip", "type=ip", "os", "type=keyword");
assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createRequest));
Settings settings = client().admin().indices().prepareGetSettings("hosts").get().getIndexToSettings().get("hosts");
assertThat(settings.get("index.mode"), equalTo("lookup"));
assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all"));
Map<String, String> allHosts = Map.of(
"192.168.1.2",
"Windows",
"192.168.1.3",
"MacOS",
"192.168.1.4",
"Linux",
"192.168.1.5",
"Android",
"192.168.1.6",
"iOS",
"192.168.1.7",
"Windows",
"192.168.1.8",
"MacOS",
"192.168.1.9",
"Linux",
"192.168.1.10",
"Linux",
"192.168.1.11",
"Windows"
);
for (Map.Entry<String, String> e : allHosts.entrySet()) {
client().prepareIndex("hosts").setSource("ip", e.getKey(), "os", e.getValue()).get();
}
refresh("hosts");
assertAcked(client().admin().indices().prepareCreate("events").setSettings(Settings.builder().put("index.mode", "logsdb")).get());
int numDocs = between(1, 10);
for (int i = 0; i < numDocs; i++) {
String ip = randomFrom(allHosts.keySet());
String message = randomFrom("login", "logout", "shutdown", "restart");
client().prepareIndex("events").setSource("@timestamp", "2024-01-01", "ip", ip, "message", message).get();
}
refresh("events");
// _search
{
SearchResponse resp = prepareSearch("events", "hosts").setQuery(new MatchQueryBuilder("_index_mode", "lookup"))
.setSize(10000)
.get();
for (SearchHit hit : resp.getHits()) {
assertThat(hit.getIndex(), equalTo("hosts"));
}
assertHitCount(resp, allHosts.size());
resp.decRef();
}
// field_caps
{
FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
request.indices("events", "hosts");
request.fields("*");
request.setMergeResults(false);
request.indexFilter(new MatchQueryBuilder("_index_mode", "lookup"));
var resp = client().fieldCaps(request).actionGet();
assertThat(resp.getIndexResponses(), hasSize(1));
FieldCapabilitiesIndexResponse indexResponse = resp.getIndexResponses().getFirst();
assertThat(indexResponse.getIndexMode(), equalTo(IndexMode.LOOKUP));
assertThat(indexResponse.getIndexName(), equalTo("hosts"));
}
}
public void testRejectMoreThanOneShard() {
int numberOfShards = between(2, 5);
IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> {
client().admin()
.indices()
.prepareCreate("hosts")
.setSettings(Settings.builder().put("index.mode", "lookup").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards))
.setMapping("ip", "type=ip", "os", "type=keyword")
.get();
});
assertThat(
error.getMessage(),
equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided " + numberOfShards)
);
}
public void testResizeLookupIndex() {
Settings.Builder createSettings = Settings.builder().put("index.mode", "lookup");
if (randomBoolean()) {
createSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
}
CreateIndexRequest createIndexRequest = new CreateIndexRequest("lookup-1").settings(createSettings);
assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createIndexRequest));
client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.WRITE, "lookup-1").get();
ResizeRequest clone = new ResizeRequest("lookup-2", "lookup-1");
clone.setResizeType(ResizeType.CLONE);
assertAcked(client().admin().indices().execute(ResizeAction.INSTANCE, clone).actionGet());
Settings settings = client().admin().indices().prepareGetSettings("lookup-2").get().getIndexToSettings().get("lookup-2");
assertThat(settings.get("index.mode"), equalTo("lookup"));
assertThat(settings.get("index.number_of_shards"), equalTo("1"));
assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all"));
ResizeRequest split = new ResizeRequest("lookup-3", "lookup-1");
split.setResizeType(ResizeType.SPLIT);
split.getTargetIndexRequest().settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3));
IllegalArgumentException error = expectThrows(
IllegalArgumentException.class,
() -> client().admin().indices().execute(ResizeAction.INSTANCE, split).actionGet()
);
assertThat(
error.getMessage(),
equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided 3")
);
}
public void testResizeRegularIndexToLookup() {
String dataNode = internalCluster().startDataOnlyNode();
assertAcked(
client().admin()
.indices()
.prepareCreate("regular-1")
.setSettings(
Settings.builder()
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2)
.put("index.routing.allocation.require._name", dataNode)
)
.setMapping("ip", "type=ip", "os", "type=keyword")
.get()
);
client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.WRITE, "regular-1").get();
client().admin()
.indices()
.prepareUpdateSettings("regular-1")
.setSettings(Settings.builder().put("index.number_of_replicas", 0))
.get();
ResizeRequest clone = new ResizeRequest("lookup-3", "regular-1");
clone.setResizeType(ResizeType.CLONE);
clone.getTargetIndexRequest().settings(Settings.builder().put("index.mode", "lookup"));
IllegalArgumentException error = expectThrows(
IllegalArgumentException.class,
() -> client().admin().indices().execute(ResizeAction.INSTANCE, clone).actionGet()
);
assertThat(
error.getMessage(),
equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided 2")
);
ResizeRequest shrink = new ResizeRequest("lookup-4", "regular-1");
shrink.setResizeType(ResizeType.SHRINK);
shrink.getTargetIndexRequest()
.settings(Settings.builder().put("index.mode", "lookup").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1));
error = expectThrows(
IllegalArgumentException.class,
() -> client().admin().indices().execute(ResizeAction.INSTANCE, shrink).actionGet()
);
assertThat(error.getMessage(), equalTo("can't change index.mode of index [regular-1] from [standard] to [lookup]"));
}
public void testDoNotOverrideAutoExpandReplicas() {
internalCluster().ensureAtLeastNumDataNodes(1);
Settings.Builder createSettings = Settings.builder().put("index.mode", "lookup");
if (randomBoolean()) {
createSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
}
createSettings.put("index.auto_expand_replicas", "3-5");
CreateIndexRequest createRequest = new CreateIndexRequest("hosts");
createRequest.settings(createSettings);
createRequest.simpleMapping("ip", "type=ip", "os", "type=keyword");
assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createRequest));
Settings settings = client().admin().indices().prepareGetSettings("hosts").get().getIndexToSettings().get("hosts");
assertThat(settings.get("index.mode"), equalTo("lookup"));
assertThat(settings.get("index.auto_expand_replicas"), equalTo("3-5"));
}
}

64
server/src/internalClusterTest/java/org/elasticsearch/search/ccs/CrossClusterSearchIT.java
View file

@ -755,6 +755,70 @@ public class CrossClusterSearchIT extends AbstractMultiClustersTestCase {
assertNotNull(ee.getCause());
}
public void testClusterDetailsWhenLocalClusterHasNoMatchingIndex() throws Exception {
Map<String, Object> testClusterInfo = setupTwoClusters();
String remoteIndex = (String) testClusterInfo.get("remote.index");
int remoteNumShards = (Integer) testClusterInfo.get("remote.num_shards");
SearchRequest searchRequest = new SearchRequest("nomatch*", REMOTE_CLUSTER + ":" + remoteIndex);
if (randomBoolean()) {
searchRequest = searchRequest.scroll(TimeValue.timeValueMinutes(1));
}
searchRequest.allowPartialSearchResults(false);
if (randomBoolean()) {
searchRequest.setBatchedReduceSize(randomIntBetween(3, 20));
}
boolean minimizeRoundtrips = false;
searchRequest.setCcsMinimizeRoundtrips(minimizeRoundtrips);
boolean dfs = randomBoolean();
if (dfs) {
searchRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH);
}
if (randomBoolean()) {
searchRequest.setPreFilterShardSize(1);
}
searchRequest.source(new SearchSourceBuilder().query(new MatchAllQueryBuilder()).size(10));
assertResponse(client(LOCAL_CLUSTER).search(searchRequest), response -> {
assertNotNull(response);
Clusters clusters = response.getClusters();
assertFalse("search cluster results should BE successful", clusters.hasPartialResults());
assertThat(clusters.getTotal(), equalTo(2));
assertThat(clusters.getClusterStateCount(Cluster.Status.SUCCESSFUL), equalTo(2));
assertThat(clusters.getClusterStateCount(Cluster.Status.SKIPPED), equalTo(0));
assertThat(clusters.getClusterStateCount(Cluster.Status.RUNNING), equalTo(0));
assertThat(clusters.getClusterStateCount(Cluster.Status.PARTIAL), equalTo(0));
assertThat(clusters.getClusterStateCount(Cluster.Status.FAILED), equalTo(0));
Cluster localClusterSearchInfo = clusters.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY);
assertNotNull(localClusterSearchInfo);
assertThat(localClusterSearchInfo.getStatus(), equalTo(Cluster.Status.SUCCESSFUL));
assertThat(localClusterSearchInfo.getIndexExpression(), equalTo("nomatch*"));
assertThat(localClusterSearchInfo.getTotalShards(), equalTo(0));
assertThat(localClusterSearchInfo.getSuccessfulShards(), equalTo(0));
assertThat(localClusterSearchInfo.getSkippedShards(), equalTo(0));
assertThat(localClusterSearchInfo.getFailedShards(), equalTo(0));
assertThat(localClusterSearchInfo.getFailures().size(), equalTo(0));
assertThat(localClusterSearchInfo.getTook().millis(), equalTo(0L));
Cluster remoteClusterSearchInfo = clusters.getCluster(REMOTE_CLUSTER);
assertNotNull(remoteClusterSearchInfo);
assertThat(remoteClusterSearchInfo.getStatus(), equalTo(Cluster.Status.SUCCESSFUL));
assertThat(remoteClusterSearchInfo.getIndexExpression(), equalTo(remoteIndex));
assertThat(remoteClusterSearchInfo.getTotalShards(), equalTo(remoteNumShards));
assertThat(remoteClusterSearchInfo.getSuccessfulShards(), equalTo(remoteNumShards));
assertThat(remoteClusterSearchInfo.getSkippedShards(), equalTo(0));
assertThat(remoteClusterSearchInfo.getFailedShards(), equalTo(0));
assertThat(remoteClusterSearchInfo.getFailures().size(), equalTo(0));
assertThat(remoteClusterSearchInfo.getTook().millis(), greaterThan(0L));
});
}
private static void assertOneFailedShard(Cluster cluster, int totalShards) {
assertNotNull(cluster);
assertThat(cluster.getStatus(), equalTo(Cluster.Status.PARTIAL));

2
server/src/main/java/org/elasticsearch/TransportVersions.java
View file

@ -181,6 +181,8 @@ public class TransportVersions {
public static final TransportVersion ESQL_FIELD_ATTRIBUTE_PARENT_SIMPLIFIED = def(8_775_00_0);
public static final TransportVersion INFERENCE_DONT_PERSIST_ON_READ = def(8_776_00_0);
public static final TransportVersion SIMULATE_MAPPING_ADDITION = def(8_777_00_0);
public static final TransportVersion INTRODUCE_ALL_APPLICABLE_SELECTOR = def(8_778_00_0);
public static final TransportVersion INDEX_MODE_LOOKUP = def(8_779_00_0);
/*
* WARNING: DO NOT MERGE INTO MAIN!

2
server/src/main/java/org/elasticsearch/action/admin/indices/get/GetIndexRequest.java
View file

@ -98,7 +98,7 @@ public class GetIndexRequest extends ClusterInfoRequest<GetIndexRequest> {
super(
DataStream.isFailureStoreFeatureFlagEnabled()
? IndicesOptions.builder(IndicesOptions.strictExpandOpen())
.selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE)
.selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
.build()
: IndicesOptions.strictExpandOpen()
);

5
server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
View file

@ -13,6 +13,7 @@ import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.IndicesRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.support.ActiveShardCount;
import org.elasticsearch.action.support.IndexComponentSelector;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.master.AcknowledgedRequest;
import org.elasticsearch.cluster.metadata.DataStream;
@ -124,8 +125,8 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
);
}
var selectors = indicesOptions.selectorOptions().defaultSelectors();
if (selectors.size() > 1) {
var selector = indicesOptions.selectorOptions().defaultSelector();
if (selector == IndexComponentSelector.ALL_APPLICABLE) {
validationException = addValidationError(
"rollover cannot be applied to both regular and failure indices at the same time",
validationException

2
server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java
View file

@ -219,7 +219,7 @@ final class BulkOperation extends ActionRunnable<BulkResponse> {
RolloverRequest rolloverRequest = new RolloverRequest(dataStream, null);
rolloverRequest.setIndicesOptions(
IndicesOptions.builder(rolloverRequest.indicesOptions())
.selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES)
.selectorOptions(IndicesOptions.SelectorOptions.FAILURES)
.build()
);
// We are executing a lazy rollover because it is an action specialised for this situation, when we want an

2
server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java
View file

@ -425,7 +425,7 @@ public class TransportBulkAction extends TransportAbstractBulkAction {
if (targetFailureStore) {
rolloverRequest.setIndicesOptions(
IndicesOptions.builder(rolloverRequest.indicesOptions())
.selectorOptions(IndicesOptions.SelectorOptions.ONLY_FAILURES)
.selectorOptions(IndicesOptions.SelectorOptions.FAILURES)
.build()
);
}

2
server/src/main/java/org/elasticsearch/action/datastreams/DataStreamsStatsAction.java
View file

@ -61,7 +61,7 @@ public class DataStreamsStatsAction extends ActionType<DataStreamsStatsAction.Re
.allowFailureIndices(true)
.build()
)
.selectorOptions(IndicesOptions.SelectorOptions.DATA_AND_FAILURE)
.selectorOptions(IndicesOptions.SelectorOptions.ALL_APPLICABLE)
.build()
);
}

23
server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java
View file

@ -1247,6 +1247,29 @@ public class TransportSearchAction extends HandledTransportAction<SearchRequest,
indicesAndAliases,
concreteLocalIndices
);
// localShardIterators is empty since there are no matching indices. In such cases,
// we update the local cluster's status from RUNNING to SUCCESSFUL right away. Before
// we attempt to do that, we must ensure that the local cluster was specified in the user's
// search request. This is done by trying to fetch the local cluster via getCluster() and
// checking for a non-null return value. If the local cluster was never specified, its status
// update can be skipped.
if (localShardIterators.isEmpty()
&& clusters != SearchResponse.Clusters.EMPTY
&& clusters.getCluster(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) != null) {
clusters.swapCluster(
RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY,
(alias, v) -> new SearchResponse.Cluster.Builder(v).setStatus(SearchResponse.Cluster.Status.SUCCESSFUL)
.setTotalShards(0)
.setSuccessfulShards(0)
.setSkippedShards(0)
.setFailedShards(0)
.setFailures(Collections.emptyList())
.setTook(TimeValue.timeValueMillis(0))
.setTimedOut(false)
.build()
);
}
}
final GroupShardsIterator<SearchShardIterator> shardIterators = mergeShardsIterators(localShardIterators, remoteShardIterators);

73
server/src/main/java/org/elasticsearch/action/support/IndexComponentSelector.java
View file

@ -9,6 +9,12 @@
package org.elasticsearch.action.support;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.core.Nullable;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@ -17,33 +23,82 @@ import java.util.Map;
* We define as index components the two different sets of indices a data stream could consist of:
* - DATA: represents the backing indices
* - FAILURES: represent the failing indices
* - ALL: represents all available in this expression components, meaning if it's a data stream both backing and failure indices and if it's
* an index only the index itself.
* Note: An index is its own DATA component, but it cannot have a FAILURE component.
*/
public enum IndexComponentSelector {
DATA("data"),
FAILURES("failures");
public enum IndexComponentSelector implements Writeable {
DATA("data", (byte) 0),
FAILURES("failures", (byte) 1),
ALL_APPLICABLE("*", (byte) 2);
private final String key;
private final byte id;
IndexComponentSelector(String key) {
IndexComponentSelector(String key, byte id) {
this.key = key;
this.id = id;
}
public String getKey() {
return key;
}
private static final Map<String, IndexComponentSelector> REGISTRY;
public byte getId() {
return id;
}
private static final Map<String, IndexComponentSelector> KEY_REGISTRY;
private static final Map<Byte, IndexComponentSelector> ID_REGISTRY;
static {
Map<String, IndexComponentSelector> registry = new HashMap<>(IndexComponentSelector.values().length);
Map<String, IndexComponentSelector> keyRegistry = new HashMap<>(IndexComponentSelector.values().length);
for (IndexComponentSelector value : IndexComponentSelector.values()) {
registry.put(value.getKey(), value);
keyRegistry.put(value.getKey(), value);
}
REGISTRY = Collections.unmodifiableMap(registry);
KEY_REGISTRY = Collections.unmodifiableMap(keyRegistry);
Map<Byte, IndexComponentSelector> idRegistry = new HashMap<>(IndexComponentSelector.values().length);
for (IndexComponentSelector value : IndexComponentSelector.values()) {
idRegistry.put(value.getId(), value);
}
ID_REGISTRY = Collections.unmodifiableMap(idRegistry);
}
/**
* Retrieves the respective selector when the suffix key is recognised
* @param key the suffix key, probably parsed from an expression
* @return the selector or null if the key was not recognised.
*/
@Nullable
public static IndexComponentSelector getByKey(String key) {
return REGISTRY.get(key);
return KEY_REGISTRY.get(key);
}
public static IndexComponentSelector read(StreamInput in) throws IOException {
return getById(in.readByte());
}
// Visible for testing
static IndexComponentSelector getById(byte id) {
IndexComponentSelector indexComponentSelector = ID_REGISTRY.get(id);
if (indexComponentSelector == null) {
throw new IllegalArgumentException(
"Unknown id of index component selector [" + id + "], available options are: " + ID_REGISTRY
);
}
return indexComponentSelector;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeByte(id);
}
public boolean shouldIncludeData() {
return this == ALL_APPLICABLE || this == DATA;
}
public boolean shouldIncludeFailures() {
return this == ALL_APPLICABLE || this == FAILURES;
}
}

133
server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java
View file

@ -421,60 +421,44 @@ public record IndicesOptions(
/**
* Defines which selectors should be used by default for an index operation in the event that no selectors are provided.
*/
public record SelectorOptions(EnumSet<IndexComponentSelector> defaultSelectors) implements Writeable {
public record SelectorOptions(IndexComponentSelector defaultSelector) implements Writeable {
public static final SelectorOptions DATA_AND_FAILURE = new SelectorOptions(
EnumSet.of(IndexComponentSelector.DATA, IndexComponentSelector.FAILURES)
);
public static final SelectorOptions ONLY_DATA = new SelectorOptions(EnumSet.of(IndexComponentSelector.DATA));
public static final SelectorOptions ONLY_FAILURES = new SelectorOptions(EnumSet.of(IndexComponentSelector.FAILURES));
public static final SelectorOptions ALL_APPLICABLE = new SelectorOptions(IndexComponentSelector.ALL_APPLICABLE);
public static final SelectorOptions DATA = new SelectorOptions(IndexComponentSelector.DATA);
public static final SelectorOptions FAILURES = new SelectorOptions(IndexComponentSelector.FAILURES);
/**
* Default instance. Uses <pre>::data</pre> as the default selector if none are present in an index expression.
*/
public static final SelectorOptions DEFAULT = ONLY_DATA;
public static final SelectorOptions DEFAULT = DATA;
public static SelectorOptions read(StreamInput in) throws IOException {
return new SelectorOptions(in.readEnumSet(IndexComponentSelector.class));
if (in.getTransportVersion().before(TransportVersions.INTRODUCE_ALL_APPLICABLE_SELECTOR)) {
EnumSet<IndexComponentSelector> set = in.readEnumSet(IndexComponentSelector.class);
if (set.isEmpty() || set.size() == 2) {
assert set.contains(IndexComponentSelector.DATA) && set.contains(IndexComponentSelector.FAILURES)
: "The enum set only supported ::data and ::failures";
return SelectorOptions.ALL_APPLICABLE;
} else if (set.contains(IndexComponentSelector.DATA)) {
return SelectorOptions.DATA;
} else {
return SelectorOptions.FAILURES;
}
} else {
return new SelectorOptions(IndexComponentSelector.read(in));
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeEnumSet(defaultSelectors);
}
public static class Builder {
private EnumSet<IndexComponentSelector> defaultSelectors;
public Builder() {
this(DEFAULT);
if (out.getTransportVersion().before(TransportVersions.INTRODUCE_ALL_APPLICABLE_SELECTOR)) {
switch (defaultSelector) {
case ALL_APPLICABLE -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.DATA, IndexComponentSelector.FAILURES));
case DATA -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.DATA));
case FAILURES -> out.writeEnumSet(EnumSet.of(IndexComponentSelector.FAILURES));
}
} else {
defaultSelector.writeTo(out);
}
Builder(SelectorOptions options) {
defaultSelectors = EnumSet.copyOf(options.defaultSelectors);
}
public Builder setDefaultSelectors(IndexComponentSelector first, IndexComponentSelector... remaining) {
defaultSelectors = EnumSet.of(first, remaining);
return this;
}
public Builder setDefaultSelectors(EnumSet<IndexComponentSelector> defaultSelectors) {
this.defaultSelectors = EnumSet.copyOf(defaultSelectors);
return this;
}
public SelectorOptions build() {
assert defaultSelectors.isEmpty() != true : "Default selectors cannot be an empty set";
return new SelectorOptions(EnumSet.copyOf(defaultSelectors));
}
}
public static Builder builder() {
return new Builder();
}
public static Builder builder(SelectorOptions selectorOptions) {
return new Builder(selectorOptions);
}
}
@ -547,7 +531,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.ONLY_DATA)
.selectorOptions(SelectorOptions.DATA)
.build();
public static final IndicesOptions STRICT_EXPAND_OPEN_FAILURE_STORE = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@ -566,7 +550,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.DATA_AND_FAILURE)
.selectorOptions(SelectorOptions.ALL_APPLICABLE)
.build();
public static final IndicesOptions LENIENT_EXPAND_OPEN = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@ -585,7 +569,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.ONLY_DATA)
.selectorOptions(SelectorOptions.DATA)
.build();
public static final IndicesOptions LENIENT_EXPAND_OPEN_NO_SELECTORS = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@ -622,7 +606,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.ONLY_DATA)
.selectorOptions(SelectorOptions.DATA)
.build();
public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@ -641,7 +625,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.ONLY_DATA)
.selectorOptions(SelectorOptions.DATA)
.build();
public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_HIDDEN = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@ -655,7 +639,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.ONLY_DATA)
.selectorOptions(SelectorOptions.DATA)
.build();
public static final IndicesOptions LENIENT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTOR = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS)
@ -687,7 +671,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.ONLY_DATA)
.selectorOptions(SelectorOptions.DATA)
.build();
public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@ -701,7 +685,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.ONLY_DATA)
.selectorOptions(SelectorOptions.DATA)
.build();
public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN_NO_SELECTORS = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@ -733,7 +717,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.DATA_AND_FAILURE)
.selectorOptions(SelectorOptions.ALL_APPLICABLE)
.build();
public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_HIDDEN_FAILURE_STORE = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@ -747,7 +731,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.DATA_AND_FAILURE)
.selectorOptions(SelectorOptions.ALL_APPLICABLE)
.build();
public static final IndicesOptions STRICT_EXPAND_OPEN_CLOSED_FAILURE_STORE = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@ -766,7 +750,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.DATA_AND_FAILURE)
.selectorOptions(SelectorOptions.ALL_APPLICABLE)
.build();
public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@ -785,7 +769,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.ONLY_DATA)
.selectorOptions(SelectorOptions.DATA)
.build();
public static final IndicesOptions STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@ -804,7 +788,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.ONLY_DATA)
.selectorOptions(SelectorOptions.DATA)
.build();
public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@ -823,7 +807,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.allowAliasToMultipleIndices(true)
)
.selectorOptions(SelectorOptions.ONLY_DATA)
.selectorOptions(SelectorOptions.DATA)
.build();
public static final IndicesOptions STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@ -842,7 +826,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.ONLY_DATA)
.selectorOptions(SelectorOptions.DATA)
.build();
public static final IndicesOptions STRICT_NO_EXPAND_FORBID_CLOSED = IndicesOptions.builder()
.concreteTargetOptions(ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@ -861,7 +845,7 @@ public record IndicesOptions(
.allowFailureIndices(true)
.ignoreThrottled(false)
)
.selectorOptions(SelectorOptions.ONLY_DATA)
.selectorOptions(SelectorOptions.DATA)
.build();
/**
@ -919,7 +903,7 @@ public record IndicesOptions(
}
/**
* @return Whether execution on closed indices is allowed.
* @return Whether execution on failure indices is allowed.
*/
public boolean allowFailureIndices() {
return gatekeeperOptions.allowFailureIndices();
@ -950,14 +934,14 @@ public record IndicesOptions(
* @return whether regular indices (stand-alone or backing indices) will be included in the response
*/
public boolean includeRegularIndices() {
return selectorOptions().defaultSelectors().contains(IndexComponentSelector.DATA);
return selectorOptions().defaultSelector().shouldIncludeData();
}
/**
* @return whether failure indices (only supported by certain data streams) will be included in the response
*/
public boolean includeFailureIndices() {
return selectorOptions().defaultSelectors().contains(IndexComponentSelector.FAILURES);
return selectorOptions().defaultSelector().shouldIncludeFailures();
}
public void writeIndicesOptions(StreamOutput out) throws IOException {
@ -1004,7 +988,7 @@ public record IndicesOptions(
out.writeBoolean(includeFailureIndices());
}
if (out.getTransportVersion().onOrAfter(TransportVersions.CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY)) {
out.writeEnumSet(selectorOptions.defaultSelectors);
selectorOptions.writeTo(out);
}
}
@ -1032,15 +1016,15 @@ public record IndicesOptions(
var includeData = in.readBoolean();
var includeFailures = in.readBoolean();
if (includeData && includeFailures) {
selectorOptions = SelectorOptions.DATA_AND_FAILURE;
selectorOptions = SelectorOptions.ALL_APPLICABLE;
} else if (includeData) {
selectorOptions = SelectorOptions.ONLY_DATA;
selectorOptions = SelectorOptions.DATA;
} else {
selectorOptions = SelectorOptions.ONLY_FAILURES;
selectorOptions = SelectorOptions.FAILURES;
}
}
if (in.getTransportVersion().onOrAfter(TransportVersions.CONVERT_FAILURE_STORE_OPTIONS_TO_SELECTOR_OPTIONS_INTERNALLY)) {
selectorOptions = new SelectorOptions(in.readEnumSet(IndexComponentSelector.class));
selectorOptions = SelectorOptions.read(in);
}
return new IndicesOptions(
options.contains(Option.ALLOW_UNAVAILABLE_CONCRETE_TARGETS)
@ -1099,11 +1083,6 @@ public record IndicesOptions(
return this;
}
public Builder selectorOptions(SelectorOptions.Builder selectorOptions) {
this.selectorOptions = selectorOptions.build();
return this;
}
public IndicesOptions build() {
return new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions, selectorOptions);
}
@ -1322,9 +1301,9 @@ public record IndicesOptions(
return defaultOptions;
}
return switch (failureStoreValue.toString()) {
case INCLUDE_ALL -> SelectorOptions.DATA_AND_FAILURE;
case INCLUDE_ONLY_REGULAR_INDICES -> SelectorOptions.ONLY_DATA;
case INCLUDE_ONLY_FAILURE_INDICES -> SelectorOptions.ONLY_FAILURES;
case INCLUDE_ALL -> SelectorOptions.ALL_APPLICABLE;
case INCLUDE_ONLY_REGULAR_INDICES -> SelectorOptions.DATA;
case INCLUDE_ONLY_FAILURE_INDICES -> SelectorOptions.FAILURES;
default -> throw new IllegalArgumentException("No valid " + FAILURE_STORE_QUERY_PARAM + " value [" + failureStoreValue + "]");
};
}
@ -1336,9 +1315,9 @@ public record IndicesOptions(
gatekeeperOptions.toXContent(builder, params);
if (DataStream.isFailureStoreFeatureFlagEnabled()) {
String displayValue;
if (SelectorOptions.DATA_AND_FAILURE.equals(selectorOptions())) {
if (SelectorOptions.ALL_APPLICABLE.equals(selectorOptions())) {
displayValue = INCLUDE_ALL;
} else if (SelectorOptions.ONLY_DATA.equals(selectorOptions())) {
} else if (SelectorOptions.DATA.equals(selectorOptions())) {
displayValue = INCLUDE_ONLY_REGULAR_INDICES;
} else {
displayValue = INCLUDE_ONLY_FAILURE_INDICES;

57
server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java
View file

@ -13,7 +13,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.TransportVersion;
import org.elasticsearch.TransportVersions;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRunnable;
import org.elasticsearch.cluster.ClusterState;
@ -31,7 +30,6 @@ import org.elasticsearch.core.AbstractRefCounted;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.core.RefCounted;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.core.UpdateForV9;
import org.elasticsearch.env.Environment;
import org.elasticsearch.node.NodeClosedException;
import org.elasticsearch.threadpool.ThreadPool;
@ -46,7 +44,6 @@ import org.elasticsearch.transport.TransportService;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
@ -162,55 +159,14 @@ public class JoinValidationService {
return;
}
if (connection.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0)) {
if (executeRefs.tryIncRef()) {
try {
execute(new JoinValidation(discoveryNode, connection, listener));
} finally {
executeRefs.decRef();
}
} else {
listener.onFailure(new NodeClosedException(transportService.getLocalNode()));
if (executeRefs.tryIncRef()) {
try {
execute(new JoinValidation(discoveryNode, connection, listener));
} finally {
executeRefs.decRef();
}
} else {
legacyValidateJoin(discoveryNode, listener, connection);
}
}
@UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION)
private void legacyValidateJoin(DiscoveryNode discoveryNode, ActionListener<Void> listener, Transport.Connection connection) {
final var responseHandler = TransportResponseHandler.empty(responseExecutor, listener.delegateResponse((l, e) -> {
logger.warn(() -> "failed to validate incoming join request from node [" + discoveryNode + "]", e);
listener.onFailure(
new IllegalStateException(
String.format(
Locale.ROOT,
"failure when sending a join validation request from [%s] to [%s]",
transportService.getLocalNode().descriptionWithoutAttributes(),
discoveryNode.descriptionWithoutAttributes()
),
e
)
);
}));
final var clusterState = clusterStateSupplier.get();
if (clusterState != null) {
assert clusterState.nodes().isLocalNodeElectedMaster();
transportService.sendRequest(
connection,
JOIN_VALIDATE_ACTION_NAME,
new ValidateJoinRequest(clusterState),
REQUEST_OPTIONS,
responseHandler
);
} else {
transportService.sendRequest(
connection,
JoinHelper.JOIN_PING_ACTION_NAME,
new JoinHelper.JoinPingRequest(),
REQUEST_OPTIONS,
responseHandler
);
listener.onFailure(new NodeClosedException(transportService.getLocalNode()));
}
}
@ -341,7 +297,6 @@ public class JoinValidationService {
@Override
protected void doRun() {
assert connection.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0) : discoveryNode.getVersion();
// NB these things never run concurrently to each other, or to the cache cleaner (see IMPLEMENTATION NOTES above) so it is safe
// to do these (non-atomic) things to the (unsynchronized) statesByVersion map.
var transportVersion = connection.getTransportVersion();

21
server/src/main/java/org/elasticsearch/cluster/coordination/ValidateJoinRequest.java
View file

@ -9,7 +9,6 @@
package org.elasticsearch.cluster.coordination;
import org.elasticsearch.TransportVersion;
import org.elasticsearch.TransportVersions;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.common.CheckedSupplier;
import org.elasticsearch.common.bytes.BytesReference;
@ -29,19 +28,12 @@ public class ValidateJoinRequest extends TransportRequest {
public ValidateJoinRequest(StreamInput in) throws IOException {
super(in);
if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0)) {
// recent versions send a BytesTransportRequest containing a compressed representation of the state
final var bytes = in.readReleasableBytesReference();
final var version = in.getTransportVersion();
final var namedWriteableRegistry = in.namedWriteableRegistry();
this.stateSupplier = () -> readCompressed(version, bytes, namedWriteableRegistry);
this.refCounted = bytes;
} else {
// older versions just contain the bare state
final var state = ClusterState.readFrom(in, null);
this.stateSupplier = () -> state;
this.refCounted = null;
}
// recent versions send a BytesTransportRequest containing a compressed representation of the state
final var bytes = in.readReleasableBytesReference();
final var version = in.getTransportVersion();
final var namedWriteableRegistry = in.namedWriteableRegistry();
this.stateSupplier = () -> readCompressed(version, bytes, namedWriteableRegistry);
this.refCounted = bytes;
}
private static ClusterState readCompressed(
@ -68,7 +60,6 @@ public class ValidateJoinRequest extends TransportRequest {
@Override
public void writeTo(StreamOutput out) throws IOException {
assert out.getTransportVersion().before(TransportVersions.V_8_3_0);
super.writeTo(out);
stateSupplier.get().writeTo(out);
}

77
server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java
View file

@ -14,7 +14,6 @@ import org.elasticsearch.TransportVersions;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
@ -22,7 +21,6 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.Processors;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.core.UpdateForV9;
import org.elasticsearch.features.NodeFeature;
import org.elasticsearch.xcontent.ConstructingObjectParser;
import org.elasticsearch.xcontent.ObjectParser;
@ -38,7 +36,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import static java.lang.String.format;
import static org.elasticsearch.node.Node.NODE_EXTERNAL_ID_SETTING;
@ -58,8 +55,6 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparabl
private static final ParseField PROCESSORS_RANGE_FIELD = new ParseField("processors_range");
private static final ParseField MEMORY_FIELD = new ParseField("memory");
private static final ParseField STORAGE_FIELD = new ParseField("storage");
@UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // Remove deprecated field
private static final ParseField VERSION_FIELD = new ParseField("node_version");
public static final ConstructingObjectParser<DesiredNode, Void> PARSER = new ConstructingObjectParser<>(
"desired_node",
@ -69,8 +64,7 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparabl
(Processors) args[1],
(ProcessorsRange) args[2],
(ByteSizeValue) args[3],
(ByteSizeValue) args[4],
(String) args[5]
(ByteSizeValue) args[4]
)
);
@ -104,12 +98,6 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparabl
STORAGE_FIELD,
ObjectParser.ValueType.STRING
);
parser.declareField(
ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> p.text(),
VERSION_FIELD,
ObjectParser.ValueType.STRING
);
}
private final Settings settings;
@ -118,21 +106,9 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparabl
private final ByteSizeValue memory;
private final ByteSizeValue storage;
@UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // Remove deprecated version field
private final String version;
private final String externalId;
private final Set<DiscoveryNodeRole> roles;
@Deprecated
public DesiredNode(Settings settings, ProcessorsRange processorsRange, ByteSizeValue memory, ByteSizeValue storage, String version) {
this(settings, null, processorsRange, memory, storage, version);
}
@Deprecated
public DesiredNode(Settings settings, double processors, ByteSizeValue memory, ByteSizeValue storage, String version) {
this(settings, Processors.of(processors), null, memory, storage, version);
}
public DesiredNode(Settings settings, ProcessorsRange processorsRange, ByteSizeValue memory, ByteSizeValue storage) {
this(settings, null, processorsRange, memory, storage);
}
@ -142,17 +118,6 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparabl
}
DesiredNode(Settings settings, Processors processors, ProcessorsRange processorsRange, ByteSizeValue memory, ByteSizeValue storage) {
this(settings, processors, processorsRange, memory, storage, null);
}
DesiredNode(
Settings settings,
Processors processors,
ProcessorsRange processorsRange,
ByteSizeValue memory,
ByteSizeValue storage,
@Deprecated String version
) {
assert settings != null;
assert memory != null;
assert storage != null;
@ -186,7 +151,6 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparabl
this.processorsRange = processorsRange;
this.memory = memory;
this.storage = storage;
this.version = version;
this.externalId = NODE_EXTERNAL_ID_SETTING.get(settings);
this.roles = Collections.unmodifiableSortedSet(new TreeSet<>(DiscoveryNode.getRolesFromSettings(settings)));
}
@ -210,19 +174,7 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparabl
} else {
version = Version.readVersion(in).toString();
}
return new DesiredNode(settings, processors, processorsRange, memory, storage, version);
}
private static final Pattern SEMANTIC_VERSION_PATTERN = Pattern.compile("^(\\d+\\.\\d+\\.\\d+)\\D?.*");
private static Version parseLegacyVersion(String version) {
if (version != null) {
var semanticVersionMatcher = SEMANTIC_VERSION_PATTERN.matcher(version);
if (semanticVersionMatcher.matches()) {
return Version.fromString(semanticVersionMatcher.group(1));
}
}
return null;
return new DesiredNode(settings, processors, processorsRange, memory, storage);
}
@Override
@ -239,15 +191,9 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparabl
memory.writeTo(out);
storage.writeTo(out);
if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) {
out.writeOptionalString(version);
out.writeOptionalString(null);
} else {
Version parsedVersion = parseLegacyVersion(version);
if (version == null) {
// Some node is from before we made the version field not required. If so, fill in with the current node version.
Version.writeVersion(Version.CURRENT, out);
} else {
Version.writeVersion(parsedVersion, out);
}
Version.writeVersion(Version.CURRENT, out);
}
}
@ -275,14 +221,6 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparabl
}
builder.field(MEMORY_FIELD.getPreferredName(), memory);
builder.field(STORAGE_FIELD.getPreferredName(), storage);
addDeprecatedVersionField(builder);
}
@UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_COORDINATION) // Remove deprecated field from response
private void addDeprecatedVersionField(XContentBuilder builder) throws IOException {
if (version != null) {
builder.field(VERSION_FIELD.getPreferredName(), version);
}
}
public boolean hasMasterRole() {
@ -366,7 +304,6 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparabl
return Objects.equals(settings, that.settings)
&& Objects.equals(memory, that.memory)
&& Objects.equals(storage, that.storage)
&& Objects.equals(version, that.version)
&& Objects.equals(externalId, that.externalId)
&& Objects.equals(roles, that.roles);
}
@ -379,7 +316,7 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparabl
@Override
public int hashCode() {
return Objects.hash(settings, processors, processorsRange, memory, storage, version, externalId, roles);
return Objects.hash(settings, processors, processorsRange, memory, storage, externalId, roles);
}
@Override
@ -408,10 +345,6 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparabl
+ '}';
}
public boolean hasVersion() {
return Strings.isNullOrBlank(version) == false;
}
public record ProcessorsRange(Processors min, @Nullable Processors max) implements Writeable, ToXContentObject {
private static final ParseField MIN_FIELD = new ParseField("min");

5
server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNodeWithStatus.java
View file

@ -44,13 +44,12 @@ public record DesiredNodeWithStatus(DesiredNode desiredNode, Status status)
(Processors) args[1],
(DesiredNode.ProcessorsRange) args[2],
(ByteSizeValue) args[3],
(ByteSizeValue) args[4],
(String) args[5]
(ByteSizeValue) args[4]
),
// An unknown status is expected during upgrades to versions >= STATUS_TRACKING_SUPPORT_VERSION
// the desired node status would be populated when a node in the newer version is elected as
// master, the desired nodes status update happens in NodeJoinExecutor.
args[6] == null ? Status.PENDING : (Status) args[6]
args[5] == null ? Status.PENDING : (Status) args[5]
)
);

186
server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
View file

@ -49,7 +49,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -307,7 +306,7 @@ public class IndexNameExpressionResolver {
} else {
return ExplicitResourceNameFilter.filterUnavailable(
context,
DateMathExpressionResolver.resolve(context, List.of(expressions))
DateMathExpressionResolver.resolve(context, Arrays.asList(expressions))
);
}
} else {
@ -318,7 +317,10 @@ public class IndexNameExpressionResolver {
} else {
return WildcardExpressionResolver.resolve(
context,
ExplicitResourceNameFilter.filterUnavailable(context, DateMathExpressionResolver.resolve(context, List.of(expressions)))
ExplicitResourceNameFilter.filterUnavailable(
context,
DateMathExpressionResolver.resolve(context, Arrays.asList(expressions))
)
);
}
}
@ -1388,34 +1390,51 @@ public class IndexNameExpressionResolver {
* </ol>
*/
public static Collection<String> resolve(Context context, List<String> expressions) {
ExpressionList expressionList = new ExpressionList(context, expressions);
// fast exit if there are no wildcards to evaluate
if (expressionList.hasWildcard() == false) {
if (context.getOptions().expandWildcardExpressions() == false) {
return expressions;
}
int firstWildcardIndex = 0;
for (; firstWildcardIndex < expressions.size(); firstWildcardIndex++) {
String expression = expressions.get(firstWildcardIndex);
if (isWildcard(expression)) {
break;
}
}
if (firstWildcardIndex == expressions.size()) {
return expressions;
}
Set<String> result = new HashSet<>();
for (ExpressionList.Expression expression : expressionList) {
if (expression.isWildcard()) {
Stream<IndexAbstraction> matchingResources = matchResourcesToWildcard(context, expression.get());
for (int i = 0; i < firstWildcardIndex; i++) {
result.add(expressions.get(i));
}
AtomicBoolean emptyWildcardExpansion = context.getOptions().allowNoIndices() ? null : new AtomicBoolean();
for (int i = firstWildcardIndex; i < expressions.size(); i++) {
String expression = expressions.get(i);
boolean isExclusion = i > firstWildcardIndex && expression.charAt(0) == '-';
if (i == firstWildcardIndex || isWildcard(expression)) {
Stream<IndexAbstraction> matchingResources = matchResourcesToWildcard(
context,
isExclusion ? expression.substring(1) : expression
);
Stream<String> matchingOpenClosedNames = expandToOpenClosed(context, matchingResources);
AtomicBoolean emptyWildcardExpansion = new AtomicBoolean(false);
if (context.getOptions().allowNoIndices() == false) {
if (emptyWildcardExpansion != null) {
emptyWildcardExpansion.set(true);
matchingOpenClosedNames = matchingOpenClosedNames.peek(x -> emptyWildcardExpansion.set(false));
}
if (expression.isExclusion()) {
matchingOpenClosedNames.forEachOrdered(result::remove);
if (isExclusion) {
matchingOpenClosedNames.forEach(result::remove);
} else {
matchingOpenClosedNames.forEachOrdered(result::add);
matchingOpenClosedNames.forEach(result::add);
}
if (emptyWildcardExpansion.get()) {
throw notFoundException(expression.get());
if (emptyWildcardExpansion != null && emptyWildcardExpansion.get()) {
throw notFoundException(expression);
}
} else {
if (expression.isExclusion()) {
result.remove(expression.get());
if (isExclusion) {
result.remove(expression.substring(1));
} else {
result.add(expression.get());
result.add(expression);
}
}
}
@ -1601,27 +1620,35 @@ public class IndexNameExpressionResolver {
// utility class
}
/**
* Resolves date math expressions. If this is a noop the given {@code expressions} list is returned without copying.
* As a result callers of this method should not mutate the returned list. Mutating it may come with unexpected side effects.
*/
public static List<String> resolve(Context context, List<String> expressions) {
List<String> result = new ArrayList<>(expressions.size());
for (ExpressionList.Expression expression : new ExpressionList(context, expressions)) {
result.add(resolveExpression(expression, context::getStartTime));
boolean wildcardSeen = false;
final boolean expandWildcards = context.getOptions().expandWildcardExpressions();
String[] result = null;
for (int i = 0, n = expressions.size(); i < n; i++) {
String expression = expressions.get(i);
// accepts date-math exclusions that are of the form "-<...{}>",f i.e. the "-" is outside the "<>" date-math template
boolean isExclusion = wildcardSeen && expression.startsWith("-");
wildcardSeen = wildcardSeen || (expandWildcards && isWildcard(expression));
String toResolve = isExclusion ? expression.substring(1) : expression;
String resolved = resolveExpression(toResolve, context::getStartTime);
if (toResolve != resolved) {
if (result == null) {
result = expressions.toArray(Strings.EMPTY_ARRAY);
}
result[i] = isExclusion ? "-" + resolved : resolved;
}
}
return result;
return result == null ? expressions : Arrays.asList(result);
}
static String resolveExpression(String expression) {
return resolveExpression(expression, System::currentTimeMillis);
}
static String resolveExpression(ExpressionList.Expression expression, LongSupplier getTime) {
if (expression.isExclusion()) {
// accepts date-math exclusions that are of the form "-<...{}>", i.e. the "-" is outside the "<>" date-math template
return "-" + resolveExpression(expression.get(), getTime);
} else {
return resolveExpression(expression.get(), getTime);
}
}
static String resolveExpression(String expression, LongSupplier getTime) {
if (expression.startsWith(EXPRESSION_LEFT_BOUND) == false || expression.endsWith(EXPRESSION_RIGHT_BOUND) == false) {
return expression;
@ -1783,14 +1810,35 @@ public class IndexNameExpressionResolver {
*/
public static List<String> filterUnavailable(Context context, List<String> expressions) {
ensureRemoteIndicesRequireIgnoreUnavailable(context.getOptions(), expressions);
List<String> result = new ArrayList<>(expressions.size());
for (ExpressionList.Expression expression : new ExpressionList(context, expressions)) {
validateAliasOrIndex(expression);
if (expression.isWildcard() || expression.isExclusion() || ensureAliasOrIndexExists(context, expression.get())) {
result.add(expression.expression());
final boolean expandWildcards = context.getOptions().expandWildcardExpressions();
boolean wildcardSeen = false;
List<String> result = null;
for (int i = 0; i < expressions.size(); i++) {
String expression = expressions.get(i);
if (Strings.isEmpty(expression)) {
throw notFoundException(expression);
}
// Expressions can not start with an underscore. This is reserved for APIs. If the check gets here, the API
// does not exist and the path is interpreted as an expression. If the expression begins with an underscore,
// throw a specific error that is different from the [[IndexNotFoundException]], which is typically thrown
// if the expression can't be found.
if (expression.charAt(0) == '_') {
throw new InvalidIndexNameException(expression, "must not start with '_'.");
}
final boolean isWildcard = expandWildcards && isWildcard(expression);
if (isWildcard || (wildcardSeen && expression.charAt(0) == '-') || ensureAliasOrIndexExists(context, expression)) {
if (result != null) {
result.add(expression);
}
} else {
if (result == null) {
result = new ArrayList<>(expressions.size() - 1);
result.addAll(expressions.subList(0, i));
}
}
wildcardSeen |= isWildcard;
}
return result;
return result == null ? expressions : result;
}
/**
@ -1830,19 +1878,6 @@ public class IndexNameExpressionResolver {
return true;
}
private static void validateAliasOrIndex(ExpressionList.Expression expression) {
if (Strings.isEmpty(expression.expression())) {
throw notFoundException(expression.expression());
}
// Expressions can not start with an underscore. This is reserved for APIs. If the check gets here, the API
// does not exist and the path is interpreted as an expression. If the expression begins with an underscore,
// throw a specific error that is different from the [[IndexNotFoundException]], which is typically thrown
// if the expression can't be found.
if (expression.expression().charAt(0) == '_') {
throw new InvalidIndexNameException(expression.expression(), "must not start with '_'.");
}
}
private static void ensureRemoteIndicesRequireIgnoreUnavailable(IndicesOptions options, List<String> indexExpressions) {
if (options.ignoreUnavailable()) {
return;
@ -1867,57 +1902,6 @@ public class IndexNameExpressionResolver {
}
}
/**
* Used to iterate expression lists and work out which expression item is a wildcard or an exclusion.
*/
public static final class ExpressionList implements Iterable<ExpressionList.Expression> {
private final List<Expression> expressionsList;
private final boolean hasWildcard;
public record Expression(String expression, boolean isWildcard, boolean isExclusion) {
public String get() {
if (isExclusion()) {
// drop the leading "-" if exclusion because it is easier for callers to handle it like this
return expression().substring(1);
} else {
return expression();
}
}
}
/**
* Creates the expression iterable that can be used to easily check which expression item is a wildcard or an exclusion (or both).
* The {@param context} is used to check if wildcards ought to be considered or not.
*/
public ExpressionList(Context context, List<String> expressionStrings) {
List<Expression> expressionsList = new ArrayList<>(expressionStrings.size());
boolean wildcardSeen = false;
for (String expressionString : expressionStrings) {
boolean isExclusion = expressionString.startsWith("-") && wildcardSeen;
if (context.getOptions().expandWildcardExpressions() && isWildcard(expressionString)) {
wildcardSeen = true;
expressionsList.add(new Expression(expressionString, true, isExclusion));
} else {
expressionsList.add(new Expression(expressionString, false, isExclusion));
}
}
this.expressionsList = expressionsList;
this.hasWildcard = wildcardSeen;
}
/**
* Returns {@code true} if the expression contains any wildcard and the options allow wildcard expansion
*/
public boolean hasWildcard() {
return this.hasWildcard;
}
@Override
public Iterator<ExpressionList.Expression> iterator() {
return expressionsList.iterator();
}
}
/**
* This is a context for the DateMathExpressionResolver which does not require {@code IndicesOptions} or {@code ClusterState}
* since it uses only the start time to resolve expressions.

Some files were not shown because too many files have changed in this diff Show more