diff --git a/docs/changelog/96035.yaml b/docs/changelog/96035.yaml new file mode 100644 index 000000000000..6fa0a6b97619 --- /dev/null +++ b/docs/changelog/96035.yaml @@ -0,0 +1,7 @@ +pr: 96035 +summary: Expand start and end time to nanoseconds during coordinator rewrite when + needed +area: TSDB +type: bug +issues: + - 96030 diff --git a/docs/changelog/96265.yaml b/docs/changelog/96265.yaml new file mode 100644 index 000000000000..653dd1ddea09 --- /dev/null +++ b/docs/changelog/96265.yaml @@ -0,0 +1,6 @@ +pr: 96265 +summary: Reduce nesting of same bool queries +area: Query Languages +type: enhancement +issues: + - 96236 diff --git a/docs/changelog/96293.yaml b/docs/changelog/96293.yaml new file mode 100644 index 000000000000..ba5d8f52e196 --- /dev/null +++ b/docs/changelog/96293.yaml @@ -0,0 +1,6 @@ +pr: 96293 +summary: Report version conflict on concurrent updates +area: Transform +type: bug +issues: + - 96311 diff --git a/docs/changelog/96317.yaml b/docs/changelog/96317.yaml new file mode 100644 index 000000000000..88d7c753ca9f --- /dev/null +++ b/docs/changelog/96317.yaml @@ -0,0 +1,5 @@ +pr: 96317 +summary: API rest compatibility for type parameter in `geo_bounding_box` query +area: Geo +type: bug +issues: [] diff --git a/docs/painless/painless-contexts/painless-context-examples.asciidoc b/docs/painless/painless-contexts/painless-context-examples.asciidoc index ab877dea7c7e..e244425087c0 100644 --- a/docs/painless/painless-contexts/painless-context-examples.asciidoc +++ b/docs/painless/painless-contexts/painless-context-examples.asciidoc @@ -28,7 +28,7 @@ Each document in the `seat` data contains the following fields: The date and time of the play as a date object. ==== Prerequisites -Start an {ref}/getting-started-install.html[{es} instance], and then access the +Start an {ref}/configuring-stack-security.html[{es} instance], and then access the {kibana-ref}/console-kibana.html[Console] in {kib}. ==== Configure the `seat` sample data diff --git a/docs/plugins/development/creating-classic-plugins.asciidoc b/docs/plugins/development/creating-classic-plugins.asciidoc index 61f9dee0749f..f3f62a11f299 100644 --- a/docs/plugins/development/creating-classic-plugins.asciidoc +++ b/docs/plugins/development/creating-classic-plugins.asciidoc @@ -47,9 +47,6 @@ Use `bin/elasticsearch-plugin install file:///path/to/your/plugin` to install your plugin for testing. The Java plugin is auto-loaded only if it's in the `plugins/` directory. -You may also load your plugin within the test framework for integration tests. -Check {ref}/integration-tests.html#changing-node-configuration[Changing Node Configuration] for more information. - [discrete] [[plugin-authors-jsm]] ==== Java Security permissions diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index 6e4089739823..3947ed71ea9a 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -2,7 +2,7 @@ === EC2 Discovery plugin The EC2 discovery plugin provides a list of seed addresses to the -{ref}/modules-discovery-hosts-providers.html[discovery process] by querying the +{ref}/discovery-hosts-providers.html[discovery process] by querying the https://github.com/aws/aws-sdk-java[AWS API] for a list of EC2 instances matching certain criteria determined by the <>. diff --git a/docs/plugins/repository-hdfs.asciidoc b/docs/plugins/repository-hdfs.asciidoc index b7f65b29fa05..94aecddbf0bc 100644 --- a/docs/plugins/repository-hdfs.asciidoc +++ b/docs/plugins/repository-hdfs.asciidoc @@ -2,7 +2,7 @@ === Hadoop HDFS repository plugin The HDFS repository plugin adds support for using HDFS File System as a repository for -{ref}/modules-snapshots.html[Snapshot/Restore]. +{ref}/snapshot-restore.html[Snapshot/Restore]. :plugin_name: repository-hdfs include::install_remove.asciidoc[] @@ -23,7 +23,7 @@ plugin folder and point `HADOOP_HOME` variable to it; this should minimize the a ==== Configuration properties Once installed, define the configuration for the `hdfs` repository through the -{ref}/modules-snapshots.html[REST API]: +{ref}/snapshot-restore.html[REST API]: [source,console] ---- diff --git a/docs/plugins/repository.asciidoc b/docs/plugins/repository.asciidoc index d90873d41660..fc0c5b3c411d 100644 --- a/docs/plugins/repository.asciidoc +++ b/docs/plugins/repository.asciidoc @@ -1,7 +1,7 @@ [[repository]] == Snapshot/restore repository plugins -Repository plugins extend the {ref}/modules-snapshots.html[Snapshot/Restore] +Repository plugins extend the {ref}/snapshot-restore.html[Snapshot/Restore] functionality in Elasticsearch by adding repositories backed by the cloud or by distributed file systems: diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc index 5778e381ca8c..f363d71a4203 100644 --- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc @@ -107,7 +107,7 @@ or <>. NOTE: By default, you cannot run a `terms` aggregation on a `text` field. Use a `keyword` <> instead. Alternatively, you can enable -<> on the `text` field to create buckets for the field's +<> on the `text` field to create buckets for the field's <> terms. Enabling `fielddata` can significantly increase memory usage. diff --git a/docs/reference/cat/allocation.asciidoc b/docs/reference/cat/allocation.asciidoc index 1ab2dde36ea9..7153e99e503a 100644 --- a/docs/reference/cat/allocation.asciidoc +++ b/docs/reference/cat/allocation.asciidoc @@ -81,7 +81,7 @@ hard-linked files. `disk.avail`:: Free disk space available to {es}. {es} retrieves this metric from the node's -OS. <> uses this metric to assign +OS. <> uses this metric to assign shards to nodes based on available disk space. `disk.total`:: diff --git a/docs/reference/cat/recovery.asciidoc b/docs/reference/cat/recovery.asciidoc index ebc233fe6e75..76fb66c6703d 100644 --- a/docs/reference/cat/recovery.asciidoc +++ b/docs/reference/cat/recovery.asciidoc @@ -135,7 +135,7 @@ measurements. [[cat-recovery-api-ex-snapshot]] ===== Example with a snapshot recovery -You can restore backups of an index using the <> API. You can use the cat recovery API retrieve information about a snapshot recovery. diff --git a/docs/reference/cat/repositories.asciidoc b/docs/reference/cat/repositories.asciidoc index ddf8e7652213..edeb66b95423 100644 --- a/docs/reference/cat/repositories.asciidoc +++ b/docs/reference/cat/repositories.asciidoc @@ -11,7 +11,7 @@ console. They are _not_ intended for use by applications. For application consumption, use the <>. ==== -Returns the <> for a cluster. +Returns the <> for a cluster. [[cat-repositories-api-request]] diff --git a/docs/reference/cat/snapshots.asciidoc b/docs/reference/cat/snapshots.asciidoc index 6c400e235cb2..059fd58630bc 100644 --- a/docs/reference/cat/snapshots.asciidoc +++ b/docs/reference/cat/snapshots.asciidoc @@ -11,7 +11,7 @@ console. They are _not_ intended for use by applications. For application consumption, use the <>. ==== -Returns information about the <> stored in one or +Returns information about the <> stored in one or more repositories. A snapshot is a backup of an index or running {es} cluster. diff --git a/docs/reference/cluster/nodes-reload-secure-settings.asciidoc b/docs/reference/cluster/nodes-reload-secure-settings.asciidoc index 7938f794c114..f86304d2e9ba 100644 --- a/docs/reference/cluster/nodes-reload-secure-settings.asciidoc +++ b/docs/reference/cluster/nodes-reload-secure-settings.asciidoc @@ -31,7 +31,7 @@ When the {es} keystore is password protected and not simply obfuscated, you must provide the password for the keystore when you reload the secure settings. Reloading the settings for the whole cluster assumes that all nodes' keystores are protected with the same password; this method is allowed only when -<>. Alternatively, you can +<>. Alternatively, you can reload the secure settings on each node by locally accessing the API and passing the node-specific {es} keystore password. diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc index b654d6c4c2aa..56e65c567de6 100644 --- a/docs/reference/cluster/stats.asciidoc +++ b/docs/reference/cluster/stats.asciidoc @@ -1294,7 +1294,7 @@ Number of selected nodes using the HTTP type. `discovery_types`:: (object) -Contains statistics about the <> used by selected nodes. + .Properties of `discovery_types` @@ -1302,7 +1302,7 @@ types>> used by selected nodes. ===== ``:: (integer) -Number of selected nodes using the <> to find other nodes. ===== diff --git a/docs/reference/commands/croneval.asciidoc b/docs/reference/commands/croneval.asciidoc index 5f2a22cd4cb9..80fd143c22a0 100644 --- a/docs/reference/commands/croneval.asciidoc +++ b/docs/reference/commands/croneval.asciidoc @@ -1,7 +1,7 @@ [[elasticsearch-croneval]] == elasticsearch-croneval -Validates and evaluates a <>. +Validates and evaluates a <>. [discrete] === Synopsis diff --git a/docs/reference/commands/node-tool.asciidoc b/docs/reference/commands/node-tool.asciidoc index e36ed42f5199..810de4a71fff 100644 --- a/docs/reference/commands/node-tool.asciidoc +++ b/docs/reference/commands/node-tool.asciidoc @@ -284,12 +284,11 @@ unsafely-bootstrapped cluster. Unsafe cluster bootstrapping is only possible if there is at least one surviving master-eligible node. If there are no remaining master-eligible nodes then the cluster metadata is completely lost. However, the individual data -nodes also contain a copy of the index metadata corresponding with their -shards. It is therefore sometimes possible to manually import these shards as -<>. For example you can sometimes recover some -indices after the loss of all master-eligible nodes in a cluster by creating a new -cluster and then using the `elasticsearch-node detach-cluster` command to move any -surviving nodes into this new cluster. Once the new cluster is fully formed, +nodes also contain a copy of the index metadata corresponding with their shards. This sometimes allows a new cluster to import these shards as +<>. You can sometimes +recover some indices after the loss of all main-eligible nodes in a cluster +by creating a new cluster and then using the `elasticsearch-node +detach-cluster` command to move any surviving nodes into this new cluster. Once the new cluster is fully formed, use the <> to list, import or delete any dangling indices. @@ -317,7 +316,7 @@ cluster formed as described above. below. Verify that the tool reported `Node was successfully detached from the cluster`. 5. If necessary, configure each data node to -<>. +<>. 6. Start each data node and verify that it has joined the new cluster. 7. Wait for all recoveries to have completed, and investigate the data in the cluster to discover if any was lost during this process. Use the diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc index d90026073990..8dba86630f24 100644 --- a/docs/reference/docs/bulk.asciidoc +++ b/docs/reference/docs/bulk.asciidoc @@ -231,7 +231,7 @@ participate in the `_bulk` request at all. [[bulk-security]] ===== Security -See <>. +See <>. [[docs-bulk-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/docs/multi-get.asciidoc b/docs/reference/docs/multi-get.asciidoc index ff839d4a662f..065e22469611 100644 --- a/docs/reference/docs/multi-get.asciidoc +++ b/docs/reference/docs/multi-get.asciidoc @@ -46,7 +46,7 @@ If you specify an index in the request URI, you only need to specify the documen [[mget-security]] ===== Security -See <>. +See <>. [[multi-get-partial-responses]] ===== Partial responses diff --git a/docs/reference/eql/eql-search-api.asciidoc b/docs/reference/eql/eql-search-api.asciidoc index 21575b5f293c..edbce021248f 100644 --- a/docs/reference/eql/eql-search-api.asciidoc +++ b/docs/reference/eql/eql-search-api.asciidoc @@ -73,7 +73,7 @@ See <>. (Optional, Boolean) + NOTE: This parameter's behavior differs from the `allow_no_indices` parameter -used in other <>. +used in other <>. + If `false`, the request returns an error if any wildcard pattern, alias, or `_all` value targets only missing or closed indices. This behavior applies even diff --git a/docs/reference/high-availability/cluster-design.asciidoc b/docs/reference/high-availability/cluster-design.asciidoc index 8fb7e47a51a6..3f8e19b47d37 100644 --- a/docs/reference/high-availability/cluster-design.asciidoc +++ b/docs/reference/high-availability/cluster-design.asciidoc @@ -69,7 +69,7 @@ cluster can report a `green` status, override the default by setting <> to `0` on every index. If the node fails, you may need to restore an older copy of any lost indices -from a <>. +from a <>. Because they are not resilient to any failures, we do not recommend using one-node clusters in production. @@ -281,7 +281,7 @@ cluster when handling such a failure. For resilience against whole-zone failures, it is important that there is a copy of each shard in more than one zone, which can be achieved by placing data -nodes in multiple zones and configuring <>. You should also ensure that client requests are sent to nodes in more than one zone. @@ -334,7 +334,7 @@ tiebreaker need not be as powerful as the other two nodes since it has no other roles and will not perform any searches nor coordinate any client requests nor be elected as the master of the cluster. -You should use <> to ensure +You should use <> to ensure that there is a copy of each shard in each zone. This means either zone remains fully available if the other zone fails. @@ -359,7 +359,7 @@ mean that the cluster can still elect a master even if one of the zones fails. As always, your indices should have at least one replica in case a node fails, unless they are <>. You -should also use <> to limit +should also use <> to limit the number of copies of each shard in each zone. For instance, if you have an index with one or two replicas configured then allocation awareness will ensure that the replicas of the shard are in a different zone from the primary. This diff --git a/docs/reference/how-to/fix-common-cluster-issues.asciidoc b/docs/reference/how-to/fix-common-cluster-issues.asciidoc index 227648a444f8..ac027f2fa771 100644 --- a/docs/reference/how-to/fix-common-cluster-issues.asciidoc +++ b/docs/reference/how-to/fix-common-cluster-issues.asciidoc @@ -181,7 +181,7 @@ For high-cardinality `text` fields, fielddata can use a large amount of JVM memory. To avoid this, {es} disables fielddata on `text` fields by default. If you've enabled fielddata and triggered the <>, consider disabling it and using a `keyword` field instead. -See <>. +See <>. **Clear the fieldata cache** diff --git a/docs/reference/how-to/indexing-speed.asciidoc b/docs/reference/how-to/indexing-speed.asciidoc index 96810a752ee4..924207bca470 100644 --- a/docs/reference/how-to/indexing-speed.asciidoc +++ b/docs/reference/how-to/indexing-speed.asciidoc @@ -107,7 +107,7 @@ that it will increase the risk of failure since the failure of any one SSD destroys the index. However this is typically the right tradeoff to make: optimize single shards for maximum performance, and then add replicas across different nodes so there's redundancy for any node failures. You can also use -<> to backup the index for further +<> to backup the index for further insurance. Directly-attached (local) storage generally performs better than remote storage diff --git a/docs/reference/how-to/use-elasticsearch-for-time-series-data.asciidoc b/docs/reference/how-to/use-elasticsearch-for-time-series-data.asciidoc index 3c275a21e115..18de2497760c 100644 --- a/docs/reference/how-to/use-elasticsearch-for-time-series-data.asciidoc +++ b/docs/reference/how-to/use-elasticsearch-for-time-series-data.asciidoc @@ -93,7 +93,7 @@ Use {kib}'s **Dashboard** feature to visualize your data in a chart, table, map, and more. See {kib}'s {kibana-ref}/dashboard.html[Dashboard documentation]. You can also search and aggregate your data using the <>. Use <> and <>. Use <> and <> to dynamically extract data from log messages and other unstructured content at search time. diff --git a/docs/reference/ilm/ilm-with-existing-indices.asciidoc b/docs/reference/ilm/ilm-with-existing-indices.asciidoc index dee764a43368..ab59935be9a6 100644 --- a/docs/reference/ilm/ilm-with-existing-indices.asciidoc +++ b/docs/reference/ilm/ilm-with-existing-indices.asciidoc @@ -47,7 +47,7 @@ to use {ilm-init} for new data. [[ilm-existing-indices-reindex]] === Reindex into a managed index -An alternative to <> is to +An alternative to <> is to reindex your data into an {ilm-init}-managed index. You might want to do this if creating periodic indices with very small amounts of data has led to excessive shard counts, or if continually indexing into the same index has led to large shards diff --git a/docs/reference/index-modules/allocation/delayed.asciidoc b/docs/reference/index-modules/allocation/delayed.asciidoc index fd199a88376d..9a35553dd605 100644 --- a/docs/reference/index-modules/allocation/delayed.asciidoc +++ b/docs/reference/index-modules/allocation/delayed.asciidoc @@ -12,7 +12,7 @@ These actions are intended to protect the cluster against data loss by ensuring that every shard is fully replicated as soon as possible. Even though we throttle concurrent recoveries both at the -<> and at the <>, this +<> and at the <>, this ``shard-shuffle'' can still put a lot of extra load on the cluster which may not be necessary if the missing node is likely to return soon. Imagine this scenario: diff --git a/docs/reference/index.asciidoc b/docs/reference/index.asciidoc index 61a4bbd65962..c7820785f8fe 100644 --- a/docs/reference/index.asciidoc +++ b/docs/reference/index.asciidoc @@ -82,4 +82,4 @@ include::release-notes.asciidoc[] include::dependencies-versions.asciidoc[] -include::redirects.asciidoc[] +include::redirects.asciidoc[] \ No newline at end of file diff --git a/docs/reference/indices/add-alias.asciidoc b/docs/reference/indices/add-alias.asciidoc index cba7d70e24e1..860e9ca46f79 100644 --- a/docs/reference/indices/add-alias.asciidoc +++ b/docs/reference/indices/add-alias.asciidoc @@ -36,7 +36,7 @@ or indices. ``:: (Required, string) Alias to update. If the alias doesn't exist, the request -creates it. Index alias names support <>. +creates it. Index alias names support <>. ``:: (Required, string) Comma-separated list of data streams or indices to add. diff --git a/docs/reference/indices/aliases.asciidoc b/docs/reference/indices/aliases.asciidoc index 5c8e3e8481af..76698501fd41 100644 --- a/docs/reference/indices/aliases.asciidoc +++ b/docs/reference/indices/aliases.asciidoc @@ -79,14 +79,14 @@ The object body contains options for the alias. Supports an empty object. ===== `alias`:: (Required*, string) Alias for the action. Index alias names support -<>. If `aliases` is not specified, the `add` +<>. If `aliases` is not specified, the `add` and `remove` actions require this parameter. For the `remove` action, this parameter supports wildcards (`*`). The `remove_index` action doesn't support this parameter. `aliases`:: (Required*, array of strings) Aliases for the action. Index alias names support -<>. If `alias` is not specified, the `add` and +<>. If `alias` is not specified, the `add` and `remove` actions require this parameter. For the `remove` action, this parameter supports wildcards (`*`). The `remove_index` action doesn't support this parameter. @@ -122,7 +122,7 @@ Only the `add` action supports this parameter. // tag::alias-options[] `is_hidden`:: -(Optional, Boolean) If `true`, the alias is <>. Defaults to +(Optional, Boolean) If `true`, the alias is <>. Defaults to `false`. All data streams or indices for the alias must have the same `is_hidden` value. // end::alias-options[] diff --git a/docs/reference/indices/create-index.asciidoc b/docs/reference/indices/create-index.asciidoc index 26c619705875..d39591a37df5 100644 --- a/docs/reference/indices/create-index.asciidoc +++ b/docs/reference/indices/create-index.asciidoc @@ -78,7 +78,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=timeoutparms] ======= ``:: (Required, object) The key is the alias name. Index alias names support -<>. +<>. + The object body contains options for the alias. Supports an empty object. + @@ -94,7 +94,7 @@ alias can access. If specified, this overwrites the `routing` value for indexing operations. `is_hidden`:: -(Optional, Boolean) If `true`, the alias is <>. Defaults to +(Optional, Boolean) If `true`, the alias is <>. Defaults to `false`. All indices for the alias must have the same `is_hidden` value. `is_write_index`:: @@ -204,7 +204,7 @@ PUT /test } -------------------------------------------------- -Index alias names also support <>. +Index alias names also support <>. [source,console] ---- diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index d3671b2c09a6..4edd4c923b55 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -205,7 +205,7 @@ policies. To retrieve the lifecycle policy for individual backing indices, use the <>. `hidden`:: -(Boolean) If `true`, the data stream is <>. +(Boolean) If `true`, the data stream is <>. `system`:: (Boolean) diff --git a/docs/reference/indices/put-index-template.asciidoc b/docs/reference/indices/put-index-template.asciidoc index ec4bfbe44237..4dfd7252a9fa 100644 --- a/docs/reference/indices/put-index-template.asciidoc +++ b/docs/reference/indices/put-index-template.asciidoc @@ -108,7 +108,7 @@ See <>. <>. Defaults to `false`. `hidden`:: -(Optional, Boolean) If `true`, the data stream is <>. Defaults to +(Optional, Boolean) If `true`, the data stream is <>. Defaults to `false`. `index_mode`:: diff --git a/docs/reference/indices/rollover-index.asciidoc b/docs/reference/indices/rollover-index.asciidoc index 3869f35b560f..5a893da552bb 100644 --- a/docs/reference/indices/rollover-index.asciidoc +++ b/docs/reference/indices/rollover-index.asciidoc @@ -75,7 +75,7 @@ index's name. .Use date math with index alias rollovers **** If you use an index alias for time series data, you can use -<> in the index name to track the rollover +<> in the index name to track the rollover date. For example, you can create an alias that points to an index named ``. If you create the index on May 6, 2099, the index's name is `my-index-2099.05.06-000001`. If you roll over the alias on May 7, 2099, @@ -98,7 +98,7 @@ Name of the data stream or index alias to roll over. ``:: (Optional, string) -Name of the index to create. Supports <>. Data +Name of the index to create. Supports <>. Data streams do not support this parameter. + If the name of the alias's current write index does not end with `-` and a diff --git a/docs/reference/indices/simulate-index.asciidoc b/docs/reference/indices/simulate-index.asciidoc index 073e5c736d8b..d4c446a58eec 100644 --- a/docs/reference/indices/simulate-index.asciidoc +++ b/docs/reference/indices/simulate-index.asciidoc @@ -112,7 +112,7 @@ access. overwrites the `routing` value for indexing operations. `is_hidden`:: -(Boolean) If `true`, the alias is <>. +(Boolean) If `true`, the alias is <>. `is_write_index`:: (Boolean) If `true`, the index is the <> for the alias. diff --git a/docs/reference/ingest.asciidoc b/docs/reference/ingest.asciidoc index 80b463804da0..c0ef4d852b84 100644 --- a/docs/reference/ingest.asciidoc +++ b/docs/reference/ingest.asciidoc @@ -861,7 +861,7 @@ PUT _ingest/pipeline/my-pipeline } ---- -You can also specify a <> as the +You can also specify a <> as the `if` condition. [source,console] diff --git a/docs/reference/ingest/common-log-format-example.asciidoc b/docs/reference/ingest/common-log-format-example.asciidoc index 9ee5a73ceb70..3fd296a167a6 100644 --- a/docs/reference/ingest/common-log-format-example.asciidoc +++ b/docs/reference/ingest/common-log-format-example.asciidoc @@ -34,7 +34,7 @@ pipeline. .. Click **Add a processor** and select the **Grok** processor type. .. Set **Field** to `message` and **Patterns** to the following -<>: +<>: + [source,grok] ---- diff --git a/docs/reference/ingest/processors/date-index-name.asciidoc b/docs/reference/ingest/processors/date-index-name.asciidoc index 77898413a37d..ae8a39f3eb9e 100644 --- a/docs/reference/ingest/processors/date-index-name.asciidoc +++ b/docs/reference/ingest/processors/date-index-name.asciidoc @@ -5,7 +5,7 @@ ++++ The purpose of this processor is to point documents to the right time based index based -on a date or timestamp field in a document by using the <>. +on a date or timestamp field in a document by using the <>. The processor sets the `_index` metadata field with a date math index name expression based on the provided index name prefix, a date or timestamp field in the documents being processed and the provided date rounding. @@ -126,7 +126,7 @@ and the result: // TESTRESPONSE[s/2016-11-08T19:43:03.850\+0000/$body.docs.0.doc._ingest.timestamp/] The above example shows that `_index` was set to ``. Elasticsearch -understands this to mean `2016-04-01` as is explained in the <> +understands this to mean `2016-04-01` as is explained in the <> [[date-index-name-options]] .Date index name options diff --git a/docs/reference/mapping/params.asciidoc b/docs/reference/mapping/params.asciidoc index 39348cb9f54f..1197d6dd5e91 100644 --- a/docs/reference/mapping/params.asciidoc +++ b/docs/reference/mapping/params.asciidoc @@ -14,7 +14,7 @@ The following mapping parameters are common to some or all field data types: * <> * <> * <> -* <> +* <> * <> * <> * <> diff --git a/docs/reference/mapping/params/eager-global-ordinals.asciidoc b/docs/reference/mapping/params/eager-global-ordinals.asciidoc index e2042e18078e..86e06f1ff00c 100644 --- a/docs/reference/mapping/params/eager-global-ordinals.asciidoc +++ b/docs/reference/mapping/params/eager-global-ordinals.asciidoc @@ -29,7 +29,7 @@ Global ordinals are used if a search contains any of the following components: * Certain bucket aggregations on `keyword`, `ip`, and `flattened` fields. This includes `terms` aggregations as mentioned above, as well as `composite`, `diversified_sampler`, and `significant_terms`. -* Bucket aggregations on `text` fields that require <> +* Bucket aggregations on `text` fields that require <> to be enabled. * Operations on parent and child documents from a `join` field, including `has_child` queries and `parent` aggregations. diff --git a/docs/reference/mapping/types/text.asciidoc b/docs/reference/mapping/types/text.asciidoc index acc9c962add9..fca6b28a76d8 100644 --- a/docs/reference/mapping/types/text.asciidoc +++ b/docs/reference/mapping/types/text.asciidoc @@ -77,7 +77,7 @@ The following parameters are accepted by `text` fields: (default). Enabling this is a good idea on fields that are frequently used for (significant) terms aggregations. -<>:: +<>:: Can the field use in-memory fielddata for sorting, aggregations, or scripting? Accepts `true` or `false` (default). diff --git a/docs/reference/migration/migrate_8_0/cluster-node-setting-changes.asciidoc b/docs/reference/migration/migrate_8_0/cluster-node-setting-changes.asciidoc index c6c925a5ffde..455f67aed38f 100644 --- a/docs/reference/migration/migrate_8_0/cluster-node-setting-changes.asciidoc +++ b/docs/reference/migration/migrate_8_0/cluster-node-setting-changes.asciidoc @@ -403,7 +403,7 @@ If you do not want to enable SSL and are currently using other * Discontinue use of other `xpack.security.http.ssl` settings If you want to enable SSL, follow the instructions in -{ref}/configuring-tls.html#tls-http[Encrypting HTTP client communications]. As part +{ref}/security-basic-setup-https.html#encrypt-http-communication[Encrypting HTTP client communications]. As part of this configuration, explicitly specify `xpack.security.http.ssl.enabled` as `true`. diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index 890945dd71f2..0240ba915e45 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -891,7 +891,7 @@ For example: "ignore_throttled": true } ``` -For more information about these options, see <>. +For more information about these options, see <>. -- end::indices-options[] diff --git a/docs/reference/modules/discovery.asciidoc b/docs/reference/modules/discovery.asciidoc index 8b3d01a65e36..b6be8409d7fe 100644 --- a/docs/reference/modules/discovery.asciidoc +++ b/docs/reference/modules/discovery.asciidoc @@ -8,7 +8,7 @@ each time it changes. The following processes and settings are part of discovery and cluster formation: -<>:: +<>:: Discovery is the process where nodes find each other when the master is unknown, such as when a node has just started up or when the previous @@ -34,7 +34,7 @@ formation: <> requires bootstrapping to be <>. -<>:: +<>:: It is recommended to have a small and fixed number of master-eligible nodes in a cluster, and to scale the cluster up and down by adding and removing diff --git a/docs/reference/modules/discovery/bootstrapping.asciidoc b/docs/reference/modules/discovery/bootstrapping.asciidoc index 850b7404a8d1..ee854f197667 100644 --- a/docs/reference/modules/discovery/bootstrapping.asciidoc +++ b/docs/reference/modules/discovery/bootstrapping.asciidoc @@ -4,11 +4,9 @@ Starting an Elasticsearch cluster for the very first time requires the initial set of <> to be explicitly defined on one or more of the master-eligible nodes in the cluster. This is known as _cluster -bootstrapping_. This is only required the first time a cluster starts up: nodes -that have already joined a cluster store this information in their data folder -for use in a <>, and freshly-started nodes -that are joining a running cluster obtain this information from the cluster's -elected master. +bootstrapping_. This is only required the first time a cluster starts up. +Freshly-started nodes that are joining a running cluster obtain this +information from the cluster's elected master. The initial set of master-eligible nodes is defined in the <>. This should be diff --git a/docs/reference/modules/discovery/discovery-settings.asciidoc b/docs/reference/modules/discovery/discovery-settings.asciidoc index 722b9382732f..401d1e7206d3 100644 --- a/docs/reference/modules/discovery/discovery-settings.asciidoc +++ b/docs/reference/modules/discovery/discovery-settings.asciidoc @@ -187,7 +187,7 @@ considered to have failed and is removed from the cluster. See `cluster.max_voting_config_exclusions`:: (<>) Sets a limit on the number of voting configuration exclusions at any one time. -The default value is `10`. See <>. +The default value is `10`. See <>. `cluster.publish.info_timeout`:: (<>) diff --git a/docs/reference/modules/discovery/quorums.asciidoc b/docs/reference/modules/discovery/quorums.asciidoc index 63c013bb4744..6f6e97889109 100644 --- a/docs/reference/modules/discovery/quorums.asciidoc +++ b/docs/reference/modules/discovery/quorums.asciidoc @@ -15,7 +15,7 @@ those of the other piece. Elasticsearch allows you to add and remove master-eligible nodes to a running cluster. In many cases you can do this simply by starting or stopping the nodes -as required. See <>. +as required. See <>. As nodes are added or removed Elasticsearch maintains an optimal level of fault tolerance by updating the cluster's <>. +see <>. The current voting configuration is stored in the cluster state so you can inspect its current contents as follows: diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc index 372af89af4a8..ec60b2bca37e 100644 --- a/docs/reference/modules/node.asciidoc +++ b/docs/reference/modules/node.asciidoc @@ -411,7 +411,7 @@ Similarly, each master-eligible node maintains the following data on disk: Each node checks the contents of its data path at startup. If it discovers unexpected data then it will refuse to start. This is to avoid importing -unwanted <> which can lead +unwanted <> which can lead to a red cluster health. To be more precise, nodes without the `data` role will refuse to start if they find any shard data on disk at startup, and nodes without both the `master` and `data` roles will refuse to start if they have any @@ -424,13 +424,13 @@ must perform some extra steps to prepare a node for repurposing when starting the node without the `data` or `master` roles. * If you want to repurpose a data node by removing the `data` role then you - should first use an <> to safely + should first use an <> to safely migrate all the shard data onto other nodes in the cluster. * If you want to repurpose a node to have neither the `data` nor `master` roles then it is simplest to start a brand-new node with an empty data path and the desired roles. You may find it safest to use an - <> to migrate the shard data elsewhere + <> to migrate the shard data elsewhere in the cluster first. If it is not possible to follow these extra steps then you may be able to use diff --git a/docs/reference/modules/transport.asciidoc b/docs/reference/modules/transport.asciidoc index 3f5a2ceecad7..2ec574544f9b 100644 --- a/docs/reference/modules/transport.asciidoc +++ b/docs/reference/modules/transport.asciidoc @@ -186,7 +186,7 @@ The `transport.compress` setting always configures local cluster request compression and is the fallback setting for remote cluster request compression. If you want to configure remote request compression differently than local request compression, you can set it on a per-remote cluster basis using the -<>. +<>. [[response-compression]] diff --git a/docs/reference/query-dsl/distance-feature-query.asciidoc b/docs/reference/query-dsl/distance-feature-query.asciidoc index 05ce0873f58e..9b22f08dee65 100644 --- a/docs/reference/query-dsl/distance-feature-query.asciidoc +++ b/docs/reference/query-dsl/distance-feature-query.asciidoc @@ -222,4 +222,4 @@ document's field value. Unlike the <> query or other ways to change <>, the `distance_feature` query efficiently skips non-competitive hits when the -<> parameter is **not** `true`. \ No newline at end of file +<> parameter is **not** `true`. \ No newline at end of file diff --git a/docs/reference/query-dsl/query_filter_context.asciidoc b/docs/reference/query-dsl/query_filter_context.asciidoc index 5cbd77423249..78e1549644aa 100644 --- a/docs/reference/query-dsl/query_filter_context.asciidoc +++ b/docs/reference/query-dsl/query_filter_context.asciidoc @@ -9,7 +9,7 @@ By default, Elasticsearch sorts matching search results by **relevance score**, which measures how well each document matches a query. The relevance score is a positive floating point number, returned in the -`_score` metadata field of the <> API. The higher the +`_score` metadata field of the <> API. The higher the `_score`, the more relevant the document. While each query type can calculate relevance scores differently, score calculation also depends on whether the query clause is run in a **query** or **filter** context. diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index 5d5b57c7c1d3..0926b8b7441a 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -569,7 +569,7 @@ instead. A regular flush has the same effect as a synced flush in 7.6 and later. [role="exclude",id="_repositories"] === Snapshot repositories -See <>. +See <>. [role="exclude",id="_snapshot"] === Snapshot diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 2ff37deba80e..6c33b9db4c59 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -274,7 +274,7 @@ Type of data stream that wildcard patterns can match. Supports comma-separated values, such as `open,hidden`. Valid values are: `all`, `hidden`:: -Match any data stream, including <> ones. +Match any data stream, including <> ones. `open`, `closed`:: Matches any non-hidden data stream. Data streams cannot be closed. @@ -295,7 +295,7 @@ streams. Supports comma-separated values, such as `open,hidden`. Valid values are: `all`:: -Match any data stream or index, including <> ones. +Match any data stream or index, including <> ones. `open`:: Match open, non-hidden indices. Also matches any non-hidden data stream. @@ -510,7 +510,7 @@ Number of documents and deleted docs, which have not yet merged out. <> can affect this statistic. `fielddata`:: -<> statistics. +<> statistics. `flush`:: <> statistics. @@ -554,9 +554,6 @@ Size of the index in <>. `translog`:: <> statistics. - -`warmer`:: -<> statistics. -- end::index-metric[] diff --git a/docs/reference/scripting/fields.asciidoc b/docs/reference/scripting/fields.asciidoc index 3812a4239b6b..c2a40d4519f9 100644 --- a/docs/reference/scripting/fields.asciidoc +++ b/docs/reference/scripting/fields.asciidoc @@ -133,7 +133,7 @@ existence of the field in mappings in an `expression` script. =================================================== The `doc['field']` syntax can also be used for <> -if <> is enabled, but *BEWARE*: enabling fielddata on a +if <> is enabled, but *BEWARE*: enabling fielddata on a `text` field requires loading all of the terms into the JVM heap, which can be very expensive both in terms of memory and CPU. It seldom makes sense to access `text` fields from scripts. diff --git a/docs/reference/search/multi-search.asciidoc b/docs/reference/search/multi-search.asciidoc index 769ea6b781f4..7f56716ee495 100644 --- a/docs/reference/search/multi-search.asciidoc +++ b/docs/reference/search/multi-search.asciidoc @@ -334,7 +334,7 @@ all search requests. [[msearch-security]] ==== Security -See <> +See <> [[multi-search-partial-responses]] diff --git a/docs/reference/search/point-in-time-api.asciidoc b/docs/reference/search/point-in-time-api.asciidoc index e1a36b84d9fd..0403f9b04b2d 100644 --- a/docs/reference/search/point-in-time-api.asciidoc +++ b/docs/reference/search/point-in-time-api.asciidoc @@ -57,7 +57,7 @@ POST /_search <1> // TEST[catch:unavailable] <1> A search request with the `pit` parameter must not specify `index`, `routing`, -and {ref}/search-request-body.html#request-body-search-preference[`preference`] +or <> as these parameters are copied from the point in time. <2> Just like regular searches, you can <>, up to the first 10,000 hits. If you diff --git a/docs/reference/search/search-vector-tile-api.asciidoc b/docs/reference/search/search-vector-tile-api.asciidoc index 22ce3ec638e2..cadd388d91a5 100644 --- a/docs/reference/search/search-vector-tile-api.asciidoc +++ b/docs/reference/search/search-vector-tile-api.asciidoc @@ -50,7 +50,7 @@ https://github.com/mapbox/vector-tile-spec[Mapbox vector tile specification]. * If the {es} {security-features} are enabled, you must have the `read` <> for the target data stream, index, -or alias. For cross-cluster search, see <>. +or alias. For cross-cluster search, see <>. [[search-vector-tile-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/search/search-your-data/retrieve-inner-hits.asciidoc b/docs/reference/search/search-your-data/retrieve-inner-hits.asciidoc index 29c06cea3087..7daf8afcf1c0 100644 --- a/docs/reference/search/search-your-data/retrieve-inner-hits.asciidoc +++ b/docs/reference/search/search-your-data/retrieve-inner-hits.asciidoc @@ -74,7 +74,7 @@ Inner hits also supports the following per document features: * <> * <> * <> -* <> +* <> * <> * <> * <> diff --git a/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc b/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc index 5869beb03406..140aeb39723f 100644 --- a/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc +++ b/docs/reference/search/search-your-data/retrieve-selected-fields.asciidoc @@ -588,7 +588,7 @@ for loading fields: parameter to get values for selected fields. This can be a good choice when returning a fairly small number of fields that support doc values, such as keywords and dates. -* Use the <> parameter to +* Use the <> parameter to get the values for specific stored fields (fields that use the <> mapping option). diff --git a/docs/reference/search/search-your-data/search-shard-routing.asciidoc b/docs/reference/search/search-your-data/search-shard-routing.asciidoc index b0286986df7f..c959ab63f99c 100644 --- a/docs/reference/search/search-your-data/search-shard-routing.asciidoc +++ b/docs/reference/search/search-your-data/search-shard-routing.asciidoc @@ -158,7 +158,7 @@ the request hits. However, hitting a large number of shards can significantly increase CPU and memory usage. TIP: For tips on preventing indices with large numbers of shards, see -<>. +<>. You can use the `max_concurrent_shard_requests` query parameter to control maximum number of concurrent shards a search request can hit per node. This diff --git a/docs/reference/search/search.asciidoc b/docs/reference/search/search.asciidoc index 41cd6d132d87..040ef0e0d78e 100644 --- a/docs/reference/search/search.asciidoc +++ b/docs/reference/search/search.asciidoc @@ -38,7 +38,7 @@ must have the `read` index privilege for the alias's data streams or indices. Allows you to execute a search query and get back search hits that match the query. You can provide search queries using the <> or <>. +query string parameter>> or <>. [[search-search-api-path-params]] ==== {api-path-parms-title} diff --git a/docs/reference/settings/security-settings.asciidoc b/docs/reference/settings/security-settings.asciidoc index f861df431acd..ab101e4a0fdf 100644 --- a/docs/reference/settings/security-settings.asciidoc +++ b/docs/reference/settings/security-settings.asciidoc @@ -2342,7 +2342,7 @@ Contents of a JSON Web Key Set (JWKS), including the secret key that the JWT realm uses to verify token signatures. This format supports multiple keys and optional attributes, and is preferred over the `hmac_key` setting. Cannot be used in conjunction with the `hmac_key` setting. Refer to -<>. +<>. // end::jwt-hmac-jwkset-tag[] // tag::jwt-hmac-key-tag[] @@ -2354,7 +2354,7 @@ without attributes, and cannot be used with the `hmac_jwkset` setting. This format is compatible with OIDC. The HMAC key must be a UNICODE string, where the key bytes are the UTF-8 encoding of the UNICODE string. The `hmac_jwkset` setting is preferred. Refer to -<>. +<>. // end::jwt-hmac-key-tag[] diff --git a/docs/reference/setup/important-settings/discovery-settings.asciidoc b/docs/reference/setup/important-settings/discovery-settings.asciidoc index c9ac91bdd42b..180121b93206 100644 --- a/docs/reference/setup/important-settings/discovery-settings.asciidoc +++ b/docs/reference/setup/important-settings/discovery-settings.asciidoc @@ -19,7 +19,7 @@ When you want to form a cluster with nodes on other hosts, use the <> `discovery.seed_hosts` setting. This setting provides a list of other nodes in the cluster that are master-eligible and likely to be live and contactable to seed -the <>. This setting +the <>. This setting accepts a YAML sequence or array of the addresses of all the master-eligible nodes in the cluster. Each address can be either an IP address or a hostname that resolves to one or more IP addresses via DNS. diff --git a/docs/reference/setup/logging-config.asciidoc b/docs/reference/setup/logging-config.asciidoc index 5575ccaeee2c..0ce2b8f1bfb5 100644 --- a/docs/reference/setup/logging-config.asciidoc +++ b/docs/reference/setup/logging-config.asciidoc @@ -143,7 +143,7 @@ documentation]. Each Java package in the {es-repo}[{es} source code] has a related logger. For example, the `org.elasticsearch.discovery` package has `logger.org.elasticsearch.discovery` for logs related to the -<> process. +<> process. To get more or less verbose logs, use the <> to change the related logger's log level. Each logger diff --git a/docs/reference/setup/restart-cluster.asciidoc b/docs/reference/setup/restart-cluster.asciidoc index e3090f9ec405..899cc9f46545 100644 --- a/docs/reference/setup/restart-cluster.asciidoc +++ b/docs/reference/setup/restart-cluster.asciidoc @@ -1,8 +1,8 @@ [[restart-cluster]] == Full-cluster restart and rolling restart -There may be {ref}/configuring-tls.html#tls-transport[situations where you want -to perform a full-cluster restart] or a rolling restart. In the case of +There may be <> or a rolling restart. In the case of <>, you shut down and restart all the nodes in the cluster while in the case of <>, you shut down only one node at a diff --git a/docs/reference/setup/starting.asciidoc b/docs/reference/setup/starting.asciidoc index 58e01ec7ceb9..1422193e851b 100644 --- a/docs/reference/setup/starting.asciidoc +++ b/docs/reference/setup/starting.asciidoc @@ -41,7 +41,7 @@ include::install/systemd.asciidoc[] If you installed a Docker image, you can start {es} from the command line. There are different methods depending on whether you're using development mode or -production mode. See <>. +production mode. See <>. [discrete] [[start-rpm]] diff --git a/docs/reference/slm/apis/slm-put.asciidoc b/docs/reference/slm/apis/slm-put.asciidoc index 8e23828eeb6a..69dfc4af4e84 100644 --- a/docs/reference/slm/apis/slm-put.asciidoc +++ b/docs/reference/slm/apis/slm-put.asciidoc @@ -62,7 +62,7 @@ include::{es-repo-dir}/snapshot-restore/apis/create-snapshot-api.asciidoc[tag=sn `name`:: (Required, string) Name automatically assigned to each snapshot created by the policy. -<> is supported. +<> is supported. To prevent conflicting snapshot names, a UUID is automatically appended to each snapshot name. @@ -70,7 +70,7 @@ snapshot name. (Required, string) Repository used to store snapshots created by this policy. This repository must exist prior to the policy's creation. You can create a repository using the -<>. +<>. [[slm-api-put-retention]] `retention`:: @@ -100,7 +100,7 @@ Minimum number of snapshots to retain, even if the snapshots have expired. ==== `schedule`:: -(Required, <>) +(Required, <>) Periodic or absolute schedule at which the policy creates snapshots. {slm-init} applies `schedule` changes immediately. diff --git a/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc index 1ff77c8af886..7c17d776e502 100644 --- a/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/clone-snapshot-api.asciidoc @@ -55,4 +55,4 @@ fails and returns an error. Defaults to `30s`. `indices`:: (Required, string) A comma-separated list of indices to include in the snapshot. -<> is supported. \ No newline at end of file +<> is supported. \ No newline at end of file diff --git a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc index 649a904786f1..56dc2e6cccd0 100644 --- a/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/create-snapshot-api.asciidoc @@ -78,7 +78,7 @@ match data streams and indices. Supports comma-separated values, such as `open,hidden`. Defaults to `all`. Valid values are: `all`::: -Match any data stream or index, including <> ones. +Match any data stream or index, including <> ones. `open`::: Match open indices and data streams. diff --git a/docs/reference/snapshot-restore/repository-azure.asciidoc b/docs/reference/snapshot-restore/repository-azure.asciidoc index abe7fff7f20d..e848ec9620cb 100644 --- a/docs/reference/snapshot-restore/repository-azure.asciidoc +++ b/docs/reference/snapshot-restore/repository-azure.asciidoc @@ -1,8 +1,7 @@ [[repository-azure]] === Azure repository -You can use https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction[Azure Blob storage] as a repository for -{ref}/modules-snapshots.html[Snapshot/Restore]. +You can use https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction[Azure Blob storage] as a repository for <>. [[repository-azure-usage]] ==== Setup diff --git a/docs/reference/snapshot-restore/repository-gcs.asciidoc b/docs/reference/snapshot-restore/repository-gcs.asciidoc index 37dfe2add0b1..d99b9bc81567 100644 --- a/docs/reference/snapshot-restore/repository-gcs.asciidoc +++ b/docs/reference/snapshot-restore/repository-gcs.asciidoc @@ -2,7 +2,7 @@ === Google Cloud Storage repository You can use the https://cloud.google.com/storage/[Google Cloud Storage] -service as a repository for {ref}/modules-snapshots.html[Snapshot/Restore]. +service as a repository for {ref}/snapshot-restore.html[Snapshot/Restore]. [[repository-gcs-usage]] ==== Getting started diff --git a/docs/reference/snapshot-restore/repository-s3.asciidoc b/docs/reference/snapshot-restore/repository-s3.asciidoc index de72511010b5..c34b295f9bd1 100644 --- a/docs/reference/snapshot-restore/repository-s3.asciidoc +++ b/docs/reference/snapshot-restore/repository-s3.asciidoc @@ -1,7 +1,7 @@ [[repository-s3]] === S3 repository -You can use AWS S3 as a repository for {ref}/modules-snapshots.html[Snapshot/Restore]. +You can use AWS S3 as a repository for {ref}/snapshot-restore.html[Snapshot/Restore]. *If you are looking for a hosted solution of Elasticsearch on AWS, please visit https://www.elastic.co/cloud/.* diff --git a/docs/reference/snapshot-restore/restore-snapshot.asciidoc b/docs/reference/snapshot-restore/restore-snapshot.asciidoc index 0fca81314391..e9f01f25036d 100644 --- a/docs/reference/snapshot-restore/restore-snapshot.asciidoc +++ b/docs/reference/snapshot-restore/restore-snapshot.asciidoc @@ -276,7 +276,7 @@ before you start. . If you <>, you can restore them to each node. This step is optional and requires a -<>. +<>. + After you shut down a node, copy the backed-up configuration files over to the node's `$ES_PATH_CONF` directory. Before restarting the node, ensure diff --git a/docs/reference/sql/endpoints/translate.asciidoc b/docs/reference/sql/endpoints/translate.asciidoc index 786d44a8f8c3..21253b4bf8e4 100644 --- a/docs/reference/sql/endpoints/translate.asciidoc +++ b/docs/reference/sql/endpoints/translate.asciidoc @@ -53,7 +53,7 @@ Which returns: Which is the request that SQL will run to provide the results. In this case, SQL will use the <> API. If the result contained an aggregation then SQL would use -the normal <> API. +the normal <>. The request body accepts the same <> as the <>, excluding `cursor`. diff --git a/docs/reference/sql/functions/date-time.asciidoc b/docs/reference/sql/functions/date-time.asciidoc index e49ed392473d..42e202d7f8f8 100644 --- a/docs/reference/sql/functions/date-time.asciidoc +++ b/docs/reference/sql/functions/date-time.asciidoc @@ -10,7 +10,7 @@ A common requirement when dealing with date/time in general revolves around the notion of `interval`, a topic that is worth exploring in the context of {es} and {es-sql}. -{es} has comprehensive support for <> both inside <> and <>. +{es} has comprehensive support for <> both inside <> and <>. Inside {es-sql} the former is supported as is by passing the expression in the table name, while the latter is supported through the standard SQL `INTERVAL`. The table below shows the mapping between {es} and {es-sql}: diff --git a/docs/reference/sql/language/syntax/commands/select.asciidoc b/docs/reference/sql/language/syntax/commands/select.asciidoc index 59a4c041b580..8ab0ca2ab8fd 100644 --- a/docs/reference/sql/language/syntax/commands/select.asciidoc +++ b/docs/reference/sql/language/syntax/commands/select.asciidoc @@ -116,7 +116,7 @@ If the table name contains special SQL characters (such as `.`,`-`,`*`,etc...) u include-tagged::{sql-specs}/docs/docs.csv-spec[fromTableQuoted] ---- -The name can be a <> pointing to multiple indices (likely requiring quoting as mentioned above) with the restriction that *all* resolved concrete tables have **exact mapping**. +The name can be a <> pointing to multiple indices (likely requiring quoting as mentioned above) with the restriction that *all* resolved concrete tables have **exact mapping**. [source, sql] ---- diff --git a/docs/reference/sql/language/syntax/lexic/index.asciidoc b/docs/reference/sql/language/syntax/lexic/index.asciidoc index dc0071582f3a..0334c2836aa5 100644 --- a/docs/reference/sql/language/syntax/lexic/index.asciidoc +++ b/docs/reference/sql/language/syntax/lexic/index.asciidoc @@ -42,7 +42,7 @@ Identifiers can be of two types: __quoted__ and __unquoted__: SELECT ip_address FROM "hosts-*" ---- -This query has two identifiers, `ip_address` and `hosts-*` (an <>). As `ip_address` does not clash with any key words it can be used verbatim, `hosts-*` on the other hand cannot as it clashes with `-` (minus operation) and `*` hence the double quotes. +This query has two identifiers, `ip_address` and `hosts-*` (an <>). As `ip_address` does not clash with any key words it can be used verbatim, `hosts-*` on the other hand cannot as it clashes with `-` (minus operation) and `*` hence the double quotes. Another example: @@ -51,7 +51,7 @@ Another example: SELECT "from" FROM "" ---- -The first identifier from needs to quoted as otherwise it clashes with the `FROM` key word (which is case insensitive as thus can be written as `from`) while the second identifier using {es} <> would have otherwise confuse the parser. +The first identifier from needs to quoted as otherwise it clashes with the `FROM` key word (which is case insensitive as thus can be written as `from`) while the second identifier using {es} <> would have otherwise confuse the parser. Hence why in general, *especially* when dealing with user input it is *highly* recommended to use quotes for identifiers. It adds minimal increase to your queries and in return offers clarity and disambiguation. diff --git a/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc b/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc index fe79ef57b6ea..fbc6fe7b42a7 100644 --- a/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc +++ b/docs/reference/troubleshooting/common-issues/circuit-breaker-errors.asciidoc @@ -80,7 +80,7 @@ For high-cardinality `text` fields, fielddata can use a large amount of JVM memory. To avoid this, {es} disables fielddata on `text` fields by default. If you've enabled fielddata and triggered the <>, consider disabling it and using a `keyword` field instead. -See <>. +See <>. **Clear the fielddata cache** diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index 507523362518..b298e0ce8614 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -45,6 +45,9 @@ public class TSDBIndexingIT extends ESSingleNodeTestCase { { "_doc":{ "properties": { + "@timestamp" : { + "type": "date" + }, "metricset": { "type": "keyword", "time_series_dimension": true @@ -86,28 +89,18 @@ public class TSDBIndexingIT extends ESSingleNodeTestCase { } public void testTimeRanges() throws Exception { - var mappingTemplate = """ - { - "_doc":{ - "properties": { - "metricset": { - "type": "keyword", - "time_series_dimension": true - } - } - } - }"""; var templateSettings = Settings.builder().put("index.mode", "time_series"); if (randomBoolean()) { templateSettings.put("index.routing_path", "metricset"); } + var mapping = new CompressedXContent(randomBoolean() ? MAPPING_TEMPLATE : MAPPING_TEMPLATE.replace("date", "date_nanos")); if (randomBoolean()) { var request = new PutComposableIndexTemplateAction.Request("id"); request.indexTemplate( new ComposableIndexTemplate( List.of("k8s*"), - new Template(templateSettings.build(), new CompressedXContent(mappingTemplate), null), + new Template(templateSettings.build(), mapping, null), null, null, null, @@ -119,9 +112,7 @@ public class TSDBIndexingIT extends ESSingleNodeTestCase { client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); } else { var putComponentTemplateRequest = new PutComponentTemplateAction.Request("1"); - putComponentTemplateRequest.componentTemplate( - new ComponentTemplate(new Template(null, new CompressedXContent(mappingTemplate), null), null, null) - ); + putComponentTemplateRequest.componentTemplate(new ComponentTemplate(new Template(null, mapping, null), null, null)); client().execute(PutComponentTemplateAction.INSTANCE, putComponentTemplateRequest).actionGet(); var putTemplateRequest = new PutComposableIndexTemplateAction.Request("id"); @@ -376,13 +367,14 @@ public class TSDBIndexingIT extends ESSingleNodeTestCase { public void testSkippingShards() throws Exception { Instant time = Instant.now(); + var mapping = new CompressedXContent(randomBoolean() ? MAPPING_TEMPLATE : MAPPING_TEMPLATE.replace("date", "date_nanos")); { var templateSettings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "metricset").build(); var request = new PutComposableIndexTemplateAction.Request("id1"); request.indexTemplate( new ComposableIndexTemplate( List.of("pattern-1"), - new Template(templateSettings, new CompressedXContent(MAPPING_TEMPLATE), null), + new Template(templateSettings, mapping, null), null, null, null, @@ -401,7 +393,7 @@ public class TSDBIndexingIT extends ESSingleNodeTestCase { request.indexTemplate( new ComposableIndexTemplate( List.of("pattern-2"), - new Template(null, new CompressedXContent(MAPPING_TEMPLATE), null), + new Template(null, mapping, null), null, null, null, diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java index 9ae76394011c..14fe7ad85562 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; @@ -213,77 +212,19 @@ public class TsdbDataStreamRestIT extends ESRestTestCase { } public void testTsdbDataStreams() throws Exception { - var bulkRequest = new Request("POST", "/k8s/_bulk"); - bulkRequest.setJsonEntity(BULK.replace("$now", formatInstant(Instant.now()))); - bulkRequest.addParameter("refresh", "true"); - var response = client().performRequest(bulkRequest); - assertOK(response); - var responseBody = entityAsMap(response); - assertThat("errors in response:\n " + responseBody, responseBody.get("errors"), equalTo(false)); - - var getDataStreamsRequest = new Request("GET", "/_data_stream"); - response = client().performRequest(getDataStreamsRequest); - assertOK(response); - var dataStreams = entityAsMap(response); - assertThat(ObjectPath.evaluate(dataStreams, "data_streams"), hasSize(1)); - assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.name"), equalTo("k8s")); - assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.generation"), equalTo(1)); - assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.template"), equalTo("1")); - assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.indices"), hasSize(1)); - String firstBackingIndex = ObjectPath.evaluate(dataStreams, "data_streams.0.indices.0.index_name"); - assertThat(firstBackingIndex, backingIndexEqualTo("k8s", 1)); - - var indices = getIndex(firstBackingIndex); - var escapedBackingIndex = firstBackingIndex.replace(".", "\\."); - assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".data_stream"), equalTo("k8s")); - assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.mode"), equalTo("time_series")); - String startTimeFirstBackingIndex = ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.start_time"); - assertThat(startTimeFirstBackingIndex, notNullValue()); - String endTimeFirstBackingIndex = ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.end_time"); - assertThat(endTimeFirstBackingIndex, notNullValue()); - List routingPaths = ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.routing_path"); - assertThat(routingPaths, containsInAnyOrder("metricset", "k8s.pod.uid", "pod.labels.*")); - - var rolloverRequest = new Request("POST", "/k8s/_rollover"); - assertOK(client().performRequest(rolloverRequest)); - - response = client().performRequest(getDataStreamsRequest); - assertOK(response); - dataStreams = entityAsMap(response); - assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.name"), equalTo("k8s")); - assertThat(ObjectPath.evaluate(dataStreams, "data_streams.0.generation"), equalTo(2)); - String secondBackingIndex = ObjectPath.evaluate(dataStreams, "data_streams.0.indices.1.index_name"); - assertThat(secondBackingIndex, backingIndexEqualTo("k8s", 2)); - - indices = getIndex(secondBackingIndex); - escapedBackingIndex = secondBackingIndex.replace(".", "\\."); - assertThat(ObjectPath.evaluate(indices, escapedBackingIndex + ".data_stream"), equalTo("k8s")); - String startTimeSecondBackingIndex = ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.start_time"); - assertThat(startTimeSecondBackingIndex, equalTo(endTimeFirstBackingIndex)); - String endTimeSecondBackingIndex = ObjectPath.evaluate(indices, escapedBackingIndex + ".settings.index.time_series.end_time"); - assertThat(endTimeSecondBackingIndex, notNullValue()); - - var indexRequest = new Request("POST", "/k8s/_doc"); - Instant time = parseInstant(startTimeFirstBackingIndex); - indexRequest.setJsonEntity(DOC.replace("$time", formatInstant(time))); - response = client().performRequest(indexRequest); - assertOK(response); - assertThat(entityAsMap(response).get("_index"), equalTo(firstBackingIndex)); - - indexRequest = new Request("POST", "/k8s/_doc"); - time = parseInstant(endTimeSecondBackingIndex).minusMillis(1); - indexRequest.setJsonEntity(DOC.replace("$time", formatInstant(time))); - response = client().performRequest(indexRequest); - assertOK(response); - assertThat(entityAsMap(response).get("_index"), equalTo(secondBackingIndex)); + assertTsdbDataStream(); } public void testTsdbDataStreamsNanos() throws Exception { - // Create a template + // Overwrite template to use date_nanos field type: var putComposableIndexTemplateRequest = new Request("POST", "/_index_template/1"); putComposableIndexTemplateRequest.setJsonEntity(TEMPLATE.replace("date", "date_nanos")); assertOK(client().performRequest(putComposableIndexTemplateRequest)); + assertTsdbDataStream(); + } + + private void assertTsdbDataStream() throws IOException { var bulkRequest = new Request("POST", "/k8s/_bulk"); bulkRequest.setJsonEntity(BULK.replace("$now", formatInstantNanos(Instant.now()))); bulkRequest.addParameter("refresh", "true"); @@ -333,6 +274,7 @@ public class TsdbDataStreamRestIT extends ESRestTestCase { assertThat(endTimeSecondBackingIndex, notNullValue()); var indexRequest = new Request("POST", "/k8s/_doc"); + indexRequest.addParameter("refresh", "true"); Instant time = parseInstant(startTimeFirstBackingIndex); indexRequest.setJsonEntity(DOC.replace("$time", formatInstantNanos(time))); response = client().performRequest(indexRequest); @@ -340,11 +282,45 @@ public class TsdbDataStreamRestIT extends ESRestTestCase { assertThat(entityAsMap(response).get("_index"), equalTo(firstBackingIndex)); indexRequest = new Request("POST", "/k8s/_doc"); + indexRequest.addParameter("refresh", "true"); time = parseInstant(endTimeSecondBackingIndex).minusMillis(1); indexRequest.setJsonEntity(DOC.replace("$time", formatInstantNanos(time))); response = client().performRequest(indexRequest); assertOK(response); assertThat(entityAsMap(response).get("_index"), equalTo(secondBackingIndex)); + + var searchRequest = new Request("GET", "k8s/_search"); + searchRequest.setJsonEntity(""" + { + "query": { + "range":{ + "@timestamp":{ + "gte": "now-7d", + "lte": "now+7d" + } + } + }, + "sort": [ + { + "@timestamp": { + "order": "desc" + } + } + ] + } + """); + response = client().performRequest(searchRequest); + assertOK(response); + responseBody = entityAsMap(response); + try { + assertThat(ObjectPath.evaluate(responseBody, "hits.total.value"), equalTo(10)); + assertThat(ObjectPath.evaluate(responseBody, "hits.total.relation"), equalTo("eq")); + assertThat(ObjectPath.evaluate(responseBody, "hits.hits.0._index"), equalTo(secondBackingIndex)); + assertThat(ObjectPath.evaluate(responseBody, "hits.hits.1._index"), equalTo(firstBackingIndex)); + } catch (Exception | AssertionError e) { + logger.error("search response body causing assertion error [" + responseBody + "]", e); + throw e; + } } public void testSimulateTsdbDataStreamTemplate() throws Exception { diff --git a/rest-api-spec/src/yamlRestTestV7Compat/resources/rest-api-spec/test/search/10_geo_bounding_box.yml b/rest-api-spec/src/yamlRestTestV7Compat/resources/rest-api-spec/test/search/10_geo_bounding_box.yml new file mode 100644 index 000000000000..964cd2d6c874 --- /dev/null +++ b/rest-api-spec/src/yamlRestTestV7Compat/resources/rest-api-spec/test/search/10_geo_bounding_box.yml @@ -0,0 +1,80 @@ +--- +setup: + - skip: + version: "9.0.0 - " + reason: "compatible from 8.x to 7.x" + features: + - "headers" + - "warnings" + - do: + indices.create: + index: locations + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + + properties: + location: + type: geo_point + - do: + bulk: + index: locations + refresh: true + body: | + {"index":{}} + {"location" : {"lat": 13.5, "lon" : 34.89}} + {"index":{}} + {"location" : {"lat": -7.9, "lon" : 120.78}} + {"index":{}} + {"location" : {"lat": 45.78, "lon" : -173.45}} + {"index":{}} + {"location" : {"lat": 32.45, "lon" : 45.6}} + {"index":{}} + {"location" : {"lat": -63.24, "lon" : 31.0}} + {"index":{}} + {"location" : {"lat": 0.0, "lon" : 0.0}} + + +--- +"geo bounding box query not compatible": + - do: + catch: /failed to parse \[geo_bounding_box\] query. unexpected field \[type\]/ + search: + index: locations + body: + query: + geo_bounding_box: + type : indexed + location: + top_left: + lat: 10 + lon: -10 + bottom_right: + lat: -10 + lon: 10 + +--- +"geo bounding box query compatible": + - do: + headers: + Content-Type: "application/vnd.elasticsearch+json;compatible-with=7" + Accept: "application/vnd.elasticsearch+json;compatible-with=7" + warnings: + - "Deprecated parameter [type] used, it should no longer be specified." + search: + index: locations + body: + query: + geo_bounding_box: + type : indexed + location: + top_left: + lat: 10 + lon: -10 + bottom_right: + lat: -10 + lon: 10 + - match: {hits.total.value: 1} + diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 09157c819577..2e069d690dd1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -43,6 +43,7 @@ import org.elasticsearch.gateway.MetadataStateFormat; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.IndexLongFieldRange; @@ -1297,14 +1298,27 @@ public class IndexMetadata implements Diffable, ToXContentFragmen } /** + * @return whether this index has a time series timestamp range + */ + public boolean hasTimeSeriesTimestampRange() { + return indexMode != null && indexMode.getTimestampBound(this) != null; + } + + /** + * @param dateFieldType the date field type of '@timestamp' field which is + * used to convert the start and end times recorded in index metadata + * to the right format that is being used by '@timestamp' field. + * For example, the '@timestamp' can be configured with nanosecond precision. * @return the time range this index represents if this index is in time series mode. * Otherwise null is returned. */ @Nullable - public IndexLongFieldRange getTimeSeriesTimestampRange() { + public IndexLongFieldRange getTimeSeriesTimestampRange(DateFieldMapper.DateFieldType dateFieldType) { var bounds = indexMode != null ? indexMode.getTimestampBound(this) : null; if (bounds != null) { - return IndexLongFieldRange.NO_SHARDS.extendWithShardRange(0, 1, ShardLongFieldRange.of(bounds.startTime(), bounds.endTime())); + long start = dateFieldType.resolution().convert(Instant.ofEpochMilli(bounds.startTime())); + long end = dateFieldType.resolution().convert(Instant.ofEpochMilli(bounds.endTime())); + return IndexLongFieldRange.NO_SHARDS.extendWithShardRange(0, 1, ShardLongFieldRange.of(start, end)); } else { return null; } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingState.java b/server/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingState.java index 3d264c7a009a..7b7433530da8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingState.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/ShardRoutingState.java @@ -31,7 +31,7 @@ public enum ShardRoutingState { */ RELOCATING((byte) 4); - private byte value; + private final byte value; ShardRoutingState(byte value) { this.value = value; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java index 5d7b69020a57..81d77b8d8fa2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java @@ -90,6 +90,7 @@ public class AllocationDeciders { } public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) { + assert shardRouting.started() : "Only started shard could be rebalanced: " + shardRouting; return withDeciders( allocation, decider -> decider.canRebalance(shardRouting, allocation), diff --git a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java index c77d4f3cc227..bb76514dddd6 100644 --- a/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java @@ -91,6 +91,7 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo private final HttpTracer httpLogger; private final Tracer tracer; + private volatile boolean gracefullyCloseConnections; private volatile long slowLogThresholdMs; @@ -454,7 +455,8 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo threadContext, corsHandler, maybeHttpLogger, - tracer + tracer, + gracefullyCloseConnections ); } catch (final IllegalArgumentException e) { badRequestCause = ExceptionsHelper.useOrSuppress(badRequestCause, e); @@ -468,7 +470,8 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo threadContext, corsHandler, httpLogger, - tracer + tracer, + gracefullyCloseConnections ); } channel = innerChannel; @@ -510,4 +513,8 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo public ThreadPool getThreadPool() { return threadPool; } + + public void gracefullyCloseConnections() { + gracefullyCloseConnections = true; + } } diff --git a/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java b/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java index 6fc6e7eb3ffb..2b4e1fdc1d58 100644 --- a/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java +++ b/server/src/main/java/org/elasticsearch/http/DefaultRestChannel.java @@ -56,6 +56,7 @@ public class DefaultRestChannel extends AbstractRestChannel implements RestChann private final HttpChannel httpChannel; private final CorsHandler corsHandler; private final Tracer tracer; + private final boolean closeConnection; @Nullable private final HttpTracer httpLogger; @@ -69,7 +70,8 @@ public class DefaultRestChannel extends AbstractRestChannel implements RestChann ThreadContext threadContext, CorsHandler corsHandler, @Nullable HttpTracer httpLogger, - Tracer tracer + Tracer tracer, + boolean closeConnection ) { super(request, settings.detailedErrorsEnabled()); this.httpChannel = httpChannel; @@ -80,6 +82,7 @@ public class DefaultRestChannel extends AbstractRestChannel implements RestChann this.corsHandler = corsHandler; this.httpLogger = httpLogger; this.tracer = tracer; + this.closeConnection = closeConnection; } @Override @@ -95,7 +98,7 @@ public class DefaultRestChannel extends AbstractRestChannel implements RestChann final SpanId spanId = SpanId.forRestRequest(request); final ArrayList toClose = new ArrayList<>(4); - if (HttpUtils.shouldCloseConnection(httpRequest)) { + if (HttpUtils.shouldCloseConnection(httpRequest) || closeConnection) { toClose.add(() -> CloseableChannel.closeChannel(httpChannel)); } toClose.add(() -> tracer.stopTrace(request)); @@ -159,6 +162,9 @@ public class DefaultRestChannel extends AbstractRestChannel implements RestChann // Add all custom headers addCustomHeaders(httpResponse, restResponse.getHeaders()); addCustomHeaders(httpResponse, restResponse.filterHeaders(threadContext.getResponseHeaders())); + if (closeConnection) { + setHeaderField(httpResponse, CONNECTION, CLOSE); + } // If our response doesn't specify a content-type header, set one setHeaderField(httpResponse, CONTENT_TYPE, restResponse.contentType(), false); diff --git a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContextProvider.java b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContextProvider.java index 9f1447d436ad..e44861b4afe8 100644 --- a/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContextProvider.java +++ b/server/src/main/java/org/elasticsearch/index/query/CoordinatorRewriteContextProvider.java @@ -49,20 +49,18 @@ public class CoordinatorRewriteContextProvider { if (indexMetadata == null) { return null; } + DateFieldMapper.DateFieldType dateFieldType = mappingSupplier.apply(index); + if (dateFieldType == null) { + return null; + } IndexLongFieldRange timestampRange = indexMetadata.getTimestampRange(); if (timestampRange.containsAllShardRanges() == false) { - timestampRange = indexMetadata.getTimeSeriesTimestampRange(); + timestampRange = indexMetadata.getTimeSeriesTimestampRange(dateFieldType); if (timestampRange == null) { return null; } } - DateFieldMapper.DateFieldType dateFieldType = mappingSupplier.apply(index); - - if (dateFieldType == null) { - return null; - } - return new CoordinatorRewriteContext(parserConfig, client, nowInMillis, timestampRange, dateFieldType); } } diff --git a/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java index 31faad19e2e2..9afadc1a8d9d 100644 --- a/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java @@ -21,6 +21,8 @@ import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.mapper.GeoShapeQueryable; @@ -42,11 +44,16 @@ import java.util.Objects; public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder { public static final String NAME = "geo_bounding_box"; + private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(GeoBoundingBoxQueryBuilder.class); + + private static final String TYPE_PARAMETER_DEPRECATION_MESSAGE = "Deprecated parameter [type] used, it should no longer be specified."; + /** * The default value for ignore_unmapped. */ public static final boolean DEFAULT_IGNORE_UNMAPPED = false; + private static final ParseField TYPE_FIELD = new ParseField("type").forRestApiVersion(RestApiVersion.equalTo(RestApiVersion.V_7)); private static final ParseField VALIDATION_METHOD_FIELD = new ParseField("validation_method"); private static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped"); @@ -349,14 +356,18 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder Decision.YES); @@ -128,29 +128,29 @@ public class AllocationDecidersTests extends ESTestCase { int expectedAllocationDecidersCalls, Decision expectedDecision ) { - IndexMetadata index = IndexMetadata.builder("index") - .settings(settings(Version.CURRENT)) - .numberOfShards(1) - .numberOfReplicas(0) - .build(); + IndexMetadata index = IndexMetadata.builder("index").settings(indexSettings(Version.CURRENT, 1, 0)).build(); + ShardId shardId = new ShardId(index.getIndex(), 0); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) .metadata(Metadata.builder().put(index, false).build()) .build(); - ShardRouting shardRouting = createShardRouting(index.getIndex()); + + ShardRouting startedShard = TestShardRouting.newShardRouting(shardId, "node", true, ShardRoutingState.STARTED); + ShardRouting unassignedShard = createUnassignedShard(index.getIndex()); + RoutingNode routingNode = RoutingNodesHelper.routingNode("node", null); - DiscoveryNode discoveryNode = TestDiscoveryNode.create("node", new TransportAddress(TransportAddress.META_ADDRESS, 0)); + DiscoveryNode discoveryNode = newNode("node"); List.>of( - (allocation, deciders) -> deciders.canAllocate(shardRouting, allocation), - (allocation, deciders) -> deciders.canAllocate(shardRouting, routingNode, allocation), + (allocation, deciders) -> deciders.canAllocate(unassignedShard, allocation), + (allocation, deciders) -> deciders.canAllocate(unassignedShard, routingNode, allocation), (allocation, deciders) -> deciders.canAllocate(index, routingNode, allocation), (allocation, deciders) -> deciders.canRebalance(allocation), - (allocation, deciders) -> deciders.canRebalance(shardRouting, allocation), - (allocation, deciders) -> deciders.canRemain(shardRouting, routingNode, allocation), + (allocation, deciders) -> deciders.canRebalance(startedShard, allocation), + (allocation, deciders) -> deciders.canRemain(unassignedShard, routingNode, allocation), (allocation, deciders) -> deciders.shouldAutoExpandToNode(index, discoveryNode, allocation), - (allocation, deciders) -> deciders.canForceAllocatePrimary(shardRouting, routingNode, allocation), - (allocation, deciders) -> deciders.canForceAllocateDuringReplace(shardRouting, routingNode, allocation), - (allocation, deciders) -> deciders.canAllocateReplicaWhenThereIsRetentionLease(shardRouting, routingNode, allocation) + (allocation, deciders) -> deciders.canForceAllocatePrimary(unassignedShard, routingNode, allocation), + (allocation, deciders) -> deciders.canForceAllocateDuringReplace(unassignedShard, routingNode, allocation), + (allocation, deciders) -> deciders.canAllocateReplicaWhenThereIsRetentionLease(unassignedShard, routingNode, allocation) ).forEach(operation -> { var decidersCalled = new int[] { 0 }; var deciders = new AllocationDeciders(decisions.stream().map(decision -> new TestAllocationDecider(() -> { @@ -180,7 +180,7 @@ public class AllocationDecidersTests extends ESTestCase { ); assertThat( - deciders.getForcedInitialShardAllocationToNodes(createShardRouting(), createRoutingAllocation(deciders)), + deciders.getForcedInitialShardAllocationToNodes(createUnassignedShard(), createRoutingAllocation(deciders)), equalTo(Optional.empty()) ); } @@ -197,7 +197,7 @@ public class AllocationDecidersTests extends ESTestCase { ); assertThat( - deciders.getForcedInitialShardAllocationToNodes(createShardRouting(), createRoutingAllocation(deciders)), + deciders.getForcedInitialShardAllocationToNodes(createUnassignedShard(), createRoutingAllocation(deciders)), equalTo(Optional.of(Set.of("node-1", "node-2"))) ); } @@ -215,12 +215,12 @@ public class AllocationDecidersTests extends ESTestCase { ); assertThat( - deciders.getForcedInitialShardAllocationToNodes(createShardRouting(), createRoutingAllocation(deciders)), + deciders.getForcedInitialShardAllocationToNodes(createUnassignedShard(), createRoutingAllocation(deciders)), equalTo(Optional.of(Set.of("node-2"))) ); } - private static ShardRouting createShardRouting(Index index) { + private static ShardRouting createUnassignedShard(Index index) { return ShardRouting.newUnassigned( new ShardId(index, 0), true, @@ -230,8 +230,8 @@ public class AllocationDecidersTests extends ESTestCase { ); } - private static ShardRouting createShardRouting() { - return createShardRouting(new Index("test", "testUUID")); + private static ShardRouting createUnassignedShard() { + return createUnassignedShard(new Index("test", "testUUID")); } private static RoutingAllocation createRoutingAllocation(AllocationDeciders deciders) { diff --git a/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java b/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java index a70373a68600..bb443d59ac11 100644 --- a/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java +++ b/server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java @@ -12,6 +12,7 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionModule; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; @@ -51,6 +52,7 @@ import org.elasticsearch.xcontent.NamedXContentRegistry; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.mockito.ArgumentCaptor; import java.net.InetSocketAddress; import java.net.UnknownHostException; @@ -69,12 +71,16 @@ import java.util.concurrent.TimeUnit; import static java.net.InetAddress.getByName; import static java.util.Arrays.asList; import static org.elasticsearch.http.AbstractHttpServerTransport.resolvePublishPort; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; public class AbstractHttpServerTransportTests extends ESTestCase { @@ -883,6 +889,66 @@ public class AbstractHttpServerTransportTests extends ESTestCase { } } + @SuppressWarnings("unchecked") + public void testSetGracefulClose() { + try ( + AbstractHttpServerTransport transport = new AbstractHttpServerTransport( + Settings.EMPTY, + networkService, + recycler, + threadPool, + xContentRegistry(), + new HttpServerTransport.Dispatcher() { + @Override + public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) { + channel.sendResponse(emptyResponse(RestStatus.OK)); + } + + @Override + public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext, Throwable cause) { + channel.sendResponse(emptyResponse(RestStatus.BAD_REQUEST)); + } + }, + new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), + Tracer.NOOP + ) { + + @Override + protected HttpServerChannel bind(InetSocketAddress hostAddress) { + return null; + } + + @Override + protected void doStart() {} + + @Override + protected void stopInternal() {} + } + ) { + final TestHttpRequest httpRequest = new TestHttpRequest(HttpRequest.HttpVersion.HTTP_1_1, RestRequest.Method.GET, "/"); + + HttpChannel httpChannel = mock(HttpChannel.class); + transport.incomingRequest(httpRequest, httpChannel); + + var response = ArgumentCaptor.forClass(TestHttpResponse.class); + var listener = ArgumentCaptor.forClass(ActionListener.class); + verify(httpChannel).sendResponse(response.capture(), listener.capture()); + + listener.getValue().onResponse(null); + assertThat(response.getValue().containsHeader(DefaultRestChannel.CONNECTION), is(false)); + verify(httpChannel, never()).close(); + + httpChannel = mock(HttpChannel.class); + transport.gracefullyCloseConnections(); + transport.incomingRequest(httpRequest, httpChannel); + verify(httpChannel).sendResponse(response.capture(), listener.capture()); + + listener.getValue().onResponse(null); + assertThat(response.getValue().headers().get(DefaultRestChannel.CONNECTION), containsInAnyOrder(DefaultRestChannel.CLOSE)); + verify(httpChannel).close(); + } + } + private static RestResponse emptyResponse(RestStatus status) { return new RestResponse(status, RestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY); } diff --git a/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java b/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java index ffe95b000474..d050c2432025 100644 --- a/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java +++ b/server/src/test/java/org/elasticsearch/http/DefaultRestChannelTests.java @@ -172,7 +172,8 @@ public class DefaultRestChannelTests extends ESTestCase { threadPool.getThreadContext(), CorsHandler.fromSettings(settings), httpTracer, - tracer + tracer, + false ); RestResponse resp = testRestResponse(); final String customHeader = "custom-header"; @@ -192,6 +193,65 @@ public class DefaultRestChannelTests extends ESTestCase { assertEquals(resp.contentType(), headers.get(DefaultRestChannel.CONTENT_TYPE).get(0)); } + public void testCloseConnection() { + Settings settings = Settings.builder().build(); + final TestHttpRequest httpRequest = new TestHttpRequest(HttpRequest.HttpVersion.HTTP_1_1, RestRequest.Method.GET, "/"); + final RestRequest request = RestRequest.request(parserConfig(), httpRequest, httpChannel); + HttpHandlingSettings handlingSettings = HttpHandlingSettings.fromSettings(settings); + // send a response + DefaultRestChannel channel = new DefaultRestChannel( + httpChannel, + httpRequest, + request, + bigArrays, + handlingSettings, + threadPool.getThreadContext(), + CorsHandler.fromSettings(settings), + httpTracer, + tracer, + true + ); + + RestResponse resp = testRestResponse(); + channel.sendResponse(resp); + // inspect what was written + ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(TestHttpResponse.class); + verify(httpChannel).sendResponse(responseCaptor.capture(), any()); + TestHttpResponse httpResponse = responseCaptor.getValue(); + Map> headers = httpResponse.headers(); + assertThat(headers.get(DefaultRestChannel.CONNECTION), containsInAnyOrder(DefaultRestChannel.CLOSE)); + } + + public void testNormallyNoConnectionClose() { + Settings settings = Settings.builder().build(); + final TestHttpRequest httpRequest = new TestHttpRequest(HttpRequest.HttpVersion.HTTP_1_1, RestRequest.Method.GET, "/"); + final RestRequest request = RestRequest.request(parserConfig(), httpRequest, httpChannel); + HttpHandlingSettings handlingSettings = HttpHandlingSettings.fromSettings(settings); + // send a response + DefaultRestChannel channel = new DefaultRestChannel( + httpChannel, + httpRequest, + request, + bigArrays, + handlingSettings, + threadPool.getThreadContext(), + CorsHandler.fromSettings(settings), + httpTracer, + tracer, + false + ); + + RestResponse resp = testRestResponse(); + channel.sendResponse(resp); + + ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(TestHttpResponse.class); + verify(httpChannel).sendResponse(responseCaptor.capture(), any()); + + TestHttpResponse httpResponse = responseCaptor.getValue(); + Map> headers = httpResponse.headers(); + assertNull(headers.get(DefaultRestChannel.CONNECTION)); + } + public void testCookiesSet() { Settings settings = Settings.builder().put(HttpTransportSettings.SETTING_HTTP_RESET_COOKIES.getKey(), true).build(); final TestHttpRequest httpRequest = new TestHttpRequest(HttpRequest.HttpVersion.HTTP_1_1, RestRequest.Method.GET, "/"); @@ -209,7 +269,8 @@ public class DefaultRestChannelTests extends ESTestCase { threadPool.getThreadContext(), CorsHandler.fromSettings(settings), httpTracer, - tracer + tracer, + false ); channel.sendResponse(testRestResponse()); @@ -238,7 +299,8 @@ public class DefaultRestChannelTests extends ESTestCase { threadPool.getThreadContext(), CorsHandler.fromSettings(settings), httpTracer, - tracer + tracer, + false ); final RestResponse response = new RestResponse( RestStatus.INTERNAL_SERVER_ERROR, @@ -306,7 +368,8 @@ public class DefaultRestChannelTests extends ESTestCase { threadPool.getThreadContext(), CorsHandler.fromSettings(settings), httpTracer, - tracer + tracer, + false ); channel.sendResponse(testRestResponse()); Class> listenerClass = (Class>) (Class) ActionListener.class; @@ -338,7 +401,8 @@ public class DefaultRestChannelTests extends ESTestCase { threadPool.getThreadContext(), CorsHandler.fromSettings(Settings.EMPTY), httpTracer, - tracer + tracer, + false ); doAnswer(invocationOnMock -> { ActionListener listener = invocationOnMock.getArgument(1); @@ -385,7 +449,8 @@ public class DefaultRestChannelTests extends ESTestCase { threadPool.getThreadContext(), CorsHandler.fromSettings(Settings.EMPTY), httpTracer, - tracer + tracer, + false ); // ESTestCase#after will invoke ensureAllArraysAreReleased which will fail if the response content was not released @@ -432,7 +497,8 @@ public class DefaultRestChannelTests extends ESTestCase { threadPool.getThreadContext(), CorsHandler.fromSettings(Settings.EMPTY), httpTracer, - tracer + tracer, + false ); // ESTestCase#after will invoke ensureAllArraysAreReleased which will fail if the response content was not released @@ -481,7 +547,8 @@ public class DefaultRestChannelTests extends ESTestCase { threadPool.getThreadContext(), CorsHandler.fromSettings(Settings.EMPTY), httpTracer, - tracer + tracer, + false ); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(HttpResponse.class); { @@ -541,7 +608,8 @@ public class DefaultRestChannelTests extends ESTestCase { threadPool.getThreadContext(), new CorsHandler(CorsHandler.buildConfig(Settings.EMPTY)), new HttpTracer(), - tracer + tracer, + false ); final MockLogAppender sendingResponseMockLog = new MockLogAppender(); @@ -603,7 +671,8 @@ public class DefaultRestChannelTests extends ESTestCase { threadPool.getThreadContext(), new CorsHandler(CorsHandler.buildConfig(Settings.EMPTY)), new HttpTracer(), - tracer + tracer, + false ); MockLogAppender mockLogAppender = new MockLogAppender(); @@ -659,7 +728,8 @@ public class DefaultRestChannelTests extends ESTestCase { threadPool.getThreadContext(), CorsHandler.fromSettings(Settings.EMPTY), new HttpTracer(), - tracer + tracer, + false ); var responseBody = new BytesArray(randomUnicodeOfLengthBetween(1, 100).getBytes(StandardCharsets.UTF_8)); @@ -729,7 +799,8 @@ public class DefaultRestChannelTests extends ESTestCase { threadPool.getThreadContext(), new CorsHandler(CorsHandler.buildConfig(settings)), httpTracer, - tracer + tracer, + false ); channel.sendResponse(testRestResponse()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 6bbb25db64f5..45a1ac2ced32 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -207,7 +207,7 @@ public abstract class NumberFieldMapperTests extends MapperTestCase { } } - protected void testNullValue() throws IOException { + public void testNullValue() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); SourceToParse source = source(b -> b.nullField("field")); ParsedDocument doc = mapper.parse(source); @@ -220,14 +220,15 @@ public abstract class NumberFieldMapperTests extends MapperTestCase { })); doc = mapper.parse(source); List fields = doc.rootDoc().getFields("field"); - assertEquals(2, fields.size()); - IndexableField pointField = fields.get(0); - assertEquals(1, pointField.fieldType().pointIndexDimensionCount()); - assertFalse(pointField.fieldType().stored()); - assertEquals(123, pointField.numericValue().doubleValue(), 0d); - IndexableField dvField = fields.get(1); - assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType()); - assertFalse(dvField.fieldType().stored()); + List pointFields = fields.stream().filter(f -> f.fieldType().pointIndexDimensionCount() != 0).toList(); + assertEquals(1, pointFields.size()); + assertEquals(1, pointFields.get(0).fieldType().pointIndexDimensionCount()); + assertFalse(pointFields.get(0).fieldType().stored()); + + List dvFields = fields.stream().filter(f -> f.fieldType().docValuesType() != DocValuesType.NONE).toList(); + assertEquals(1, dvFields.size()); + assertEquals(DocValuesType.SORTED_NUMERIC, dvFields.get(0).fieldType().docValuesType()); + assertFalse(dvFields.get(0).fieldType().stored()); } public void testOutOfRangeValues() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 43ed85675511..ede03b2a64f4 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -302,7 +302,7 @@ public class RemoteClusterConnectionTests extends ESTestCase { service.acceptIncomingRequests(); String clusterAlias = "test-cluster"; Settings settings = buildRandomSettings(clusterAlias, seedNodes); - try (RemoteClusterConnection connection = new RemoteClusterConnection(settings, clusterAlias, service, randomBoolean())) { + try (RemoteClusterConnection connection = new RemoteClusterConnection(settings, clusterAlias, service, false)) { int numThreads = randomIntBetween(4, 10); Thread[] threads = new Thread[numThreads]; CyclicBarrier barrier = new CyclicBarrier(numThreads + 1); diff --git a/x-pack/docs/en/rest-api/security/saml-authenticate-api.asciidoc b/x-pack/docs/en/rest-api/security/saml-authenticate-api.asciidoc index c8bf40b2d9d6..aa556a42d699 100644 --- a/x-pack/docs/en/rest-api/security/saml-authenticate-api.asciidoc +++ b/x-pack/docs/en/rest-api/security/saml-authenticate-api.asciidoc @@ -8,7 +8,7 @@ Submits a SAML `Response` message to {es} for consumption. NOTE: This API is intended for use by custom web applications other than {kib}. -If you are using {kib}, see the <>. +If you are using {kib}, see the <>. [[security-api-saml-authenticate-request]] ==== {api-request-title} diff --git a/x-pack/docs/en/rest-api/security/saml-complete-logout-api.asciidoc b/x-pack/docs/en/rest-api/security/saml-complete-logout-api.asciidoc index 7ebb49494d9b..1fb4ab1581ab 100644 --- a/x-pack/docs/en/rest-api/security/saml-complete-logout-api.asciidoc +++ b/x-pack/docs/en/rest-api/security/saml-complete-logout-api.asciidoc @@ -8,7 +8,7 @@ Verifies the logout response sent from the SAML IdP. NOTE: This API is intended for use by custom web applications other than {kib}. -If you are using {kib}, see the <>. +If you are using {kib}, see the <>. [[security-api-saml-complete-logout-request]] ==== {api-request-title} diff --git a/x-pack/docs/en/rest-api/security/saml-invalidate-api.asciidoc b/x-pack/docs/en/rest-api/security/saml-invalidate-api.asciidoc index dc663ce78786..21c10341c6fe 100644 --- a/x-pack/docs/en/rest-api/security/saml-invalidate-api.asciidoc +++ b/x-pack/docs/en/rest-api/security/saml-invalidate-api.asciidoc @@ -8,7 +8,7 @@ Submits a SAML LogoutRequest message to {es} for consumption. NOTE: This API is intended for use by custom web applications other than {kib}. -If you are using {kib}, see the <>. +If you are using {kib}, see the <>. [[security-api-saml-invalidate-request]] ==== {api-request-title} diff --git a/x-pack/docs/en/rest-api/security/saml-logout-api.asciidoc b/x-pack/docs/en/rest-api/security/saml-logout-api.asciidoc index 5333ae0a717b..71729365865d 100644 --- a/x-pack/docs/en/rest-api/security/saml-logout-api.asciidoc +++ b/x-pack/docs/en/rest-api/security/saml-logout-api.asciidoc @@ -8,7 +8,7 @@ Submits a request to invalidate an access token and refresh token. NOTE: This API is intended for use by custom web applications other than {kib}. -If you are using {kib}, see the <>. +If you are using {kib}, see the <>. [[security-api-saml-logout-request]] ==== {api-request-title} diff --git a/x-pack/docs/en/rest-api/security/saml-prepare-authentication-api.asciidoc b/x-pack/docs/en/rest-api/security/saml-prepare-authentication-api.asciidoc index 7e64c7f9c6d1..b62d3d2ac9f7 100644 --- a/x-pack/docs/en/rest-api/security/saml-prepare-authentication-api.asciidoc +++ b/x-pack/docs/en/rest-api/security/saml-prepare-authentication-api.asciidoc @@ -8,7 +8,7 @@ Creates a SAML authentication request (``) as a URL string, based on the configuration of the respective SAML realm in {es}. NOTE: This API is intended for use by custom web applications other than {kib}. -If you are using {kib}, see the <>. +If you are using {kib}, see the <>. [[security-api-saml-prepare-authentication-request]] ==== {api-request-title} diff --git a/x-pack/docs/en/rest-api/watcher/query-watches.asciidoc b/x-pack/docs/en/rest-api/watcher/query-watches.asciidoc index 4a469af852b5..7a006243ed7f 100644 --- a/x-pack/docs/en/rest-api/watcher/query-watches.asciidoc +++ b/x-pack/docs/en/rest-api/watcher/query-watches.asciidoc @@ -39,9 +39,9 @@ This API supports the following fields: | `query` | no | null | Optional, <> filter watches to be returned. -| `sort` | no | null | Optional <>. +| `sort` | no | null | Optional <>. -| `search_after` | no | null | Optional <> to do pagination +| `search_after` | no | null | Optional <> to do pagination using last hit's sort values. |====== diff --git a/x-pack/docs/en/security/authentication/configuring-kerberos-realm.asciidoc b/x-pack/docs/en/security/authentication/configuring-kerberos-realm.asciidoc index a42660a76d95..c177ae24da90 100644 --- a/x-pack/docs/en/security/authentication/configuring-kerberos-realm.asciidoc +++ b/x-pack/docs/en/security/authentication/configuring-kerberos-realm.asciidoc @@ -89,7 +89,7 @@ default realm, the Key Distribution Center (KDC), and other configuration detail required for Kerberos authentication. When the JVM needs some configuration properties, it tries to find those values by locating and loading this file. The JVM system property to configure the file path is `java.security.krb5.conf`. To -configure JVM system properties see <>. +configure JVM system properties see <>. If this system property is not specified, Java tries to locate the file based on the conventions. diff --git a/x-pack/docs/en/security/authentication/oidc-guide.asciidoc b/x-pack/docs/en/security/authentication/oidc-guide.asciidoc index 47512797d6f8..41cead20789b 100644 --- a/x-pack/docs/en/security/authentication/oidc-guide.asciidoc +++ b/x-pack/docs/en/security/authentication/oidc-guide.asciidoc @@ -12,7 +12,7 @@ Elastic Stack Relying Party will be registered. NOTE: The OpenID Connect realm support in {kib} is designed with the expectation that it will be the primary authentication method for the users of that {kib} instance. The -<> section describes what this entails and how you can set it up to support +<> section describes what this entails and how you can set it up to support other realms if necessary. [[oidc-guide-op]] @@ -591,7 +591,7 @@ client with the OpenID Connect Provider. Note that when registering the ==== OpenID Connect Realm An OpenID Connect realm needs to be created and configured accordingly -in {es}. See <> +in {es}. See <> ==== Service Account user for accessing the APIs diff --git a/x-pack/docs/en/security/authentication/realms.asciidoc b/x-pack/docs/en/security/authentication/realms.asciidoc index a8694e6639e2..5136737e96c4 100644 --- a/x-pack/docs/en/security/authentication/realms.asciidoc +++ b/x-pack/docs/en/security/authentication/realms.asciidoc @@ -51,7 +51,7 @@ A realm that facilitates authentication using OpenID Connect. It enables {es} to _jwt_:: A realm that facilitates using JWT identity tokens as authentication bearer tokens. Compatible tokens are OpenID Connect ID Tokens, or custom JWTs containing the same claims. -See <>. +See <>. The {security-features} also support custom realms. If you need to integrate with another authentication system, you can build a custom realm plugin. For diff --git a/x-pack/docs/en/security/authentication/saml-guide.asciidoc b/x-pack/docs/en/security/authentication/saml-guide.asciidoc index a7c194489896..c0cdd6bc01dc 100644 --- a/x-pack/docs/en/security/authentication/saml-guide.asciidoc +++ b/x-pack/docs/en/security/authentication/saml-guide.asciidoc @@ -231,7 +231,7 @@ The recommended steps for configuring these SAML attributes are as follows: This varies greatly between providers, but you should be able to obtain a list from the documentation, or from your local admin. -. Read through the list of <> that {es} +. Read through the list of <> that {es} supports, and decide which of them are useful to you, and can be provided by your IdP. At a _minimum_, the `principal` attribute is required. @@ -244,7 +244,7 @@ The recommended steps for configuring these SAML attributes are as follows: URIs are used. . Configure the SAML realm in {es} to associate the {es} user properties (see - <> below), to the URIs that you configured + <> below), to the URIs that you configured in your IdP. In the example above, we have configured the `principal` and `groups` attributes. @@ -281,7 +281,7 @@ NOTE: Identity Providers can be either statically configured to release a `NameI with a specific format, or they can be configured to try to conform with the requirements of the SP. The SP declares its requirements as part of the Authentication Request, using an element which is called the `NameIDPolicy`. If -this is needed, you can set the relevant <> named +this is needed, you can set the relevant <> named `nameid_format` in order to request that the IdP releases a `NameID` with a specific format. @@ -925,7 +925,7 @@ access tokens after the current one expires. ==== SAML realm You must create a SAML realm and configure it accordingly -in {es}. See <> +in {es}. See <> [[saml-no-kibana-user]] ==== Service Account user for accessing the APIs diff --git a/x-pack/docs/en/security/authentication/token-authentication-services.asciidoc b/x-pack/docs/en/security/authentication/token-authentication-services.asciidoc index 8e49ab678f08..94fd5f96ea56 100644 --- a/x-pack/docs/en/security/authentication/token-authentication-services.asciidoc +++ b/x-pack/docs/en/security/authentication/token-authentication-services.asciidoc @@ -81,6 +81,6 @@ you to invalidate the tokens. See <>. IMPORTANT: Authentication support for JWT bearer tokens was introduced in {es} -8.2 through the <>, which cannot be enabled through +8.2 through the <>, which cannot be enabled through token-authentication services. Realms offer flexible order and configurations of zero, one, or multiple JWT realms. diff --git a/x-pack/docs/en/security/authorization/alias-privileges.asciidoc b/x-pack/docs/en/security/authorization/alias-privileges.asciidoc index c10cd0556599..91cd3aef6a40 100644 --- a/x-pack/docs/en/security/authorization/alias-privileges.asciidoc +++ b/x-pack/docs/en/security/authorization/alias-privileges.asciidoc @@ -67,7 +67,7 @@ GET .ds-my-data-stream-2099.03.09-000003/_doc/2 Use <> to control access to an <>. Privileges on an index or data stream do not grant privileges -on its aliases. For information about managing aliases, see <>. +on its aliases. For information about managing aliases, see <>. IMPORTANT: Don't use <> in place of <>. {es} doesn't always apply diff --git a/x-pack/docs/en/security/authorization/privileges.asciidoc b/x-pack/docs/en/security/authorization/privileges.asciidoc index 670f1e3b628a..e6dd78b0b07e 100644 --- a/x-pack/docs/en/security/authorization/privileges.asciidoc +++ b/x-pack/docs/en/security/authorization/privileges.asciidoc @@ -200,7 +200,7 @@ on all {es} API keys. `transport_client`:: All privileges necessary for a transport client to connect. Required by the remote -cluster to enable <>. +cluster to enable <>. [[privileges-list-indices]] ==== Indices privileges @@ -318,7 +318,7 @@ more like this, multi percolate/search/termvector, percolate, scroll, clear_scroll, search, suggest, tv). `read_cross_cluster`:: -Read-only access to the search action from a <>. +Read-only access to the search action from a <>. `view_index_metadata`:: Read-only access to index and data stream metadata (aliases, exists, diff --git a/x-pack/docs/en/security/troubleshooting.asciidoc b/x-pack/docs/en/security/troubleshooting.asciidoc index 3f647f8834c7..1c084ca1d82a 100644 --- a/x-pack/docs/en/security/troubleshooting.asciidoc +++ b/x-pack/docs/en/security/troubleshooting.asciidoc @@ -444,7 +444,7 @@ Kerberos/SPNEGO debug logging on JVM, add following JVM system properties: `-Dsun.security.spnego.debug=true` -For more information about JVM system properties, see <>. +For more information about JVM system properties, see <>. [[trb-security-saml]] === Common SAML issues @@ -589,7 +589,7 @@ Identity Provider sent. In this example, {es} is configured as follows: xpack.security.authc.realms.saml..attributes.principal: AttributeName0 .... This configuration means that {es} expects to find a SAML Attribute with the name `AttributeName0` or a `NameID` with the appropriate format in the SAML -response so that <> to the `principal` user property. The `principal` user property is a +response so that <> to the `principal` user property. The `principal` user property is a mandatory one, so if this mapping can't happen, the authentication fails. If you are attempting to map a `NameID`, make sure that the expected `NameID` format matches the one that is sent. diff --git a/x-pack/docs/en/watcher/input/search.asciidoc b/x-pack/docs/en/watcher/input/search.asciidoc index cd73c5cbbeac..9ea7a7d5da72 100644 --- a/x-pack/docs/en/watcher/input/search.asciidoc +++ b/x-pack/docs/en/watcher/input/search.asciidoc @@ -173,7 +173,7 @@ accurately. | `request.indices` | no | - | The indices to search. If omitted, all indices are searched, which is the default behaviour in Elasticsearch. -| `request.body` | no | - | The body of the request. The <> +| `request.body` | no | - | The body of the request. The <> follows the same structure you normally send in the body of a REST `_search` request. The body can be static text or include `mustache` <>. @@ -181,13 +181,13 @@ accurately. for more information. | `request.indices_options.expand_wildcards` | no | `open` | How to expand wildcards. Valid values are: `all`, `open`, `closed`, and `none` - See <> for more information. + See <> for more information. | `request.indices_options.ignore_unavailable` | no | `true` | Whether the search should ignore unavailable indices. See - <> for more information. + <> for more information. | `request.indices_options.allow_no_indices` | no | `true` | Whether to allow a search where a wildcard indices expression results in no - concrete indices. See <> + concrete indices. See <> for more information. | `extract` | no | - | A array of JSON keys to extract from the search response and load as the payload. diff --git a/x-pack/docs/en/watcher/transform/search.asciidoc b/x-pack/docs/en/watcher/transform/search.asciidoc index 9ed60ceda2df..deef57783ea9 100644 --- a/x-pack/docs/en/watcher/transform/search.asciidoc +++ b/x-pack/docs/en/watcher/transform/search.asciidoc @@ -63,7 +63,7 @@ The following table lists all available settings for the search | `request.indices` | no | all indices | One or more indices to search on. | `request.body` | no | `match_all` query | The body of the request. The - <> follows + <> follows the same structure you normally send in the body of a REST `_search` request. The body can be static text or include `mustache` <>. @@ -71,15 +71,15 @@ The following table lists all available settings for the search | `request.indices_options.expand_wildcards` | no | `open` | Determines how to expand indices wildcards. An array consisting of a combination of `open`, `closed`, and `hidden`. Alternatively a value of `none` or `all`. - (see <>) + (see <>) | `request.indices_options.ignore_unavailable` | no | `true` | A boolean value that determines whether the search should leniently ignore unavailable indices - (see <>) + (see <>) | `request.indices_options.allow_no_indices` | no | `true` | A boolean value that determines whether the search should leniently return no results when no indices - are resolved (see <>) + are resolved (see <>) | `request.template` | no | - | The body of the search template. See <> for more information. diff --git a/x-pack/docs/en/watcher/trigger/schedule/cron.asciidoc b/x-pack/docs/en/watcher/trigger/schedule/cron.asciidoc index 459f3dab0a61..673f350435c5 100644 --- a/x-pack/docs/en/watcher/trigger/schedule/cron.asciidoc +++ b/x-pack/docs/en/watcher/trigger/schedule/cron.asciidoc @@ -4,8 +4,10 @@ Cron schedule ++++ -Defines a <> using a <> -that specifies when to execute a watch. + +Defines a <> using a <> +that specifiues when to execute a watch. + TIP: While cron expressions are powerful, a regularly occurring schedule is easier to configure with the other schedule types. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java index f64b654cdd28..1f9bf7d3ec40 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/SecurityContext.java @@ -23,7 +23,8 @@ import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.ParentActionAuthorization; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; -import org.elasticsearch.xpack.core.security.user.SystemUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; import java.io.IOException; @@ -148,8 +149,7 @@ public class SecurityContext { * Sets the user forcefully to the provided user. There must not be an existing user in the ThreadContext otherwise an exception * will be thrown. This method is package private for testing. */ - public void setInternalUser(User internalUser, TransportVersion version) { - assert User.isInternal(internalUser); + public void setInternalUser(InternalUser internalUser, TransportVersion version) { setAuthentication(Authentication.newInternalAuthentication(internalUser, version, nodeName)); } @@ -157,8 +157,7 @@ public class SecurityContext { * Runs the consumer in a new context as the provided user. The original context is provided to the consumer. When this method * returns, the original context is restored. */ - public void executeAsInternalUser(User internalUser, TransportVersion version, Consumer consumer) { - assert User.isInternal(internalUser); + public void executeAsInternalUser(InternalUser internalUser, TransportVersion version, Consumer consumer) { final StoredContext original = threadContext.newStoredContextPreservingResponseHeaders(); try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { setInternalUser(internalUser, version); @@ -171,7 +170,7 @@ public class SecurityContext { } public void executeAsSystemUser(TransportVersion version, Consumer consumer) { - executeAsInternalUser(SystemUser.INSTANCE, version, consumer); + executeAsInternalUser(InternalUsers.SYSTEM_USER, version, consumer); } /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CrossClusterApiKeyRoleDescriptorBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CrossClusterApiKeyRoleDescriptorBuilder.java index 92c12763143d..4014162af605 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CrossClusterApiKeyRoleDescriptorBuilder.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CrossClusterApiKeyRoleDescriptorBuilder.java @@ -23,9 +23,9 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstr public class CrossClusterApiKeyRoleDescriptorBuilder { - private static final String[] CCS_CLUSTER_PRIVILEGE_NAMES = { "cross_cluster_search" }; - private static final String[] CCR_CLUSTER_PRIVILEGE_NAMES = { "cross_cluster_replication" }; - private static final String[] CCS_AND_CCR_CLUSTER_PRIVILEGE_NAMES = { "cross_cluster_search", "cross_cluster_replication" }; + public static final String[] CCS_CLUSTER_PRIVILEGE_NAMES = { "cross_cluster_search" }; + public static final String[] CCR_CLUSTER_PRIVILEGE_NAMES = { "cross_cluster_replication" }; + public static final String[] CCS_AND_CCR_CLUSTER_PRIVILEGE_NAMES = { "cross_cluster_search", "cross_cluster_replication" }; private static final String[] CCS_INDICES_PRIVILEGE_NAMES = { "read", "read_cross_cluster", "view_index_metadata" }; private static final String[] CCR_INDICES_PRIVILEGE_NAMES = { "cross_cluster_replication", "cross_cluster_replication_internal" }; private static final String ROLE_DESCRIPTOR_NAME = "cross_cluster"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java index d41de69ba3a8..1c3a1fa86906 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/GetUsersResponse.java @@ -14,6 +14,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.user.InternalUser; import org.elasticsearch.xpack.core.security.user.User; import java.io.IOException; @@ -38,7 +39,7 @@ public class GetUsersResponse extends ActionResponse implements ToXContentObject users = new User[size]; for (int i = 0; i < size; i++) { final User user = Authentication.AuthenticationSerializationHelper.readUserFrom(in); - assert false == User.isInternal(user) : "should not get internal users"; + assert false == user instanceof InternalUser : "should not get internal user [" + user + "]"; users[i] = user; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java index 2902962e5cf3..e9b925fbcd61 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountSetting import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.user.AnonymousUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; @@ -126,7 +127,7 @@ public final class Authentication implements ToXContentObject { // Read the user(s) final User outerUser = AuthenticationSerializationHelper.readUserWithoutTrailingBoolean(in); final boolean hasInnerUser; - if (User.isInternal(outerUser)) { + if (outerUser instanceof InternalUser) { hasInnerUser = false; } else { hasInnerUser = in.readBoolean(); @@ -134,7 +135,7 @@ public final class Authentication implements ToXContentObject { final User innerUser; if (hasInnerUser) { innerUser = AuthenticationSerializationHelper.readUserFrom(in); - assert false == User.isInternal(innerUser) : "internal users cannot participate in run-as"; + assert false == innerUser instanceof InternalUser : "internal users cannot participate in run-as [" + innerUser + "]"; } else { innerUser = null; } @@ -384,7 +385,7 @@ public final class Authentication implements ToXContentObject { final boolean shouldAddAnonymousRoleNames = anonymousUser != null && anonymousUser.enabled() && false == anonymousUser.equals(getEffectiveSubject().getUser()) - && false == User.isInternal(getEffectiveSubject().getUser()) + && false == getEffectiveSubject().getUser() instanceof InternalUser && false == isApiKey() && false == isCrossClusterAccess() && false == isServiceAccount(); @@ -507,7 +508,7 @@ public final class Authentication implements ToXContentObject { // There is no reason for internal users to run-as. This check prevents either internal user itself // or a token created for it (though no such thing in current code) to run-as. - if (User.isInternal(getEffectiveSubject().getUser())) { + if (getEffectiveSubject().getUser() instanceof InternalUser) { return false; } @@ -582,8 +583,8 @@ public final class Authentication implements ToXContentObject { if (isRunAs) { final User outerUser = effectiveSubject.getUser(); final User innerUser = authenticatingSubject.getUser(); - assert false == User.isInternal(outerUser) && false == User.isInternal(innerUser) - : "internal users cannot participate in run-as"; + assert false == outerUser instanceof InternalUser && false == innerUser instanceof InternalUser + : "internal users cannot participate in run-as (outer=[" + outerUser + "] inner=[" + innerUser + "])"; User.writeUser(outerUser, out); out.writeBoolean(true); User.writeUser(innerUser, out); @@ -836,7 +837,7 @@ public final class Authentication implements ToXContentObject { ); } checkNoDomain(authenticatingRealm, "Internal"); - if (false == User.isInternal(authenticatingSubject.getUser())) { + if (false == authenticatingSubject.getUser() instanceof InternalUser) { throw new IllegalArgumentException("Internal authentication must have internal user"); } checkNoRunAs(this, "Internal"); @@ -943,7 +944,7 @@ public final class Authentication implements ToXContentObject { } private static void checkNoInternalUser(Subject subject, String prefixMessage) { - if (User.isInternal(subject.getUser())) { + if (subject.getUser() instanceof InternalUser) { throw new IllegalArgumentException( Strings.format(prefixMessage + " authentication cannot have internal user [%s]", subject.getUser().principal()) ); @@ -1186,9 +1187,7 @@ public final class Authentication implements ToXContentObject { } // TODO is a newer version than the node's a valid value? - public static Authentication newInternalAuthentication(User internalUser, TransportVersion version, String nodeName) { - // TODO create a system user class, so that the type system guarantees that this is only invoked for internal users - assert User.isInternal(internalUser); + public static Authentication newInternalAuthentication(InternalUser internalUser, TransportVersion version, String nodeName) { final Authentication.RealmRef authenticatedBy = newInternalAttachRealmRef(nodeName); Authentication authentication = new Authentication( new Subject(internalUser, authenticatedBy, version, Map.of()), @@ -1445,7 +1444,7 @@ public final class Authentication implements ToXContentObject { */ public static User readUserFrom(StreamInput input) throws IOException { final User user = readUserWithoutTrailingBoolean(input); - if (false == User.isInternal(user)) { + if (false == user instanceof InternalUser) { boolean hasInnerUser = input.readBoolean(); assert false == hasInnerUser : "inner user is not allowed"; if (hasInnerUser) { @@ -1456,8 +1455,8 @@ public final class Authentication implements ToXContentObject { } public static void writeUserTo(User user, StreamOutput output) throws IOException { - if (User.isInternal(user)) { - writeInternalUser(user, output); + if (user instanceof InternalUser internal) { + writeInternalUser(internal, output); } else { User.writeUser(user, output); output.writeBoolean(false); // simple user, no inner user possible @@ -1478,10 +1477,9 @@ public final class Authentication implements ToXContentObject { return new User(username, roles, fullName, email, metadata, enabled); } - private static void writeInternalUser(User user, StreamOutput output) throws IOException { - assert User.isInternal(user); + private static void writeInternalUser(InternalUser user, StreamOutput output) throws IOException { output.writeBoolean(true); - output.writeString(InternalUsers.getInternalUserName(user)); + output.writeString(user.principal()); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/CrossClusterAccessSubjectInfo.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/CrossClusterAccessSubjectInfo.java index 7f925490ba62..c8cbab57f2fa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/CrossClusterAccessSubjectInfo.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/CrossClusterAccessSubjectInfo.java @@ -27,7 +27,8 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountSettings; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection; -import org.elasticsearch.xpack.core.security.user.CrossClusterAccessUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; import java.io.IOException; @@ -204,7 +205,7 @@ public final class CrossClusterAccessSubjectInfo { assert false == authentication.isCrossClusterAccess(); authentication.checkConsistency(); final User user = authentication.getEffectiveSubject().getUser(); - if (CrossClusterAccessUser.is(user)) { + if (user == InternalUsers.CROSS_CLUSTER_ACCESS_USER) { if (false == getRoleDescriptorsBytesList().isEmpty()) { logger.warn( "Received non-empty role descriptors bytes list for internal cross cluster access user. " @@ -212,7 +213,7 @@ public final class CrossClusterAccessSubjectInfo { ); assert false : "role descriptors bytes list for internal cross cluster access user must be empty"; } - } else if (User.isInternal(user)) { + } else if (user instanceof InternalUser) { throw new IllegalArgumentException( "received cross cluster request from an unexpected internal user [" + user.principal() + "]" ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Subject.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Subject.java index 126a978802b8..9d81872a99fe 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Subject.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Subject.java @@ -16,10 +16,11 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.xpack.core.security.action.apikey.ApiKey; import org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo.RoleDescriptorsBytes; import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountSettings; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.store.RoleReference; import org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection; import org.elasticsearch.xpack.core.security.user.AnonymousUser; -import org.elasticsearch.xpack.core.security.user.CrossClusterAccessUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; import org.elasticsearch.xpack.core.security.user.User; import java.util.ArrayList; @@ -294,12 +295,16 @@ public class Subject { ); final var innerAuthentication = (Authentication) metadata.get(CROSS_CLUSTER_ACCESS_AUTHENTICATION_KEY); final User innerUser = innerAuthentication.getEffectiveSubject().getUser(); - if (CrossClusterAccessUser.is(innerUser)) { + if (innerUser instanceof InternalUser internalUser) { assert crossClusterAccessRoleDescriptorsBytes.isEmpty() : "role descriptors bytes list for internal cross cluster access user must be empty"; - roleReferences.add( - new RoleReference.FixedRoleReference(CrossClusterAccessUser.ROLE_DESCRIPTOR, "cross_cluster_access_internal") - ); + final RoleDescriptor internalRoleDescriptor = internalUser.getRemoteAccessRoleDescriptor() + .orElseThrow( + () -> new ElasticsearchSecurityException( + "The internal user [" + internalUser + "] is not permitted to perform cross cluster actions" + ) + ); + roleReferences.add(new RoleReference.FixedRoleReference(internalRoleDescriptor, "cross_cluster_access_internal")); } else if (crossClusterAccessRoleDescriptorsBytes.isEmpty()) { // If the cross cluster access role descriptors are empty, the remote user has no privileges. We need to add an empty role to // restrict access of the overall intersection accordingly diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java index dec16ba3b867..33f81d0f000f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java @@ -266,6 +266,11 @@ public class SimpleRole implements Role { return result; } + @Override + public String toString() { + return getClass().getName() + "{" + String.join(",", names) + "}"; + } + private final AtomicReference> hasPrivilegesCacheReference = new AtomicReference<>(); public void cacheHasPrivileges(Settings settings, PrivilegesToCheck privilegesToCheck, PrivilegesCheckResult privilegesCheckResult) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/APMSystemUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/APMSystemUser.java index 4e4081d23ccd..764115979d9a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/APMSystemUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/APMSystemUser.java @@ -6,17 +6,14 @@ */ package org.elasticsearch.xpack.core.security.user; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; - /** * Built in user for APM server internals. Currently used for APM server monitoring. */ -public class APMSystemUser extends User { +public class APMSystemUser extends ReservedUser { public static final String NAME = UsernamesField.APM_NAME; - public static final String ROLE_NAME = UsernamesField.APM_ROLE; public APMSystemUser(boolean enabled) { - super(NAME, new String[] { ROLE_NAME }, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, enabled); + super(NAME, UsernamesField.APM_ROLE, enabled); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AnonymousUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AnonymousUser.java index 3be2f6a44571..9c1281a616ff 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AnonymousUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AnonymousUser.java @@ -10,16 +10,16 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; import java.util.List; +import java.util.Optional; import static org.elasticsearch.xpack.core.security.SecurityField.setting; /** * The user object for the anonymous user. */ -public class AnonymousUser extends User { +public class AnonymousUser extends ReservedUser { public static final String DEFAULT_ANONYMOUS_USERNAME = "_anonymous"; public static final Setting USERNAME_SETTING = new Setting<>( @@ -37,10 +37,8 @@ public class AnonymousUser extends User { super( USERNAME_SETTING.get(settings), ROLES_SETTING.get(settings).toArray(Strings.EMPTY_ARRAY), - null, - null, - MetadataUtils.DEFAULT_RESERVED_METADATA, - isAnonymousEnabled(settings) + isAnonymousEnabled(settings), + Optional.empty() ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AsyncSearchUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AsyncSearchUser.java deleted file mode 100644 index 9de534f7c5a8..000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/AsyncSearchUser.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.core.security.user; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.xpack.core.XPackPlugin; -import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; - -public class AsyncSearchUser extends User { - - public static final String NAME = UsernamesField.ASYNC_SEARCH_NAME; - public static final AsyncSearchUser INSTANCE = new AsyncSearchUser(); - public static final RoleDescriptor ROLE_DESCRIPTOR = new RoleDescriptor( - UsernamesField.ASYNC_SEARCH_ROLE, - new String[] { "cancel_task" }, - new RoleDescriptor.IndicesPrivileges[] { - RoleDescriptor.IndicesPrivileges.builder() - .indices(XPackPlugin.ASYNC_RESULTS_INDEX + "*") - .privileges("all") - .allowRestrictedIndices(true) - .build(), }, - null, - null, - null, - MetadataUtils.DEFAULT_RESERVED_METADATA, - null - ); - - private AsyncSearchUser() { - super(NAME, Strings.EMPTY_ARRAY); - // the following traits, and especially the run-as one, go with all the internal users - // TODO abstract in a base `InternalUser` class - assert enabled(); - assert roles() != null && roles().length == 0; - } - - @Override - public boolean equals(Object o) { - return INSTANCE == o; - } - - @Override - public int hashCode() { - return System.identityHashCode(this); - } - - public static boolean is(User user) { - return INSTANCE.equals(user); - } - -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/BeatsSystemUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/BeatsSystemUser.java index 839dadb0ec5b..94264ef0f960 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/BeatsSystemUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/BeatsSystemUser.java @@ -6,17 +6,14 @@ */ package org.elasticsearch.xpack.core.security.user; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; - /** * Built in user for beats internals. Currently used for Beats monitoring. */ -public class BeatsSystemUser extends User { +public class BeatsSystemUser extends ReservedUser { public static final String NAME = UsernamesField.BEATS_NAME; - public static final String ROLE_NAME = UsernamesField.BEATS_ROLE; public BeatsSystemUser(boolean enabled) { - super(NAME, new String[] { ROLE_NAME }, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, enabled); + super(NAME, UsernamesField.BEATS_ROLE, enabled); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/CrossClusterAccessUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/CrossClusterAccessUser.java index c28c649d11bd..b238387e44b4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/CrossClusterAccessUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/CrossClusterAccessUser.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.core.security.user; import org.elasticsearch.TransportVersion; -import org.elasticsearch.common.Strings; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo; import org.elasticsearch.xpack.core.security.authc.Subject; @@ -17,11 +16,11 @@ import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.Optional; -public class CrossClusterAccessUser extends User { - public static final String NAME = UsernamesField.CROSS_CLUSTER_ACCESS_NAME; +public class CrossClusterAccessUser extends InternalUser { - public static final RoleDescriptor ROLE_DESCRIPTOR = new RoleDescriptor( + private static final RoleDescriptor REMOTE_ACCESS_ROLE_DESCRIPTOR = new RoleDescriptor( UsernamesField.CROSS_CLUSTER_ACCESS_ROLE, new String[] { "cross_cluster_search", "cross_cluster_replication" }, // Needed for CCR background jobs (with system user) @@ -39,28 +38,21 @@ public class CrossClusterAccessUser extends User { null ); - public static final User INSTANCE = new CrossClusterAccessUser(); + /** + * Package protected to enforce a singleton (private constructor) - use {@link InternalUsers#CROSS_CLUSTER_ACCESS_USER} instead + */ + static final InternalUser INSTANCE = new CrossClusterAccessUser(); private CrossClusterAccessUser() { - super(NAME, Strings.EMPTY_ARRAY); - // the following traits, and especially the run-as one, go with all the internal users - // TODO abstract in a base `InternalUser` class - assert enabled(); - assert roles() != null && roles().length == 0; - } - - @Override - public boolean equals(Object o) { - return INSTANCE == o; - } - - @Override - public int hashCode() { - return System.identityHashCode(this); - } - - public static boolean is(User user) { - return INSTANCE.equals(user); + super( + UsernamesField.CROSS_CLUSTER_ACCESS_NAME, + /** + * this user is not permitted to execute actions that originate on the local cluster, + * its only purpose is to execute actions from a remote cluster + */ + Optional.empty(), + Optional.of(REMOTE_ACCESS_ROLE_DESCRIPTOR) + ); } /** @@ -71,7 +63,7 @@ public class CrossClusterAccessUser extends User { public static CrossClusterAccessSubjectInfo subjectInfo(TransportVersion transportVersion, String nodeName) { try { return new CrossClusterAccessSubjectInfo( - Authentication.newInternalAuthentication(INSTANCE, transportVersion, nodeName), + Authentication.newInternalAuthentication(InternalUsers.CROSS_CLUSTER_ACCESS_USER, transportVersion, nodeName), RoleDescriptorsIntersection.EMPTY ); } catch (IOException e) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ElasticUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ElasticUser.java index afd971fb370c..d1860512a1be 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ElasticUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ElasticUser.java @@ -6,19 +6,17 @@ */ package org.elasticsearch.xpack.core.security.user; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; - /** * The reserved {@code elastic} superuser. Has full permission/access to the cluster/indices and can * run as any other user. */ -public class ElasticUser extends User { +public class ElasticUser extends ReservedUser { public static final String NAME = UsernamesField.ELASTIC_NAME; // used for testing in a different package public static final String ROLE_NAME = UsernamesField.ELASTIC_ROLE; public ElasticUser(boolean enabled) { - super(NAME, new String[] { ROLE_NAME }, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, enabled); + super(NAME, ROLE_NAME, enabled); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUser.java new file mode 100644 index 000000000000..fe15513f5188 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUser.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.user; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; + +import java.util.Objects; +import java.util.Optional; + +public class InternalUser extends User { + + private final Optional localClusterRoleDescriptor; + private final Optional remoteAccessRoleDescriptor; + + InternalUser(String username, @Nullable RoleDescriptor localClusterRole) { + this(username, Optional.ofNullable(localClusterRole), Optional.empty()); + } + + InternalUser(String username, Optional localClusterRole, Optional remoteAccessRole) { + super(username, Strings.EMPTY_ARRAY); + assert enabled(); + assert roles() != null && roles().length == 0; + this.localClusterRoleDescriptor = Objects.requireNonNull(localClusterRole); + this.localClusterRoleDescriptor.ifPresent(rd -> { assert rd.getName().equals(username); }); + this.remoteAccessRoleDescriptor = Objects.requireNonNull(remoteAccessRole); + } + + @Override + public boolean equals(Object o) { + return o == this; + } + + @Override + public int hashCode() { + return System.identityHashCode(this); + } + + /** + * The local-cluster role descriptor assigned to this internal user, or {@link Optional#empty()} if this user does not have a role. + * This {@link RoleDescriptor} defines the privileges that the internal-user has for requests that originate from a node within the + * local cluster. + * @see #getRemoteAccessRoleDescriptor() + */ + public Optional getLocalClusterRoleDescriptor() { + return localClusterRoleDescriptor; + } + + /** + * The remote-access role descriptor assigned to this internal user, or {@link Optional#empty()} if this user is not permitted to + * make cross-cluster requests. + * This {@link RoleDescriptor} defines the privileges that the internal-user has for requests that run on the current cluster, but + * originate from a node within an external cluster (via CCS/CCR). + * @see #getLocalClusterRoleDescriptor() + */ + public Optional getRemoteAccessRoleDescriptor() { + return remoteAccessRoleDescriptor; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java index 84ad857c1752..dd689da23249 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/InternalUsers.java @@ -7,94 +7,147 @@ package org.elasticsearch.xpack.core.security.user; -import org.elasticsearch.core.Nullable; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; +import org.elasticsearch.xpack.core.security.support.MetadataUtils; -import java.util.HashMap; +import java.util.Collection; +import java.util.Collections; import java.util.Map; -import java.util.function.Predicate; +import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; public class InternalUsers { - private record UserInstance(User user, @Nullable RoleDescriptor role, Predicate isUserPredicate) { - private UserInstance { - // Check that the parameters align - assert User.isInternal(user) : "User " + user + " is not internal"; - // The role descriptor should match the user name as well - assert role == null || user.principal().equals(role.getName()) - : "Internal user " + user + " should have a role named [" + user.principal() + "] but was [" + role.getName() + "]"; - // : The user object should match as an instance of the user. - assert isUserPredicate.test(user) : "User " + user + " does not match provided predicate"; - } + /** + * "Async Search" internal user - used to manage async search tasks and write results to the internal results system index + */ + public static final InternalUser ASYNC_SEARCH_USER = new InternalUser( + UsernamesField.ASYNC_SEARCH_NAME, + new RoleDescriptor( + UsernamesField.ASYNC_SEARCH_ROLE, + new String[] { "cancel_task" }, + new RoleDescriptor.IndicesPrivileges[] { + RoleDescriptor.IndicesPrivileges.builder() + .indices(XPackPlugin.ASYNC_RESULTS_INDEX + "*") + .privileges("all") + .allowRestrictedIndices(true) + .build(), }, + null, + null, + null, + MetadataUtils.DEFAULT_RESERVED_METADATA, + null + ) + ); - public boolean is(User u) { - return isUserPredicate.test(u); - } - } + /** + * internal user that manages the security profile index. Has no cluster permission. + */ + public static final InternalUser SECURITY_PROFILE_USER = new InternalUser( + UsernamesField.SECURITY_PROFILE_NAME, + new RoleDescriptor( + UsernamesField.SECURITY_PROFILE_ROLE, + null, + new RoleDescriptor.IndicesPrivileges[] { + RoleDescriptor.IndicesPrivileges.builder() + .indices(".security-profile", "/\\.security-profile-[0-9].*/") + .privileges("all") + .allowRestrictedIndices(true) + .build() }, + null, + null, + null, + MetadataUtils.DEFAULT_RESERVED_METADATA, + Map.of() + ) + ); + + /** + * "Storage" internal user - used when the indexing/storage subsystem needs to perform actions on specific indices + * (that may not be permitted by the authenticated user) + */ + public static final InternalUser STORAGE_USER = new InternalUser( + UsernamesField.STORAGE_USER_NAME, + new RoleDescriptor( + UsernamesField.STORAGE_ROLE_NAME, + new String[] {}, + new RoleDescriptor.IndicesPrivileges[] { + RoleDescriptor.IndicesPrivileges.builder() + .indices("*") + .privileges(RefreshAction.NAME + "*") + .allowRestrictedIndices(true) + .build() }, + new String[] {}, + MetadataUtils.DEFAULT_RESERVED_METADATA + ) + ); + + /** + * XPack internal user that manages xpack. Has all cluster/indices permissions for x-pack to operate excluding security permissions. + */ + public static final InternalUser XPACK_USER = new InternalUser( + UsernamesField.XPACK_NAME, + new RoleDescriptor( + UsernamesField.XPACK_ROLE, + new String[] { "all" }, + new RoleDescriptor.IndicesPrivileges[] { + RoleDescriptor.IndicesPrivileges.builder() + .indices("/@&~(\\.security.*)&~(\\.async-search.*)/") + .privileges("all") + .allowRestrictedIndices(true) + .build() }, + new String[] { "*" }, + MetadataUtils.DEFAULT_RESERVED_METADATA + ) + ); + + /** + * internal user that manages xpack security. Has all cluster/indices permissions. + */ + public static final InternalUser XPACK_SECURITY_USER = new InternalUser( + UsernamesField.XPACK_SECURITY_NAME, + new RoleDescriptor( + UsernamesField.XPACK_SECURITY_ROLE, + new String[] { "all" }, + new RoleDescriptor.IndicesPrivileges[] { + RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("all").allowRestrictedIndices(true).build() }, + null, + null, + new String[] { "*" }, + MetadataUtils.DEFAULT_RESERVED_METADATA, + Map.of() + ) + ); + + public static final SystemUser SYSTEM_USER = SystemUser.INSTANCE; + public static final InternalUser CROSS_CLUSTER_ACCESS_USER = CrossClusterAccessUser.INSTANCE; + + private static final Map INTERNAL_USERS; - private static final Map INTERNAL_USERS = new HashMap<>(); static { - defineUser( - SystemUser.NAME, - SystemUser.INSTANCE, - null /* SystemUser relies on a simple action predicate rather than a role descriptor */, - SystemUser::is - ); - defineUser(XPackUser.NAME, XPackUser.INSTANCE, XPackUser.ROLE_DESCRIPTOR, XPackUser::is); - defineUser(XPackSecurityUser.NAME, XPackSecurityUser.INSTANCE, XPackSecurityUser.ROLE_DESCRIPTOR, XPackSecurityUser::is); - defineUser(SecurityProfileUser.NAME, SecurityProfileUser.INSTANCE, SecurityProfileUser.ROLE_DESCRIPTOR, SecurityProfileUser::is); - defineUser(AsyncSearchUser.NAME, AsyncSearchUser.INSTANCE, AsyncSearchUser.ROLE_DESCRIPTOR, AsyncSearchUser::is); - defineUser( - CrossClusterAccessUser.NAME, - CrossClusterAccessUser.INSTANCE, - null /* CrossClusterAccessUser has a role descriptor, but it should never be resolved by this class */, - CrossClusterAccessUser::is - ); - defineUser(StorageInternalUser.NAME, StorageInternalUser.INSTANCE, StorageInternalUser.ROLE_DESCRIPTOR, StorageInternalUser::is); + INTERNAL_USERS = Stream.of( + SYSTEM_USER, + XPACK_USER, + XPACK_SECURITY_USER, + SECURITY_PROFILE_USER, + ASYNC_SEARCH_USER, + CROSS_CLUSTER_ACCESS_USER, + STORAGE_USER + ).collect(Collectors.toUnmodifiableMap(InternalUser::principal, Function.identity())); } - private static void defineUser(String name, User user, @Nullable RoleDescriptor roleDescriptor, Predicate predicate) { - assert name.equals(user.principal()) - : "User " + user + " has a principal [" + user.principal() + "] that does not match the provided name [" + name + "]"; - INTERNAL_USERS.put(name, new UserInstance(user, roleDescriptor, predicate)); + public static Collection get() { + return Collections.unmodifiableCollection(INTERNAL_USERS.values()); } - private static UserInstance findInternalUser(User user) { - final UserInstance instance = INTERNAL_USERS.get(user.principal()); - if (instance != null && instance.is(user)) { - return instance; - } - throw new IllegalStateException("user [" + user + "] is not internal"); - } - - public static User getUser(String username) { - final UserInstance instance = INTERNAL_USERS.get(username); + public static InternalUser getUser(String username) { + final var instance = INTERNAL_USERS.get(username); if (instance == null) { throw new IllegalStateException("user [" + username + "] is not internal"); } - return instance.user; + return instance; } - - public static String getInternalUserName(User user) { - assert User.isInternal(user); - return findInternalUser(user).user.principal(); - } - - public static RoleDescriptor getRoleDescriptor(User user) { - assert User.isInternal(user); - UserInstance instance = findInternalUser(user); - if (instance.role == null) { - throw new IllegalArgumentException("should never try to get the roles for internal user [" + user.principal() + "]"); - } - return instance.role; - } - - public static Map getRoleDescriptors() { - return INTERNAL_USERS.values() - .stream() - .filter(instance -> instance.role != null) - .collect(Collectors.toMap(instance -> instance.user().principal(), UserInstance::role)); - } - } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaSystemUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaSystemUser.java index e8da4e40fc51..931f369efa14 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaSystemUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaSystemUser.java @@ -6,17 +6,15 @@ */ package org.elasticsearch.xpack.core.security.user; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; - /** * Built in user for the kibana server */ -public class KibanaSystemUser extends User { +public class KibanaSystemUser extends ReservedUser { public static final String NAME = UsernamesField.KIBANA_NAME; public static final String ROLE_NAME = UsernamesField.KIBANA_ROLE; public KibanaSystemUser(boolean enabled) { - super(NAME, new String[] { ROLE_NAME }, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, enabled); + super(NAME, ROLE_NAME, enabled); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaUser.java index 507a9a408906..e5cad0d054ea 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/KibanaUser.java @@ -6,26 +6,19 @@ */ package org.elasticsearch.xpack.core.security.user; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; +import java.util.Optional; /** * Built in user for the kibana server * @deprecated use KibanaSystemUser */ @Deprecated -public class KibanaUser extends User { +public class KibanaUser extends ReservedUser { public static final String NAME = UsernamesField.DEPRECATED_KIBANA_NAME; public static final String ROLE_NAME = UsernamesField.KIBANA_ROLE; public KibanaUser(boolean enabled) { - super( - NAME, - new String[] { ROLE_NAME }, - null, - null, - MetadataUtils.getDeprecatedReservedMetadata("Please use the [kibana_system] user instead."), - enabled - ); + super(NAME, new String[] { ROLE_NAME }, enabled, Optional.of("Please use the [kibana_system] user instead.")); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/LogstashSystemUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/LogstashSystemUser.java index 8be90564827e..e402d7ab787b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/LogstashSystemUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/LogstashSystemUser.java @@ -6,17 +6,14 @@ */ package org.elasticsearch.xpack.core.security.user; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; - /** * Built in user for logstash internals. Currently used for Logstash monitoring. */ -public class LogstashSystemUser extends User { +public class LogstashSystemUser extends ReservedUser { public static final String NAME = UsernamesField.LOGSTASH_NAME; - public static final String ROLE_NAME = UsernamesField.LOGSTASH_ROLE; public LogstashSystemUser(boolean enabled) { - super(NAME, new String[] { ROLE_NAME }, null, null, MetadataUtils.DEFAULT_RESERVED_METADATA, enabled); + super(NAME, UsernamesField.LOGSTASH_ROLE, enabled); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/RemoteMonitoringUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/RemoteMonitoringUser.java index 4a12f63921c1..cda3af9c82c5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/RemoteMonitoringUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/RemoteMonitoringUser.java @@ -6,25 +6,18 @@ */ package org.elasticsearch.xpack.core.security.user; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; +import java.util.Optional; /** * Built in user for remote monitoring: collection as well as indexing. */ -public class RemoteMonitoringUser extends User { +public class RemoteMonitoringUser extends ReservedUser { public static final String NAME = UsernamesField.REMOTE_MONITORING_NAME; public static final String COLLECTION_ROLE_NAME = UsernamesField.REMOTE_MONITORING_COLLECTION_ROLE; public static final String INDEXING_ROLE_NAME = UsernamesField.REMOTE_MONITORING_INDEXING_ROLE; public RemoteMonitoringUser(boolean enabled) { - super( - NAME, - new String[] { COLLECTION_ROLE_NAME, INDEXING_ROLE_NAME }, - null, - null, - MetadataUtils.DEFAULT_RESERVED_METADATA, - enabled - ); + super(NAME, new String[] { COLLECTION_ROLE_NAME, INDEXING_ROLE_NAME }, enabled, Optional.empty()); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ReservedUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ReservedUser.java new file mode 100644 index 000000000000..e0998a1c6eb4 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/ReservedUser.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.security.user; + +import org.elasticsearch.xpack.core.security.support.MetadataUtils; + +import java.util.Optional; + +public abstract class ReservedUser extends User { + protected ReservedUser(String username, String role, boolean enabled) { + this(username, new String[] { role }, enabled, Optional.empty()); + } + + protected ReservedUser(String username, String[] roles, boolean enabled, Optional deprecation) { + super( + username, + roles, + null, + null, + deprecation.map(MetadataUtils::getDeprecatedReservedMetadata).orElse(MetadataUtils.DEFAULT_RESERVED_METADATA), + enabled + ); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SecurityProfileUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SecurityProfileUser.java deleted file mode 100644 index a91d4eb46558..000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SecurityProfileUser.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.core.security.user; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; - -import java.util.Map; - -/** - * internal user that manages the security profile index. Has no cluster permission. - */ -public class SecurityProfileUser extends User { - - public static final String NAME = UsernamesField.SECURITY_PROFILE_NAME; - public static final SecurityProfileUser INSTANCE = new SecurityProfileUser(); - public static final RoleDescriptor ROLE_DESCRIPTOR = new RoleDescriptor( - UsernamesField.SECURITY_PROFILE_ROLE, - null, - new RoleDescriptor.IndicesPrivileges[] { - RoleDescriptor.IndicesPrivileges.builder() - .indices(".security-profile", "/\\.security-profile-[0-9].*/") - .privileges("all") - .allowRestrictedIndices(true) - .build() }, - null, - null, - null, - MetadataUtils.DEFAULT_RESERVED_METADATA, - Map.of() - ); - - private SecurityProfileUser() { - super(NAME, Strings.EMPTY_ARRAY); - // the following traits, and especially the run-as one, go with all the internal users - // TODO abstract in a base `InternalUser` class - assert enabled(); - assert roles() != null && roles().length == 0; - } - - @Override - public boolean equals(Object o) { - return INSTANCE == o; - } - - @Override - public int hashCode() { - return System.identityHashCode(this); - } - - public static boolean is(User user) { - return INSTANCE.equals(user); - } - -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/StorageInternalUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/StorageInternalUser.java deleted file mode 100644 index d1f3566f4d2f..000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/StorageInternalUser.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.core.security.user; - -import org.elasticsearch.action.admin.indices.refresh.RefreshAction; -import org.elasticsearch.common.Strings; -import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; - -/** - * "Storage" internal user - used when the indexing/storage subsystem needs to perform actions on specific indices - * (that may not be permitted by the authenticated user) - */ -public class StorageInternalUser extends User { - - public static final String NAME = UsernamesField.STORAGE_USER_NAME; - public static final RoleDescriptor ROLE_DESCRIPTOR = new RoleDescriptor( - UsernamesField.STORAGE_ROLE_NAME, - new String[] {}, - new RoleDescriptor.IndicesPrivileges[] { - RoleDescriptor.IndicesPrivileges.builder() - .indices("*") - .privileges(RefreshAction.NAME + "*") - .allowRestrictedIndices(true) - .build() }, - new String[] {}, - MetadataUtils.DEFAULT_RESERVED_METADATA - ); - public static final StorageInternalUser INSTANCE = new StorageInternalUser(); - - private StorageInternalUser() { - super(NAME, Strings.EMPTY_ARRAY); - assert enabled(); - assert roles() != null && roles().length == 0; - } - - @Override - public boolean equals(Object o) { - return INSTANCE == o; - } - - @Override - public int hashCode() { - return System.identityHashCode(this); - } - - public static boolean is(User user) { - return INSTANCE.equals(user); - } - -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SystemUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SystemUser.java index 8f6737c1fa51..b5f08d3c44a5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SystemUser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/SystemUser.java @@ -6,43 +6,45 @@ */ package org.elasticsearch.xpack.core.security.user; -import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.privilege.SystemPrivilege; +import java.util.Optional; import java.util.function.Predicate; /** * Internal user that is applied to all requests made elasticsearch itself */ -public class SystemUser extends User { +public class SystemUser extends InternalUser { public static final String NAME = UsernamesField.SYSTEM_NAME; + + @Deprecated public static final String ROLE_NAME = UsernamesField.SYSTEM_ROLE; - public static final User INSTANCE = new SystemUser(); + /** + * Package protected to enforce a singleton (private constructor) - use {@link InternalUsers#SYSTEM_USER} instead + */ + static final SystemUser INSTANCE = new SystemUser(); private static final Predicate PREDICATE = SystemPrivilege.INSTANCE.predicate(); private SystemUser() { - super(NAME, Strings.EMPTY_ARRAY); - // the following traits, and especially the run-as one, go with all the internal users - // TODO abstract in a base `InternalUser` class - assert enabled(); - assert roles() != null && roles().length == 0; + super(NAME, Optional.empty(), Optional.empty()); } + /** + * @return {@link Optional#empty()} because the {@code _system} user does not use role based security + * @see #isAuthorized(String) + */ @Override - public boolean equals(Object o) { - return o == INSTANCE; - } - - @Override - public int hashCode() { - return System.identityHashCode(this); + public Optional getLocalClusterRoleDescriptor() { + return Optional.empty(); } + @Deprecated public static boolean is(User user) { - return INSTANCE.equals(user); + return InternalUsers.SYSTEM_USER.equals(user); } public static boolean isAuthorized(String action) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/User.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/User.java index a4ae698a489c..e089b21433bd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/User.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/User.java @@ -147,19 +147,9 @@ public class User implements ToXContentObject { builder.field(Fields.ENABLED.getPreferredName(), enabled()); } - public static boolean isInternal(User user) { - // TODO : Drop this method and rely entirely on the InternalUsers class - return SystemUser.is(user) - || XPackUser.is(user) - || XPackSecurityUser.is(user) - || SecurityProfileUser.is(user) - || AsyncSearchUser.is(user) - || CrossClusterAccessUser.is(user) - || StorageInternalUser.is(user); - } - /** Write the given {@link User} */ public static void writeUser(User user, StreamOutput output) throws IOException { + // TODO : reject `InternalUser` output.writeBoolean(false); // not a system user output.writeString(user.username); output.writeStringArray(user.roles); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackSecurityUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackSecurityUser.java deleted file mode 100644 index d7c39edd8c87..000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackSecurityUser.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.core.security.user; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; - -import java.util.Map; - -/** - * internal user that manages xpack security. Has all cluster/indices permissions. - */ -public class XPackSecurityUser extends User { - - public static final String NAME = UsernamesField.XPACK_SECURITY_NAME; - public static final XPackSecurityUser INSTANCE = new XPackSecurityUser(); - public static final RoleDescriptor ROLE_DESCRIPTOR = new RoleDescriptor( - UsernamesField.XPACK_SECURITY_ROLE, - new String[] { "all" }, - new RoleDescriptor.IndicesPrivileges[] { - RoleDescriptor.IndicesPrivileges.builder().indices("*").privileges("all").allowRestrictedIndices(true).build() }, - null, - null, - new String[] { "*" }, - MetadataUtils.DEFAULT_RESERVED_METADATA, - Map.of() - ); - - private XPackSecurityUser() { - super(NAME, Strings.EMPTY_ARRAY); - // the following traits, and especially the run-as one, go with all the internal users - // TODO abstract in a base `InternalUser` class - assert enabled(); - assert roles() != null && roles().length == 0; - } - - @Override - public boolean equals(Object o) { - return INSTANCE == o; - } - - @Override - public int hashCode() { - return System.identityHashCode(this); - } - - public static boolean is(User user) { - return INSTANCE.equals(user); - } - -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java deleted file mode 100644 index d15b494cee29..000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/user/XPackUser.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.core.security.user; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; -import org.elasticsearch.xpack.core.security.support.MetadataUtils; - -/** - * XPack internal user that manages xpack. Has all cluster/indices permissions for x-pack to operate excluding security permissions. - */ -public class XPackUser extends User { - - public static final String NAME = UsernamesField.XPACK_NAME; - public static final RoleDescriptor ROLE_DESCRIPTOR = new RoleDescriptor( - UsernamesField.XPACK_ROLE, - new String[] { "all" }, - new RoleDescriptor.IndicesPrivileges[] { - RoleDescriptor.IndicesPrivileges.builder() - .indices("/@&~(\\.security.*)&~(\\.async-search.*)/") - .privileges("all") - .allowRestrictedIndices(true) - .build() }, - new String[] { "*" }, - MetadataUtils.DEFAULT_RESERVED_METADATA - ); - public static final XPackUser INSTANCE = new XPackUser(); - - private XPackUser() { - super(NAME, Strings.EMPTY_ARRAY); - // the following traits, and especially the run-as one, go with all the internal users - // TODO abstract in a base `InternalUser` class - assert enabled(); - assert roles() != null && roles().length == 0; - } - - @Override - public boolean equals(Object o) { - return INSTANCE == o; - } - - @Override - public int hashCode() { - return System.identityHashCode(this); - } - - public static boolean is(User user) { - return INSTANCE.equals(user); - } - -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java index 0fa9a0846e63..9f50cc43db4c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java @@ -16,6 +16,7 @@ public class TransformMessages { "Timed out after [{0}] while waiting for transform [{1}] to stop"; public static final String REST_STOP_TRANSFORM_WAIT_FOR_COMPLETION_INTERRUPT = "Interrupted while waiting for transform [{0}] to stop"; public static final String REST_PUT_TRANSFORM_EXISTS = "Transform with id [{0}] already exists"; + public static final String REST_UPDATE_TRANSFORM_CONFLICT = "Transform with id [{0}] got updated in the meantime. Please try again"; public static final String REST_UNKNOWN_TRANSFORM = "Transform with id [{0}] could not be found"; public static final String REST_STOP_TRANSFORM_WITHOUT_CONFIG = "Detected transforms with no config [{0}]. Use force to stop/delete them."; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationConsistencyTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationConsistencyTests.java index 72d984fc6237..47c4c1c7c120 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationConsistencyTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationConsistencyTests.java @@ -11,8 +11,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountSettings; import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.core.security.user.XPackUser; import java.io.IOException; import java.util.List; @@ -73,7 +73,7 @@ public class AuthenticationConsistencyTests extends ESTestCase { entry( "Anonymous authentication cannot have internal user [_xpack]", encodeAuthentication( - new Subject(XPackUser.INSTANCE, Authentication.RealmRef.newAnonymousRealmRef("node")), + new Subject(InternalUsers.XPACK_USER, Authentication.RealmRef.newAnonymousRealmRef("node")), Authentication.AuthenticationType.ANONYMOUS ) ), @@ -151,7 +151,7 @@ public class AuthenticationConsistencyTests extends ESTestCase { entry( "API key authentication cannot have internal user [_xpack]", encodeAuthentication( - new Subject(XPackUser.INSTANCE, Authentication.RealmRef.newApiKeyRealmRef("node")), + new Subject(InternalUsers.XPACK_USER, Authentication.RealmRef.newApiKeyRealmRef("node")), Authentication.AuthenticationType.API_KEY ) ), @@ -248,7 +248,7 @@ public class AuthenticationConsistencyTests extends ESTestCase { // Authentication type: Token entry( "Token authentication cannot have internal user [_xpack]", - encodeAuthentication(new Subject(XPackUser.INSTANCE, realm1), Authentication.AuthenticationType.TOKEN) + encodeAuthentication(new Subject(InternalUsers.XPACK_USER, realm1), Authentication.AuthenticationType.TOKEN) ), entry( "Service account authentication cannot have domain", diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java index 36350a22242a..a8e021675dfa 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationSerializationTests.java @@ -12,13 +12,11 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.TransportVersionUtils; import org.elasticsearch.transport.RemoteClusterPortSettings; -import org.elasticsearch.xpack.core.security.user.AsyncSearchUser; import org.elasticsearch.xpack.core.security.user.ElasticUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.KibanaSystemUser; import org.elasticsearch.xpack.core.security.user.KibanaUser; -import org.elasticsearch.xpack.core.security.user.SystemUser; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.core.security.user.XPackUser; import java.util.Arrays; @@ -114,28 +112,28 @@ public class AuthenticationSerializationTests extends ESTestCase { public void testSystemUserReadAndWrite() throws Exception { BytesStreamOutput output = new BytesStreamOutput(); - AuthenticationSerializationHelper.writeUserTo(SystemUser.INSTANCE, output); + AuthenticationSerializationHelper.writeUserTo(InternalUsers.SYSTEM_USER, output); User readFrom = AuthenticationSerializationHelper.readUserFrom(output.bytes().streamInput()); - assertThat(readFrom, is(sameInstance(SystemUser.INSTANCE))); + assertThat(readFrom, is(sameInstance(InternalUsers.SYSTEM_USER))); } public void testXPackUserReadAndWrite() throws Exception { BytesStreamOutput output = new BytesStreamOutput(); - AuthenticationSerializationHelper.writeUserTo(XPackUser.INSTANCE, output); + AuthenticationSerializationHelper.writeUserTo(InternalUsers.XPACK_USER, output); User readFrom = AuthenticationSerializationHelper.readUserFrom(output.bytes().streamInput()); - assertThat(readFrom, is(sameInstance(XPackUser.INSTANCE))); + assertThat(readFrom, is(sameInstance(InternalUsers.XPACK_USER))); } public void testAsyncSearchUserReadAndWrite() throws Exception { BytesStreamOutput output = new BytesStreamOutput(); - AuthenticationSerializationHelper.writeUserTo(AsyncSearchUser.INSTANCE, output); + AuthenticationSerializationHelper.writeUserTo(InternalUsers.ASYNC_SEARCH_USER, output); User readFrom = AuthenticationSerializationHelper.readUserFrom(output.bytes().streamInput()); - assertThat(readFrom, is(sameInstance(AsyncSearchUser.INSTANCE))); + assertThat(readFrom, is(sameInstance(InternalUsers.ASYNC_SEARCH_USER))); } public void testFakeInternalUserSerialization() throws Exception { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java index a9d0691f985e..b147f574a14c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/AuthenticationTestHelper.java @@ -30,19 +30,17 @@ import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountSetting import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection; import org.elasticsearch.xpack.core.security.user.AnonymousUser; -import org.elasticsearch.xpack.core.security.user.AsyncSearchUser; import org.elasticsearch.xpack.core.security.user.CrossClusterAccessUser; -import org.elasticsearch.xpack.core.security.user.SecurityProfileUser; -import org.elasticsearch.xpack.core.security.user.SystemUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.security.user.UsernamesField; -import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; -import org.elasticsearch.xpack.core.security.user.XPackUser; import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -78,14 +76,10 @@ public class AuthenticationTestHelper { AuthenticationField.CROSS_CLUSTER_ACCESS_REALM_TYPE ); - private static final Set INTERNAL_USERS = Set.of( - SystemUser.INSTANCE, - XPackUser.INSTANCE, - XPackSecurityUser.INSTANCE, - AsyncSearchUser.INSTANCE, - SecurityProfileUser.INSTANCE, - CrossClusterAccessUser.INSTANCE - ); + private static final List INTERNAL_USERS_WITH_ROLE_DESCRIPTOR = InternalUsers.get() + .stream() + .filter(u -> u.getLocalClusterRoleDescriptor().isPresent()) + .toList(); public static AuthenticationTestBuilder builder() { return new AuthenticationTestBuilder(); @@ -98,8 +92,16 @@ public class AuthenticationTestHelper { ); } - public static User randomInternalUser() { - return ESTestCase.randomFrom(INTERNAL_USERS); + public static InternalUser randomInternalUser() { + return ESTestCase.randomFrom(InternalUsers.get()); + } + + public static Collection internalUsersWithLocalRoleDescriptor() { + return INTERNAL_USERS_WITH_ROLE_DESCRIPTOR; + } + + public static InternalUser randomInternalUserWithLocalRoleDescriptor() { + return ESTestCase.randomFrom(INTERNAL_USERS_WITH_ROLE_DESCRIPTOR); } public static User userWithRandomMetadataAndDetails(final String username, final String... roles) { @@ -228,7 +230,7 @@ public class AuthenticationTestHelper { * @return non-empty collection of internal usernames */ public static List randomInternalUsernames() { - return ESTestCase.randomNonEmptySubsetOf(INTERNAL_USERS.stream().map(User::principal).toList()); + return ESTestCase.randomNonEmptySubsetOf(InternalUsers.get().stream().map(User::principal).toList()); } public static String randomInternalRoleName() { @@ -254,7 +256,7 @@ public class AuthenticationTestHelper { } public static CrossClusterAccessSubjectInfo crossClusterAccessSubjectInfoForInternalUser() { - final Authentication authentication = AuthenticationTestHelper.builder().internal(CrossClusterAccessUser.INSTANCE).build(); + final Authentication authentication = AuthenticationTestHelper.builder().internal(InternalUsers.CROSS_CLUSTER_ACCESS_USER).build(); return CrossClusterAccessUser.subjectInfo( authentication.getEffectiveSubject().getTransportVersion(), authentication.getEffectiveSubject().getRealm().getNodeName() @@ -270,7 +272,7 @@ public class AuthenticationTestHelper { return switch (type) { case "realm" -> AuthenticationTestHelper.builder().realm().build(); case "apikey" -> AuthenticationTestHelper.builder().apiKey().build(); - case "internal" -> AuthenticationTestHelper.builder().internal(CrossClusterAccessUser.INSTANCE).build(); + case "internal" -> AuthenticationTestHelper.builder().internal(InternalUsers.CROSS_CLUSTER_ACCESS_USER).build(); case "service_account" -> AuthenticationTestHelper.builder().serviceAccount().build(); default -> throw new UnsupportedOperationException("unknown type " + type); }; @@ -286,7 +288,7 @@ public class AuthenticationTestHelper { } public static CrossClusterAccessSubjectInfo randomCrossClusterAccessSubjectInfo(final Authentication authentication) { - if (CrossClusterAccessUser.is(authentication.getEffectiveSubject().getUser())) { + if (InternalUsers.CROSS_CLUSTER_ACCESS_USER == authentication.getEffectiveSubject().getUser()) { return crossClusterAccessSubjectInfoForInternalUser(); } final int numberOfRoleDescriptors; @@ -419,12 +421,12 @@ public class AuthenticationTestHelper { } public AuthenticationTestBuilder internal() { - return internal(ESTestCase.randomFrom(INTERNAL_USERS)); + return internal(ESTestCase.randomFrom(InternalUsers.get())); } - public AuthenticationTestBuilder internal(User user) { + public AuthenticationTestBuilder internal(InternalUser user) { assert authenticatingAuthentication == null : "shortcut method cannot be used for effective authentication"; - assert User.isInternal(user) : "user must be internal for internal authentication"; + assert user instanceof InternalUser : "user must be internal for internal authentication"; resetShortcutRelatedVariables(); this.user = user; realmRef = null; @@ -433,8 +435,8 @@ public class AuthenticationTestHelper { } public AuthenticationTestBuilder user(User user) { - if (User.isInternal(user)) { - return internal(user); + if (user instanceof InternalUser internalUser) { + return internal(internalUser); } else if (user instanceof AnonymousUser) { return anonymous(user); } else { @@ -515,7 +517,7 @@ public class AuthenticationTestHelper { if (user == null) { user = randomUser(); } - assert false == User.isInternal(user) && false == user instanceof AnonymousUser + assert false == user instanceof InternalUser && false == user instanceof AnonymousUser : "cannot run-as internal or anonymous user"; if (realmRef == null) { realmRef = randomRealmRef(isRealmUnderDomain == null ? ESTestCase.randomBoolean() : isRealmUnderDomain); @@ -626,18 +628,31 @@ public class AuthenticationTestHelper { } case INTERNAL -> { if (user == null) { - user = ESTestCase.randomFrom(INTERNAL_USERS); + user = ESTestCase.randomFrom(InternalUsers.get()); } - assert User.isInternal(user) : "user must be internal for internal authentication"; - assert realmRef == null : "cannot specify realm type for internal authentication"; - String nodeName = ESTestCase.randomAlphaOfLengthBetween(3, 8); - if (user == SystemUser.INSTANCE) { - authentication = ESTestCase.randomFrom( - Authentication.newInternalAuthentication(user, TransportVersion.CURRENT, nodeName), - Authentication.newInternalFallbackAuthentication(user, nodeName) - ); + if (user instanceof InternalUser internalUser) { + assert realmRef == null : "cannot specify realm type for internal authentication"; + String nodeName = ESTestCase.randomAlphaOfLengthBetween(3, 8); + if (internalUser == InternalUsers.SYSTEM_USER) { + authentication = ESTestCase.randomFrom( + Authentication.newInternalAuthentication(internalUser, TransportVersion.CURRENT, nodeName), + Authentication.newInternalFallbackAuthentication(user, nodeName) + ); + } else { + authentication = Authentication.newInternalAuthentication(internalUser, TransportVersion.CURRENT, nodeName); + } } else { - authentication = Authentication.newInternalAuthentication(user, TransportVersion.CURRENT, nodeName); + throw new IllegalArgumentException( + "Cannot have authentication type [" + + authenticationType + + "] (" + + candidateAuthenticationTypes + + ") with non-internal user [" + + user + + "] (" + + user.getClass().getName() + + ")" + ); } } default -> throw new IllegalArgumentException("unknown authentication type [" + authenticationType + "]"); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/SubjectTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/SubjectTests.java index b7212cd049bc..36793b91f89b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/SubjectTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authc/SubjectTests.java @@ -27,7 +27,7 @@ import org.elasticsearch.xpack.core.security.authz.store.RoleReference.NamedRole import org.elasticsearch.xpack.core.security.authz.store.RoleReference.ServiceAccountRoleReference; import org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersection; import org.elasticsearch.xpack.core.security.user.AnonymousUser; -import org.elasticsearch.xpack.core.security.user.CrossClusterAccessUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; import java.util.Arrays; @@ -217,9 +217,8 @@ public class SubjectTests extends ESTestCase { authMetadata ); - final boolean isInternalUser = CrossClusterAccessUser.is( - crossClusterAccessSubjectInfo.getAuthentication().getEffectiveSubject().getUser() - ); + final Authentication authentication = crossClusterAccessSubjectInfo.getAuthentication(); + final boolean isInternalUser = authentication.getEffectiveSubject().getUser() == InternalUsers.CROSS_CLUSTER_ACCESS_USER; final RoleReferenceIntersection roleReferenceIntersection = subject.getRoleReferenceIntersection(getAnonymousUser()); // Number of role references depends on the authentication and its number of roles. // Test setup can randomly authentication with 0, 1 or 2 (in case of API key) role descriptors, @@ -276,10 +275,11 @@ public class SubjectTests extends ESTestCase { private static void expectFixedReferenceAtIndex(int index, List roleReferences) { final FixedRoleReference fixedRoleReference = (FixedRoleReference) roleReferences.get(index); - assertThat( - fixedRoleReference.id(), - equalTo(new RoleKey(Set.of(CrossClusterAccessUser.ROLE_DESCRIPTOR.getName()), "cross_cluster_access_internal")) + final RoleKey expectedKey = new RoleKey( + Set.of(InternalUsers.CROSS_CLUSTER_ACCESS_USER.getRemoteAccessRoleDescriptor().get().getName()), + "cross_cluster_access_internal" ); + assertThat(fixedRoleReference.id(), equalTo(expectedKey)); } public void testGetRoleReferencesForApiKeyBwc() { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index 0857a29cb003..9c51cdb8c977 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -187,9 +187,6 @@ import org.elasticsearch.xpack.core.security.authz.permission.Role; import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor; import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeTests; import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices; -import org.elasticsearch.xpack.core.security.user.APMSystemUser; -import org.elasticsearch.xpack.core.security.user.BeatsSystemUser; -import org.elasticsearch.xpack.core.security.user.LogstashSystemUser; import org.elasticsearch.xpack.core.security.user.RemoteMonitoringUser; import org.elasticsearch.xpack.core.security.user.SystemUser; import org.elasticsearch.xpack.core.security.user.UsernamesField; @@ -267,9 +264,9 @@ public class ReservedRolesStoreTests extends ESTestCase { assertThat(ReservedRolesStore.isReserved("watcher_user"), is(true)); assertThat(ReservedRolesStore.isReserved("watcher_admin"), is(true)); assertThat(ReservedRolesStore.isReserved("beats_admin"), is(true)); - assertThat(ReservedRolesStore.isReserved(LogstashSystemUser.ROLE_NAME), is(true)); - assertThat(ReservedRolesStore.isReserved(BeatsSystemUser.ROLE_NAME), is(true)); - assertThat(ReservedRolesStore.isReserved(APMSystemUser.ROLE_NAME), is(true)); + assertThat(ReservedRolesStore.isReserved(UsernamesField.LOGSTASH_ROLE), is(true)); + assertThat(ReservedRolesStore.isReserved(UsernamesField.BEATS_ROLE), is(true)); + assertThat(ReservedRolesStore.isReserved(UsernamesField.APM_ROLE), is(true)); assertThat(ReservedRolesStore.isReserved(RemoteMonitoringUser.COLLECTION_ROLE_NAME), is(true)); assertThat(ReservedRolesStore.isReserved(RemoteMonitoringUser.INDEXING_ROLE_NAME), is(true)); assertThat(ReservedRolesStore.isReserved("snapshot_user"), is(true)); @@ -2233,7 +2230,7 @@ public class ReservedRolesStoreTests extends ESTestCase { final TransportRequest request = mock(TransportRequest.class); final Authentication authentication = AuthenticationTestHelper.builder().build(); - RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor(BeatsSystemUser.ROLE_NAME); + RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor(UsernamesField.BEATS_ROLE); assertNotNull(roleDescriptor); assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true)); @@ -2275,7 +2272,7 @@ public class ReservedRolesStoreTests extends ESTestCase { final TransportRequest request = mock(TransportRequest.class); final Authentication authentication = AuthenticationTestHelper.builder().build(); - RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor(APMSystemUser.ROLE_NAME); + RoleDescriptor roleDescriptor = new ReservedRolesStore().roleDescriptor(UsernamesField.APM_ROLE); assertNotNull(roleDescriptor); assertThat(roleDescriptor.getMetadata(), hasEntry("_reserved", true)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java index 3d61538136e5..07bce6c70a48 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/InternalUsersTests.java @@ -7,67 +7,253 @@ package org.elasticsearch.xpack.core.security.user; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.apache.lucene.util.automaton.Operations; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksAction; +import org.elasticsearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.TransportUnpromotableShardRefreshAction; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; +import org.elasticsearch.action.bulk.BulkAction; +import org.elasticsearch.action.get.GetAction; +import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; +import org.elasticsearch.xpack.core.security.authc.Authentication; +import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; +import org.elasticsearch.xpack.core.security.authz.permission.ApplicationPermission; +import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache; +import org.elasticsearch.xpack.core.security.authz.permission.RemoteIndicesPermission; +import org.elasticsearch.xpack.core.security.authz.permission.Role; +import org.elasticsearch.xpack.core.security.authz.permission.RunAsPermission; +import org.elasticsearch.xpack.core.security.authz.permission.SimpleRole; +import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices; -import static org.elasticsearch.test.TestMatchers.throwableWithMessage; +import java.util.List; + +import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7; +import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_TOKENS_INDEX_7; +import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.SECURITY_MAIN_ALIAS; +import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.SECURITY_TOKENS_ALIAS; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; public class InternalUsersTests extends ESTestCase { public void testSystemUser() { - assertThat(InternalUsers.getUser("_system"), is(SystemUser.INSTANCE)); - assertThat(InternalUsers.getInternalUserName(SystemUser.INSTANCE), is("_system")); - final IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> InternalUsers.getRoleDescriptor(SystemUser.INSTANCE) - ); - assertThat(e, throwableWithMessage("should never try to get the roles for internal user [_system]")); + assertThat(InternalUsers.getUser("_system"), is(InternalUsers.SYSTEM_USER)); } public void testXPackUser() { - assertThat(InternalUsers.getUser("_xpack"), is(XPackUser.INSTANCE)); - assertThat(InternalUsers.getInternalUserName(XPackUser.INSTANCE), is("_xpack")); - assertThat(InternalUsers.getRoleDescriptor(XPackUser.INSTANCE), is(XPackUser.ROLE_DESCRIPTOR)); + assertThat(InternalUsers.getUser("_xpack"), is(InternalUsers.XPACK_USER)); + + final SimpleRole role = getLocalClusterRole(InternalUsers.XPACK_USER); + + assertThat(role.runAs().toString(), Operations.isTotal(role.runAs().getPrivilege().getAutomaton()), is(true)); + assertThat(role.application(), is(ApplicationPermission.NONE)); + assertThat(role.remoteIndices(), is(RemoteIndicesPermission.NONE)); + + final List sampleClusterActions = List.of( + ClusterStateAction.NAME, + PutComponentTemplateAction.NAME, + DeleteStoredScriptAction.NAME, + UpdateJobAction.NAME, + CleanupRepositoryAction.NAME + ); + checkClusterAccess(InternalUsers.XPACK_USER, role, randomFrom(sampleClusterActions), true); + + final List sampleIndexActions = List.of( + GetAction.NAME, + BulkAction.NAME, + RefreshAction.NAME, + CreateIndexAction.NAME, + PutMappingAction.NAME, + DeleteIndexAction.NAME + ); + checkIndexAccess(role, randomFrom(sampleIndexActions), randomAlphaOfLengthBetween(3, 12), true); + checkIndexAccess( + role, + randomFrom(sampleIndexActions), + randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7, SECURITY_TOKENS_ALIAS, INTERNAL_SECURITY_TOKENS_INDEX_7), + false + ); } public void testXPackSecurityUser() { - assertThat(InternalUsers.getUser("_xpack_security"), is(XPackSecurityUser.INSTANCE)); - assertThat(InternalUsers.getInternalUserName(XPackSecurityUser.INSTANCE), is("_xpack_security")); - assertThat(InternalUsers.getRoleDescriptor(XPackSecurityUser.INSTANCE), is(XPackSecurityUser.ROLE_DESCRIPTOR)); + assertThat(InternalUsers.getUser("_xpack_security"), is(InternalUsers.XPACK_SECURITY_USER)); + + final SimpleRole role = getLocalClusterRole(InternalUsers.XPACK_SECURITY_USER); + + assertThat(role.runAs().toString(), Operations.isTotal(role.runAs().getPrivilege().getAutomaton()), is(true)); + assertThat(role.application(), is(ApplicationPermission.NONE)); + assertThat(role.remoteIndices(), is(RemoteIndicesPermission.NONE)); + + final List sampleClusterActions = List.of( + ClusterStateAction.NAME, + PutComponentTemplateAction.NAME, + DeleteStoredScriptAction.NAME, + UpdateJobAction.NAME, + CleanupRepositoryAction.NAME + ); + checkClusterAccess(InternalUsers.XPACK_SECURITY_USER, role, randomFrom(sampleClusterActions), true); + + final List sampleIndexActions = List.of( + GetAction.NAME, + BulkAction.NAME, + RefreshAction.NAME, + CreateIndexAction.NAME, + PutMappingAction.NAME, + DeleteIndexAction.NAME + ); + checkIndexAccess( + role, + randomFrom(sampleIndexActions), + randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7, SECURITY_TOKENS_ALIAS, INTERNAL_SECURITY_TOKENS_INDEX_7), + true + ); + checkIndexAccess(role, randomFrom(sampleIndexActions), randomAlphaOfLengthBetween(3, 12), true); } public void testSecurityProfileUser() { - assertThat(InternalUsers.getUser("_security_profile"), is(SecurityProfileUser.INSTANCE)); - assertThat(InternalUsers.getInternalUserName(SecurityProfileUser.INSTANCE), is("_security_profile")); - assertThat(InternalUsers.getRoleDescriptor(SecurityProfileUser.INSTANCE), is(SecurityProfileUser.ROLE_DESCRIPTOR)); + assertThat(InternalUsers.getUser("_security_profile"), is(InternalUsers.SECURITY_PROFILE_USER)); + + final SimpleRole role = getLocalClusterRole(InternalUsers.SECURITY_PROFILE_USER); + + assertThat(role.cluster().privileges(), hasSize(0)); + assertThat(role.runAs(), is(RunAsPermission.NONE)); + assertThat(role.application(), is(ApplicationPermission.NONE)); + assertThat(role.remoteIndices(), is(RemoteIndicesPermission.NONE)); + + final List sampleAllowedActions = List.of( + GetAction.NAME, + BulkAction.NAME, + RefreshAction.NAME, + CreateIndexAction.NAME, + PutMappingAction.NAME, + DeleteIndexAction.NAME + ); + checkIndexAccess(role, randomFrom(sampleAllowedActions), ".security-profile", true); + checkIndexAccess(role, randomFrom(sampleAllowedActions), ".security-profile-" + randomIntBetween(1, 9), true); + checkIndexAccess( + role, + randomFrom(sampleAllowedActions), + randomFrom(SECURITY_MAIN_ALIAS, INTERNAL_SECURITY_MAIN_INDEX_7, SECURITY_TOKENS_ALIAS, INTERNAL_SECURITY_TOKENS_INDEX_7), + false + ); + checkIndexAccess(role, randomFrom(sampleAllowedActions), randomAlphaOfLengthBetween(3, 12), false); } public void testAsyncSearchUser() { - assertThat(InternalUsers.getUser("_async_search"), is(AsyncSearchUser.INSTANCE)); - assertThat(InternalUsers.getInternalUserName(AsyncSearchUser.INSTANCE), is("_async_search")); - assertThat(InternalUsers.getRoleDescriptor(AsyncSearchUser.INSTANCE), is(AsyncSearchUser.ROLE_DESCRIPTOR)); + assertThat(InternalUsers.getUser("_async_search"), is(InternalUsers.ASYNC_SEARCH_USER)); + + final SimpleRole role = getLocalClusterRole(InternalUsers.ASYNC_SEARCH_USER); + + assertThat(role.runAs(), is(RunAsPermission.NONE)); + assertThat(role.application(), is(ApplicationPermission.NONE)); + assertThat(role.remoteIndices(), is(RemoteIndicesPermission.NONE)); + + checkClusterAccess(InternalUsers.ASYNC_SEARCH_USER, role, CancelTasksAction.NAME, true); + checkClusterAccess(InternalUsers.ASYNC_SEARCH_USER, role, ClusterStateAction.NAME, false); + + final List sampleAllowedActions = List.of( + GetAction.NAME, + BulkAction.NAME, + RefreshAction.NAME, + CreateIndexAction.NAME, + PutMappingAction.NAME, + DeleteIndexAction.NAME + ); + checkIndexAccess(role, randomFrom(sampleAllowedActions), XPackPlugin.ASYNC_RESULTS_INDEX, true); + checkIndexAccess( + role, + randomFrom(sampleAllowedActions), + XPackPlugin.ASYNC_RESULTS_INDEX + "-" + randomAlphaOfLengthBetween(4, 8), + true + ); + checkIndexAccess(role, randomFrom(sampleAllowedActions), randomAlphaOfLengthBetween(3, 12), false); } public void testCrossClusterAccessUser() { - assertThat(InternalUsers.getUser("_cross_cluster_access"), is(CrossClusterAccessUser.INSTANCE)); - assertThat(InternalUsers.getInternalUserName(CrossClusterAccessUser.INSTANCE), is("_cross_cluster_access")); - final IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> InternalUsers.getRoleDescriptor(CrossClusterAccessUser.INSTANCE) - ); - assertThat(e, throwableWithMessage("should never try to get the roles for internal user [_cross_cluster_access]")); + assertThat(InternalUsers.getUser("_cross_cluster_access"), is(InternalUsers.CROSS_CLUSTER_ACCESS_USER)); } public void testStorageUser() { - assertThat(InternalUsers.getUser("_storage"), is(StorageInternalUser.INSTANCE)); - assertThat(InternalUsers.getInternalUserName(StorageInternalUser.INSTANCE), is("_storage")); - assertThat(InternalUsers.getRoleDescriptor(StorageInternalUser.INSTANCE), is(StorageInternalUser.ROLE_DESCRIPTOR)); + assertThat(InternalUsers.getUser("_storage"), is(InternalUsers.STORAGE_USER)); + + final SimpleRole role = getLocalClusterRole(InternalUsers.STORAGE_USER); + + assertThat(role.cluster().privileges(), hasSize(0)); + assertThat(role.runAs(), is(RunAsPermission.NONE)); + assertThat(role.application(), is(ApplicationPermission.NONE)); + assertThat(role.remoteIndices(), is(RemoteIndicesPermission.NONE)); + + final List sampleAllowedActions = List.of(RefreshAction.NAME, TransportUnpromotableShardRefreshAction.NAME); + checkIndexAccess(role, randomFrom(sampleAllowedActions), randomAlphaOfLengthBetween(4, 8), true); + checkIndexAccess(role, randomFrom(sampleAllowedActions), ".ds-" + randomAlphaOfLengthBetween(4, 8), true); + checkIndexAccess(role, randomFrom(sampleAllowedActions), INTERNAL_SECURITY_MAIN_INDEX_7, true); + + final List sampleDeniedActions = List.of(GetAction.NAME, BulkAction.NAME, PutMappingAction.NAME, DeleteIndexAction.NAME); + checkIndexAccess(role, randomFrom(sampleDeniedActions), randomAlphaOfLengthBetween(4, 8), false); + checkIndexAccess(role, randomFrom(sampleDeniedActions), ".ds-" + randomAlphaOfLengthBetween(4, 8), false); + checkIndexAccess(role, randomFrom(sampleDeniedActions), INTERNAL_SECURITY_MAIN_INDEX_7, false); } public void testRegularUser() { var username = randomAlphaOfLengthBetween(4, 12); expectThrows(IllegalStateException.class, () -> InternalUsers.getUser(username)); - // Can't test other methods because they have an assert that the provided user is internal + } + + private static SimpleRole getLocalClusterRole(InternalUser internalUser) { + final FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); + return Role.buildFromRoleDescriptor( + internalUser.getLocalClusterRoleDescriptor().get(), + fieldPermissionsCache, + TestRestrictedIndices.RESTRICTED_INDICES + ); + } + + private static void checkClusterAccess(InternalUser user, SimpleRole role, String action, boolean expectedValue) { + Authentication authentication = AuthenticationTestHelper.builder().internal(user).build(); + assertThat( + "Role [" + role + "] for user [" + user + "] should grant " + action, + role.cluster().check(action, mock(TransportRequest.class), authentication), + is(expectedValue) + ); + + } + + private static void checkIndexAccess(SimpleRole role, String action, String indexName, boolean expectedValue) { + if (expectedValue) { + // Can't check this if "expectedValue" is false, because the role might grant the action for a different index + assertThat("Role " + role + " should grant " + action, role.indices().check(action), is(true)); + } + + final Automaton automaton = role.indices().allowedActionsMatcher(indexName); + assertThat( + "Role " + role + ", action " + action + " access to " + indexName, + new CharacterRunAutomaton(automaton).run(action), + is(expectedValue) + ); + + final IndexMetadata metadata = IndexMetadata.builder(indexName).settings(indexSettings(Version.CURRENT, 1, 1)).build(); + final IndexAbstraction.ConcreteIndex index = new IndexAbstraction.ConcreteIndex(metadata); + assertThat( + "Role " + role + ", action " + action + " access to " + indexName, + role.allowedIndicesMatcher(action).test(index), + is(expectedValue) + ); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/StorageInternalUserTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/StorageInternalUserTests.java deleted file mode 100644 index 801524c59e11..000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/user/StorageInternalUserTests.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core.security.user; - -import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.automaton.CharacterRunAutomaton; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; -import org.elasticsearch.action.admin.indices.refresh.RefreshAction; -import org.elasticsearch.action.admin.indices.refresh.TransportUnpromotableShardRefreshAction; -import org.elasticsearch.action.bulk.BulkAction; -import org.elasticsearch.action.get.GetAction; -import org.elasticsearch.cluster.metadata.IndexAbstraction; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.security.authz.permission.ApplicationPermission; -import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache; -import org.elasticsearch.xpack.core.security.authz.permission.RemoteIndicesPermission; -import org.elasticsearch.xpack.core.security.authz.permission.Role; -import org.elasticsearch.xpack.core.security.authz.permission.RunAsPermission; -import org.elasticsearch.xpack.core.security.authz.permission.SimpleRole; -import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices; - -import java.util.List; - -import static org.elasticsearch.xpack.core.security.test.TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; - -public class StorageInternalUserTests extends ESTestCase { - - public void testRoleDescriptor() { - final FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); - final SimpleRole role = Role.buildFromRoleDescriptor( - StorageInternalUser.ROLE_DESCRIPTOR, - fieldPermissionsCache, - TestRestrictedIndices.RESTRICTED_INDICES - ); - - assertThat(role.cluster().privileges(), hasSize(0)); - assertThat(role.runAs(), is(RunAsPermission.NONE)); - assertThat(role.application(), is(ApplicationPermission.NONE)); - assertThat(role.remoteIndices(), is(RemoteIndicesPermission.NONE)); - - final List sampleAllowedActions = List.of(RefreshAction.NAME, TransportUnpromotableShardRefreshAction.NAME); - checkIndexAccess(role, randomFrom(sampleAllowedActions), randomAlphaOfLengthBetween(4, 8), true); - checkIndexAccess(role, randomFrom(sampleAllowedActions), ".ds-" + randomAlphaOfLengthBetween(4, 8), true); - checkIndexAccess(role, randomFrom(sampleAllowedActions), INTERNAL_SECURITY_MAIN_INDEX_7, true); - - final List sampleDeniedActions = List.of(GetAction.NAME, BulkAction.NAME, PutMappingAction.NAME, DeleteIndexAction.NAME); - checkIndexAccess(role, randomFrom(sampleDeniedActions), randomAlphaOfLengthBetween(4, 8), false); - checkIndexAccess(role, randomFrom(sampleDeniedActions), ".ds-" + randomAlphaOfLengthBetween(4, 8), false); - checkIndexAccess(role, randomFrom(sampleDeniedActions), INTERNAL_SECURITY_MAIN_INDEX_7, false); - } - - private static void checkIndexAccess(SimpleRole role, String action, String indexName, boolean expectedValue) { - assertThat("Role " + role + " should grant " + action, role.indices().check(action), is(expectedValue)); - - final Automaton automaton = role.indices().allowedActionsMatcher(indexName); - assertThat( - "Role " + role + " should grant " + action + " access to " + indexName, - new CharacterRunAutomaton(automaton).run(action), - is(expectedValue) - ); - - final IndexMetadata metadata = IndexMetadata.builder(indexName).settings(indexSettings(Version.CURRENT, 1, 1)).build(); - final IndexAbstraction.ConcreteIndex index = new IndexAbstraction.ConcreteIndex(metadata); - assertThat( - "Role " + role + " should grant " + action + " access to " + indexName, - role.allowedIndicesMatcher(action).test(index), - is(expectedValue) - ); - } - -} diff --git a/x-pack/plugin/eql/src/test/resources/querytranslator_tests.txt b/x-pack/plugin/eql/src/test/resources/querytranslator_tests.txt index e72260fbffc5..75df2796848a 100644 --- a/x-pack/plugin/eql/src/test/resources/querytranslator_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/querytranslator_tests.txt @@ -208,14 +208,14 @@ twoFunctionsEqualsBooleanLiterals-caseSensitive process where endsWith(process_path, "x") == true and endsWith(process_path, "yx") != true ; {"bool":{"must":[{"wildcard":{"process_path":{"wildcard":"*x","boost":1.0}}}, -{"bool":{"must_not":[{"wildcard":{"process_path":{"wildcard":"*yx","boost":1.0}}}],"boost":1.0}}],"boost":1.0}} +{"bool":{"must_not":[{"wildcard":{"process_path":{"wildcard":"*yx","boost":1.0}}}],"boost":1.0}} ; twoFunctionsEqualsBooleanLiterals-insensitive process where endsWith~(process_path, "x") == true and endsWith~(process_path, "yx") != true ; {"bool":{"must":[{"wildcard":{"process_path":{"wildcard":"*x","case_insensitive":true,"boost":1.0}}}, -{"bool":{"must_not":[{"wildcard":{"process_path":{"wildcard":"*yx","case_insensitive":true,"boost":1.0}}}],"boost":1.0}}],"boost":1.0}} +{"bool":{"must_not":[{"wildcard":{"process_path":{"wildcard":"*yx","case_insensitive":true,"boost":1.0}}}],"boost":1.0}} ; endsWithKeywordFieldFunction-caseSensitive diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java index 9b58cbe3a70d..6ff46c0f098b 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java @@ -41,7 +41,7 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapsho import org.elasticsearch.xpack.core.ml.job.results.Bucket; import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats; import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.junit.After; import org.junit.Before; @@ -404,10 +404,14 @@ public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase { assertThatNumberOfAnnotationsIsEqualTo(1); // The following 4 annotations are created by the system and the 2 oldest ones *will* be deleted - client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(1)), XPackUser.NAME)).actionGet(); - client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(2)), XPackUser.NAME)).actionGet(); - client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(3)), XPackUser.NAME)).actionGet(); - client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(4)), XPackUser.NAME)).actionGet(); + client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(1)), InternalUsers.XPACK_USER.principal())) + .actionGet(); + client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(2)), InternalUsers.XPACK_USER.principal())) + .actionGet(); + client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(3)), InternalUsers.XPACK_USER.principal())) + .actionGet(); + client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(4)), InternalUsers.XPACK_USER.principal())) + .actionGet(); // The following 4 annotations are created by the user and *will not* be deleted client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(1)), USER_NAME)).actionGet(); client().index(randomAnnotationIndexRequest(jobId, now.minus(Duration.ofDays(2)), USER_NAME)).actionGet(); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteJobIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteJobIT.java index a05b0b646d35..40f0167e5121 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteJobIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DeleteJobIT.java @@ -19,7 +19,7 @@ import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.junit.After; import org.junit.Before; @@ -55,14 +55,14 @@ public class DeleteJobIT extends MlNativeAutodetectIntegTestCase { assertThatNumberOfAnnotationsIsEqualTo(0); runJob(jobIdA, datafeedIdA); - client().index(randomAnnotationIndexRequest(jobIdA, XPackUser.NAME)).actionGet(); - client().index(randomAnnotationIndexRequest(jobIdA, XPackUser.NAME)).actionGet(); + client().index(randomAnnotationIndexRequest(jobIdA, InternalUsers.XPACK_USER.principal())).actionGet(); + client().index(randomAnnotationIndexRequest(jobIdA, InternalUsers.XPACK_USER.principal())).actionGet(); client().index(randomAnnotationIndexRequest(jobIdA, "real_user")).actionGet(); // 3 jobA annotations (2 _xpack, 1 real_user) assertThatNumberOfAnnotationsIsEqualTo(3); runJob(jobIdB, datafeedIdB); - client().index(randomAnnotationIndexRequest(jobIdB, XPackUser.NAME)).actionGet(); + client().index(randomAnnotationIndexRequest(jobIdB, InternalUsers.XPACK_USER.principal())).actionGet(); client().index(randomAnnotationIndexRequest(jobIdB, "other_real_user")).actionGet(); // 3 jobA annotations (2 _xpack, 1 real_user) and 2 jobB annotations (1 _xpack, 1 real_user) assertThatNumberOfAnnotationsIsEqualTo(5); diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java index 9effc822590c..ceafc9c8f349 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java @@ -34,7 +34,7 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapsho import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles; import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; import org.elasticsearch.xpack.core.ml.job.results.Bucket; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.junit.After; import java.io.IOException; @@ -312,7 +312,7 @@ public class RevertModelSnapshotIT extends MlNativeAutodetectIntegTestCase { private static IndexRequest randomAnnotationIndexRequest(String jobId, Instant timestamp, Event event) throws IOException { Annotation annotation = new Annotation.Builder(randomAnnotation(jobId)).setTimestamp(Date.from(timestamp)) - .setCreateUsername(XPackUser.NAME) + .setCreateUsername(InternalUsers.XPACK_USER.principal()) .setEvent(event) .build(); try (XContentBuilder xContentBuilder = annotation.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java index fc5c8b5b35fe..132b982ba13c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java @@ -31,7 +31,7 @@ import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.job.results.Bucket; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.ml.annotations.AnnotationPersister; import org.elasticsearch.xpack.ml.datafeed.delayeddatacheck.DelayedDataDetector; import org.elasticsearch.xpack.ml.datafeed.delayeddatacheck.DelayedDataDetectorFactory.BucketWithMissingData; @@ -300,12 +300,12 @@ class DatafeedJob { Date currentTime = new Date(currentTimeSupplier.get()); return new Annotation.Builder().setAnnotation(msg) .setCreateTime(currentTime) - .setCreateUsername(XPackUser.NAME) + .setCreateUsername(InternalUsers.XPACK_USER.principal()) .setTimestamp(startTime) .setEndTimestamp(endTime) .setJobId(jobId) .setModifiedTime(currentTime) - .setModifiedUsername(XPackUser.NAME) + .setModifiedUsername(InternalUsers.XPACK_USER.principal()) .setType(Annotation.Type.ANNOTATION) .setEvent(Annotation.Event.DELAYED_DATA) .build(); @@ -316,7 +316,7 @@ class DatafeedJob { .setTimestamp(annotation.getTimestamp()) .setEndTimestamp(annotation.getEndTimestamp()) .setModifiedTime(new Date(currentTimeSupplier.get())) - .setModifiedUsername(XPackUser.NAME) + .setModifiedUsername(InternalUsers.XPACK_USER.principal()) .build(); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersister.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersister.java index 21c79276d854..91f7f3878ebd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersister.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersister.java @@ -28,7 +28,7 @@ import org.elasticsearch.xpack.core.ml.inference.TrainedModelType; import org.elasticsearch.xpack.core.ml.inference.preprocessing.PreProcessor; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.metadata.TrainedModelMetadata; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.ml.dataframe.process.results.ModelMetadata; import org.elasticsearch.xpack.ml.dataframe.process.results.TrainedModelDefinitionChunk; import org.elasticsearch.xpack.ml.extractor.ExtractedField; @@ -294,7 +294,7 @@ public class ChunkedTrainedModelPersister { return TrainedModelConfig.builder() .setModelId(modelId) .setModelType(trainedModelType) - .setCreatedBy(XPackUser.NAME) + .setCreatedBy(InternalUsers.XPACK_USER.principal()) .setVersion(Version.CURRENT) .setCreateTime(createTime) // NOTE: GET _cat/ml/trained_models relies on the creating analytics ID being in the tags diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java index 7415740bcbb1..5d674ff232c2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java @@ -63,7 +63,7 @@ import org.elasticsearch.xpack.core.ml.job.results.Influencer; import org.elasticsearch.xpack.core.ml.job.results.ModelPlot; import org.elasticsearch.xpack.core.ml.job.results.Result; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.ml.utils.MlIndicesUtils; import java.util.ArrayList; @@ -169,7 +169,7 @@ public class JobDataDeleter { ) { BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId)); if (deleteUserAnnotations == false) { - boolQuery.filter(QueryBuilders.termQuery(Annotation.CREATE_USERNAME.getPreferredName(), XPackUser.NAME)); + boolQuery.filter(QueryBuilders.termQuery(Annotation.CREATE_USERNAME.getPreferredName(), InternalUsers.XPACK_USER.principal())); } if (fromEpochMs != null || toEpochMs != null) { boolQuery.filter(QueryBuilders.rangeQuery(Annotation.TIMESTAMP.getPreferredName()).gte(fromEpochMs).lt(toEpochMs)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java index ff32f0cc72d6..0843bac10367 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessor.java @@ -35,7 +35,7 @@ import org.elasticsearch.xpack.core.ml.job.results.Forecast; import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats; import org.elasticsearch.xpack.core.ml.job.results.Influencer; import org.elasticsearch.xpack.core.ml.job.results.ModelPlot; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.ml.annotations.AnnotationPersister; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; import org.elasticsearch.xpack.ml.job.persistence.TimingStatsReporter; @@ -425,12 +425,12 @@ public class AutodetectResultProcessor { Messages.getMessage(Messages.JOB_AUDIT_SNAPSHOT_STORED, modelSnapshot.getSnapshotId()) ) .setCreateTime(currentTime) - .setCreateUsername(XPackUser.NAME) + .setCreateUsername(InternalUsers.XPACK_USER.principal()) .setTimestamp(modelSnapshot.getLatestResultTimeStamp()) .setEndTimestamp(modelSnapshot.getLatestResultTimeStamp()) .setJobId(jobId) .setModifiedTime(currentTime) - .setModifiedUsername(XPackUser.NAME) + .setModifiedUsername(InternalUsers.XPACK_USER.principal()) .setType(Annotation.Type.ANNOTATION) .setEvent(Annotation.Event.MODEL_SNAPSHOT_STORED) .build(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java index 2ef38e3e3693..427b7c9defa5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredAnnotationsRemover.java @@ -25,7 +25,7 @@ import org.elasticsearch.xpack.core.ml.annotations.Annotation; import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.messages.Messages; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.notifications.AnomalyDetectionAuditor; @@ -109,7 +109,7 @@ public class ExpiredAnnotationsRemover extends AbstractExpiredJobDataRemover { QueryBuilder query = QueryBuilders.boolQuery() .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), job.getId())) .filter(QueryBuilders.rangeQuery(Annotation.TIMESTAMP.getPreferredName()).lt(cutoffEpochMs).format("epoch_millis")) - .filter(QueryBuilders.termQuery(Annotation.CREATE_USERNAME.getPreferredName(), XPackUser.NAME)); + .filter(QueryBuilders.termQuery(Annotation.CREATE_USERNAME.getPreferredName(), InternalUsers.XPACK_USER.principal())); DeleteByQueryRequest request = new DeleteByQueryRequest(AnnotationIndex.READ_ALIAS_NAME).setSlices( AbstractBulkByScrollRequest.AUTO_SLICES ) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java index e8719bd8d6db..ed18a88e39a1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatTrainedModelsAction.java @@ -30,7 +30,7 @@ import org.elasticsearch.xpack.core.ml.action.GetTrainedModelsStatsAction; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; import org.elasticsearch.xpack.core.ml.dataframe.analyses.DataFrameAnalysis; import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import java.util.Collection; import java.util.Collections; @@ -99,7 +99,7 @@ public class RestCatTrainedModelsAction extends AbstractCatAction { Set potentialAnalyticsIds = new HashSet<>(); // Analytics Configs are created by the XPackUser trainedModelConfigs.stream() - .filter(c -> XPackUser.NAME.equals(c.getCreatedBy())) + .filter(c -> InternalUsers.XPACK_USER.principal().equals(c.getCreatedBy())) .forEach(c -> potentialAnalyticsIds.addAll(c.getTags())); // Find the related DataFrameAnalyticsConfigs diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobTests.java index d409bd722aea..afaea47c8a6b 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobTests.java @@ -42,7 +42,7 @@ import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.job.results.Bucket; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.ml.annotations.AnnotationPersister; import org.elasticsearch.xpack.ml.datafeed.delayeddatacheck.DelayedDataDetector; import org.elasticsearch.xpack.ml.datafeed.delayeddatacheck.DelayedDataDetectorFactory.BucketWithMissingData; @@ -316,12 +316,12 @@ public class DatafeedJobTests extends ESTestCase { { // What we expect the created annotation to be indexed as Annotation expectedAnnotation = new Annotation.Builder().setAnnotation(msg) .setCreateTime(new Date(annotationCreateTime)) - .setCreateUsername(XPackUser.NAME) + .setCreateUsername(InternalUsers.XPACK_USER.principal()) .setTimestamp(bucket.getTimestamp()) .setEndTimestamp(new Date((bucket.getEpoch() + bucket.getBucketSpan()) * 1000)) .setJobId(jobId) .setModifiedTime(new Date(annotationCreateTime)) - .setModifiedUsername(XPackUser.NAME) + .setModifiedUsername(InternalUsers.XPACK_USER.principal()) .setType(Annotation.Type.ANNOTATION) .setEvent(Annotation.Event.DELAYED_DATA) .build(); @@ -365,12 +365,12 @@ public class DatafeedJobTests extends ESTestCase { { // What we expect the updated annotation to be indexed as Annotation expectedUpdatedAnnotation = new Annotation.Builder().setAnnotation(msg) .setCreateTime(new Date(annotationCreateTime)) - .setCreateUsername(XPackUser.NAME) + .setCreateUsername(InternalUsers.XPACK_USER.principal()) .setTimestamp(bucket.getTimestamp()) .setEndTimestamp(new Date((bucket2.getEpoch() + bucket2.getBucketSpan()) * 1000)) .setJobId(jobId) .setModifiedTime(new Date(annotationUpdateTime)) - .setModifiedUsername(XPackUser.NAME) + .setModifiedUsername(InternalUsers.XPACK_USER.principal()) .setType(Annotation.Type.ANNOTATION) .setEvent(Annotation.Event.DELAYED_DATA) .build(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersisterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersisterTests.java index 5cd2af66058f..b981325ef7f7 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersisterTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/ChunkedTrainedModelPersisterTests.java @@ -25,7 +25,7 @@ import org.elasticsearch.xpack.core.ml.inference.trainedmodel.metadata.FeatureIm import org.elasticsearch.xpack.core.ml.inference.trainedmodel.metadata.HyperparametersTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.metadata.TotalFeatureImportanceTests; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.metadata.TrainedModelMetadata; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.ml.dataframe.process.results.ModelMetadata; import org.elasticsearch.xpack.ml.dataframe.process.results.TrainedModelDefinitionChunk; import org.elasticsearch.xpack.ml.extractor.DocValueField; @@ -135,7 +135,7 @@ public class ChunkedTrainedModelPersisterTests extends ESTestCase { assertThat(storedModel.getLicenseLevel(), equalTo(License.OperationMode.PLATINUM)); assertThat(storedModel.getModelId(), containsString(JOB_ID)); assertThat(storedModel.getVersion(), equalTo(Version.CURRENT)); - assertThat(storedModel.getCreatedBy(), equalTo(XPackUser.NAME)); + assertThat(storedModel.getCreatedBy(), equalTo(InternalUsers.XPACK_USER.principal())); assertThat(storedModel.getTags(), contains(JOB_ID)); assertThat(storedModel.getDescription(), equalTo(JOB_DESCRIPTION)); assertThat(storedModel.getModelDefinition(), is(nullValue())); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java index d49596a4811f..9a4489c4857d 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultProcessorTests.java @@ -38,7 +38,7 @@ import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition; import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats; import org.elasticsearch.xpack.core.ml.job.results.Influencer; import org.elasticsearch.xpack.core.ml.job.results.ModelPlot; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.ml.annotations.AnnotationPersister; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess; @@ -373,7 +373,7 @@ public class AutodetectResultProcessorTests extends ESTestCase { .setAnnotation("Categorization status changed to 'warn' for partition 'foo'") .setEvent(Annotation.Event.CATEGORIZATION_STATUS_CHANGE) .setCreateTime(new Date()) - .setCreateUsername(XPackUser.NAME) + .setCreateUsername(InternalUsers.XPACK_USER.principal()) .setTimestamp(new Date()) .setPartitionFieldName("part") .setPartitionFieldValue("foo") @@ -405,12 +405,12 @@ public class AutodetectResultProcessorTests extends ESTestCase { Annotation expectedAnnotation = new Annotation.Builder().setAnnotation("Job model snapshot with id [a_snapshot_id] stored") .setCreateTime(Date.from(CURRENT_TIME)) - .setCreateUsername(XPackUser.NAME) + .setCreateUsername(InternalUsers.XPACK_USER.principal()) .setTimestamp(Date.from(Instant.ofEpochMilli(1000_000_000))) .setEndTimestamp(Date.from(Instant.ofEpochMilli(1000_000_000))) .setJobId(JOB_ID) .setModifiedTime(Date.from(CURRENT_TIME)) - .setModifiedUsername(XPackUser.NAME) + .setModifiedUsername(InternalUsers.XPACK_USER.principal()) .setType(Annotation.Type.ANNOTATION) .setEvent(Annotation.Event.MODEL_SNAPSHOT_STORED) .build(); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java index 8a658b628bec..cbc433b12807 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/planner/ExpressionTranslators.java @@ -53,12 +53,14 @@ import org.elasticsearch.xpack.ql.querydsl.query.WildcardQuery; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.ql.util.Check; +import org.elasticsearch.xpack.ql.util.CollectionUtils; import java.time.OffsetTime; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.temporal.TemporalAccessor; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; @@ -464,6 +466,15 @@ public final class ExpressionTranslators { if (right == null) { return left; } - return new BoolQuery(source, isAnd, left, right); + List queries; + // check if either side is already a bool query to an extra bool query + if (left instanceof BoolQuery bool && bool.isAnd() == isAnd) { + queries = CollectionUtils.combine(bool.queries(), right); + } else if (right instanceof BoolQuery bool && bool.isAnd() == isAnd) { + queries = CollectionUtils.combine(bool.queries(), left); + } else { + queries = Arrays.asList(left, right); + } + return new BoolQuery(source, isAnd, queries); } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/BoolQuery.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/BoolQuery.java index 18b4ba2d0339..5792a63cdf73 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/BoolQuery.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/BoolQuery.java @@ -9,9 +9,15 @@ package org.elasticsearch.xpack.ql.querydsl.query; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.sort.NestedSortBuilder; +import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.tree.Source; +import org.elasticsearch.xpack.ql.util.CollectionUtils; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Objects; +import java.util.StringJoiner; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; @@ -23,52 +29,59 @@ public class BoolQuery extends Query { * {@code true} for boolean {@code AND}, {@code false} for boolean {@code OR}. */ private final boolean isAnd; - private final Query left; - private final Query right; + private final List queries; public BoolQuery(Source source, boolean isAnd, Query left, Query right) { + this(source, isAnd, Arrays.asList(left, right)); + } + + public BoolQuery(Source source, boolean isAnd, List queries) { super(source); - if (left == null) { - throw new IllegalArgumentException("left is required"); - } - if (right == null) { - throw new IllegalArgumentException("right is required"); + if (CollectionUtils.isEmpty(queries) || queries.size() < 2) { + throw new QlIllegalArgumentException("At least two queries required by bool query"); } this.isAnd = isAnd; - this.left = left; - this.right = right; + this.queries = queries; } @Override public boolean containsNestedField(String path, String field) { - return left.containsNestedField(path, field) || right.containsNestedField(path, field); + for (Query query : queries) { + if (query.containsNestedField(path, field)) { + return true; + } + } + return false; } @Override public Query addNestedField(String path, String field, String format, boolean hasDocValues) { - Query rewrittenLeft = left.addNestedField(path, field, format, hasDocValues); - Query rewrittenRight = right.addNestedField(path, field, format, hasDocValues); - if (rewrittenLeft == left && rewrittenRight == right) { - return this; + boolean unchanged = true; + List rewritten = new ArrayList<>(queries.size()); + for (Query query : queries) { + var rewrittenQuery = query.addNestedField(path, field, format, hasDocValues); + unchanged &= rewrittenQuery == query; + rewritten.add(rewrittenQuery); } - return new BoolQuery(source(), isAnd, rewrittenLeft, rewrittenRight); + return unchanged ? this : new BoolQuery(source(), isAnd, rewritten); } @Override public void enrichNestedSort(NestedSortBuilder sort) { - left.enrichNestedSort(sort); - right.enrichNestedSort(sort); + for (Query query : queries) { + query.enrichNestedSort(sort); + } } @Override public QueryBuilder asBuilder() { BoolQueryBuilder boolQuery = boolQuery(); - if (isAnd) { - boolQuery.must(left.asBuilder()); - boolQuery.must(right.asBuilder()); - } else { - boolQuery.should(left.asBuilder()); - boolQuery.should(right.asBuilder()); + for (Query query : queries) { + if (isAnd) { + boolQuery.must(query.asBuilder()); + } else { + boolQuery.should(query.asBuilder()); + } } return boolQuery; } @@ -77,17 +90,13 @@ public class BoolQuery extends Query { return isAnd; } - public Query left() { - return left; - } - - public Query right() { - return right; + public List queries() { + return queries; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), isAnd, left, right); + return Objects.hash(super.hashCode(), isAnd, queries); } @Override @@ -96,11 +105,15 @@ public class BoolQuery extends Query { return false; } BoolQuery other = (BoolQuery) obj; - return isAnd == other.isAnd && left.equals(other.left) && right.equals(other.right); + return isAnd == other.isAnd && queries.equals(other.queries); } @Override protected String innerToString() { - return left + (isAnd ? " AND " : " OR ") + right; + StringJoiner sb = new StringJoiner(isAnd ? " AND " : " OR "); + for (Query query : queries) { + sb.add(query.toString()); + } + return sb.toString(); } } diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/querydsl/query/BoolQueryTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/querydsl/query/BoolQueryTests.java index 74249793b72b..6af539e015bf 100644 --- a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/querydsl/query/BoolQueryTests.java +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/querydsl/query/BoolQueryTests.java @@ -19,6 +19,8 @@ import java.util.function.Function; import static java.util.Collections.singletonMap; import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; public class BoolQueryTests extends ESTestCase { static BoolQuery randomBoolQuery(int depth) { @@ -35,15 +37,15 @@ public class BoolQueryTests extends ESTestCase { } private static BoolQuery copy(BoolQuery query) { - return new BoolQuery(query.source(), query.isAnd(), query.left(), query.right()); + return new BoolQuery(query.source(), query.isAnd(), query.queries()); } private static BoolQuery mutate(BoolQuery query) { List> options = Arrays.asList( - q -> new BoolQuery(SourceTests.mutate(q.source()), q.isAnd(), q.left(), q.right()), - q -> new BoolQuery(q.source(), false == q.isAnd(), q.left(), q.right()), - q -> new BoolQuery(q.source(), q.isAnd(), randomValueOtherThan(q.left(), () -> NestedQueryTests.randomQuery(5)), q.right()), - q -> new BoolQuery(q.source(), q.isAnd(), q.left(), randomValueOtherThan(q.right(), () -> NestedQueryTests.randomQuery(5))) + q -> new BoolQuery(SourceTests.mutate(q.source()), q.isAnd(), left(q), right(q)), + q -> new BoolQuery(q.source(), false == q.isAnd(), left(q), right(q)), + q -> new BoolQuery(q.source(), q.isAnd(), randomValueOtherThan(left(q), () -> NestedQueryTests.randomQuery(5)), right(q)), + q -> new BoolQuery(q.source(), q.isAnd(), left(q), randomValueOtherThan(right(q), () -> NestedQueryTests.randomQuery(5))) ); return randomFrom(options).apply(query); } @@ -124,4 +126,19 @@ public class BoolQueryTests extends ESTestCase { ).toString() ); } + + public static Query left(BoolQuery bool) { + return indexOf(bool, 0); + } + + public static Query right(BoolQuery bool) { + return indexOf(bool, 1); + } + + private static Query indexOf(BoolQuery bool, int index) { + List queries = bool.queries(); + assertThat(queries, hasSize(greaterThanOrEqualTo(2))); + return queries.get(index); + } + } diff --git a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java index 06b7062b45e2..0ac3f4b573b8 100644 --- a/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java +++ b/x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessHeadersForCcsRestIT.java @@ -758,6 +758,72 @@ public class CrossClusterAccessHeadersForCcsRestIT extends SecurityOnTrialLicens } } + public void testCrossClusterApiKeyUsage() throws IOException { + // Randomly create REST API keys + for (int i = 0; i < randomIntBetween(0, 2); i++) { + createOrGrantApiKey(Strings.format(""" + { + "name": "my-rest-key-%s" + } + """, i)); + } + + // Randomly create cross-cluster API keys + final int ccsKeys = randomIntBetween(0, 2); + for (int i = 0; i < ccsKeys; i++) { + final Request request = new Request("POST", "/_security/cross_cluster/api_key"); + request.setJsonEntity(Strings.format(""" + { + "name": "my-ccs-key-%s", + "access": { + "search": [ { + "names": [ "logs*" ] + } ] + } + }""", i)); + assertOK(adminClient().performRequest(request)); + } + final int ccrKeys = randomIntBetween(0, 2); + for (int i = 0; i < ccrKeys; i++) { + final Request request = new Request("POST", "/_security/cross_cluster/api_key"); + request.setJsonEntity(Strings.format(""" + { + "name": "my-ccr-key-%s", + "access": { + "replication": [ { + "names": [ "archive*" ] + } ] + } + }""", i)); + assertOK(adminClient().performRequest(request)); + } + final int ccsCcrKeys = randomIntBetween(0, 2); + for (int i = 0; i < ccsCcrKeys; i++) { + final Request request = new Request("POST", "/_security/cross_cluster/api_key"); + request.setJsonEntity(Strings.format(""" + { + "name": "my-ccs-ccr-key-%s", + "access": { + "search": [ { + "names": [ "logs*" ] + } ], + "replication": [ { + "names": [ "archive*" ] + } ] + } + }""", i)); + assertOK(adminClient().performRequest(request)); + } + + final Request xPackUsageRequest = new Request("GET", "/_xpack/usage"); + final ObjectPath path = assertOKAndCreateObjectPath(adminClient().performRequest(xPackUsageRequest)); + + assertThat(path.evaluate("security.remote_cluster_server.api_keys.ccs"), equalTo(ccsKeys)); + assertThat(path.evaluate("security.remote_cluster_server.api_keys.ccr"), equalTo(ccrKeys)); + assertThat(path.evaluate("security.remote_cluster_server.api_keys.ccs_ccr"), equalTo(ccsCcrKeys)); + assertThat(path.evaluate("security.remote_cluster_server.api_keys.total"), equalTo(ccsKeys + ccrKeys + ccsCcrKeys)); + } + private void testCcsWithApiKeyCrossClusterAccessAuthenticationAgainstSingleCluster( String cluster, String apiKeyEncoded, diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/InternalUserAndRoleIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/InternalUserAndRoleIntegTests.java index 4dc165941aa8..7c5a590da579 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/InternalUserAndRoleIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/InternalUserAndRoleIntegTests.java @@ -13,34 +13,18 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.SecuritySettingsSourceField; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.security.authc.support.Hasher; -import org.elasticsearch.xpack.core.security.user.AsyncSearchUser; -import org.elasticsearch.xpack.core.security.user.CrossClusterAccessUser; -import org.elasticsearch.xpack.core.security.user.SecurityProfileUser; -import org.elasticsearch.xpack.core.security.user.SystemUser; -import org.elasticsearch.xpack.core.security.user.UsernamesField; -import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; +import org.elasticsearch.xpack.core.security.user.User; import org.junit.AfterClass; import org.junit.BeforeClass; import java.nio.file.Path; public class InternalUserAndRoleIntegTests extends AbstractPrivilegeTestCase { - private static final String[] INTERNAL_USERNAMES = new String[] { - SystemUser.NAME, - XPackUser.NAME, - XPackSecurityUser.NAME, - AsyncSearchUser.NAME, - SecurityProfileUser.NAME, - CrossClusterAccessUser.NAME }; - private static final String[] INTERNAL_ROLE_NAMES = new String[] { - UsernamesField.SYSTEM_ROLE, - UsernamesField.XPACK_ROLE, - UsernamesField.XPACK_SECURITY_ROLE, - UsernamesField.ASYNC_SEARCH_ROLE, - UsernamesField.SECURITY_PROFILE_ROLE, - UsernamesField.CROSS_CLUSTER_ACCESS_ROLE }; + private static final String[] INTERNAL_USERNAMES = InternalUsers.get().stream().map(User::principal).toArray(String[]::new); + private static final String[] INTERNAL_ROLE_NAMES = INTERNAL_USERNAMES; + public static final String NON_INTERNAL_USERNAME = "user"; public static final String NON_INTERNAL_ROLE_NAME = "role"; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index d4d25d77c475..085865b7abd6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -646,7 +646,7 @@ public class Security extends Plugin ) throws Exception { logger.info("Security is {}", enabled ? "enabled" : "disabled"); if (enabled == false) { - return Collections.singletonList(new SecurityUsageServices(null, null, null, null, null)); + return Collections.singletonList(new SecurityUsageServices(null, null, null, null, null, null)); } systemIndices.init(client, clusterService); @@ -995,7 +995,9 @@ public class Security extends Plugin ) ); - components.add(new SecurityUsageServices(realms, allRolesStore, nativeRoleMappingStore, ipFilter.get(), profileService)); + components.add( + new SecurityUsageServices(realms, allRolesStore, nativeRoleMappingStore, ipFilter.get(), profileService, apiKeyService) + ); reservedRoleMappingAction.set(new ReservedRoleMappingAction(nativeRoleMappingStore)); systemIndices.getMainIndexManager().onStateRecovered(state -> reservedRoleMappingAction.get().securityIndexRecovered()); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageServices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageServices.java index 215130ea33b9..677457296028 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageServices.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageServices.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.security; +import org.elasticsearch.xpack.security.authc.ApiKeyService; import org.elasticsearch.xpack.security.authc.Realms; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; @@ -23,18 +24,21 @@ class SecurityUsageServices { final NativeRoleMappingStore roleMappingStore; final IPFilter ipFilter; final ProfileService profileService; + final ApiKeyService apiKeyService; SecurityUsageServices( Realms realms, CompositeRolesStore rolesStore, NativeRoleMappingStore roleMappingStore, IPFilter ipFilter, - ProfileService profileService + ProfileService profileService, + ApiKeyService apiKeyService ) { this.realms = realms; this.rolesStore = rolesStore; this.roleMappingStore = roleMappingStore; this.ipFilter = ipFilter; this.profileService = profileService; + this.apiKeyService = apiKeyService; } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java index d03279a834e5..973388cc7604 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/SecurityUsageTransportAction.java @@ -30,6 +30,7 @@ import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; import org.elasticsearch.xpack.core.security.SecurityFeatureSetUsage; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail; +import org.elasticsearch.xpack.security.authc.ApiKeyService; import org.elasticsearch.xpack.security.authc.Realms; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; @@ -61,6 +62,7 @@ public class SecurityUsageTransportAction extends XPackUsageFeatureTransportActi private final NativeRoleMappingStore roleMappingStore; private final IPFilter ipFilter; private final ProfileService profileService; + private final ApiKeyService apiKeyService; @Inject public SecurityUsageTransportAction( @@ -88,6 +90,7 @@ public class SecurityUsageTransportAction extends XPackUsageFeatureTransportActi this.roleMappingStore = securityServices.roleMappingStore; this.ipFilter = securityServices.ipFilter; this.profileService = securityServices.profileService; + this.apiKeyService = securityServices.apiKeyService; } @Override @@ -116,9 +119,10 @@ public class SecurityUsageTransportAction extends XPackUsageFeatureTransportActi final AtomicReference> realmsUsageRef = new AtomicReference<>(); final AtomicReference> domainsUsageRef = new AtomicReference<>(); final AtomicReference> userProfileUsageRef = new AtomicReference<>(); + final AtomicReference> remoteClusterServerUsageRef = new AtomicReference<>(); final boolean enabled = XPackSettings.SECURITY_ENABLED.get(settings); - final CountDown countDown = new CountDown(4); + final CountDown countDown = new CountDown(5); final Runnable doCountDown = () -> { if (countDown.countDown()) { var usage = new SecurityFeatureSetUsage( @@ -136,7 +140,7 @@ public class SecurityUsageTransportAction extends XPackUsageFeatureTransportActi operatorPrivilegesUsage, domainsUsageRef.get(), userProfileUsageRef.get(), - remoteClusterServerUsage() + remoteClusterServerUsageRef.get() ); listener.onResponse(new XPackUsageFeatureResponse(usage)); } @@ -163,6 +167,11 @@ public class SecurityUsageTransportAction extends XPackUsageFeatureTransportActi doCountDown.run(); }, listener::onFailure); + final ActionListener> remoteClusterServerUsageListener = ActionListener.wrap(remoteClusterServerUsage -> { + remoteClusterServerUsageRef.set(remoteClusterServerUsage); + doCountDown.run(); + }, listener::onFailure); + if (rolesStore == null || enabled == false) { rolesStoreUsageListener.onResponse(Collections.emptyMap()); } else { @@ -185,18 +194,32 @@ public class SecurityUsageTransportAction extends XPackUsageFeatureTransportActi } else { profileService.usageStats(userProfileUsageListener); } + if (apiKeyService == null || enabled == false) { + remoteClusterServerUsageListener.onResponse(Map.of()); + } else { + remoteClusterServerUsage(remoteClusterServerUsageListener); + } } - private Map remoteClusterServerUsage() { - if (TcpTransport.isUntrustedRemoteClusterEnabled() && XPackSettings.SECURITY_ENABLED.get(settings)) { - return Map.of( - "available", - ADVANCED_REMOTE_CLUSTER_SECURITY_FEATURE.checkWithoutTracking(licenseState), - "enabled", - RemoteClusterPortSettings.REMOTE_CLUSTER_SERVER_ENABLED.get(settings) + private void remoteClusterServerUsage(ActionListener> listener) { + if (TcpTransport.isUntrustedRemoteClusterEnabled()) { + apiKeyService.crossClusterApiKeyUsageStats( + ActionListener.wrap( + usage -> listener.onResponse( + Map.of( + "available", + ADVANCED_REMOTE_CLUSTER_SECURITY_FEATURE.checkWithoutTracking(licenseState), + "enabled", + RemoteClusterPortSettings.REMOTE_CLUSTER_SERVER_ENABLED.get(settings), + "api_keys", + usage + ) + ), + listener::onFailure + ) ); } else { - return Map.of(); + listener.onResponse(Map.of()); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java index e8cb854c80ce..fdc651d7273f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilter.java @@ -29,7 +29,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.authz.privilege.HealthAndStatsPrivilege; -import org.elasticsearch.xpack.core.security.user.SystemUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.security.action.SecurityActionMapper; import org.elasticsearch.xpack.security.audit.AuditTrailService; import org.elasticsearch.xpack.security.audit.AuditUtil; @@ -152,7 +152,7 @@ public class SecurityActionFilter implements ActionFilter { here if a request is not associated with any other user. */ final String securityAction = SecurityActionMapper.action(action, request); - authcService.authenticate(securityAction, request, SystemUser.INSTANCE, ActionListener.wrap((authc) -> { + authcService.authenticate(securityAction, request, InternalUsers.SYSTEM_USER, ActionListener.wrap((authc) -> { if (authc != null) { final String requestId = AuditUtil.extractRequestId(threadContext); assert Strings.hasText(requestId); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java index 94de6b6c8c48..0ca85ada4b5e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportAuthenticateAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.core.security.action.user.AuthenticateRequest; import org.elasticsearch.xpack.core.security.action.user.AuthenticateResponse; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.user.AnonymousUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; import org.elasticsearch.xpack.core.security.user.User; public class TransportAuthenticateAction extends HandledTransportAction { @@ -50,9 +51,9 @@ public class TransportAuthenticateAction extends HandledTransportAction indices = Optional.ofNullable(indices(msg)); if (eventFilterPolicyRegistry.ignorePredicate() @@ -818,7 +819,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener { assert eventType == ACCESS_DENIED || eventType == AuditLevel.ACCESS_GRANTED || eventType == SYSTEM_ACCESS_GRANTED; final String[] indices = index == null ? null : new String[] { index }; final User user = authentication.getEffectiveSubject().getUser(); - if (User.isInternal(user) && eventType == ACCESS_GRANTED) { + if (user instanceof InternalUser && eventType == ACCESS_GRANTED) { eventType = SYSTEM_ACCESS_GRANTED; } if (events.contains(eventType)) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index cdb1a95480ec..290a4f1fed3b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -144,6 +144,9 @@ import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; +import static org.elasticsearch.xpack.core.security.action.apikey.CrossClusterApiKeyRoleDescriptorBuilder.CCR_CLUSTER_PRIVILEGE_NAMES; +import static org.elasticsearch.xpack.core.security.action.apikey.CrossClusterApiKeyRoleDescriptorBuilder.CCS_AND_CCR_CLUSTER_PRIVILEGE_NAMES; +import static org.elasticsearch.xpack.core.security.action.apikey.CrossClusterApiKeyRoleDescriptorBuilder.CCS_CLUSTER_PRIVILEGE_NAMES; import static org.elasticsearch.xpack.security.Security.SECURITY_CRYPTO_THREAD_POOL_NAME; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; @@ -1234,6 +1237,48 @@ public class ApiKeyService { } } + public void crossClusterApiKeyUsageStats(ActionListener> listener) { + if (false == isEnabled()) { + listener.onResponse(Map.of()); + return; + } + final SecurityIndexManager frozenSecurityIndex = securityIndex.freeze(); + if (frozenSecurityIndex.indexExists() == false) { + logger.debug("security index does not exist"); + listener.onResponse(Map.of("total", 0, "ccs", 0, "ccr", 0, "ccs_ccr", 0)); + } else if (frozenSecurityIndex.isAvailable() == false) { + listener.onFailure(frozenSecurityIndex.getUnavailableReason()); + } else { + final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery("doc_type", "api_key")) + .filter(QueryBuilders.termQuery("type", ApiKey.Type.CROSS_CLUSTER.value())); + findApiKeys(boolQuery, true, true, this::convertSearchHitToApiKeyInfo, ActionListener.wrap(apiKeyInfos -> { + int ccsKeys = 0, ccrKeys = 0, ccsCcrKeys = 0; + for (ApiKey apiKeyInfo : apiKeyInfos) { + assert apiKeyInfo.getType() == ApiKey.Type.CROSS_CLUSTER; + assert apiKeyInfo.getRoleDescriptors().size() == 1; + final String[] clusterPrivileges = apiKeyInfo.getRoleDescriptors().iterator().next().getClusterPrivileges(); + if (Arrays.equals(clusterPrivileges, CCS_CLUSTER_PRIVILEGE_NAMES)) { + ccsKeys += 1; + } else if (Arrays.equals(clusterPrivileges, CCR_CLUSTER_PRIVILEGE_NAMES)) { + ccrKeys += 1; + } else if (Arrays.equals(clusterPrivileges, CCS_AND_CCR_CLUSTER_PRIVILEGE_NAMES)) { + ccsCcrKeys += 1; + } else { + final String message = "invalid cluster privileges [" + + Strings.arrayToCommaDelimitedString(clusterPrivileges) + + "] for cross-cluster API key [" + + apiKeyInfo.getId() + + "]"; + assert false : message; + listener.onFailure(new IllegalStateException(message)); + } + } + listener.onResponse(Map.of("total", apiKeyInfos.size(), "ccs", ccsKeys, "ccr", ccrKeys, "ccs_ccr", ccsCcrKeys)); + }, listener::onFailure)); + } + } + // public class for testing public static final class ApiKeyCredentials implements AuthenticationToken, Closeable { private final String id; @@ -1462,7 +1507,7 @@ public class ApiKeyService { } if (filterOutExpiredKeys) { final BoolQueryBuilder expiredQuery = QueryBuilders.boolQuery(); - expiredQuery.should(QueryBuilders.rangeQuery("expiration_time").gt(Instant.now().toEpochMilli())); + expiredQuery.should(QueryBuilders.rangeQuery("expiration_time").gt(clock.instant().toEpochMilli())); expiredQuery.should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("expiration_time"))); boolQuery.filter(expiredQuery); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java index 6180594c1430..080d5d5e39a6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealm.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.core.security.user.KibanaSystemUser; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.LogstashSystemUser; import org.elasticsearch.xpack.core.security.user.RemoteMonitoringUser; +import org.elasticsearch.xpack.core.security.user.ReservedUser; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore.ReservedUserInfo; import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm; @@ -202,7 +203,7 @@ public class ReservedRealm extends CachingUsernamePasswordRealm { } } - private User getUser(String username, ReservedUserInfo userInfo) { + private ReservedUser getUser(String username, ReservedUserInfo userInfo) { assert username != null; switch (username) { case ElasticUser.NAME: @@ -232,7 +233,7 @@ public class ReservedRealm extends CachingUsernamePasswordRealm { listener.onResponse(anonymousEnabled ? Collections.singletonList(anonymousUser) : Collections.emptyList()); } else { nativeUsersStore.getAllReservedUserInfo(ActionListener.wrap((reservedUserInfos) -> { - List users = new ArrayList<>(4); + final List users = new ArrayList<>(8); ReservedUserInfo userInfo = reservedUserInfos.get(ElasticUser.NAME); users.add(new ElasticUser(userInfo == null || userInfo.enabled)); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index e19f88215015..75c0cb83f006 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -70,6 +70,7 @@ import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivileg import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilegeResolver; import org.elasticsearch.xpack.core.security.authz.privilege.IndexPrivilege; import org.elasticsearch.xpack.core.security.user.AnonymousUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; import org.elasticsearch.xpack.core.security.user.SystemUser; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.Security; @@ -101,7 +102,6 @@ import static org.elasticsearch.xpack.core.security.authz.AuthorizationServiceFi import static org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField.ORIGINATING_ACTION_KEY; import static org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl.allowAll; import static org.elasticsearch.xpack.core.security.support.Exceptions.authorizationError; -import static org.elasticsearch.xpack.core.security.user.User.isInternal; import static org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.PRINCIPAL_ROLES_FIELD_NAME; public class AuthorizationService { @@ -216,7 +216,7 @@ public class AuthorizationService { final Subject subject, final ActionListener listener ) { - if (isInternal(subject.getUser())) { + if (subject.getUser() instanceof InternalUser) { final String message = "the user [" + subject.getUser().principal() + "] is an internal user and we should never try to retrieve its roles descriptors towards a remote cluster"; @@ -352,7 +352,7 @@ public class AuthorizationService { if (auditId == null) { // We would like to assert that there is an existing request-id, but if this is a system action, then that might not be // true because the request-id is generated during authentication - if (isInternal(authentication.getEffectiveSubject().getUser())) { + if (authentication.getEffectiveSubject().getUser() instanceof InternalUser) { auditId = AuditUtil.getOrGenerateRequestId(threadContext); } else { auditTrailService.get().tamperedRequest(null, authentication, action, originalRequest); @@ -630,7 +630,7 @@ public class AuthorizationService { private AuthorizationEngine getAuthorizationEngineForUser(final User user) { if (rbacEngine != authorizationEngine && Security.AUTHORIZATION_ENGINE_FEATURE.check(licenseState)) { - if (ClientReservedRealm.isReserved(user.principal(), settings) || isInternal(user)) { + if (ClientReservedRealm.isReserved(user.principal(), settings) || user instanceof InternalUser) { return rbacEngine; } else { return authorizationEngine; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java index 3aca6e19cbe2..22bd46d4442a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java @@ -14,11 +14,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; import org.elasticsearch.xpack.core.security.support.Automatons; -import org.elasticsearch.xpack.core.security.user.AsyncSearchUser; -import org.elasticsearch.xpack.core.security.user.SecurityProfileUser; -import org.elasticsearch.xpack.core.security.user.StorageInternalUser; -import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import java.util.function.Consumer; import java.util.function.Predicate; @@ -123,13 +119,13 @@ public final class AuthorizationUtils { switch (actionOrigin) { case SECURITY_ORIGIN: - securityContext.executeAsInternalUser(XPackSecurityUser.INSTANCE, version, consumer); + securityContext.executeAsInternalUser(InternalUsers.XPACK_SECURITY_USER, version, consumer); break; case SECURITY_PROFILE_ORIGIN: - securityContext.executeAsInternalUser(SecurityProfileUser.INSTANCE, version, consumer); + securityContext.executeAsInternalUser(InternalUsers.SECURITY_PROFILE_USER, version, consumer); break; case POST_WRITE_REFRESH_ORIGIN: - securityContext.executeAsInternalUser(StorageInternalUser.INSTANCE, version, consumer); + securityContext.executeAsInternalUser(InternalUsers.STORAGE_USER, version, consumer); break; case WATCHER_ORIGIN: case ML_ORIGIN: @@ -150,10 +146,10 @@ public final class AuthorizationUtils { case FLEET_ORIGIN: case ENT_SEARCH_ORIGIN: case TASKS_ORIGIN: // TODO use a more limited user for tasks - securityContext.executeAsInternalUser(XPackUser.INSTANCE, version, consumer); + securityContext.executeAsInternalUser(InternalUsers.XPACK_USER, version, consumer); break; case ASYNC_SEARCH_ORIGIN: - securityContext.executeAsInternalUser(AsyncSearchUser.INSTANCE, version, consumer); + securityContext.executeAsInternalUser(InternalUsers.ASYNC_SEARCH_USER, version, consumer); break; default: assert false : "action.origin [" + actionOrigin + "] is unknown!"; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java index 2f2d4e767494..d927dec4b1c6 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java @@ -46,6 +46,7 @@ import org.elasticsearch.xpack.core.security.authz.store.RoleReferenceIntersecti import org.elasticsearch.xpack.core.security.authz.store.RolesRetrievalResult; import org.elasticsearch.xpack.core.security.support.CacheIteratorHelper; import org.elasticsearch.xpack.core.security.user.AnonymousUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.authc.ApiKeyService; @@ -151,13 +152,17 @@ public class CompositeRolesStore { fieldPermissionsCache, this.restrictedIndices ); - this.internalUserRoles = InternalUsers.getRoleDescriptors() - .entrySet() + this.internalUserRoles = InternalUsers.get() .stream() + .filter(u -> u.getLocalClusterRoleDescriptor().isPresent()) .collect( Collectors.toMap( - Map.Entry::getKey, - e -> Role.buildFromRoleDescriptor(e.getValue(), fieldPermissionsCache, this.restrictedIndices) + u -> u.principal(), + u -> Role.buildFromRoleDescriptor( + u.getLocalClusterRoleDescriptor().get(), + fieldPermissionsCache, + this.restrictedIndices + ) ) ); this.roleReferenceResolver = new RoleDescriptorStore( @@ -195,7 +200,7 @@ public class CompositeRolesStore { return; } - assert false == User.isInternal(subject.getUser()) : "Internal user should not pass here"; + assert false == subject.getUser() instanceof InternalUser : "Internal user [" + subject.getUser() + "] should not pass here"; final RoleReferenceIntersection roleReferenceIntersection = subject.getRoleReferenceIntersection(anonymousUser); roleReferenceIntersection.buildRole(this::buildRoleFromRoleReference, roleActionListener); @@ -210,18 +215,18 @@ public class CompositeRolesStore { // method. // The other internal users have directly assigned roles that are handled with special cases here final User user = subject.getUser(); - if (User.isInternal(user)) { - return getInternalUserRole(user); + if (user instanceof InternalUser internal) { + return getInternalUserRole(internal); } return null; } // Accessible for testing - protected Role getInternalUserRole(User user) { - String name = InternalUsers.getInternalUserName(user); + protected Role getInternalUserRole(InternalUser user) { + String name = user.principal(); final Role role = this.internalUserRoles.get(name); if (role == null) { - throw new IllegalArgumentException("the internal user [" + user.principal() + "] should never have its roles resolved"); + throw new IllegalArgumentException("the internal user [" + name + "] should never have its roles resolved"); } return role; } @@ -356,9 +361,14 @@ public class CompositeRolesStore { // Package private for testing static Optional tryGetRoleDescriptorForInternalUser(Subject subject) { - final User user = subject.getUser(); - if (User.isInternal(user)) { - return Optional.of(InternalUsers.getRoleDescriptor(user)); + if (subject.getUser() instanceof InternalUser internalUser) { + final Optional roleDescriptor = internalUser.getLocalClusterRoleDescriptor(); + if (roleDescriptor.isEmpty()) { + throw new IllegalArgumentException( + "should never try to get the roles for internal user [" + internalUser.principal() + "]" + ); + } + return roleDescriptor; } else { return Optional.empty(); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivileges.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivileges.java index 0a41b7e36bd9..67349dc021ae 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivileges.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorPrivileges.java @@ -17,6 +17,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; +import org.elasticsearch.xpack.core.security.user.InternalUser; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.Security; @@ -78,7 +79,7 @@ public class OperatorPrivileges { final User user = authentication.getEffectiveSubject().getUser(); // Let internal users pass, they are exempt from marking and checking // Also check run_as, it is impossible to run_as internal users, but just to be extra safe - if (User.isInternal(user) && false == authentication.isRunAs()) { + if (user instanceof InternalUser && false == authentication.isRunAs()) { return; } // The header is already set by previous authentication either on this node or a remote node @@ -105,7 +106,7 @@ public class OperatorPrivileges { } final User user = authentication.getEffectiveSubject().getUser(); // Let internal users pass (also check run_as, it is impossible to run_as internal users, but just to be extra safe) - if (User.isInternal(user) && false == authentication.isRunAs()) { + if (user instanceof InternalUser && false == authentication.isRunAs()) { return null; } if (false == AuthenticationField.PRIVILEGE_CATEGORY_VALUE_OPERATOR.equals( diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/profile/ProfileService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/profile/ProfileService.java index b2fcdf320474..0544a82448bc 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/profile/ProfileService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/profile/ProfileService.java @@ -69,6 +69,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.DomainConfig; import org.elasticsearch.xpack.core.security.authc.RealmDomain; import org.elasticsearch.xpack.core.security.authc.Subject; +import org.elasticsearch.xpack.core.security.user.InternalUser; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.support.SecurityIndexManager; @@ -196,7 +197,7 @@ public class ProfileService { return; } - if (User.isInternal(subject.getUser())) { + if (subject.getUser() instanceof InternalUser) { listener.onFailure( new IllegalStateException("profile should not be created for internal user [" + subject.getUser().principal() + "]") ); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java index 0c73ebcec43b..e0b9f942be3b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptor.java @@ -41,6 +41,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo; import org.elasticsearch.xpack.core.security.transport.ProfileConfigurations; import org.elasticsearch.xpack.core.security.user.CrossClusterAccessUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; import org.elasticsearch.xpack.core.security.user.SystemUser; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.ssl.SSLService; @@ -335,7 +336,7 @@ public class SecurityServerTransportInterceptor implements TransportInterceptor assert authentication != null : "authentication must be present in security context"; final User user = authentication.getEffectiveSubject().getUser(); - if (User.isInternal(user) && false == SystemUser.is(user)) { + if (user instanceof InternalUser && false == SystemUser.is(user)) { final String message = "Internal user [" + user.principal() + "] should not be used for cross cluster requests"; assert false : message; throw illegalArgumentExceptionWithDebugLog(message); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java index 145532d31637..c1103d8cdf55 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityContextTests.java @@ -27,11 +27,9 @@ import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.Authoriza import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.ParentActionAuthorization; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; import org.elasticsearch.xpack.core.security.user.AnonymousUser; -import org.elasticsearch.xpack.core.security.user.AsyncSearchUser; -import org.elasticsearch.xpack.core.security.user.SystemUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; -import org.elasticsearch.xpack.core.security.user.XPackUser; import org.junit.Before; import org.mockito.Mockito; @@ -88,7 +86,7 @@ public class SecurityContextTests extends ESTestCase { } public void testSetInternalUser() { - final User internalUser = randomFrom(SystemUser.INSTANCE, XPackUser.INSTANCE, XPackSecurityUser.INSTANCE, AsyncSearchUser.INSTANCE); + final InternalUser internalUser = AuthenticationTestHelper.randomInternalUser(); assertNull(securityContext.getAuthentication()); assertNull(securityContext.getUser()); securityContext.setInternalUser(internalUser, TransportVersion.CURRENT); @@ -97,7 +95,7 @@ public class SecurityContextTests extends ESTestCase { IllegalStateException e = expectThrows( IllegalStateException.class, - () -> securityContext.setInternalUser(randomFrom(internalUser, SystemUser.INSTANCE), TransportVersion.CURRENT) + () -> securityContext.setInternalUser(randomFrom(internalUser, InternalUsers.SYSTEM_USER), TransportVersion.CURRENT) ); assertEquals("authentication ([_xpack_security_authentication]) is already present in the context", e.getMessage()); } @@ -116,12 +114,7 @@ public class SecurityContextTests extends ESTestCase { original = null; } - final User executionUser = randomFrom( - SystemUser.INSTANCE, - XPackUser.INSTANCE, - XPackSecurityUser.INSTANCE, - AsyncSearchUser.INSTANCE - ); + final InternalUser executionUser = AuthenticationTestHelper.randomInternalUserWithLocalRoleDescriptor(); final AtomicReference contextAtomicReference = new AtomicReference<>(); securityContext.executeAsInternalUser(executionUser, TransportVersion.CURRENT, (originalCtx) -> { assertEquals(executionUser, securityContext.getUser()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java index bacf0b571002..5016539a62bf 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityInfoTransportActionTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.core.security.SecurityFeatureSetUsage; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.watcher.support.xcontent.XContentSource; import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail; +import org.elasticsearch.xpack.security.authc.ApiKeyService; import org.elasticsearch.xpack.security.authc.Realms; import org.elasticsearch.xpack.security.authc.support.mapper.NativeRoleMappingStore; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; @@ -43,6 +44,7 @@ import java.util.Map; import static org.elasticsearch.test.ActionListenerUtils.anyActionListener; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.emptyIterable; import static org.hamcrest.Matchers.notNullValue; @@ -61,6 +63,7 @@ public class SecurityInfoTransportActionTests extends ESTestCase { private NativeRoleMappingStore roleMappingStore; private ProfileService profileService; private SecurityUsageServices securityServices; + private ApiKeyService apiKeyService; @Before public void init() throws Exception { @@ -71,7 +74,8 @@ public class SecurityInfoTransportActionTests extends ESTestCase { rolesStore = mock(CompositeRolesStore.class); roleMappingStore = mock(NativeRoleMappingStore.class); profileService = mock(ProfileService.class); - securityServices = new SecurityUsageServices(realms, rolesStore, roleMappingStore, ipFilter, profileService); + apiKeyService = mock(ApiKeyService.class); + securityServices = new SecurityUsageServices(realms, rolesStore, roleMappingStore, ipFilter, profileService, apiKeyService); } public void testAvailable() { @@ -196,6 +200,25 @@ public class SecurityInfoTransportActionTests extends ESTestCase { return null; }).when(profileService).usageStats(anyActionListener()); + final int ccsKeys = randomIntBetween(0, 50); + final int ccrKeys = randomIntBetween(0, 50); + final int ccsCcrKeys = randomIntBetween(0, 50); + final Map crossClusterApiKeyUsage = Map.of( + "total", + ccsKeys + ccrKeys + ccsCcrKeys, + "ccs", + ccsKeys, + "ccr", + ccrKeys, + "ccs_ccr", + ccsCcrKeys + ); + doAnswer(invocation -> { + final ActionListener> listener = invocation.getArgument(0); + listener.onResponse(apiKeyServiceEnabled ? crossClusterApiKeyUsage : Map.of()); + return null; + }).when(apiKeyService).crossClusterApiKeyUsageStats(anyActionListener()); + var usageAction = newUsageAction(settings.build()); PlainActionFuture future = new PlainActionFuture<>(); usageAction.masterOperation(null, null, null, future); @@ -281,6 +304,14 @@ public class SecurityInfoTransportActionTests extends ESTestCase { assertThat(source.getValue("ssl.remote_cluster_client.enabled"), is(remoteClusterClientSslEnabled)); assertThat(source.getValue("remote_cluster_server.available"), is(remoteClusterServerAvailable)); assertThat(source.getValue("remote_cluster_server.enabled"), is(remoteClusterServerEnabled)); + if (apiKeyServiceEnabled) { + assertThat(source.getValue("remote_cluster_server.api_keys.total"), equalTo(crossClusterApiKeyUsage.get("total"))); + assertThat(source.getValue("remote_cluster_server.api_keys.ccs"), equalTo(ccsKeys)); + assertThat(source.getValue("remote_cluster_server.api_keys.ccr"), equalTo(ccrKeys)); + assertThat(source.getValue("remote_cluster_server.api_keys.ccs_ccr"), equalTo(ccsCcrKeys)); + } else { + assertThat(source.getValue("remote_cluster_server.api_keys"), anEmptyMap()); + } } } else { assertThat(source.getValue("ssl"), is(nullValue())); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java index ec229fde9940..e64fb7c409bb 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/filter/SecurityActionFilterTests.java @@ -37,7 +37,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationField; import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; -import org.elasticsearch.xpack.core.security.user.SystemUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.audit.AuditTrail; import org.elasticsearch.xpack.security.audit.AuditTrailService; @@ -196,7 +196,7 @@ public class SecurityActionFilterTests extends ESTestCase { requestIdFromAuthn.set(AuditUtil.generateRequestId(threadContext)); callback.onResponse(threadContext.getTransient(AuthenticationField.AUTHENTICATION_KEY)); return Void.TYPE; - }).when(authcService).authenticate(eq(action), eq(request), eq(SystemUser.INSTANCE), anyActionListener()); + }).when(authcService).authenticate(eq(action), eq(request), eq(InternalUsers.SYSTEM_USER), anyActionListener()); IndicesAccessControl authzAccessControl = mock(IndicesAccessControl.class); when(authzAccessControl.isGranted()).thenReturn(true); mockAuthorize(authzAccessControl); @@ -213,7 +213,7 @@ public class SecurityActionFilterTests extends ESTestCase { } assertNotNull(authenticationSetOnce.get()); assertNotEquals(authentication, authenticationSetOnce.get()); - assertEquals(SystemUser.INSTANCE, authenticationSetOnce.get().getEffectiveSubject().getUser()); + assertEquals(InternalUsers.SYSTEM_USER, authenticationSetOnce.get().getEffectiveSubject().getUser()); assertThat(accessControlSetOnce.get(), sameInstance(authzAccessControl)); assertThat(requestIdOnActionHandler.get(), is(requestIdFromAuthn.get())); } @@ -243,7 +243,7 @@ public class SecurityActionFilterTests extends ESTestCase { threadContext.putHeader(AuthenticationField.AUTHENTICATION_KEY, authentication.encode()); callback.onResponse(authentication); return Void.TYPE; - }).when(authcService).authenticate(eq(action), eq(request), eq(SystemUser.INSTANCE), anyActionListener()); + }).when(authcService).authenticate(eq(action), eq(request), eq(InternalUsers.SYSTEM_USER), anyActionListener()); doAnswer((i) -> { ActionListener callback = (ActionListener) i.getArguments()[3]; callback.onResponse(null); @@ -285,7 +285,7 @@ public class SecurityActionFilterTests extends ESTestCase { AuditUtil.generateRequestId(threadContext); callback.onResponse(authentication); return Void.TYPE; - }).when(authcService).authenticate(eq("_action"), eq(request), eq(SystemUser.INSTANCE), anyActionListener()); + }).when(authcService).authenticate(eq("_action"), eq(request), eq(InternalUsers.SYSTEM_USER), anyActionListener()); if (randomBoolean()) { doThrow(exception).when(authzService).authorize(eq(authentication), eq("_action"), eq(request), anyActionListener()); } else { @@ -311,7 +311,7 @@ public class SecurityActionFilterTests extends ESTestCase { threadContext.putHeader("_xpack_audit_request_id", requestId); callback.onResponse(authentication); return Void.TYPE; - }).when(authcService).authenticate(eq("_action"), eq(request), eq(SystemUser.INSTANCE), anyActionListener()); + }).when(authcService).authenticate(eq("_action"), eq(request), eq(InternalUsers.SYSTEM_USER), anyActionListener()); } private void mockAuthorize() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledActionTests.java index 11a479e79ae9..1b7d2ba19d64 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportSetEnabledActionTests.java @@ -26,8 +26,8 @@ import org.elasticsearch.xpack.core.security.authc.esnative.NativeRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.ElasticUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.KibanaUser; -import org.elasticsearch.xpack.core.security.user.SystemUser; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import org.elasticsearch.xpack.security.authc.esnative.ReservedRealm; @@ -110,7 +110,7 @@ public class TransportSetEnabledActionTests extends ESTestCase { public void testValidUser() throws Exception { testValidUser( - randomFrom(new ElasticUser(true), new KibanaUser(true), new User("joe"), new User(SystemUser.INSTANCE.principal())), + randomFrom(new ElasticUser(true), new KibanaUser(true), new User("joe"), new User(InternalUsers.SYSTEM_USER.principal())), defaultAuthentication() ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java index 38fa920c8c22..cf17c6071618 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailFilterTests.java @@ -41,7 +41,8 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationField; import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.AuthorizationInfo; -import org.elasticsearch.xpack.core.security.user.SystemUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.audit.AuditUtil; import org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail.AuditEventMetaInfo; @@ -2855,10 +2856,10 @@ public class LoggingAuditTrailFilterTests extends ESTestCase { private static Authentication createSystemUserAuthentication(boolean isFallback) { if (isFallback) { - return Authentication.newInternalFallbackAuthentication(SystemUser.INSTANCE, randomAlphaOfLengthBetween(3, 8)); + return Authentication.newInternalFallbackAuthentication(InternalUsers.SYSTEM_USER, randomAlphaOfLengthBetween(3, 8)); } else { return Authentication.newInternalAuthentication( - SystemUser.INSTANCE, + InternalUsers.SYSTEM_USER, TransportVersion.CURRENT, randomAlphaOfLengthBetween(3, 8) ); @@ -2870,7 +2871,7 @@ public class LoggingAuditTrailFilterTests extends ESTestCase { } private static Authentication createAuthentication(User effectiveUser, @Nullable User authenticatingUser, String effectiveRealmName) { - assert false == User.isInternal(effectiveUser); + assert false == effectiveUser instanceof InternalUser; if (authenticatingUser != null) { return AuthenticationTestHelper.builder() .user(authenticatingUser) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index fcf5addfd1aa..d09c81e51e2a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -113,13 +113,10 @@ import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableCluster import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges; import org.elasticsearch.xpack.core.security.support.ValidationTests; import org.elasticsearch.xpack.core.security.user.AnonymousUser; -import org.elasticsearch.xpack.core.security.user.AsyncSearchUser; -import org.elasticsearch.xpack.core.security.user.CrossClusterAccessUser; -import org.elasticsearch.xpack.core.security.user.SystemUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.core.security.user.UsernamesField; -import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; -import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.audit.AuditLevel; import org.elasticsearch.xpack.security.audit.AuditTrail; import org.elasticsearch.xpack.security.audit.AuditUtil; @@ -2019,7 +2016,7 @@ public class LoggingAuditTrailTests extends ESTestCase { final TransportRequest request = randomBoolean() ? new MockRequest(threadContext) : new MockIndicesRequest(threadContext); final String[] expectedRoles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); final AuthorizationInfo authorizationInfo = () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, expectedRoles); - final User systemUser = randomFrom(SystemUser.INSTANCE, XPackUser.INSTANCE, XPackSecurityUser.INSTANCE, AsyncSearchUser.INSTANCE); + final InternalUser systemUser = AuthenticationTestHelper.randomInternalUser(); final Authentication authentication = AuthenticationTestHelper.builder().internal(systemUser).build(); final String requestId = randomRequestId(); @@ -2098,7 +2095,7 @@ public class LoggingAuditTrailTests extends ESTestCase { final TransportRequest request = randomBoolean() ? new MockRequest(threadContext) : new MockIndicesRequest(threadContext); final String[] expectedRoles = randomArray(0, 4, String[]::new, () -> randomBoolean() ? null : randomAlphaOfLengthBetween(1, 4)); final AuthorizationInfo authorizationInfo = () -> Collections.singletonMap(PRINCIPAL_ROLES_FIELD_NAME, expectedRoles); - final User systemUser = randomFrom(SystemUser.INSTANCE, XPackUser.INSTANCE, XPackSecurityUser.INSTANCE, AsyncSearchUser.INSTANCE); + final InternalUser systemUser = AuthenticationTestHelper.randomInternalUser(); final Authentication authentication = AuthenticationTestHelper.builder().internal(systemUser).build(); final String requestId = randomRequestId(); auditTrail.accessGranted(requestId, authentication, "internal:_action", request, authorizationInfo); @@ -2681,7 +2678,7 @@ public class LoggingAuditTrailTests extends ESTestCase { final String requestId = randomRequestId(); final Authentication remoteAuthentication = randomFrom( AuthenticationTestHelper.builder().realm(false), - AuthenticationTestHelper.builder().internal(CrossClusterAccessUser.INSTANCE), + AuthenticationTestHelper.builder().internal(InternalUsers.CROSS_CLUSTER_ACCESS_USER), AuthenticationTestHelper.builder().serviceAccount(), AuthenticationTestHelper.builder().apiKey().metadata(Map.of(AuthenticationField.API_KEY_NAME_KEY, randomAlphaOfLength(42))) ).build(false); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java index 4bf61112bd94..dc4e5bc1a1a4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java @@ -128,6 +128,7 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.Collection; @@ -705,6 +706,157 @@ public class ApiKeyServiceTests extends ESTestCase { assertThat(ex.getMessage(), containsString("authentication via API key not supported: only the owner user can update an API key")); } + public void testCrossClusterApiKeyUsageStats() { + final Instant now = Instant.now(); + when(clock.instant()).thenReturn(now); + when(client.threadPool()).thenReturn(threadPool); + SearchRequestBuilder searchRequestBuilder = Mockito.spy(new SearchRequestBuilder(client, SearchAction.INSTANCE)); + when(client.prepareSearch(eq(SECURITY_MAIN_ALIAS))).thenReturn(searchRequestBuilder); + + final List searchHits = new ArrayList<>(); + final int ccsKeys = randomIntBetween(0, 2); + for (int i = 0; i < ccsKeys; i++) { + searchHits.add(searchHitForCrossClusterApiKey(0)); + } + final int ccrKeys = randomIntBetween(0, 2); + for (int i = 0; i < ccrKeys; i++) { + searchHits.add(searchHitForCrossClusterApiKey(1)); + } + final int ccsCcrKeys = randomIntBetween(0, 2); + for (int i = 0; i < ccsCcrKeys; i++) { + searchHits.add(searchHitForCrossClusterApiKey(2)); + } + + final AtomicReference searchRequest = new AtomicReference<>(); + doAnswer(invocationOnMock -> { + searchRequest.set(invocationOnMock.getArgument(0)); + final var searchResponse = new SearchResponse( + new InternalSearchResponse( + new SearchHits( + searchHits.toArray(SearchHit[]::new), + new TotalHits(searchHits.size(), TotalHits.Relation.EQUAL_TO), + randomFloat(), + null, + null, + null + ), + null, + null, + null, + false, + null, + 0 + ), + randomAlphaOfLengthBetween(3, 8), + 1, + 1, + 0, + 10, + null, + null + ); + final ActionListener listener = invocationOnMock.getArgument(1); + listener.onResponse(searchResponse); + return null; + }).when(client).search(any(SearchRequest.class), anyActionListener()); + + final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery() + .filter(QueryBuilders.termQuery("doc_type", "api_key")) + .filter(QueryBuilders.termQuery("type", ApiKey.Type.CROSS_CLUSTER.value())) + .filter(QueryBuilders.termQuery("api_key_invalidated", false)) + .filter( + QueryBuilders.boolQuery() + .should(QueryBuilders.rangeQuery("expiration_time").gt(now.toEpochMilli())) + .should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("expiration_time"))) + ); + + final ApiKeyService apiKeyService = createApiKeyService(); + final PlainActionFuture> future = new PlainActionFuture<>(); + apiKeyService.crossClusterApiKeyUsageStats(future); + + verify(searchRequestBuilder).setQuery(eq(boolQuery)); + assertThat(searchRequest.get().source().query(), is(boolQuery)); + + assertThat( + future.actionGet(), + equalTo(Map.of("total", ccsKeys + ccrKeys + ccsCcrKeys, "ccs", ccsKeys, "ccr", ccrKeys, "ccs_ccr", ccsCcrKeys)) + ); + } + + private SearchHit searchHitForCrossClusterApiKey(int crossClusterAccessLevel) { + assert crossClusterAccessLevel >= 0 && crossClusterAccessLevel <= 2; + final String roleDescriptor = switch (crossClusterAccessLevel) { + case 0 -> """ + { + "cluster": ["cross_cluster_search"] + }"""; + case 1 -> """ + { + "cluster": ["cross_cluster_replication"] + }"""; + default -> """ + { + "cluster": ["cross_cluster_search", "cross_cluster_replication"] + }"""; + }; + final int docId = randomIntBetween(0, Integer.MAX_VALUE); + final String apiKeyId = randomAlphaOfLength(20); + final var searchHit = new SearchHit(docId, apiKeyId); + try (XContentBuilder builder = JsonXContent.contentBuilder()) { + builder.map(XContentHelper.convertToMap(JsonXContent.jsonXContent, Strings.format(""" + { + "doc_type": "api_key", + "type": "cross_cluster", + "creation_time": 1591919944598, + "expiration_time": null, + "api_key_invalidated": false, + "api_key_hash": "{PBKDF2}10000$abc", + "role_descriptors": { "cross_cluster": %s }, + "limited_by_role_descriptors": { }, + "name": null, + "version": 8090099, + "creator": { + "principal": "admin", + "metadata": {}, + "realm": "file1" + } + }""", roleDescriptor), randomBoolean())); + searchHit.sourceRef(BytesReference.bytes(builder)); + return searchHit; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public void testCrossClusterApiKeyUsageStatsAreZerosWhenServiceNotEnabled() { + final Settings settings = Settings.builder().put(XPackSettings.API_KEY_SERVICE_ENABLED_SETTING.getKey(), false).build(); + final ApiKeyService service = createApiKeyService(settings); + final PlainActionFuture> future = new PlainActionFuture<>(); + service.crossClusterApiKeyUsageStats(future); + assertThat(future.actionGet(), anEmptyMap()); + } + + public void testCrossClusterApiKeyUsageStatsAreZerosWhenIndexDoesNotExist() { + securityIndex = SecurityMocks.mockSecurityIndexManager(".security", false, false); + final ApiKeyService apiKeyService = createApiKeyService(); + + final PlainActionFuture> future = new PlainActionFuture<>(); + apiKeyService.crossClusterApiKeyUsageStats(future); + assertThat(future.actionGet(), equalTo(Map.of("total", 0, "ccs", 0, "ccr", 0, "ccs_ccr", 0))); + } + + public void testCrossClusterApiKeyUsageFailsWhenIndexNotAvailable() { + securityIndex = SecurityMocks.mockSecurityIndexManager(".security", true, false); + final ElasticsearchException expectedException = new ElasticsearchException("not available"); + when(securityIndex.getUnavailableReason()).thenReturn(expectedException); + final ApiKeyService apiKeyService = createApiKeyService(); + + final PlainActionFuture> future = new PlainActionFuture<>(); + apiKeyService.crossClusterApiKeyUsageStats(future); + final ElasticsearchException e = expectThrows(ElasticsearchException.class, future::actionGet); + assertThat(e, sameInstance(expectedException)); + } + private Map mockKeyDocument( String id, String key, diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index 24f0c4670d19..33734f8f26df 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -89,6 +89,7 @@ import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.EmptyAuthorizationInfo; import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices; import org.elasticsearch.xpack.core.security.user.AnonymousUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.SystemUser; import org.elasticsearch.xpack.core.security.user.User; import org.elasticsearch.xpack.security.Security; @@ -876,7 +877,7 @@ public class AuthenticationServiceTests extends ESTestCase { reqId.set(AuditUtil.getOrGenerateRequestId(threadContext)); } User user = new User("username", new String[] { "r1", "r2" }, null, null, Map.of(), false); - User fallback = randomBoolean() ? SystemUser.INSTANCE : null; + User fallback = randomBoolean() ? InternalUsers.SYSTEM_USER : null; when(firstRealm.token(threadContext)).thenReturn(token); when(firstRealm.supports(token)).thenReturn(true); mockAuthenticate(firstRealm, token, user); @@ -922,7 +923,7 @@ public class AuthenticationServiceTests extends ESTestCase { final User user = new User("username", "r1", "r2"); final Consumer> authenticate; if (randomBoolean()) { - authenticate = listener -> service.authenticate("_action", transportRequest, SystemUser.INSTANCE, listener); + authenticate = listener -> service.authenticate("_action", transportRequest, InternalUsers.SYSTEM_USER, listener); } else { authenticate = listener -> service.authenticate("_action", transportRequest, true, listener); } @@ -990,7 +991,7 @@ public class AuthenticationServiceTests extends ESTestCase { if (requestIdAlreadyPresent) { reqId.set(AuditUtil.getOrGenerateRequestId(threadContext)); } - service.authenticate("_action", transportRequest, SystemUser.INSTANCE, ActionListener.wrap(authentication -> { + service.authenticate("_action", transportRequest, InternalUsers.SYSTEM_USER, ActionListener.wrap(authentication -> { if (requestIdAlreadyPresent) { assertThat(expectAuditRequestId(threadContext), is(reqId.get())); } else { @@ -1036,7 +1037,7 @@ public class AuthenticationServiceTests extends ESTestCase { threadContext1.putTransient(AuthenticationField.AUTHENTICATION_KEY, authRef.get()); threadContext1.putHeader(AuthenticationField.AUTHENTICATION_KEY, authHeaderRef.get()); - service.authenticate("_action", message1, SystemUser.INSTANCE, ActionListener.wrap(ctxAuth -> { + service.authenticate("_action", message1, InternalUsers.SYSTEM_USER, ActionListener.wrap(ctxAuth -> { if (requestIdAlreadyPresent) { assertThat(expectAuditRequestId(threadContext1), is(reqId.get())); } else { @@ -1101,7 +1102,7 @@ public class AuthenticationServiceTests extends ESTestCase { serviceAccountService, operatorPrivilegesService ); - service.authenticate("_action", new InternalRequest(), SystemUser.INSTANCE, ActionListener.wrap(result -> { + service.authenticate("_action", new InternalRequest(), InternalUsers.SYSTEM_USER, ActionListener.wrap(result -> { if (requestIdAlreadyPresent) { assertThat(expectAuditRequestId(threadPool2.getThreadContext()), is(reqId.get())); } else { @@ -1129,7 +1130,7 @@ public class AuthenticationServiceTests extends ESTestCase { reqId.set(AuditUtil.getOrGenerateRequestId(threadContext)); } try { - authenticateBlocking("_action", message, randomBoolean() ? SystemUser.INSTANCE : null, null); + authenticateBlocking("_action", message, randomBoolean() ? InternalUsers.SYSTEM_USER : null, null); } catch (Exception e) { // expected if (requestIdAlreadyPresent) { @@ -1381,13 +1382,13 @@ public class AuthenticationServiceTests extends ESTestCase { reqId.set(AuditUtil.getOrGenerateRequestId(threadContext)); } - authenticateBlocking("_action", message, SystemUser.INSTANCE, result -> { + authenticateBlocking("_action", message, InternalUsers.SYSTEM_USER, result -> { if (requestIdAlreadyPresent) { assertThat(expectAuditRequestId(threadContext), is(reqId.get())); } assertThat(result, notNullValue()); assertThat(expectAuditRequestId(threadContext), is(result.v2())); - assertThat(result.v1().getEffectiveSubject().getUser(), sameInstance(SystemUser.INSTANCE)); + assertThat(result.v1().getEffectiveSubject().getUser(), sameInstance(InternalUsers.SYSTEM_USER)); assertThat(result.v1().getAuthenticationType(), is(AuthenticationType.INTERNAL)); assertThat(result.v1().getEffectiveSubject().getRealm().getDomain(), nullValue()); assertThreadContextContainsAuthentication(result.v1()); @@ -1825,7 +1826,7 @@ public class AuthenticationServiceTests extends ESTestCase { listener.onResponse(new User("looked up user", new String[] { "some role" }, null, null, Map.of(), false)); return null; }).when(secondRealm).lookupUser(eq("run_as"), anyActionListener()); - User fallback = randomBoolean() ? SystemUser.INSTANCE : null; + User fallback = randomBoolean() ? InternalUsers.SYSTEM_USER : null; ElasticsearchSecurityException e = expectThrows( ElasticsearchSecurityException.class, () -> authenticateBlocking("_action", transportRequest, fallback, null) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java index 9405a6ee63c4..791697e11e95 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/CrossClusterAccessAuthenticationServiceTests.java @@ -23,7 +23,7 @@ import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; import org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo; import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection; -import org.elasticsearch.xpack.core.security.user.XPackUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.junit.Before; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; @@ -155,7 +155,7 @@ public class CrossClusterAccessAuthenticationServiceTests extends ESTestCase { CrossClusterAccessHeadersTests.randomEncodedApiKeyHeader(), new CrossClusterAccessSubjectInfo( // Invalid internal user - AuthenticationTestHelper.builder().internal(XPackUser.INSTANCE).build(), + AuthenticationTestHelper.builder().internal(InternalUsers.XPACK_USER).build(), new RoleDescriptorsIntersection( new RoleDescriptor( "invalid_role", @@ -205,7 +205,7 @@ public class CrossClusterAccessAuthenticationServiceTests extends ESTestCase { assertThat(actual.getCause().getCause(), instanceOf(IllegalArgumentException.class)); assertThat( actual.getCause().getCause().getMessage(), - containsString("received cross cluster request from an unexpected internal user [" + XPackUser.NAME + "]") + containsString("received cross cluster request from an unexpected internal user [" + InternalUsers.XPACK_USER.principal() + "]") ); verify(auditableRequest).exceptionProcessingRequest( any(Exception.class), diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index e8500a03e9d6..0c812b1a5aff 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -151,11 +151,11 @@ import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableCluster import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; import org.elasticsearch.xpack.core.security.user.AnonymousUser; import org.elasticsearch.xpack.core.security.user.ElasticUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.KibanaUser; import org.elasticsearch.xpack.core.security.user.SystemUser; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; -import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.audit.AuditLevel; import org.elasticsearch.xpack.security.audit.AuditTrail; @@ -461,7 +461,7 @@ public class AuthorizationServiceTests extends ESTestCase { final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); // A failure would throw an exception - final Authentication authentication = createAuthentication(SystemUser.INSTANCE); + final Authentication authentication = createAuthentication(InternalUsers.SYSTEM_USER); final String[] actions = { "indices:monitor/whatever", "internal:whatever", @@ -530,7 +530,7 @@ public class AuthorizationServiceTests extends ESTestCase { } public void testSystemUserActionMatchingCustomRoleNameDenied() { - final Authentication authentication = createAuthentication(SystemUser.INSTANCE); + final Authentication authentication = createAuthentication(InternalUsers.SYSTEM_USER); final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); RoleDescriptor role = new RoleDescriptor( @@ -593,12 +593,12 @@ public class AuthorizationServiceTests extends ESTestCase { public void testIndicesActionsForSystemUserWhichAreNotAuthorized() { final TransportRequest request = mock(TransportRequest.class); - final Authentication authentication = createAuthentication(SystemUser.INSTANCE); + final Authentication authentication = createAuthentication(InternalUsers.SYSTEM_USER); final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); assertThrowsAuthorizationException( () -> authorize(authentication, "indices:", request), "indices:", - SystemUser.INSTANCE.principal() + InternalUsers.SYSTEM_USER.principal() ); verify(auditTrail).accessDenied( eq(requestId), @@ -612,12 +612,12 @@ public class AuthorizationServiceTests extends ESTestCase { public void testClusterAdminActionsForSystemUserWhichAreNotAuthorized() { final TransportRequest request = mock(TransportRequest.class); - final Authentication authentication = createAuthentication(SystemUser.INSTANCE); + final Authentication authentication = createAuthentication(InternalUsers.SYSTEM_USER); final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); assertThrowsAuthorizationException( () -> authorize(authentication, "cluster:admin/whatever", request), "cluster:admin/whatever", - SystemUser.INSTANCE.principal() + InternalUsers.SYSTEM_USER.principal() ); verify(auditTrail).accessDenied( eq(requestId), @@ -631,12 +631,12 @@ public class AuthorizationServiceTests extends ESTestCase { public void testClusterAdminSnapshotStatusActionForSystemUserWhichIsNotAuthorized() { final TransportRequest request = mock(TransportRequest.class); - final Authentication authentication = createAuthentication(SystemUser.INSTANCE); + final Authentication authentication = createAuthentication(InternalUsers.SYSTEM_USER); final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); assertThrowsAuthorizationException( () -> authorize(authentication, "cluster:admin/snapshot/status", request), "cluster:admin/snapshot/status", - SystemUser.INSTANCE.principal() + InternalUsers.SYSTEM_USER.principal() ); verify(auditTrail).accessDenied( eq(requestId), @@ -2582,9 +2582,9 @@ public class AuthorizationServiceTests extends ESTestCase { private Authentication createAuthentication(User user, @Nullable User authenticatingUser) { final Authentication authentication; - if (User.isInternal(user)) { + if (user instanceof InternalUser internalUser) { assert authenticatingUser == null; - authentication = AuthenticationTestHelper.builder().internal(user).build(); + authentication = AuthenticationTestHelper.builder().internal(internalUser).build(); } else if (user instanceof AnonymousUser) { assert authenticatingUser == null; authentication = AuthenticationTestHelper.builder().anonymous(user).build(false); @@ -2997,7 +2997,7 @@ public class AuthorizationServiceTests extends ESTestCase { when(licenseState.isAllowed(Security.AUTHORIZATION_ENGINE_FEATURE)).thenReturn(true); try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { authentication = createAuthentication( - randomFrom(XPackUser.INSTANCE, XPackSecurityUser.INSTANCE, new ElasticUser(true), new KibanaUser(true)) + randomFrom(InternalUsers.XPACK_USER, InternalUsers.XPACK_SECURITY_USER, new ElasticUser(true), new KibanaUser(true)) ); assertNotEquals(engine, authorizationService.getRunAsAuthorizationEngine(authentication)); assertThat(authorizationService.getRunAsAuthorizationEngine(authentication), instanceOf(RBACEngine.class)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java index b0d2736c4eb3..1056746da941 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationUtilsTests.java @@ -20,12 +20,8 @@ import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; -import org.elasticsearch.xpack.core.security.user.AsyncSearchUser; -import org.elasticsearch.xpack.core.security.user.SecurityProfileUser; -import org.elasticsearch.xpack.core.security.user.SystemUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; -import org.elasticsearch.xpack.core.security.user.XPackUser; import org.junit.Before; import java.util.Arrays; @@ -54,7 +50,7 @@ public class AuthorizationUtilsTests extends ESTestCase { public void testSystemUserSwitchWithSystemUser() { threadContext.putTransient( AuthenticationField.AUTHENTICATION_KEY, - AuthenticationTestHelper.builder().internal(SystemUser.INSTANCE).build() + AuthenticationTestHelper.builder().internal(InternalUsers.SYSTEM_USER).build() ); assertThat(AuthorizationUtils.shouldReplaceUserWithSystem(threadContext, "internal:something"), is(false)); } @@ -111,11 +107,15 @@ public class AuthorizationUtilsTests extends ESTestCase { } public void testSwitchAndExecuteXpackSecurityUser() throws Exception { - assertSwitchBasedOnOriginAndExecute(ClientHelper.SECURITY_ORIGIN, XPackSecurityUser.INSTANCE, randomTransportVersion()); + assertSwitchBasedOnOriginAndExecute(ClientHelper.SECURITY_ORIGIN, InternalUsers.XPACK_SECURITY_USER, randomTransportVersion()); } public void testSwitchAndExecuteSecurityProfileUser() throws Exception { - assertSwitchBasedOnOriginAndExecute(ClientHelper.SECURITY_PROFILE_ORIGIN, SecurityProfileUser.INSTANCE, randomTransportVersion()); + assertSwitchBasedOnOriginAndExecute( + ClientHelper.SECURITY_PROFILE_ORIGIN, + InternalUsers.SECURITY_PROFILE_USER, + randomTransportVersion() + ); } public void testSwitchAndExecuteXpackUser() throws Exception { @@ -127,17 +127,17 @@ public class AuthorizationUtilsTests extends ESTestCase { PersistentTasksService.PERSISTENT_TASK_ORIGIN, ClientHelper.INDEX_LIFECYCLE_ORIGIN )) { - assertSwitchBasedOnOriginAndExecute(origin, XPackUser.INSTANCE, randomTransportVersion()); + assertSwitchBasedOnOriginAndExecute(origin, InternalUsers.XPACK_USER, randomTransportVersion()); } } public void testSwitchAndExecuteAsyncSearchUser() throws Exception { String origin = ClientHelper.ASYNC_SEARCH_ORIGIN; - assertSwitchBasedOnOriginAndExecute(origin, AsyncSearchUser.INSTANCE, randomTransportVersion()); + assertSwitchBasedOnOriginAndExecute(origin, InternalUsers.ASYNC_SEARCH_USER, randomTransportVersion()); } public void testSwitchWithTaskOrigin() throws Exception { - assertSwitchBasedOnOriginAndExecute(TASKS_ORIGIN, XPackUser.INSTANCE, randomTransportVersion()); + assertSwitchBasedOnOriginAndExecute(TASKS_ORIGIN, InternalUsers.XPACK_USER, randomTransportVersion()); } private void assertSwitchBasedOnOriginAndExecute(String origin, User user, TransportVersion version) throws Exception { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index e22cd73abed9..82f244452dee 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -66,10 +66,9 @@ import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCa import org.elasticsearch.xpack.core.security.authz.permission.Role; import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; import org.elasticsearch.xpack.core.security.authz.store.RoleReference; -import org.elasticsearch.xpack.core.security.user.AsyncSearchUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; -import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; import org.elasticsearch.xpack.security.test.SecurityTestUtils; import org.junit.Before; @@ -352,23 +351,18 @@ public class IndicesAndAliasesResolverTests extends ESTestCase { User user = ((Subject) i.getArguments()[0]).getUser(); @SuppressWarnings("unchecked") ActionListener listener = (ActionListener) i.getArguments()[1]; - if (XPackUser.is(user)) { - listener.onResponse(Role.buildFromRoleDescriptor(XPackUser.ROLE_DESCRIPTOR, fieldPermissionsCache, RESTRICTED_INDICES)); - return Void.TYPE; + if (user instanceof InternalUser internalUser) { + if (internalUser.getLocalClusterRoleDescriptor().isPresent()) { + listener.onResponse( + Role.buildFromRoleDescriptor( + internalUser.getLocalClusterRoleDescriptor().get(), + fieldPermissionsCache, + RESTRICTED_INDICES + ) + ); + return Void.TYPE; + } } - if (XPackSecurityUser.is(user)) { - listener.onResponse( - Role.buildFromRoleDescriptor(ReservedRolesStore.SUPERUSER_ROLE_DESCRIPTOR, fieldPermissionsCache, RESTRICTED_INDICES) - ); - return Void.TYPE; - } - if (AsyncSearchUser.is(user)) { - listener.onResponse( - Role.buildFromRoleDescriptor(AsyncSearchUser.ROLE_DESCRIPTOR, fieldPermissionsCache, RESTRICTED_INDICES) - ); - return Void.TYPE; - } - i.callRealMethod(); return Void.TYPE; }).when(rolesStore).getRole(any(Subject.class), anyActionListener()); @@ -1560,14 +1554,17 @@ public class IndicesAndAliasesResolverTests extends ESTestCase { public void testXPackSecurityUserHasAccessToSecurityIndex() { SearchRequest request = new SearchRequest(); { - final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, SearchAction.NAME); + final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(InternalUsers.XPACK_SECURITY_USER, SearchAction.NAME); List indices = resolveIndices(request, authorizedIndices).getLocal(); assertThat(indices, hasItem(SECURITY_MAIN_ALIAS)); } { IndicesAliasesRequest aliasesRequest = new IndicesAliasesRequest(); aliasesRequest.addAliasAction(AliasActions.add().alias("security_alias").index(SECURITY_MAIN_ALIAS)); - final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackSecurityUser.INSTANCE, IndicesAliasesAction.NAME); + final AuthorizedIndices authorizedIndices = buildAuthorizedIndices( + InternalUsers.XPACK_SECURITY_USER, + IndicesAliasesAction.NAME + ); List indices = resolveIndices(aliasesRequest, authorizedIndices).getLocal(); assertThat(indices, hasItem(SECURITY_MAIN_ALIAS)); } @@ -1575,7 +1572,7 @@ public class IndicesAndAliasesResolverTests extends ESTestCase { public void testXPackUserDoesNotHaveAccessToSecurityIndex() { SearchRequest request = new SearchRequest(); - final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(XPackUser.INSTANCE, SearchAction.NAME); + final AuthorizedIndices authorizedIndices = buildAuthorizedIndices(InternalUsers.XPACK_USER, SearchAction.NAME); List indices = resolveIndices(request, authorizedIndices).getLocal(); assertThat(indices, not(hasItem(SECURITY_MAIN_ALIAS))); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index c26e2d0e6ea4..4ce8ed3ad1a4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -42,7 +42,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.license.LicenseStateListener; import org.elasticsearch.license.MockLicenseState; @@ -92,14 +91,10 @@ import org.elasticsearch.xpack.core.security.support.Automatons; import org.elasticsearch.xpack.core.security.support.MetadataUtils; import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices; import org.elasticsearch.xpack.core.security.user.AnonymousUser; -import org.elasticsearch.xpack.core.security.user.AsyncSearchUser; -import org.elasticsearch.xpack.core.security.user.CrossClusterAccessUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; import org.elasticsearch.xpack.core.security.user.InternalUsers; -import org.elasticsearch.xpack.core.security.user.SecurityProfileUser; import org.elasticsearch.xpack.core.security.user.SystemUser; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; -import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchAction; import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.audit.AuditUtil; @@ -142,11 +137,13 @@ import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AP import static org.elasticsearch.xpack.security.authc.ApiKeyServiceTests.Utils.createApiKeyAuthentication; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.arrayContaining; +import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @@ -1703,12 +1700,7 @@ public class CompositeRolesStoreTests extends ESTestCase { ); verify(fileRolesStore).addListener(anyConsumer()); // adds a listener in ctor - for (var internalUser : List.of( - XPackUser.INSTANCE, - AsyncSearchUser.INSTANCE, - XPackSecurityUser.INSTANCE, - SecurityProfileUser.INSTANCE - )) { + for (var internalUser : AuthenticationTestHelper.internalUsersWithLocalRoleDescriptor()) { Role expectedRole = compositeRolesStore.getInternalUserRole(internalUser); Subject subject = new Subject(internalUser, new RealmRef("name", "type", "node")); PlainActionFuture rolesFuture = new PlainActionFuture<>(); @@ -1742,20 +1734,21 @@ public class CompositeRolesStoreTests extends ESTestCase { assertThat(role.cluster().privileges(), empty()); assertThat(role.indices(), is(IndicesPermission.NONE)); - final User internalUser = InternalUsers.getUser(roleName); + final InternalUser internalUser = InternalUsers.getUser(roleName); assertThat(internalUser, notNullValue()); - if (InternalUsers.getRoleDescriptors().containsKey(internalUser.principal())) { + if (internalUser.getLocalClusterRoleDescriptor().isPresent()) { Role internalRole = compositeRolesStore.getInternalUserRole(internalUser); assertThat(internalRole, notNullValue()); assertThat(role, not(internalRole)); } - final Role[] internalRoles = InternalUsers.getRoleDescriptors() - .keySet() + final Role[] internalRoles = InternalUsers.get() .stream() - .map(InternalUsers::getUser) + .filter(u -> u.getLocalClusterRoleDescriptor().isPresent()) .map(compositeRolesStore::getInternalUserRole) .toArray(Role[]::new); + // Check that we're actually testing something here... + assertThat(internalRoles, arrayWithSize(greaterThan(1))); assertThat(role, not(is(oneOf(internalRoles)))); } @@ -1790,7 +1783,7 @@ public class CompositeRolesStoreTests extends ESTestCase { IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, () -> compositeRolesStore.getRole( - new Subject(SystemUser.INSTANCE, new RealmRef("__attach", "__attach", randomAlphaOfLengthBetween(3, 8))), + new Subject(InternalUsers.SYSTEM_USER, new RealmRef("__attach", "__attach", randomAlphaOfLengthBetween(3, 8))), null ) ); @@ -1829,7 +1822,10 @@ public class CompositeRolesStoreTests extends ESTestCase { IllegalArgumentException iae = expectThrows( IllegalArgumentException.class, () -> compositeRolesStore.getRole( - new Subject(CrossClusterAccessUser.INSTANCE, new RealmRef("__attach", "__attach", randomAlphaOfLengthBetween(3, 8))), + new Subject( + InternalUsers.CROSS_CLUSTER_ACCESS_USER, + new RealmRef("__attach", "__attach", randomAlphaOfLengthBetween(3, 8)) + ), null ) ); @@ -2412,7 +2408,7 @@ public class CompositeRolesStoreTests extends ESTestCase { } final Subject subject = mock(Subject.class); - when(subject.getUser()).thenReturn(SecurityProfileUser.INSTANCE); + when(subject.getUser()).thenReturn(InternalUsers.SECURITY_PROFILE_USER); assertThat(CompositeRolesStore.tryGetRoleDescriptorForInternalUser(subject).get().getClusterPrivileges(), emptyArray()); } @@ -2503,32 +2499,28 @@ public class CompositeRolesStoreTests extends ESTestCase { ); final Subject subject = mock(Subject.class); - when(subject.getUser()).thenReturn(SystemUser.INSTANCE); + when(subject.getUser()).thenReturn(InternalUsers.SYSTEM_USER); final IllegalArgumentException e1 = expectThrows( IllegalArgumentException.class, () -> compositeRolesStore.getRoleDescriptorsList(subject, new PlainActionFuture<>()) ); assertThat(e1.getMessage(), equalTo("should never try to get the roles for internal user [" + SystemUser.NAME + "]")); - when(subject.getUser()).thenReturn(CrossClusterAccessUser.INSTANCE); + when(subject.getUser()).thenReturn(InternalUsers.CROSS_CLUSTER_ACCESS_USER); final IllegalArgumentException e2 = expectThrows( IllegalArgumentException.class, () -> compositeRolesStore.getRoleDescriptorsList(subject, new PlainActionFuture<>()) ); - assertThat(e2.getMessage(), equalTo("should never try to get the roles for internal user [" + CrossClusterAccessUser.NAME + "]")); + assertThat( + e2.getMessage(), + equalTo("should never try to get the roles for internal user [" + InternalUsers.CROSS_CLUSTER_ACCESS_USER.principal() + "]") + ); - for (var userAndDescriptor : List.of( - new Tuple<>(XPackUser.INSTANCE, XPackUser.ROLE_DESCRIPTOR), - new Tuple<>(AsyncSearchUser.INSTANCE, AsyncSearchUser.ROLE_DESCRIPTOR), - new Tuple<>(XPackSecurityUser.INSTANCE, XPackSecurityUser.ROLE_DESCRIPTOR), - new Tuple<>(SecurityProfileUser.INSTANCE, SecurityProfileUser.ROLE_DESCRIPTOR) - )) { - User internalUser = userAndDescriptor.v1(); + for (var internalUser : AuthenticationTestHelper.internalUsersWithLocalRoleDescriptor()) { when(subject.getUser()).thenReturn(internalUser); final PlainActionFuture>> future = new PlainActionFuture<>(); compositeRolesStore.getRoleDescriptorsList(subject, future); - RoleDescriptor expectedRoleDescriptor = userAndDescriptor.v2(); - assertThat(future.actionGet(), equalTo(List.of(Set.of(expectedRoleDescriptor)))); + assertThat(future.actionGet(), equalTo(List.of(Set.of(internalUser.getLocalClusterRoleDescriptor().get())))); } } @@ -2561,19 +2553,19 @@ public class CompositeRolesStoreTests extends ESTestCase { } private Role getXPackSecurityRole() { - return getInternalUserRole(XPackSecurityUser.INSTANCE); + return getInternalUserRole(InternalUsers.XPACK_SECURITY_USER); } private Role getSecurityProfileRole() { - return getInternalUserRole(SecurityProfileUser.INSTANCE); + return getInternalUserRole(InternalUsers.SECURITY_PROFILE_USER); } private Role getXPackUserRole() { - return getInternalUserRole(XPackUser.INSTANCE); + return getInternalUserRole(InternalUsers.XPACK_USER); } private Role getAsyncSearchUserRole() { - return getInternalUserRole(AsyncSearchUser.INSTANCE); + return getInternalUserRole(InternalUsers.ASYNC_SEARCH_USER); } private Role getInternalUserRole(User internalUser) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessAuthenticationServiceIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessAuthenticationServiceIntegTests.java index fa4b6de8b58b..d0279eec1d7b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessAuthenticationServiceIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessAuthenticationServiceIntegTests.java @@ -23,7 +23,7 @@ import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; import org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo; import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection; -import org.elasticsearch.xpack.core.security.user.CrossClusterAccessUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.security.authc.ApiKeyService; import org.elasticsearch.xpack.security.authc.CrossClusterAccessAuthenticationService; import org.elasticsearch.xpack.security.authc.CrossClusterAccessHeaders; @@ -99,7 +99,10 @@ public class CrossClusterAccessAuthenticationServiceIntegTests extends SecurityI } try (var ignored = threadContext.stashContext()) { - final var internalUser = randomValueOtherThan(CrossClusterAccessUser.INSTANCE, AuthenticationTestHelper::randomInternalUser); + final var internalUser = randomValueOtherThan( + InternalUsers.CROSS_CLUSTER_ACCESS_USER, + AuthenticationTestHelper::randomInternalUser + ); new CrossClusterAccessHeaders( encodedCrossClusterAccessApiKey, new CrossClusterAccessSubjectInfo( diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java index 61b5a3319ee1..e0a58d031b8d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/SecurityServerTransportInterceptorTests.java @@ -54,13 +54,10 @@ import org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection; import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore; -import org.elasticsearch.xpack.core.security.user.AsyncSearchUser; import org.elasticsearch.xpack.core.security.user.CrossClusterAccessUser; -import org.elasticsearch.xpack.core.security.user.SecurityProfileUser; -import org.elasticsearch.xpack.core.security.user.SystemUser; +import org.elasticsearch.xpack.core.security.user.InternalUser; +import org.elasticsearch.xpack.core.security.user.InternalUsers; import org.elasticsearch.xpack.core.security.user.User; -import org.elasticsearch.xpack.core.security.user.XPackSecurityUser; -import org.elasticsearch.xpack.core.security.user.XPackUser; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.audit.AuditUtil; @@ -184,7 +181,7 @@ public class SecurityServerTransportInterceptorTests extends ESTestCase { assertTrue(calledWrappedSender.get()); assertEquals(user, sendingUser.get()); assertEquals(user, securityContext.getUser()); - verify(securityContext, never()).executeAsInternalUser(any(User.class), any(TransportVersion.class), anyConsumer()); + verify(securityContext, never()).executeAsInternalUser(any(InternalUser.class), any(TransportVersion.class), anyConsumer()); } public void testSendAsyncSwitchToSystem() throws Exception { @@ -235,9 +232,9 @@ public class SecurityServerTransportInterceptorTests extends ESTestCase { sender.sendRequest(connection, "internal:foo", null, null, null); assertTrue(calledWrappedSender.get()); assertNotEquals(user, sendingUser.get()); - assertEquals(SystemUser.INSTANCE, sendingUser.get()); + assertEquals(InternalUsers.SYSTEM_USER, sendingUser.get()); assertEquals(user, securityContext.getUser()); - verify(securityContext).executeAsInternalUser(any(User.class), eq(TransportVersion.CURRENT), anyConsumer()); + verify(securityContext).executeAsInternalUser(any(InternalUser.class), eq(TransportVersion.CURRENT), anyConsumer()); } public void testSendWithoutUser() throws Exception { @@ -282,7 +279,7 @@ public class SecurityServerTransportInterceptorTests extends ESTestCase { ); assertEquals("there should always be a user when sending a message for action [indices:foo]", e.getMessage()); assertNull(securityContext.getUser()); - verify(securityContext, never()).executeAsInternalUser(any(User.class), any(TransportVersion.class), anyConsumer()); + verify(securityContext, never()).executeAsInternalUser(any(InternalUser.class), any(TransportVersion.class), anyConsumer()); } public void testSendToNewerVersionSetsCorrectVersion() throws Exception { @@ -426,13 +423,13 @@ public class SecurityServerTransportInterceptorTests extends ESTestCase { public void testSetUserBasedOnActionOrigin() { final Map originToUserMap = Map.of( SECURITY_ORIGIN, - XPackSecurityUser.INSTANCE, + InternalUsers.XPACK_SECURITY_USER, SECURITY_PROFILE_ORIGIN, - SecurityProfileUser.INSTANCE, + InternalUsers.SECURITY_PROFILE_USER, TRANSFORM_ORIGIN, - XPackUser.INSTANCE, + InternalUsers.XPACK_USER, ASYNC_SEARCH_ORIGIN, - AsyncSearchUser.INSTANCE + InternalUsers.ASYNC_SEARCH_USER ); final String origin = randomFrom(originToUserMap.keySet()); @@ -678,7 +675,7 @@ public class SecurityServerTransportInterceptorTests extends ESTestCase { true, action, request, - AuthenticationTestHelper.builder().internal(SystemUser.INSTANCE).build() + AuthenticationTestHelper.builder().internal(InternalUsers.SYSTEM_USER).build() ); } @@ -693,7 +690,7 @@ public class SecurityServerTransportInterceptorTests extends ESTestCase { true, action, request, - AuthenticationTestHelper.builder().internal(SystemUser.INSTANCE).build() + AuthenticationTestHelper.builder().internal(InternalUsers.SYSTEM_USER).build() ); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java index 48457b757fff..0c20a69d2186 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java @@ -95,6 +95,8 @@ import java.util.stream.Stream; import static java.util.Arrays.asList; import static org.elasticsearch.xpack.ql.expression.Literal.TRUE; +import static org.elasticsearch.xpack.ql.querydsl.query.BoolQueryTests.left; +import static org.elasticsearch.xpack.ql.querydsl.query.BoolQueryTests.right; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE; import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; @@ -111,6 +113,7 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.everyItem; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.startsWith; @@ -740,11 +743,11 @@ public class QueryTranslatorTests extends ESTestCase { assertTrue(query instanceof BoolQuery); BoolQuery bq = (BoolQuery) query; assertFalse(bq.isAnd()); - assertTrue(bq.left() instanceof RangeQuery); - assertTrue(bq.right() instanceof RangeQuery); + assertTrue(left(bq) instanceof RangeQuery); + assertTrue(right(bq) instanceof RangeQuery); List> tuples = asList( - new Tuple<>(dates[0], (RangeQuery) bq.left()), - new Tuple<>(dates[1], (RangeQuery) bq.right()) + new Tuple<>(dates[0], (RangeQuery) left(bq)), + new Tuple<>(dates[1], (RangeQuery) right(bq)) ); for (Tuple t : tuples) { @@ -834,12 +837,12 @@ public class QueryTranslatorTests extends ESTestCase { assertEquals(BoolQuery.class, qt.query.getClass()); BoolQuery bq = ((BoolQuery) qt.query); assertTrue(bq.isAnd()); - assertTrue(bq.left() instanceof WildcardQuery); - assertTrue(bq.right() instanceof NotQuery); + assertTrue(left(bq) instanceof WildcardQuery); + assertTrue(right(bq) instanceof NotQuery); - NotQuery nq = (NotQuery) bq.right(); + NotQuery nq = (NotQuery) right(bq); assertTrue(nq.child() instanceof WildcardQuery); - WildcardQuery lqsq = (WildcardQuery) bq.left(); + WildcardQuery lqsq = (WildcardQuery) left(bq); WildcardQuery rqsq = (WildcardQuery) nq.child(); assertEquals("X*", lqsq.query()); @@ -879,12 +882,12 @@ public class QueryTranslatorTests extends ESTestCase { assertEquals(BoolQuery.class, qt.query.getClass()); BoolQuery bq = ((BoolQuery) qt.query); assertTrue(bq.isAnd()); - assertTrue(bq.left() instanceof RegexQuery); - assertTrue(bq.right() instanceof NotQuery); + assertTrue(left(bq) instanceof RegexQuery); + assertTrue(right(bq) instanceof NotQuery); - NotQuery nq = (NotQuery) bq.right(); + NotQuery nq = (NotQuery) right(bq); assertTrue(nq.child() instanceof RegexQuery); - RegexQuery lqsq = (RegexQuery) bq.left(); + RegexQuery lqsq = (RegexQuery) left(bq); RegexQuery rqsq = (RegexQuery) nq.child(); assertEquals(firstPattern, lqsq.regex()); @@ -906,14 +909,14 @@ public class QueryTranslatorTests extends ESTestCase { BoolQuery bq = (BoolQuery) translation.query; assertFalse(bq.isAnd()); - assertTrue(bq.left() instanceof PrefixQuery); - assertTrue(bq.right() instanceof PrefixQuery); + assertTrue(left(bq) instanceof PrefixQuery); + assertTrue(right(bq) instanceof PrefixQuery); - PrefixQuery pqr = (PrefixQuery) bq.right(); + PrefixQuery pqr = (PrefixQuery) right(bq); assertEquals("keyword", pqr.field()); assertEquals("y", pqr.query()); - PrefixQuery pql = (PrefixQuery) bq.left(); + PrefixQuery pql = (PrefixQuery) left(bq); assertEquals("keyword", pql.field()); assertEquals("x", pql.query()); } @@ -934,20 +937,21 @@ public class QueryTranslatorTests extends ESTestCase { BoolQuery bq = (BoolQuery) translation.query; assertTrue(bq.isAnd()); - assertTrue(bq.left() instanceof BoolQuery); - assertTrue(bq.right() instanceof ScriptQuery); + List queries = bq.queries(); + assertThat(queries, hasSize(3)); + assertTrue(queries.get(0) instanceof PrefixQuery); + assertTrue(queries.get(1) instanceof PrefixQuery); + assertTrue(queries.get(2) instanceof ScriptQuery); - BoolQuery bbq = (BoolQuery) bq.left(); - assertTrue(bbq.isAnd()); - PrefixQuery pqr = (PrefixQuery) bbq.right(); + PrefixQuery pqr = (PrefixQuery) queries.get(0); assertEquals("keyword", pqr.field()); - assertEquals("xy", pqr.query()); + assertEquals("x", pqr.query()); - PrefixQuery pql = (PrefixQuery) bbq.left(); + PrefixQuery pql = (PrefixQuery) queries.get(1); assertEquals("keyword", pql.field()); - assertEquals("x", pql.query()); + assertEquals("xy", pql.query()); - ScriptQuery sq = (ScriptQuery) bq.right(); + ScriptQuery sq = (ScriptQuery) queries.get(2); assertEquals( "InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.startsWith(" + "InternalSqlScriptUtils.lcase(InternalQlScriptUtils.docValue(doc,params.v0)), " diff --git a/x-pack/plugin/sql/src/test/resources/org/elasticsearch/xpack/sql/planner/querytranslator_tests.txt b/x-pack/plugin/sql/src/test/resources/org/elasticsearch/xpack/sql/planner/querytranslator_tests.txt index f33ed4aa643e..27a56ec29c3e 100644 --- a/x-pack/plugin/sql/src/test/resources/org/elasticsearch/xpack/sql/planner/querytranslator_tests.txt +++ b/x-pack/plugin/sql/src/test/resources/org/elasticsearch/xpack/sql/planner/querytranslator_tests.txt @@ -350,6 +350,21 @@ SELECT * FROM test WHERE some.string LIKE '%a%'; "query":{"wildcard":{"some.string.typical":{"wildcard":"*a*", ; +LikeOnInexactWithOR +SELECT * FROM test WHERE some.string LIKE '%a%' OR some.string LIKE '%b%' OR some.string LIKE '%c%'; +{"should":[{"wildcard":{"some.string.typical":{"wildcard":"*a*","boost":1.0}}},{"wildcard":{"some.string.typical":{"wildcard":"*b*","boost":1.0}}},{"wildcard":{"some.string.typical":{"wildcard":"*c*","boost":1.0}}}],"boost":1.0}} +; + +LikeOnInexactWithORandAND +SELECT * FROM test WHERE some.string LIKE '%a%' OR some.string LIKE '%b%' AND some.string LIKE '%c%' OR some.string LIKE '%d%' OR some.string LIKE '%e%'; +{"bool":{"should":[{"wildcard":{"some.string.typical":{"wildcard":"*a*","boost":1.0}}},{"bool":{"must":[{"wildcard":{"some.string.typical":{"wildcard":"*b*","boost":1.0}}},{"wildcard":{"some.string.typical":{"wildcard":"*c*","boost":1.0}}}],"boost":1.0}},{"wildcard":{"some.string.typical":{"wildcard":"*d*","boost":1.0}}},{"wildcard":{"some.string.typical":{"wildcard":"*e*","boost":1.0}}}],"boost":1.0}} +; + +LikeOnInexactWithANDandGroupedOR +SELECT * FROM test WHERE (some.string LIKE '%a%' OR some.string LIKE '%b%') AND (some.string LIKE '%c%' OR some.string LIKE '%d%' OR some.string LIKE '%e%'); +{"bool":{"must":[{"bool":{"should":[{"wildcard":{"some.string.typical":{"wildcard":"*a*","boost":1.0}}},{"wildcard":{"some.string.typical":{"wildcard":"*b*","boost":1.0}}}],"boost":1.0}},{"bool":{"should":[{"wildcard":{"some.string.typical":{"wildcard":"*c*","boost":1.0}}},{"wildcard":{"some.string.typical":{"wildcard":"*d*","boost":1.0}}},{"wildcard":{"some.string.typical":{"wildcard":"*e*","boost":1.0}}}],"boost":1.0}}],"boost":1.0}} +; + RLikeOnInexact SELECT * FROM test WHERE some.string RLIKE '.*a.*'; "query":{"regexp":{"some.string.typical":{"value":".*a.*", diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_crud.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_crud.yml index 0feaf48dde6b..729094ba7e9d 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_crud.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_crud.yml @@ -105,6 +105,41 @@ setup: } } - match: { acknowledged: true } + +--- +"Test put transform with invalid source index pattern": + - do: + # The transform can be created even though the source index pattern resolves to empty set of concrete indices. + transform.put_transform: + transform_id: "missing-source-transform-pattern" + body: > + { + "source": { "index": "missing-index*" }, + "dest": { "index": "missing-source-dest" }, + "pivot": { + "group_by": { "airline": {"terms": {"field": "airline"}}}, + "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} + } + } + - match: { acknowledged: true } + + - do: + # The transform can be updated even though the source index pattern resolves to empty set of concrete indices. + transform.update_transform: + transform_id: "missing-source-transform-pattern" + body: > + { + "description": "updated description" + } + - match: { id: "missing-source-transform-pattern" } + - match: { description: "updated description" } + + - do: + # The transform can be started even though the source index pattern resolves to empty set of concrete indices. + transform.start_transform: + transform_id: "missing-source-transform-pattern" + - match: { acknowledged: true } + --- "Test basic transform crud": - do: diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpdateIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpdateIT.java index b554fe3173b7..44a03f20373c 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpdateIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpdateIT.java @@ -9,21 +9,33 @@ package org.elasticsearch.xpack.transform.integration; import org.apache.http.HttpHost; import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Strings; +import org.elasticsearch.threadpool.TestThreadPool; +import org.junit.After; import org.junit.Before; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThan; public class TransformUpdateIT extends TransformRestTestCase { @@ -47,6 +59,8 @@ public class TransformUpdateIT extends TransformRestTestCase { private static final String DATA_ACCESS_ROLE = "test_data_access"; private static final String DATA_ACCESS_ROLE_2 = "test_data_access_2"; + private TestThreadPool threadPool; + // preserve indices in order to reuse source indices in several test cases @Override protected boolean preserveIndicesUponCompletion() { @@ -76,6 +90,15 @@ public class TransformUpdateIT extends TransformRestTestCase { setupUser(TEST_ADMIN_USER_NAME_2, List.of("transform_admin", DATA_ACCESS_ROLE_2)); setupUser(TEST_ADMIN_USER_NAME_NO_DATA, List.of("transform_admin")); createReviewsIndex(); + + threadPool = new TestThreadPool(getTestName()); + } + + @After + public void shutdownThreadPool() { + if (threadPool != null) { + threadPool.shutdown(); + } } @SuppressWarnings("unchecked") @@ -167,6 +190,50 @@ public class TransformUpdateIT extends TransformRestTestCase { assertThat(updatedConfig.get("settings"), is(equalTo(Map.of("max_page_search_size", 123)))); } + public void testConcurrentUpdates() throws Exception { + String transformId = "test_concurrent_updates"; + String destIndex = transformId + "-dest"; + + // Create the transform + createPivotReviewsTransform(transformId, destIndex, null, null, null); + + // Create a number of concurrent threads competing to update the transform with different settings. + int minMaxPageSearchSize = 10; + int maxMaxPageSearchSize = 20; + List> concurrentUpdates = new ArrayList<>(10); + for (int maxPageSearchSize = minMaxPageSearchSize; maxPageSearchSize < maxMaxPageSearchSize; ++maxPageSearchSize) { + Request updateTransformRequest = createRequestWithAuth("POST", getTransformEndpoint() + transformId + "/_update", null); + updateTransformRequest.setJsonEntity(Strings.format(""" + { "settings": { "max_page_search_size": %s } }""", maxPageSearchSize)); + + // Schedule a thread to update the transform's settings + concurrentUpdates.add(() -> client().performRequest(updateTransformRequest)); + } + + // Gather the results. + List> futures = threadPool.generic().invokeAll(concurrentUpdates); + for (Future future : futures) { + try { // The update may succeed... + future.get(); + } catch (ExecutionException e) { // ... but if it fails, it's due to conflict + assertThat(e.getCause(), instanceOf(ResponseException.class)); + ResponseException re = (ResponseException) e.getCause(); + assertThat(re.getResponse().getStatusLine().getStatusCode(), is(equalTo(409))); + assertThat( + re.getMessage(), + containsString("Transform with id [" + transformId + "] got updated in the meantime. Please try again") + ); + } + } + + // Verify that the settings got updated. Any of the concurrent threads could have won the competition. + Map finalConfig = getTransformConfig(transformId, null); + assertThat( + (int) XContentMapValues.extractValue(finalConfig, "settings", "max_page_search_size"), + is(both(greaterThanOrEqualTo(minMaxPageSearchSize)).and(lessThan(maxMaxPageSearchSize))) + ); + } + private void updateTransferRightsTester(boolean useSecondaryAuthHeaders) throws Exception { String transformId = "transform1"; // Note: Due to a bug the transform does not fail to start after deleting the user and role, therefore invalidating diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java index 5d7e706447f9..30577a41998a 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java @@ -319,14 +319,16 @@ public class IndexBasedTransformConfigManager implements TransformConfigManager private void putTransformConfiguration( TransformConfig transformConfig, - DocWriteRequest.OpType optType, + DocWriteRequest.OpType opType, SeqNoPrimaryTermAndIndex seqNoPrimaryTermAndIndex, ActionListener listener ) { + assert DocWriteRequest.OpType.CREATE.equals(opType) || DocWriteRequest.OpType.INDEX.equals(opType); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { XContentBuilder source = transformConfig.toXContent(builder, new ToXContent.MapParams(TO_XCONTENT_PARAMS)); - IndexRequest indexRequest = new IndexRequest(TransformInternalIndexConstants.LATEST_INDEX_NAME).opType(optType) + IndexRequest indexRequest = new IndexRequest(TransformInternalIndexConstants.LATEST_INDEX_NAME).opType(opType) .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .id(TransformConfig.documentId(transformConfig.getId())) .source(source); @@ -340,12 +342,20 @@ public class IndexBasedTransformConfigManager implements TransformConfigManager indexRequest, ActionListener.wrap(r -> { listener.onResponse(true); }, e -> { if (e instanceof VersionConflictEngineException) { - // the transform already exists - listener.onFailure( - new ResourceAlreadyExistsException( - TransformMessages.getMessage(TransformMessages.REST_PUT_TRANSFORM_EXISTS, transformConfig.getId()) - ) - ); + if (DocWriteRequest.OpType.CREATE.equals(opType)) { // we want to create the transform but it already exists + listener.onFailure( + new ResourceAlreadyExistsException( + TransformMessages.getMessage(TransformMessages.REST_PUT_TRANSFORM_EXISTS, transformConfig.getId()) + ) + ); + } else { // we want to update the transform but it got updated in the meantime, report version conflict + listener.onFailure( + new ElasticsearchStatusException( + TransformMessages.getMessage(TransformMessages.REST_UPDATE_TRANSFORM_CONFLICT, transformConfig.getId()), + RestStatus.CONFLICT + ) + ); + } } else { listener.onFailure(new RuntimeException(TransformMessages.REST_PUT_FAILED_PERSIST_TRANSFORM_CONFIGURATION, e)); } diff --git a/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/usage/10_basic.yml b/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/usage/10_basic.yml index 754f5281d853..17031abf39e0 100644 --- a/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/usage/10_basic.yml +++ b/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/usage/10_basic.yml @@ -1,5 +1,8 @@ --- "Test watcher usage stats output": + - skip: + version: "all" + reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/65547" - do: catch: missing watcher.delete_watch: