mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-06-28 17:34:17 -04:00
1fe3b77a2a
* Add multi-project support for more stats APIs This affects the following APIs: - `GET _nodes/stats`: - For `indices`, it now prefixes the index name with the project ID (for non-default projects). Previously, it didn't tell you which project an index was in, and it failed if two projects had the same index name. - For `ingest`, it now gets the pipeline and processor stats for all projects, and prefixes the pipeline ID with the project ID. Previously, it only got them for the default project. - `GET /_cluster/stats`: - For `ingest`, it now aggregates the pipeline and processor stats for all projects. Previously, it only got them for the default project. - `GET /_info`: - For `ingest`, same as for `GET /_nodes/stats`. This is done by making `IndicesService.stats()` and `IngestService.stats()` include project IDs in the `NodeIndicesStats` and `IngestStats` objects they return, and making those stats objects incorporate the project IDs when converting to XContent. The transitive callers of these two methods are rather extensive (including all callers to `NodeService.stats()`, all callers of `TransportNodesStatsAction`, and so on). To ensure the change is safe, the callers were all checked out, and they fall into the following cases: - The behaviour change is one of the desired enhancements described above. - There is no behaviour change because it was getting node stats but neither `indices` nor `ingest` stats were requested. - There is no behaviour change because it was getting `indices` and/or `ingest` stats but only using aggregate values. - In `MachineLearningUsageTransportAction` and `TransportGetTrainedModelsStatsAction`, the `IngestStats` returned will return stats from all projects instead of just the default with this change, but they have been changed to filter the non-default project stats out, so this change is a noop there. (These actions are not MP-ready yet.) - `MonitoringService` will be affected, but this is the legacy monitoring module which is not in use anywhere that MP is going to be enabled. (If anything, the behaviour is probably improved by this change, as it will now include project IDs, rather than producing ambiguous unqualified results and failing in the case of duplicates.) * Update test/external-modules/multi-project/build.gradle Change suggested by Niels. Co-authored-by: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> * Respond to review comments * fix merge weirdness * [CI] Auto commit changes from spotless * Fix test compilation following upstream change to base class * Update x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTierUsageFixtures.java Co-authored-by: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> * Make projects-by-index map nullable and omit in single-project; always include project prefix in XContent in multip-project, even if default; also incorporate one other review comment * Add a TODO * update IT to reflect changed behaviour * Switch to using XContent.Params to indicate whether it is multi-project or not * Refactor NodesStatsMultiProjectIT to common up repeated assertions * Defer use of ProjectIdResolver in REST handlers to keep tests happy * Include index UUID in "unknown project" case * Make the index-to-project map empty rather than null in the BWC deserialization case. This works out fine, for the reasons given in the comment. As it happens, I'd already forgotten to do the null check in the one place it's actively used. * remove a TODO that is done, and add a comment * fix typo * Get REST YAML tests working with project ID prefix TODO finish this * As a drive-by, fix and un-suppress one of the health REST tests * [CI] Auto commit changes from spotless * TODO ugh * Experiment with different stashing behaviour * [CI] Auto commit changes from spotless * Try a more sensible stash behaviour for assertions * clarify comment * Make checkstyle happy * Make the way `Assertion` works more consistent, and simplify implementation * [CI] Auto commit changes from spotless * In RestNodesStatsAction, make the XContent params to channel.request(), which is the value it would have had before this change --------- Co-authored-by: Niels Bauman <33722607+nielsbauman@users.noreply.github.com> Co-authored-by: elasticsearchmachine <infra-root+elasticsearchmachine@elastic.co>
82 lines
3.6 KiB
Groovy
82 lines
3.6 KiB
Groovy
apply plugin: 'elasticsearch.internal-yaml-rest-test'
|
|
import org.elasticsearch.gradle.util.GradleUtils
|
|
|
|
dependencies {
|
|
testImplementation project(':x-pack:qa')
|
|
testImplementation project(':x-pack:qa:multi-project:yaml-test-framework')
|
|
clusterModules project(':modules:mapper-extras')
|
|
clusterModules project(':modules:rank-eval')
|
|
clusterModules project(':modules:ingest-common')
|
|
clusterModules project(':modules:reindex')
|
|
clusterModules project(':modules:analysis-common')
|
|
clusterModules project(':modules:health-shards-availability')
|
|
clusterModules project(':modules:data-streams')
|
|
clusterModules project(':modules:lang-mustache')
|
|
clusterModules project(':modules:parent-join')
|
|
clusterModules project(xpackModule('stack'))
|
|
clusterModules project(xpackModule('ilm'))
|
|
clusterModules project(xpackModule('mapper-constant-keyword'))
|
|
clusterModules project(xpackModule('wildcard'))
|
|
clusterModules project(':test:external-modules:test-multi-project')
|
|
restTestConfig project(path: ':modules:data-streams', configuration: "basicRestSpecs")
|
|
restTestConfig project(path: ':modules:ingest-common', configuration: "basicRestSpecs")
|
|
restTestConfig project(path: ':modules:reindex', configuration: "basicRestSpecs")
|
|
}
|
|
|
|
// let the yamlRestTests see the classpath of test
|
|
GradleUtils.extendSourceSet(project, "test", "yamlRestTest", tasks.named("yamlRestTest"))
|
|
|
|
restResources {
|
|
restTests {
|
|
includeCore '*'
|
|
}
|
|
}
|
|
|
|
tasks.named("yamlRestTest").configure {
|
|
ArrayList<String> blacklist = [
|
|
/* These tests don't work on multi-project yet - we need to go through each of them and make them work */
|
|
'^cat.recovery/*/*',
|
|
'^cat.snapshots/*/*',
|
|
'^cluster.desired_balance/10_basic/*',
|
|
'^data_stream/40_supported_apis/Verify shard stores api', // uses _shard_stores API
|
|
'^health/10_basic/*',
|
|
'^indices.get_alias/10_basic/Get alias against closed indices', // Does NOT work with security enabled, see also core-rest-tests-with-security
|
|
'^indices.recovery/*/*',
|
|
'^indices.resolve_cluster/*/*',
|
|
'^indices.resolve_cluster/*/*/*',
|
|
'^indices.shard_stores/*/*',
|
|
'^migration/*/*',
|
|
'^snapshot.clone/*/*',
|
|
'^snapshot.create/*/*',
|
|
'^snapshot.delete/*/*',
|
|
'^snapshot.get/*/*',
|
|
'^snapshot.get_repository/20_repository_uuid/*',
|
|
'^snapshot.restore/*/*',
|
|
'^snapshot.status/*/*',
|
|
'^synonyms/*/*',
|
|
'^tsdb/30_snapshot/*',
|
|
'^update_by_query/80_scripting/Update all docs with one deletion and one noop using a stored script', // scripting is not project aware yet
|
|
|
|
// The following tests are muted because the functionality that they are testing is not available in a multi-project setup
|
|
|
|
// The node removal prevalidation is only available for a Stateful deployment.
|
|
'^cluster.prevalidate_node_removal/*/*',
|
|
|
|
// This behaviour doesn't work with security: https://github.com/elastic/elasticsearch/issues/120252
|
|
'^reindex/30_search/Sorting deprecated wait_for_completion false',
|
|
|
|
// These tests assume we are running on a single node
|
|
'^delete_by_query/50_wait_for_active_shards/can override wait_for_active_shards',
|
|
'^update_by_query/50_consistency/can override wait_for_active_shards',
|
|
|
|
// Reindex from remote is not supported on Serverless and required additional testing setup
|
|
'^reindex/60_wait_for_active_shards/can override wait_for_active_shards', // <- Requires a single shard
|
|
'^reindex/90_remote/*',
|
|
'^reindex/95_parent_join/Reindex from remote*',
|
|
];
|
|
if (buildParams.snapshotBuild == false) {
|
|
blacklist += [];
|
|
}
|
|
systemProperty 'tests.rest.blacklist', blacklist.join(',')
|
|
systemProperty "tests.multi_project.enabled", true
|
|
}
|