mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-06-28 09:28:55 -04:00
19550a838f
This change enhances the dense_vector section of the Nodes stats and Index stats APIs so that they report the desired size of off-heap memory for all indexed vectors. The dense_vector section of the Custer stats API remains unchanged.
The retrieval mechanism and structure of the new stats is the same across the various three stats APIs, but more fine-grained information is disclosed as when moving from Cluster -> Node -> Index API.
For Node stats, we aggregate the total byte sizes for all vectors, categorised by the data type. For example:
"dense_vector" : {
"value_count" : 5,
"off_heap" : {
"total_size_in_bytes" : 27,
"total_veb_size_in_bytes" : 3,
"total_vec_size_in_bytes" : 23,
"total_veq_size_in_bytes" : 0,
"total_vex_size_in_bytes" : 1
}
}
Index stats: same as Node stats with included field break down . For example:
"dense_vector" : {
"value_count" : 5,
"off_heap" : {
"total_size_in_bytes" : 27,
"total_veb_size_in_bytes" : 3,
"total_vec_size_in_bytes" : 23,
"total_veq_size_in_bytes" : 0,
"total_vex_size_in_bytes" : 1,
"fielddata" : {
"bar" : {
"veb_size_in_bytes" : 3,
"vec_size_in_bytes" : 14,
"vex_size_in_bytes" : 1
},
"foo" : {
"vec_size_in_bytes" : 9
}
}
}
The implementation accesses the actual statistics through reflection. This will be completely removed when Lucene exposes this, which is expected in Lucene 10.3
46 lines
2.4 KiB
Groovy
46 lines
2.4 KiB
Groovy
apply plugin: 'elasticsearch.internal-yaml-rest-test'
|
|
|
|
// This subproject copies a subset of the rest-api-spec rest tests and runs them in a slightly
|
|
// modified way on two clusters connected via CCS. All operations except searches and a few other
|
|
// APIs that support CCS are run against the remote "write" cluster where all indexed data will live.
|
|
// All search requests however are run against the local cluster connected via CCS to the remote
|
|
// cluster. The test runner modifies index names on these API calls to route to the remote cluster
|
|
// and also modifies certain "match" sections to expect index names with the remote cluster prefix
|
|
// on the fly while running these tests.
|
|
|
|
restResources {
|
|
restApi {
|
|
include 'capabilities', 'cat.shards', '_common', 'bulk', 'count', 'cluster', 'field_caps', 'get', 'knn_search', 'index', 'indices', 'msearch',
|
|
"nodes.stats", 'search', 'async_search', 'graph', '*_point_in_time', 'info', 'scroll', 'clear_scroll', 'search_mvt', 'eql', 'sql'
|
|
}
|
|
restTests {
|
|
includeCore 'field_caps', 'msearch', 'search', 'suggest', 'scroll', "indices.resolve_index"
|
|
includeXpack 'async_search', 'vector-tile', 'sql'
|
|
}
|
|
}
|
|
|
|
dependencies {
|
|
clusterModules project(':x-pack:plugin:async-search')
|
|
clusterModules project(':modules:mapper-extras')
|
|
clusterModules project(':modules:aggregations')
|
|
clusterModules project(':modules:analysis-common')
|
|
clusterModules project(':x-pack:plugin:analytics')
|
|
clusterModules project(':x-pack:plugin:vector-tile')
|
|
clusterModules project(':modules:legacy-geo')
|
|
clusterModules project(':x-pack:plugin:eql')
|
|
clusterModules project(':x-pack:plugin:sql')
|
|
}
|
|
|
|
tasks.named("yamlRestTest") {
|
|
systemProperty 'tests.rest.blacklist',
|
|
[
|
|
'search/150_rewrite_on_coordinator/Ensure that we fetch the document only once', // terms lookup query with index
|
|
'search/170_terms_query/Terms Query with No.of terms exceeding index.max_terms_count should FAIL', // terms lookup query with index
|
|
'search.aggregation/220_filters_bucket/cache busting', // node_selector?
|
|
'search.aggregation/220_filters_bucket/cache hits', // node_selector?
|
|
'search.aggregation/50_filter/Standard queries get cached',
|
|
'search.aggregation/50_filter/Terms lookup gets cached', // terms lookup by "index" doesn't seem to work correctly
|
|
'search.aggregation/70_adjacency_matrix/Terms lookup' // terms lookup by "index" doesn't seem to work correctly
|
|
].join(',')
|
|
}
|
|
|