mirror of
https://github.com/elastic/elasticsearch.git
synced 2025-04-26 08:07:27 -04:00
c1a2d390ef
Currently, the data stream lifecycle telemetry has the following
structure:
```
{
....
"data_lifecycle" : {
"available": true,
"enabled": true,
"count": 0,
"default_rollover_used": true,
"retention": {
"minimum_millis": 0,
"maximum_millis": 0,
"average_millis": 0.0
}
}....
```
In the snippet above you can see that we track:
- The amount of data streams managed by the data stream lifecycle by `count`
- If the default rollover has been overwritten by `default_rollover_used`
- The min, max and average of the `data_retention` configured on a data stream level.
In this PR we propose the following extention:
```
....
"data_lifecycle" : {
"available": true,
"enabled": true,
"count": 0,
"default_rollover_used": true,
"effective_retention": { #https://github.com/elastic/dev/issues/2537
"retained_data_streams": 5,
"minimum_millis": 0, # Only if retained data streams > 1
"maximum_millis": 0,
"average_millis": 0.0
},
"data_retention": {
"configured_data_streams": 5,
"minimum_millis": 0, # Only if retained data streams > 1
"maximum_millis": 0,
"average_millis": 0.0
},
"global_retention": {
"default": {
"defined": true/false,
"affected_data_streams": 0,
"millis": 0
},
"max": {
"defined": true/false,
"affected_data_streams": 0,
"millis": 0
}
}
```
With this extension we are tracking:
- The amount of data streams managed by the data stream lifecycle by `count`
- If the default rollover has been overwritten by `default_rollover_used`
- The min, max and average of the `data_retention` configured on a data stream level and the number of data streams that have it configured. We add the min, max and avg only if there are data streams with data retention configuration to avoid messing with the stats in a dashboard.
- The min, max and average of the `effective_retention` and the number of data streams that are retained. We add the min, max and avg only if there are retained data streams to avoid messing with the stats in a dashboard.
- Global retention stats, if they are defined, if the number of the affected data streams and the actual value.
The above metrics allow us to answer questions like:
- How many data streams are affected by global retention.
- How big is the difference between the longest data retention compared to max global retention.
- How much does the effective retention diverging from the data retention, this will show the impact of the global retention.
532 lines
12 KiB
Text
532 lines
12 KiB
Text
[role="xpack"]
|
|
[[usage-api]]
|
|
== Usage API
|
|
|
|
Provides usage information about the installed {xpack} features.
|
|
|
|
[discrete]
|
|
[[usage-api-request]]
|
|
=== {api-request-title}
|
|
|
|
`GET /_xpack/usage`
|
|
|
|
[discrete]
|
|
[[usage-api-prereqs]]
|
|
=== {api-prereq-title}
|
|
|
|
* If the {es} {security-features} are enabled, you must have the `monitor` or
|
|
`manage` <<privileges-list-cluster,cluster privilege>> to use this API.
|
|
|
|
[discrete]
|
|
[[usage-api-desc]]
|
|
=== {api-description-title}
|
|
|
|
This API provides information about which features are currently enabled and
|
|
available under the current license and some usage statistics.
|
|
|
|
[discrete]
|
|
[[usage-api-query-parms]]
|
|
=== {api-query-parms-title}
|
|
|
|
include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
|
|
|
|
[discrete]
|
|
[[usage-api-example]]
|
|
=== {api-examples-title}
|
|
|
|
[source,console]
|
|
------------------------------------------------------------
|
|
GET /_xpack/usage
|
|
------------------------------------------------------------
|
|
// TEST[s/usage/usage?filter_path=-watcher.execution.actions.index*\,-watcher.execution.actions.logging*,-watcher.execution.actions.email*/]
|
|
// This response filter removes watcher logging results if they are included
|
|
// to avoid errors in the CI builds.
|
|
|
|
[source,console-result]
|
|
------------------------------------------------------------
|
|
{
|
|
"security" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
...
|
|
},
|
|
"monitoring" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
"collection_enabled" : false,
|
|
"enabled_exporters" : {
|
|
"local" : 1
|
|
}
|
|
},
|
|
"watcher" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
"execution" : {
|
|
"actions" : {
|
|
"_all" : {
|
|
"total" : 0,
|
|
"total_time_in_ms" : 0
|
|
}
|
|
}
|
|
},
|
|
"watch" : {
|
|
"input" : {
|
|
"_all" : {
|
|
"total" : 0,
|
|
"active" : 0
|
|
}
|
|
},
|
|
"trigger" : {
|
|
"_all" : {
|
|
"total" : 0,
|
|
"active" : 0
|
|
}
|
|
}
|
|
},
|
|
"count" : {
|
|
"total" : 0,
|
|
"active" : 0
|
|
}
|
|
},
|
|
"graph" : {
|
|
"available" : true,
|
|
"enabled" : true
|
|
},
|
|
"ml" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
"jobs" : {
|
|
"_all" : {
|
|
"count" : 0,
|
|
"detectors" : {
|
|
...
|
|
},
|
|
"created_by" : { },
|
|
"model_size" : {
|
|
...
|
|
},
|
|
"forecasts" : {
|
|
"total" : 0,
|
|
"forecasted_jobs" : 0
|
|
}
|
|
}
|
|
},
|
|
"datafeeds" : {
|
|
"_all" : {
|
|
"count" : 0
|
|
}
|
|
},
|
|
"data_frame_analytics_jobs" : {
|
|
"_all" : {
|
|
"count" : 0
|
|
},
|
|
"analysis_counts": { },
|
|
"memory_usage": {
|
|
"peak_usage_bytes": {
|
|
"min": 0.0,
|
|
"max": 0.0,
|
|
"avg": 0.0,
|
|
"total": 0.0
|
|
}
|
|
}
|
|
},
|
|
"inference" : {
|
|
"ingest_processors" : {
|
|
"_all" : {
|
|
"num_docs_processed" : {
|
|
"max" : 0,
|
|
"sum" : 0,
|
|
"min" : 0
|
|
},
|
|
"pipelines" : {
|
|
"count" : 0
|
|
},
|
|
"num_failures" : {
|
|
"max" : 0,
|
|
"sum" : 0,
|
|
"min" : 0
|
|
},
|
|
"time_ms" : {
|
|
"max" : 0,
|
|
"sum" : 0,
|
|
"min" : 0
|
|
}
|
|
}
|
|
},
|
|
"trained_models" : {
|
|
"_all" : {
|
|
"count": 1
|
|
},
|
|
"count": {
|
|
"total": 1,
|
|
"prepackaged": 1,
|
|
"other": 0
|
|
},
|
|
"model_size_bytes": {
|
|
"min": 0.0,
|
|
"max": 0.0,
|
|
"avg": 0.0,
|
|
"total": 0.0
|
|
},
|
|
"estimated_operations": {
|
|
"min": 0.0,
|
|
"max": 0.0,
|
|
"avg": 0.0,
|
|
"total": 0.0
|
|
}
|
|
},
|
|
"deployments": {
|
|
"count": 0,
|
|
"inference_counts": {
|
|
"total": 0.0,
|
|
"min": 0.0,
|
|
"avg": 0.0,
|
|
"max": 0.0
|
|
},
|
|
"stats_by_model": [],
|
|
"model_sizes_bytes": {
|
|
"total": 0.0,
|
|
"min": 0.0,
|
|
"avg": 0.0,
|
|
"max": 0.0
|
|
},
|
|
"time_ms": {
|
|
"avg": 0.0
|
|
}
|
|
}
|
|
},
|
|
"node_count" : 1
|
|
},
|
|
"inference": {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
"models" : []
|
|
},
|
|
"logstash" : {
|
|
"available" : true,
|
|
"enabled" : true
|
|
},
|
|
"eql" : {
|
|
"available" : true,
|
|
"enabled" : true
|
|
},
|
|
"esql" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
"features" : {
|
|
"eval" : 0,
|
|
"stats" : 0,
|
|
"dissect": 0,
|
|
"grok" : 0,
|
|
"limit" : 0,
|
|
"where" : 0,
|
|
"sort" : 0,
|
|
"drop" : 0,
|
|
"show" : 0,
|
|
"rename" : 0,
|
|
"mv_expand" : 0,
|
|
"keep" : 0,
|
|
"enrich" : 0,
|
|
"from" : 0,
|
|
"row" : 0,
|
|
"meta" : 0
|
|
},
|
|
"queries" : {
|
|
"rest" : {
|
|
"total" : 0,
|
|
"failed" : 0
|
|
},
|
|
"kibana" : {
|
|
"total" : 0,
|
|
"failed" : 0
|
|
},
|
|
"_all" : {
|
|
"total" : 0,
|
|
"failed" : 0
|
|
}
|
|
}
|
|
},
|
|
"sql" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
"features" : {
|
|
"having" : 0,
|
|
"subselect" : 0,
|
|
"limit" : 0,
|
|
"orderby" : 0,
|
|
"where" : 0,
|
|
"join" : 0,
|
|
"groupby" : 0,
|
|
"command" : 0,
|
|
"local" : 0
|
|
},
|
|
"queries" : {
|
|
"rest" : {
|
|
"total" : 0,
|
|
"paging" : 0,
|
|
"failed" : 0
|
|
},
|
|
"cli" : {
|
|
"total" : 0,
|
|
"paging" : 0,
|
|
"failed" : 0
|
|
},
|
|
"canvas" : {
|
|
"total" : 0,
|
|
"paging" : 0,
|
|
"failed" : 0
|
|
},
|
|
"odbc" : {
|
|
"total" : 0,
|
|
"paging" : 0,
|
|
"failed" : 0
|
|
},
|
|
"jdbc" : {
|
|
"total" : 0,
|
|
"paging" : 0,
|
|
"failed" : 0
|
|
},
|
|
"odbc32" : {
|
|
"total" : 0,
|
|
"paging" : 0,
|
|
"failed" : 0
|
|
},
|
|
"odbc64" : {
|
|
"total" : 0,
|
|
"paging" : 0,
|
|
"failed" : 0
|
|
},
|
|
"_all" : {
|
|
"total" : 0,
|
|
"paging" : 0,
|
|
"failed" : 0
|
|
},
|
|
"translate" : {
|
|
"count" : 0
|
|
}
|
|
}
|
|
},
|
|
"rollup" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
...
|
|
},
|
|
"ilm" : {
|
|
"policy_count" : 3,
|
|
"policy_stats" : [
|
|
...
|
|
]
|
|
},
|
|
"slm" : {
|
|
"available" : true,
|
|
"enabled" : true
|
|
},
|
|
"ccr" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
"follower_indices_count" : 0,
|
|
"auto_follow_patterns_count" : 0
|
|
},
|
|
"transform" : {
|
|
"available" : true,
|
|
"enabled" : true
|
|
},
|
|
"voting_only" : {
|
|
"available" : true,
|
|
"enabled" : true
|
|
},
|
|
"searchable_snapshots" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
"indices_count" : 0,
|
|
"full_copy_indices_count" : 0,
|
|
"shared_cache_indices_count" : 0
|
|
},
|
|
"frozen_indices" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
"indices_count" : 0
|
|
},
|
|
"spatial" : {
|
|
"available" : true,
|
|
"enabled" : true
|
|
},
|
|
"analytics" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
"stats": {
|
|
"boxplot_usage" : 0,
|
|
"top_metrics_usage" : 0,
|
|
"normalize_usage" : 0,
|
|
"cumulative_cardinality_usage" : 0,
|
|
"t_test_usage" : 0,
|
|
"rate_usage" : 0,
|
|
"string_stats_usage" : 0,
|
|
"moving_percentiles_usage" : 0,
|
|
"multi_terms_usage" : 0
|
|
}
|
|
},
|
|
"data_streams" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
"data_streams" : 0,
|
|
"indices_count" : 0
|
|
},
|
|
"data_lifecycle" : {
|
|
"available": true,
|
|
"enabled": true,
|
|
"count": 0,
|
|
"default_rollover_used": true,
|
|
"data_retention": {
|
|
"configured_data_streams": 0
|
|
},
|
|
"effective_retention": {
|
|
"retained_data_streams": 0
|
|
},
|
|
"global_retention": {
|
|
"default": {
|
|
"defined": false
|
|
},
|
|
"max": {
|
|
"defined": false
|
|
}
|
|
}
|
|
},
|
|
"data_tiers" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
"data_warm" : {
|
|
"node_count" : 0,
|
|
"index_count" : 0,
|
|
"total_shard_count" : 0,
|
|
"primary_shard_count" : 0,
|
|
"doc_count" : 0,
|
|
"total_size_bytes" : 0,
|
|
"primary_size_bytes" : 0,
|
|
"primary_shard_size_avg_bytes" : 0,
|
|
"primary_shard_size_median_bytes" : 0,
|
|
"primary_shard_size_mad_bytes" : 0
|
|
},
|
|
"data_frozen" : {
|
|
"node_count" : 1,
|
|
"index_count" : 0,
|
|
"total_shard_count" : 0,
|
|
"primary_shard_count" : 0,
|
|
"doc_count" : 0,
|
|
"total_size_bytes" : 0,
|
|
"primary_size_bytes" : 0,
|
|
"primary_shard_size_avg_bytes" : 0,
|
|
"primary_shard_size_median_bytes" : 0,
|
|
"primary_shard_size_mad_bytes" : 0
|
|
},
|
|
"data_cold" : {
|
|
"node_count" : 0,
|
|
"index_count" : 0,
|
|
"total_shard_count" : 0,
|
|
"primary_shard_count" : 0,
|
|
"doc_count" : 0,
|
|
"total_size_bytes" : 0,
|
|
"primary_size_bytes" : 0,
|
|
"primary_shard_size_avg_bytes" : 0,
|
|
"primary_shard_size_median_bytes" : 0,
|
|
"primary_shard_size_mad_bytes" : 0
|
|
},
|
|
"data_content" : {
|
|
"node_count" : 0,
|
|
"index_count" : 0,
|
|
"total_shard_count" : 0,
|
|
"primary_shard_count" : 0,
|
|
"doc_count" : 0,
|
|
"total_size_bytes" : 0,
|
|
"primary_size_bytes" : 0,
|
|
"primary_shard_size_avg_bytes" : 0,
|
|
"primary_shard_size_median_bytes" : 0,
|
|
"primary_shard_size_mad_bytes" : 0
|
|
},
|
|
"data_hot" : {
|
|
"node_count" : 0,
|
|
"index_count" : 0,
|
|
"total_shard_count" : 0,
|
|
"primary_shard_count" : 0,
|
|
"doc_count" : 0,
|
|
"total_size_bytes" : 0,
|
|
"primary_size_bytes" : 0,
|
|
"primary_shard_size_avg_bytes" : 0,
|
|
"primary_shard_size_median_bytes" : 0,
|
|
"primary_shard_size_mad_bytes" : 0
|
|
}
|
|
},
|
|
"aggregate_metric" : {
|
|
"available" : true,
|
|
"enabled" : true
|
|
},
|
|
"archive" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
"indices_count" : 0
|
|
},
|
|
"health_api" : {
|
|
"available" : true,
|
|
"enabled" : true,
|
|
"invocations": {
|
|
"total": 0
|
|
}
|
|
},
|
|
"remote_clusters": {
|
|
"size": 0,
|
|
"mode": {
|
|
"proxy": 0,
|
|
"sniff": 0
|
|
},
|
|
"security": {
|
|
"cert": 0,
|
|
"api_key": 0
|
|
}
|
|
},
|
|
"enterprise_search" : {
|
|
"available": true,
|
|
"enabled": true,
|
|
"search_applications" : {
|
|
"count": 0
|
|
},
|
|
"analytics_collections": {
|
|
"count": 0
|
|
},
|
|
"query_rulesets": {
|
|
"total_rule_count": 0,
|
|
"total_count": 0,
|
|
"min_rule_count": 0,
|
|
"max_rule_count": 0
|
|
}
|
|
},
|
|
"universal_profiling" : {
|
|
"available" : true,
|
|
"enabled" : true
|
|
}
|
|
}
|
|
------------------------------------------------------------
|
|
// TESTRESPONSE[s/"security" : \{[^\}]*\},/"security" : $body.$_path,/]
|
|
// TESTRESPONSE[s/"rollup" : \{[^\}]*\},/"rollup" : $body.$_path,/]
|
|
// TESTRESPONSE[s/"detectors" : \{[^\}]*\},/"detectors" : $body.$_path,/]
|
|
// TESTRESPONSE[s/"model_size" : \{[^\}]*\},/"model_size" : $body.$_path,/]
|
|
// TESTRESPONSE[s/"eql" : \{[^\}]*\},/"eql" : $body.$_path,/]
|
|
// TESTRESPONSE[s/"policy_stats" : \[[^\]]*\]/"policy_stats" : $body.$_path/]
|
|
// TESTRESPONSE[s/"slm" : \{[^\}]*\},/"slm" : $body.$_path,/]
|
|
// TESTRESPONSE[s/"health_api" : \{[^\}]*\}\s*\}/"health_api" : $body.$_path/]
|
|
// TESTRESPONSE[s/"data_streams" : \{[^\}]*\},/"data_streams" : $body.$_path,/]
|
|
// TESTRESPONSE[s/ : true/ : $body.$_path/]
|
|
// TESTRESPONSE[s/ : false/ : $body.$_path/]
|
|
// TESTRESPONSE[s/ : (\-)?[0-9]+/ : $body.$_path/]
|
|
|
|
// These replacements do a few things:
|
|
// 1. Ignore the contents of the `policy_stats` object because we don't know all
|
|
// of the policies that will be in it. And because we figure folks don't need
|
|
// to see an exhaustive list anyway.
|
|
// 2. Handling eql, which is disabled by default on release builds and enabled
|
|
// everywhere else during the initial implementation phase until its release
|
|
// 3. Ignore the contents of the `slm` object because it might contain policies
|
|
// 4. Ignore the contents of the `health_api` object because we do not know if the api has been called or not
|
|
// 5. All of the numbers and strings on the right hand side of *every* field in
|
|
// the response are ignored. So we're really only asserting things about the
|
|
// the shape of this response, not the values in it.
|
|
// 6. Ignore the contents of data streams until the failure store is tech preview.
|