From 3abc8524cb891b4c6b1af145e8c63f1544200bad Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Fri, 7 Feb 2025 09:29:13 -1000 Subject: [PATCH] [ES|QL] Support subset of metrics in agg metric double (#121805) This commit adds adds support for mappings containing only a subset of metrics in aggregate metric double (i.e. only sum and value_count, or just max, etc) as well as tests for grouped aggregations on aggregate metric double. --- docs/changelog/121805.yaml | 5 + .../xpack/esql/action/EsqlCapabilities.java | 7 +- .../xpack/esql/planner/AggregateMapper.java | 4 - .../AggregateMetricDoubleFieldMapper.java | 19 +- .../rest-api-spec/test/esql/40_tsdb.yml | 202 +++++++++++++++++- 5 files changed, 213 insertions(+), 24 deletions(-) create mode 100644 docs/changelog/121805.yaml diff --git a/docs/changelog/121805.yaml b/docs/changelog/121805.yaml new file mode 100644 index 000000000000..7d0f3a96221a --- /dev/null +++ b/docs/changelog/121805.yaml @@ -0,0 +1,5 @@ +pr: 121805 +summary: Support subset of metrics in aggregate metric double +area: "ES|QL" +type: enhancement +issues: [] diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 57ac363adee8..dc1d41117c41 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -791,7 +791,12 @@ public class EsqlCapabilities { /** * Support for aggregate_metric_double type */ - AGGREGATE_METRIC_DOUBLE(AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG.isEnabled()); + AGGREGATE_METRIC_DOUBLE(AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG), + + /** + * Support for partial subset of metrics in aggregate_metric_double type + */ + AGGREGATE_METRIC_DOUBLE_PARTIAL_SUBMETRICS(AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG); private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java index 56ab4652bd48..ede5f76930f8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java @@ -48,7 +48,6 @@ import java.lang.invoke.MethodType; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -200,9 +199,6 @@ final class AggregateMapper { if (clazz.isAssignableFrom(Rate.class)) { // rate doesn't support non-grouping aggregations return Stream.of(new AggDef(clazz, type, extra, true)); - } else if (Objects.equals(type, "AggregateMetricDouble")) { - // TODO: support grouping aggregations for aggregate metric double - return Stream.of(new AggDef(clazz, type, extra, false)); } else { return Stream.of(new AggDef(clazz, type, extra, true), new AggDef(clazz, type, extra, false)); } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java index 3ab49126ecb0..70adabf414f6 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateMetricDoubleFieldMapper.java @@ -540,9 +540,6 @@ public class AggregateMetricDoubleFieldMapper extends FieldMapper { NumericDocValues sumValues = getNumericDocValues(sumFieldType, context.reader()); NumericDocValues valueCountValues = getNumericDocValues(countFieldType, context.reader()); - if (minValues == null || maxValues == null || sumValues == null || valueCountValues == null) { - throw new UnsupportedOperationException("Must have all subfields to use aggregate double metric in ESQL"); - } return new BlockDocValuesReader() { private int docID = -1; @@ -576,13 +573,13 @@ public class AggregateMetricDoubleFieldMapper extends FieldMapper { if (doc < lastDoc) { throw new IllegalStateException("docs within same block must be in order"); } - if (values.advanceExact(doc)) { + if (values == null || values.advanceExact(doc) == false) { + builder.appendNull(); + } else { double value = NumericUtils.sortableLongToDouble(values.longValue()); lastDoc = doc; this.docID = doc; builder.appendDouble(value); - } else { - builder.appendNull(); } } } @@ -595,13 +592,13 @@ public class AggregateMetricDoubleFieldMapper extends FieldMapper { if (doc < lastDoc) { throw new IllegalStateException("docs within same block must be in order"); } - if (values.advanceExact(doc)) { + if (values == null || values.advanceExact(doc) == false) { + builder.appendNull(); + } else { int value = Math.toIntExact(values.longValue()); lastDoc = doc; this.docID = doc; builder.appendInt(value); - } else { - builder.appendNull(); } } } @@ -610,10 +607,10 @@ public class AggregateMetricDoubleFieldMapper extends FieldMapper { public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { var blockBuilder = (AggregateMetricDoubleBuilder) builder; this.docID = docId; - read(docId, blockBuilder); + readSingleRow(docId, blockBuilder); } - private void read(int docId, AggregateMetricDoubleBuilder builder) throws IOException { + private void readSingleRow(int docId, AggregateMetricDoubleBuilder builder) throws IOException { if (minValues.advanceExact(docId)) { builder.min().appendDouble(NumericUtils.sortableLongToDouble(minValues.longValue())); } else { diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index 5bdd2baf6050..512ea155144b 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -80,7 +80,6 @@ setup: time_series_dimension: true agg_metric: type: aggregate_metric_double - # TODO: tests with a subset of metrics metrics: [ min, max, sum, value_count ] default_metric: max k8s: @@ -102,6 +101,80 @@ setup: - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": "A", "agg_metric": {"max": 10, "min": -1, "sum": 20, "value_count": 5}}' - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:24.467Z", "dim": "B", "agg_metric": {"max": 20, "min": 3, "sum": 50, "value_count": 7}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:44.467Z", "dim": "B", "agg_metric": {"max": 17, "min": -5, "sum": 33, "value_count": 9}}' + + - do: + indices.create: + index: test3 + body: + settings: + index: + mode: time_series + routing_path: [ k8s.pod.uid ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + agg_metric: + type: aggregate_metric_double + metrics: [ min, max ] + default_metric: min + k8s: + properties: + pod: + properties: + uid: + type: keyword + time_series_dimension: true + - do: + bulk: + refresh: true + index: test3 + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T19:50:04.467Z", "agg_metric": {"max": 1, "min": -3}, "k8s": {"pod": {"uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T19:50:24.467Z", "agg_metric": {"max": 10, "min": 3}, "k8s": {"pod": {"uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T19:50:44.467Z", "agg_metric": {"max": 17, "min": 2}, "k8s": {"pod": {"uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}}}' + + - do: + indices.create: + index: test4 + body: + settings: + index: + mode: time_series + routing_path: [ k8s.pod.uid ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + agg_metric: + type: aggregate_metric_double + metrics: [ sum, value_count ] + default_metric: sum + k8s: + properties: + pod: + properties: + uid: + type: keyword + time_series_dimension: true + - do: + bulk: + refresh: true + index: test4 + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T23:50:04.467Z", "agg_metric": {"sum": 1, "value_count": 10}, "k8s": {"pod": {"uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}}}' --- load everything: @@ -226,7 +299,7 @@ from doc with aggregate_metric_double: - match: {columns.3.type: "ip"} - match: {columns.4.name: "k8s.pod.network.tx"} - match: {columns.4.type: "long"} - - length: {values: 2} + - length: {values: 3} --- stats on aggregate_metric_double: @@ -255,9 +328,122 @@ stats on aggregate_metric_double: - match: {columns.3.name: "count(agg_metric)"} - match: {columns.3.type: "long"} - match: {values.0.0: 20.0} + - match: {values.0.1: -5.0} + - match: {values.0.2: 103.0} + - match: {values.0.3: 21.0} + +--- +grouping stats on aggregate_metric_double: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [aggregate_metric_double] + reason: "Support for aggregate_metric_double" + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: "FROM test2 + | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) BY dim + | SORT dim" + - length: {values: 2} + - length: {values.0: 5} + - match: {columns.0.name: "max(agg_metric)"} + - match: {columns.0.type: "double"} + - match: {columns.1.name: "min(agg_metric)"} + - match: {columns.1.type: "double"} + - match: {columns.2.name: "sum(agg_metric)"} + - match: {columns.2.type: "double"} + - match: {columns.3.name: "count(agg_metric)"} + - match: {columns.3.type: "long"} + - match: {columns.4.name: "dim"} + - match: {columns.4.type: "keyword"} + - match: {values.0.0: 10.0} - match: {values.0.1: -1.0} - - match: {values.0.2: 70.0} - - match: {values.0.3: 12.0} + - match: {values.0.2: 20.0} + - match: {values.0.3: 5.0} + - match: {values.0.4: "A"} + - match: {values.1.0: 20.0} + - match: {values.1.1: -5.0} + - match: {values.1.2: 83.0} + - match: {values.1.3: 16.0} + - match: {values.1.4: "B"} + +--- +stats on aggregate_metric_double with partial submetrics: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [aggregate_metric_double_partial_submetrics] + reason: "Support for partial submetrics in aggregate_metric_double" + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test3 | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric) BY k8s.pod.uid | SORT k8s.pod.uid' + + - length: {values: 2} + - length: {values.0: 5} + - match: {columns.0.name: "max(agg_metric)"} + - match: {columns.0.type: "double"} + - match: {columns.1.name: "min(agg_metric)"} + - match: {columns.1.type: "double"} + - match: {columns.2.name: "sum(agg_metric)"} + - match: {columns.2.type: "double"} + - match: {columns.3.name: "count(agg_metric)"} + - match: {columns.3.type: "long"} + - match: {columns.4.name: "k8s.pod.uid"} + - match: {columns.4.type: "keyword"} + - match: {values.0.0: 10.0} + - match: {values.0.1: -3.0} + - match: {values.0.2: null} + - match: {values.0.3: null} + - match: {values.0.4: "947e4ced-1786-4e53-9e0c-5c447e959507"} + - match: {values.1.0: 17.0} + - match: {values.1.1: 2.0} + - match: {values.1.2: null} + - match: {values.1.3: null} + - match: {values.1.4: "df3145b3-0563-4d3b-a0f7-897eb2876ea9"} + +--- +stats on aggregate_metric_double missing min and max: + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: POST + path: /_query + parameters: [ ] + capabilities: [ aggregate_metric_double_partial_submetrics ] + reason: "Support for partial submetrics in aggregate_metric_double" + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test4 | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric)' + + - length: {values: 1} + - length: {values.0: 4} + - match: {columns.0.name: "max(agg_metric)"} + - match: {columns.0.type: "double"} + - match: {columns.1.name: "min(agg_metric)"} + - match: {columns.1.type: "double"} + - match: {columns.2.name: "sum(agg_metric)"} + - match: {columns.2.type: "double"} + - match: {columns.3.name: "count(agg_metric)"} + - match: {columns.3.type: "long"} + - match: {values.0.0: null} + - match: {values.0.1: null} + - match: {values.0.2: 1.0} + - match: {values.0.3: 10} --- from index pattern unsupported counter: @@ -267,8 +453,8 @@ from index pattern unsupported counter: - method: POST path: /_query parameters: [] - capabilities: [aggregate_metric_double] - reason: "Support for aggregate_metric_double" + capabilities: [aggregate_metric_double_partial_submetrics] + reason: "Support for partial submetrics in aggregate_metric_double" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" @@ -294,7 +480,7 @@ from index pattern unsupported counter: - match: {columns.7.type: "keyword"} - match: {columns.8.name: "metricset"} - match: {columns.8.type: "keyword"} - - length: {values: 10} + - length: {values: 15} --- from index pattern explicit counter use: @@ -315,7 +501,7 @@ from index pattern explicit counter use: query: 'FROM test* | keep *.tx' - match: {columns.0.name: "k8s.pod.network.tx"} - match: {columns.0.type: "unsupported"} - - length: {values: 10} + - length: {values: 15} --- _source: