[Metrics UI] Calculate interval based on the dataset's period (#50194)

* Calculate interval based on the dataset's period

* Remove unused import

* Handle empty data case

* Update x-pack/legacy/plugins/infra/server/utils/calculate_metric_interval.ts

Co-Authored-By: Chris Cowan <chris@chriscowan.us>

* Update x-pack/legacy/plugins/infra/server/routes/metrics_explorer/lib/populate_series_with_tsvb_data.ts

Co-Authored-By: Chris Cowan <chris@chriscowan.us>

x-pack/legacy/plugins/infra/server/lib/adapters/metrics/kibana_metrics_adapter.ts

@@ -14,6 +14,7 @@ import { checkValidNode } from './lib/check_valid_node';
 import { InvalidNodeError } from './lib/errors';
 import { metrics } from '../../../../common/inventory_models';
 import { TSVBMetricModelCreator } from '../../../../common/inventory_models/types';
+import { calculateMetricInterval } from '../../../utils/calculate_metric_interval';
 
 export class KibanaMetricsAdapter implements InfraMetricsAdapter {
   private framework: InfraBackendFrameworkAdapter;
@@ -32,14 +33,7 @@ export class KibanaMetricsAdapter implements InfraMetricsAdapter {
       [InfraNodeType.pod]: options.sourceConfiguration.fields.pod,
     };
     const indexPattern = `${options.sourceConfiguration.metricAlias},${options.sourceConfiguration.logAlias}`;
-    const timeField = options.sourceConfiguration.fields.timestamp;
-    const interval = options.timerange.interval;
     const nodeField = fields[options.nodeType];
-    const timerange = {
-      min: options.timerange.from,
-      max: options.timerange.to,
-    };
-
     const search = <Aggregation>(searchOptions: object) =>
       this.framework.callWithRequest<{}, Aggregation>(req, 'search', searchOptions);
 
@@ -55,41 +49,10 @@ export class KibanaMetricsAdapter implements InfraMetricsAdapter {
       );
     }
 
-    const requests = options.metrics.map(metricId => {
-      const createTSVBModel = get(metrics, ['tsvb', metricId]) as
-        | TSVBMetricModelCreator
-        | undefined;
-      if (!createTSVBModel) {
-        throw new Error(
-          i18n.translate('xpack.infra.metrics.missingTSVBModelError', {
-            defaultMessage: 'The TSVB model for {metricId} does not exist for {nodeType}',
-            values: {
-              metricId,
-              nodeType: options.nodeType,
-            },
-          })
-        );
-      }
-      const model = createTSVBModel(timeField, indexPattern, interval);
-      if (model.id_type === 'cloud' && !options.nodeIds.cloudId) {
-        throw new InvalidNodeError(
-          i18n.translate('xpack.infra.kibanaMetrics.cloudIdMissingErrorMessage', {
-            defaultMessage:
-              'Model for {metricId} requires a cloudId, but none was given for {nodeId}.',
-            values: {
-              metricId,
-              nodeId: options.nodeIds.nodeId,
-            },
-          })
-        );
-      }
-      const id =
-        model.id_type === 'cloud' ? (options.nodeIds.cloudId as string) : options.nodeIds.nodeId;
-      const filters = model.map_field_to
-        ? [{ match: { [model.map_field_to]: id } }]
-        : [{ match: { [nodeField]: id } }];
-      return this.framework.makeTSVBRequest(req, model, timerange, filters);
-    });
+    const requests = options.metrics.map(metricId =>
+      this.makeTSVBRequest(metricId, options, req, nodeField)
+    );
+
     return Promise.all(requests)
       .then(results => {
         return results.map(result => {
@@ -125,4 +88,70 @@ export class KibanaMetricsAdapter implements InfraMetricsAdapter {
       })
       .then(result => flatten(result));
   }
+
+  async makeTSVBRequest(
+    metricId: InfraMetric,
+    options: InfraMetricsRequestOptions,
+    req: InfraFrameworkRequest,
+    nodeField: string
+  ) {
+    const createTSVBModel = get(metrics, ['tsvb', metricId]) as TSVBMetricModelCreator | undefined;
+    if (!createTSVBModel) {
+      throw new Error(
+        i18n.translate('xpack.infra.metrics.missingTSVBModelError', {
+          defaultMessage: 'The TSVB model for {metricId} does not exist for {nodeType}',
+          values: {
+            metricId,
+            nodeType: options.nodeType,
+          },
+        })
+      );
+    }
+
+    const indexPattern = `${options.sourceConfiguration.metricAlias},${options.sourceConfiguration.logAlias}`;
+    const timerange = {
+      min: options.timerange.from,
+      max: options.timerange.to,
+    };
+
+    const model = createTSVBModel(
+      options.sourceConfiguration.fields.timestamp,
+      indexPattern,
+      options.timerange.interval
+    );
+    const calculatedInterval = await calculateMetricInterval(
+      this.framework,
+      req,
+      {
+        indexPattern: `${options.sourceConfiguration.logAlias},${options.sourceConfiguration.metricAlias}`,
+        timestampField: options.sourceConfiguration.fields.timestamp,
+        timerange: options.timerange,
+      },
+      model.requires
+    );
+
+    if (calculatedInterval) {
+      model.interval = `>=${calculatedInterval}s`;
+    }
+
+    if (model.id_type === 'cloud' && !options.nodeIds.cloudId) {
+      throw new InvalidNodeError(
+        i18n.translate('xpack.infra.kibanaMetrics.cloudIdMissingErrorMessage', {
+          defaultMessage:
+            'Model for {metricId} requires a cloudId, but none was given for {nodeId}.',
+          values: {
+            metricId,
+            nodeId: options.nodeIds.nodeId,
+          },
+        })
+      );
+    }
+    const id =
+      model.id_type === 'cloud' ? (options.nodeIds.cloudId as string) : options.nodeIds.nodeId;
+    const filters = model.map_field_to
+      ? [{ match: { [model.map_field_to]: id } }]
+      : [{ match: { [nodeField]: id } }];
+
+    return this.framework.makeTSVBRequest(req, model, timerange, filters);
+  }
 }

x-pack/legacy/plugins/infra/server/routes/metrics_explorer/lib/populate_series_with_tsvb_data.ts

@@ -18,6 +18,7 @@ import {
 } from '../types';
 import { createMetricModel } from './create_metrics_model';
 import { JsonObject } from '../../../../common/typed_json';
+import { calculateMetricInterval } from '../../../utils/calculate_metric_interval';
 
 export const populateSeriesWithTSVBData = (
   req: InfraFrameworkRequest<MetricsExplorerWrappedRequest>,
@@ -54,6 +55,27 @@ export const populateSeriesWithTSVBData = (
 
   // Create the TSVB model based on the request options
   const model = createMetricModel(options);
+  const calculatedInterval = await calculateMetricInterval(
+    framework,
+    req,
+    {
+      indexPattern: options.indexPattern,
+      timestampField: options.timerange.field,
+      timerange: options.timerange,
+    },
+    options.metrics
+      .filter(metric => metric.field)
+      .map(metric => {
+        return metric
+          .field!.split(/\./)
+          .slice(0, 2)
+          .join('.');
+      })
+  );
+
+  if (calculatedInterval) {
+    model.interval = `>=${calculatedInterval}s`;
+  }
 
   // Get TSVB results using the model, timerange and filters
   const tsvbResults = await framework.makeTSVBRequest(req, model, timerange, filters);

x-pack/legacy/plugins/infra/server/utils/calculate_metric_interval.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { InfraBackendFrameworkAdapter, InfraFrameworkRequest } from '../lib/adapters/framework';
+
+interface Options {
+  indexPattern: string;
+  timestampField: string;
+  timerange: {
+    from: number;
+    to: number;
+  };
+}
+
+/**
+ * Look at the data from metricbeat and get the max period for a given timerange.
+ * This is useful for visualizing metric modules like s3 that only send metrics once per day.
+ */
+export const calculateMetricInterval = async (
+  framework: InfraBackendFrameworkAdapter,
+  request: InfraFrameworkRequest,
+  options: Options,
+  modules: string[]
+) => {
+  const query = {
+    allowNoIndices: true,
+    index: options.indexPattern,
+    ignoreUnavailable: true,
+    body: {
+      query: {
+        bool: {
+          filter: [
+            {
+              range: {
+                [options.timestampField]: {
+                  gte: options.timerange.from,
+                  lte: options.timerange.to,
+                  format: 'epoch_millis',
+                },
+              },
+            },
+          ],
+        },
+      },
+      size: 0,
+      aggs: {
+        modules: {
+          terms: {
+            field: 'event.dataset',
+            include: modules,
+          },
+          aggs: {
+            period: {
+              max: {
+                field: 'metricset.period',
+              },
+            },
+          },
+        },
+      },
+    },
+  };
+
+  const resp = await framework.callWithRequest<{}, PeriodAggregationData>(request, 'search', query);
+
+  // if ES doesn't return an aggregations key, something went seriously wrong.
+  if (!resp.aggregations) {
+    return;
+  }
+
+  const intervals = resp.aggregations.modules.buckets.map(a => a.period.value).filter(v => !!v);
+  if (!intervals.length) {
+    return;
+  }
+
+  return Math.max(...intervals) / 1000;
+};
+
+interface PeriodAggregationData {
+  modules: {
+    buckets: Array<{
+      key: string;
+      doc_count: number;
+      period: {
+        value: number;
+      };
+    }>;
+  };
+}