[Monitoring] Add tests to ensure mappings exist on metric fields (#23958) (#24333)

* Ensure mappings exist

* Add APM

* Remove unnecessary eslint disable comment

* Update snapshots

x-pack/plugins/monitoring/server/lib/metrics/__test__/__snapshots__/metrics.test.js.snap

@@ -1796,6 +1796,7 @@ Object {
     "description": "The number of operations the follower index is lagging behind the leader.",
     "field": "",
     "format": "0,0.[00]",
+    "getFields": [Function],
     "label": "Ops delay",
     "metricAgg": "sum",
     "timestampField": "timestamp",
@@ -2277,7 +2278,7 @@ Object {
     "app": "elasticsearch",
     "derivative": true,
     "description": "Time spent on Elasticsearch refresh for primary and replica shards.",
-    "field": "total.refresh.total_time_in_millis",
+    "field": "index_stats.total.refresh.total_time_in_millis",
     "format": "0,0.[00]",
     "label": "Total Refresh Time",
     "metricAgg": "max",

x-pack/plugins/monitoring/server/lib/metrics/classes/metric.js

@@ -54,6 +54,19 @@ export class Metric {
     };
   }
 
+  getFields() {
+    return [this.field];
+  }
+
+  getDocType() {
+    return this.docType || this.getInferredDocType();
+  }
+
+  getInferredDocType() {
+    const fields = this.getFields();
+    return fields && fields.length ? fields[0].split('.')[0] : null;
+  }
+
   static calculateLatency(timeInMillis, totalEvents) {
     if (timeInMillis === null || totalEvents === null) {
       return null;

x-pack/plugins/monitoring/server/lib/metrics/elasticsearch/classes.js

@@ -56,6 +56,8 @@ export class DifferenceMetric extends ElasticsearchMetric {
       },
     };
 
+    this.getFields = () => [`${fieldSource}.${metric}`, `${fieldSource}.${metric2}`];
+
     this.calculation = (bucket) => {
       return _.get(bucket, 'metric_max.value') - _.get(bucket, 'metric2_max.value');
     };

x-pack/plugins/monitoring/server/lib/metrics/elasticsearch/metrics.js

@@ -937,7 +937,7 @@ export const metrics = {
     type: 'index'
   }),
   index_refresh_time: new ElasticsearchMetric({
-    field: 'total.refresh.total_time_in_millis',
+    field: 'index_stats.total.refresh.total_time_in_millis',
     label: 'Total Refresh Time',
     description:
       'Time spent on Elasticsearch refresh for primary and replica shards.',

x-pack/test/api_integration/apis/monitoring/common/index.js

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export default function ({ loadTestFile }) {
+  describe('common', () => {
+    loadTestFile(require.resolve('./mappings_exist'));
+  });
+}

x-pack/test/api_integration/apis/monitoring/common/mappings_exist.js

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from 'expect.js';
+import { get } from 'lodash';
+import * as esMetrics from '../../../../../plugins/monitoring/server/lib/metrics/elasticsearch/metrics';
+import * as kibanaMetrics from '../../../../../plugins/monitoring/server/lib/metrics/kibana/metrics';
+import * as logstashMetrics from '../../../../../plugins/monitoring/server/lib/metrics/logstash/metrics';
+import * as beatsMetrics from '../../../../../plugins/monitoring/server/lib/metrics/beats/metrics';
+import * as apmMetrics from '../../../../../plugins/monitoring/server/lib/metrics/apm/metrics';
+
+export default function ({ getService }) {
+  const es = getService('es');
+
+  const metricSets = [
+    {
+      metrics: esMetrics.metrics,
+      name: 'es metrics',
+      indexTemplate: '.monitoring-es'
+    },
+    {
+      metrics: kibanaMetrics.metrics,
+      name: 'kibana metrics',
+      indexTemplate: '.monitoring-kibana'
+    },
+    {
+      metrics: logstashMetrics.metrics,
+      name: 'logstash metrics',
+      indexTemplate: '.monitoring-logstash'
+    },
+    {
+      metrics: beatsMetrics.metrics,
+      name: 'beats metrics',
+      indexTemplate: '.monitoring-beats'
+    },
+    {
+      metrics: apmMetrics.metrics,
+      name: 'apm metrics',
+      indexTemplate: '.monitoring-beats' // apm uses the same as beats
+    },
+  ];
+
+  describe('mappings', () => {
+    for (const { indexTemplate, metrics, name } of metricSets) {
+      let mappings;
+
+      before('load mappings', async () => {
+        const template = await es.indices.getTemplate({ name: indexTemplate });
+        mappings = get(template, [indexTemplate, 'mappings', 'doc', 'properties']);
+      });
+
+      describe(`for ${name}`, () => { // eslint-disable-line no-loop-func
+        for (const metric of Object.values(metrics)) {
+          for (const field of metric.getFields()) {
+            it(`${field} should exist in the mappings`, () => { // eslint-disable-line no-loop-func
+              const propertyGetter = field.split('.').reduce((list, field) => {
+                list.push(field);
+                list.push('properties');
+                return list;
+              }, []).slice(0, -1); // Remove the trailing 'properties'
+
+              const foundMapping = get(mappings, propertyGetter, null);
+              expect(foundMapping).to.not.equal(null);
+            });
+          }
+        }
+      });
+    }
+  });
+}

x-pack/test/api_integration/apis/monitoring/index.js

@@ -13,5 +13,6 @@ export default function ({ loadTestFile }) {
     loadTestFile(require.resolve('./elasticsearch_settings'));
     loadTestFile(require.resolve('./kibana'));
     loadTestFile(require.resolve('./logstash'));
+    loadTestFile(require.resolve('./common'));
   });
 }