[Stack Monitoring] api tests for cluster and elasticsearch (#145138)

### Summary
Part of https://github.com/elastic/kibana/issues/119658

Add api integration tests for cluster and elasticsearch routes to
validate behavior when reading data ingested by elastic-agent.

We currently have a testing suite for legacy and another one for
metricbeat. Since metricbeat and agent documents only differ in their
metadata, for example agent will populate a `data_stream.*` property to
identify the document types while metricbeat uses `metricset.*`, the
tests assertion validating _business_ data should pass regardless of the
documents source. With this in mind the metricbeat tests were updated to
run the tests twice, one time with metricbeat data and a second time
with package data.

To generate the archives the `metrics-*` mappings were extracted with
esArchiver from an elasticsearch with the package installed, and the
documents were transformed from the metricbeat documents with [this
script](https://gist.github.com/klacabane/654497ff86053c60af6df15fa6f6f657).

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr.ts

@@ -6,8 +6,10 @@
  */
 
 import { get, groupBy } from 'lodash';
-import { prefixIndexPatternWithCcs } from '../../../../../common/ccs_utils';
-import { INDEX_PATTERN_ELASTICSEARCH } from '../../../../../common/constants';
+import {
+  getIndexPatterns,
+  getElasticsearchDataset,
+} from '../../../../lib/cluster/get_index_patterns';
 import {
   postElasticsearchCcrRequestParamsRT,
   postElasticsearchCcrRequestPayloadRT,
@@ -140,6 +142,13 @@ function buildRequest(
                       },
                     },
                   },
+                  {
+                    term: {
+                      'data_stream.dataset': {
+                        value: getElasticsearchDataset('ccr'),
+                      },
+                    },
+                  },
                 ],
               },
             },
@@ -215,7 +224,14 @@ export function ccrRoute(server: MonitoringCore) {
     async handler(req) {
       const config = server.config;
       const ccs = req.payload.ccs;
-      const esIndexPattern = prefixIndexPatternWithCcs(config, INDEX_PATTERN_ELASTICSEARCH, ccs);
+      const dataset = 'ccr';
+      const moduleType = 'elasticsearch';
+      const esIndexPattern = getIndexPatterns({
+        config,
+        moduleType,
+        dataset,
+        ccs,
+      });
 
       try {
         const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring');

x-pack/test/api_integration/apis/monitoring/cluster/list_mb.js

@@ -13,33 +13,38 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('list mb', function () {
-    // Archive contains non-cgroup data which collides with the in-cgroup services present by default on cloud deployments
-    this.tags(['skipCloud']);
+  describe('list - metricbeat and package', function () {
+    ['mb', 'package'].forEach((source) => {
+      describe(`list ${source}`, function () {
+        // Archive contains non-cgroup data which collides with the in-cgroup services present by default on cloud deployments
+        this.tags(['skipCloud']);
 
-    describe('with trial license clusters', () => {
-      const archive = 'x-pack/test/functional/es_archives/monitoring/multicluster_mb';
-      const timeRange = {
-        min: '2017-08-15T21:00:00Z',
-        max: '2017-08-16T00:00:00Z',
-      };
-      const codePaths = ['all'];
+        const archive = `x-pack/test/functional/es_archives/monitoring/multicluster_${source}`;
 
-      before('load clusters archive', () => {
-        return setup(archive);
-      });
+        describe('with trial license clusters', () => {
+          const timeRange = {
+            min: '2017-08-15T21:00:00Z',
+            max: '2017-08-16T00:00:00Z',
+          };
+          const codePaths = ['all'];
 
-      after('unload clusters archive', () => {
-        return tearDown();
-      });
+          before('load clusters archive', () => {
+            return setup(archive);
+          });
 
-      it('should load multiple clusters', async () => {
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, codePaths })
-          .expect(200);
-        expect(body).to.eql(multiclusterFixture);
+          after('unload clusters archive', () => {
+            return tearDown(archive);
+          });
+
+          it('should load multiple clusters', async () => {
+            const { body } = await supertest
+              .post('/api/monitoring/v1/clusters')
+              .set('kbn-xsrf', 'xxx')
+              .send({ timeRange, codePaths })
+              .expect(200);
+            expect(body).to.eql(multiclusterFixture);
+          });
+        });
       });
     });
   });

x-pack/test/api_integration/apis/monitoring/cluster/overview_mb.js

@@ -13,33 +13,38 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('overview mb', function () {
-    // TODO: https://github.com/elastic/stack-monitoring/issues/31
-    this.tags(['skipCloud']);
+  describe('overview - metricbeat and package', function () {
+    ['mb', 'package'].forEach((source) => {
+      describe(`overview ${source}`, function () {
+        // TODO: https://github.com/elastic/stack-monitoring/issues/31
+        this.tags(['skipCloud']);
 
-    describe('with trial license clusters', () => {
-      const archive = 'x-pack/test/functional/es_archives/monitoring/singlecluster_green_gold_mb';
-      const timeRange = {
-        min: '2017-08-23T21:29:35Z',
-        max: '2017-08-23T21:47:25Z',
-      };
-      const codePaths = ['all'];
+        const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_green_gold_${source}`;
 
-      before('load clusters archive', () => {
-        return setup(archive);
-      });
+        describe('with trial license clusters', () => {
+          const timeRange = {
+            min: '2017-08-23T21:29:35Z',
+            max: '2017-08-23T21:47:25Z',
+          };
+          const codePaths = ['all'];
 
-      after('unload clusters archive', () => {
-        return tearDown();
-      });
+          before('load clusters archive', () => {
+            return setup(archive);
+          });
 
-      it('should load multiple clusters', async () => {
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/y1qOsQPiRrGtmdEuM3APJw')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, codePaths })
-          .expect(200);
-        expect(body).to.eql(overviewFixture);
+          after('unload clusters archive', () => {
+            return tearDown(archive);
+          });
+
+          it('should load multiple clusters', async () => {
+            const { body } = await supertest
+              .post('/api/monitoring/v1/clusters/y1qOsQPiRrGtmdEuM3APJw')
+              .set('kbn-xsrf', 'xxx')
+              .send({ timeRange, codePaths })
+              .expect(200);
+            expect(body).to.eql(overviewFixture);
+          });
+        });
       });
     });
   });

x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_mb.js

@@ -12,32 +12,36 @@ import { getLifecycleMethods } from '../data_stream';
 export default function ({ getService }) {
   const supertest = getService('supertest');
 
-  describe('ccr mb', () => {
-    const archive = 'x-pack/test/functional/es_archives/monitoring/ccr_mb';
-    const { setup, tearDown } = getLifecycleMethods(getService);
-    const timeRange = {
-      min: '2018-09-19T00:00:00.000Z',
-      max: '2018-09-19T23:59:59.000Z',
-    };
+  describe('ccr - metricbeat and package', () => {
+    ['mb', 'package'].forEach((source) => {
+      describe(`ccr ${source}`, () => {
+        const archive = `x-pack/test/functional/es_archives/monitoring/ccr_${source}`;
+        const { setup, tearDown } = getLifecycleMethods(getService);
+        const timeRange = {
+          min: '2018-09-19T00:00:00.000Z',
+          max: '2018-09-19T23:59:59.000Z',
+        };
 
-    before('load archive', () => {
-      return setup(archive);
-    });
+        before('load archive', () => {
+          return setup(archive);
+        });
 
-    after('unload archive', () => {
-      return tearDown();
-    });
+        after('unload archive', () => {
+          return tearDown(archive);
+        });
 
-    it('should return all followers and a grouping of stats by follower index', async () => {
-      const { body } = await supertest
-        .post('/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr')
-        .set('kbn-xsrf', 'xxx')
-        .send({
-          timeRange,
-        })
-        .expect(200);
+        it('should return all followers and a grouping of stats by follower index', async () => {
+          const { body } = await supertest
+            .post('/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr')
+            .set('kbn-xsrf', 'xxx')
+            .send({
+              timeRange,
+            })
+            .expect(200);
 
-      expect(body).to.eql(ccrFixture);
+          expect(body).to.eql(ccrFixture);
+        });
+      });
     });
   });
 }

x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_shard_mb.js

@@ -14,35 +14,40 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('ccr shard mb', () => {
-    const archive = 'x-pack/test/functional/es_archives/monitoring/ccr_mb';
-    const timeRange = {
-      min: '2018-09-19T00:00:00.000Z',
-      max: '2018-09-19T23:59:59.000Z',
-    };
+  describe('ccr shard - metricbeat and package', () => {
+    ['mb', 'package'].forEach((source) => {
+      const archive = `x-pack/test/functional/es_archives/monitoring/ccr_${source}`;
 
-    before('load archive', () => {
-      return setup(archive);
-    });
+      describe(`ccr shard ${source}`, () => {
+        const timeRange = {
+          min: '2018-09-19T00:00:00.000Z',
+          max: '2018-09-19T23:59:59.000Z',
+        };
 
-    after('unload archive', () => {
-      return tearDown();
-    });
+        before('load archive', () => {
+          return setup(archive);
+        });
 
-    it('should return specific shard details', async () => {
-      const { body } = await supertest
-        .post(
-          '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr/follower/shard/0'
-        )
-        .set('kbn-xsrf', 'xxx')
-        .send({
-          timeRange,
-        })
-        .expect(200);
+        after('unload archive', () => {
+          return tearDown(archive);
+        });
 
-      // These will be inherently different, but they are only shown in JSON format in the UI so that's okay
-      const keysToIgnore = ['stat', 'oldestStat'];
-      expect(omit(body, keysToIgnore)).to.eql(omit(ccrShardFixture, keysToIgnore));
+        it('should return specific shard details', async () => {
+          const { body } = await supertest
+            .post(
+              '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr/follower/shard/0'
+            )
+            .set('kbn-xsrf', 'xxx')
+            .send({
+              timeRange,
+            })
+            .expect(200);
+
+          // These will be inherently different, but they are only shown in JSON format in the UI so that's okay
+          const keysToIgnore = ['stat', 'oldestStat'];
+          expect(omit(body, keysToIgnore)).to.eql(omit(ccrShardFixture, keysToIgnore));
+        });
+      });
     });
   });
 }

x-pack/test/api_integration/apis/monitoring/elasticsearch/index_detail_mb.js

@@ -14,50 +14,53 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('index detail mb', () => {
-    const archive =
-      'x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_mb';
-    const timeRange = {
-      min: '2017-10-05T20:31:48.000Z',
-      max: '2017-10-05T20:35:12.000Z',
-    };
+  describe('index detail - metricbeat and package', () => {
+    ['mb', 'package'].forEach((source) => {
+      describe(`index detail ${source}`, () => {
+        const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_${source}`;
+        const timeRange = {
+          min: '2017-10-05T20:31:48.000Z',
+          max: '2017-10-05T20:35:12.000Z',
+        };
 
-    before('load archive', () => {
-      return setup(archive);
-    });
+        before('load archive', () => {
+          return setup(archive);
+        });
 
-    after('unload archive', () => {
-      return tearDown();
-    });
+        after('unload archive', () => {
+          return tearDown(archive);
+        });
 
-    it('should summarize index with chart metrics data for the non-advanced view', async () => {
-      const { body } = await supertest
-        .post(
-          '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
-        )
-        .set('kbn-xsrf', 'xxx')
-        .send({
-          timeRange,
-          is_advanced: false,
-        })
-        .expect(200);
+        it('should summarize index with chart metrics data for the non-advanced view', async () => {
+          const { body } = await supertest
+            .post(
+              '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
+            )
+            .set('kbn-xsrf', 'xxx')
+            .send({
+              timeRange,
+              is_advanced: false,
+            })
+            .expect(200);
 
-      expect(body).to.eql(indexDetailFixture);
-    });
+          expect(body).to.eql(indexDetailFixture);
+        });
 
-    it('should summarize index with chart metrics data for the advanced view', async () => {
-      const { body } = await supertest
-        .post(
-          '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
-        )
-        .set('kbn-xsrf', 'xxx')
-        .send({
-          timeRange,
-          is_advanced: true,
-        })
-        .expect(200);
+        it('should summarize index with chart metrics data for the advanced view', async () => {
+          const { body } = await supertest
+            .post(
+              '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
+            )
+            .set('kbn-xsrf', 'xxx')
+            .send({
+              timeRange,
+              is_advanced: true,
+            })
+            .expect(200);
 
-      expect(body).to.eql(indexDetailAdvancedFixture);
+          expect(body).to.eql(indexDetailAdvancedFixture);
+        });
+      });
     });
   });
 }

x-pack/test/api_integration/apis/monitoring/elasticsearch/indices_mb.js

@@ -14,86 +14,88 @@ import { getLifecycleMethods } from '../data_stream';
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
-  const esArchiver = getService('esArchiver');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('indices mb', () => {
-    describe('shard-relocation', () => {
-      const archive =
-        'x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_mb';
-      const timeRange = {
-        min: '2017-10-05T20:31:48.000Z',
-        max: '2017-10-05T20:35:12.000Z',
-      };
+  describe('indices - metricbeat and package', () => {
+    ['mb', 'package'].forEach((source) => {
+      describe(`indices ${source}`, () => {
+        describe('shard-relocation', () => {
+          const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_${source}`;
+          const timeRange = {
+            min: '2017-10-05T20:31:48.000Z',
+            max: '2017-10-05T20:35:12.000Z',
+          };
 
-      before('load archive', () => {
-        return setup(archive);
-      });
+          before('load archive', () => {
+            return setup(archive);
+          });
 
-      after('unload archive', () => {
-        return tearDown();
-      });
+          after('unload archive', () => {
+            return tearDown(archive);
+          });
 
-      it('should summarize the non-system indices with stats', async () => {
-        const { body } = await supertest
-          .post(
-            '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices?show_system_indices=false'
-          )
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange })
-          .expect(200);
-        expect(body).to.eql(relocatingShardsFixture);
-      });
+          it('should summarize the non-system indices with stats', async () => {
+            const { body } = await supertest
+              .post(
+                '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices?show_system_indices=false'
+              )
+              .set('kbn-xsrf', 'xxx')
+              .send({ timeRange })
+              .expect(200);
+            expect(body).to.eql(relocatingShardsFixture);
+          });
 
-      it('should summarize all indices with stats', async () => {
-        const { body } = await supertest
-          .post(
-            '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices?show_system_indices=true'
-          )
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange })
-          .expect(200);
+          it('should summarize all indices with stats', async () => {
+            const { body } = await supertest
+              .post(
+                '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices?show_system_indices=true'
+              )
+              .set('kbn-xsrf', 'xxx')
+              .send({ timeRange })
+              .expect(200);
 
-        expect(body).to.eql(relocationShardsAllFixture);
-      });
-    });
+            expect(body).to.eql(relocationShardsAllFixture);
+          });
+        });
 
-    describe('health-red', () => {
-      const archive = 'x-pack/test/functional/es_archives/monitoring/singlecluster_red_platinum';
-      const timeRange = {
-        min: '2017-10-06T19:53:06.000Z',
-        max: '2017-10-06T20:15:30.000Z',
-      };
+        describe('health-red', () => {
+          const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_red_platinum_${source}`;
+          const timeRange = {
+            min: '2017-10-06T19:53:06.000Z',
+            max: '2017-10-06T20:15:30.000Z',
+          };
 
-      before('load clusters archive', () => {
-        return esArchiver.load(archive);
-      });
+          before('load clusters archive', () => {
+            return setup(archive);
+          });
 
-      after('unload clusters archive', () => {
-        return esArchiver.unload(archive);
-      });
+          after('unload clusters archive', () => {
+            return tearDown(archive);
+          });
 
-      it('should summarize the non-system indices with stats', async () => {
-        const { body } = await supertest
-          .post(
-            '/api/monitoring/v1/clusters/1LYuyvCCQFS3FAO_h65PQw/elasticsearch/indices?show_system_indices=false'
-          )
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange })
-          .expect(200);
-        expect(body).to.eql(indicesRedClusterFixture);
-      });
+          it('should summarize the non-system indices with stats', async () => {
+            const { body } = await supertest
+              .post(
+                '/api/monitoring/v1/clusters/1LYuyvCCQFS3FAO_h65PQw/elasticsearch/indices?show_system_indices=false'
+              )
+              .set('kbn-xsrf', 'xxx')
+              .send({ timeRange })
+              .expect(200);
+            expect(body).to.eql(indicesRedClusterFixture);
+          });
 
-      it('should summarize all indices with stats', async () => {
-        const { body } = await supertest
-          .post(
-            '/api/monitoring/v1/clusters/1LYuyvCCQFS3FAO_h65PQw/elasticsearch/indices?show_system_indices=true'
-          )
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange })
-          .expect(200);
+          it('should summarize all indices with stats', async () => {
+            const { body } = await supertest
+              .post(
+                '/api/monitoring/v1/clusters/1LYuyvCCQFS3FAO_h65PQw/elasticsearch/indices?show_system_indices=true'
+              )
+              .set('kbn-xsrf', 'xxx')
+              .send({ timeRange })
+              .expect(200);
 
-        expect(body).to.eql(indicesRedClusterAllFixture);
+            expect(body).to.eql(indicesRedClusterAllFixture);
+          });
+        });
       });
     });
   });

x-pack/test/api_integration/apis/monitoring/elasticsearch/node_detail_advanced_mb.js

@@ -13,35 +13,38 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('node detail advanced mb', () => {
-    const archive =
-      'x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_mb';
-    const timeRange = {
-      min: '2017-10-05T20:31:48.000Z',
-      max: '2017-10-05T20:35:12.000Z',
-    };
+  describe('node detail advanced - metricbeat and package', () => {
+    ['mb', 'package'].forEach((source) => {
+      describe(`node detail advanced ${source}`, () => {
+        const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_${source}`;
+        const timeRange = {
+          min: '2017-10-05T20:31:48.000Z',
+          max: '2017-10-05T20:35:12.000Z',
+        };
 
-    before('load archive', () => {
-      return setup(archive);
-    });
+        before('load archive', () => {
+          return setup(archive);
+        });
 
-    after('unload archive', () => {
-      return tearDown();
-    });
+        after('unload archive', () => {
+          return tearDown(archive);
+        });
 
-    it('should summarize node with metrics', async () => {
-      const { body } = await supertest
-        .post(
-          '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/nodes/jUT5KdxfRbORSCWkb5zjmA'
-        )
-        .set('kbn-xsrf', 'xxx')
-        .send({
-          timeRange,
-          is_advanced: true,
-        })
-        .expect(200);
+        it('should summarize node with metrics', async () => {
+          const { body } = await supertest
+            .post(
+              '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/nodes/jUT5KdxfRbORSCWkb5zjmA'
+            )
+            .set('kbn-xsrf', 'xxx')
+            .send({
+              timeRange,
+              is_advanced: true,
+            })
+            .expect(200);
 
-      expect(body).to.eql(nodeDetailFixture);
+          expect(body).to.eql(nodeDetailFixture);
+        });
+      });
     });
   });
 }

x-pack/test/api_integration/apis/monitoring/elasticsearch/node_detail_mb.js

@@ -13,39 +13,42 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('node detail mb', function () {
-    // TODO: https://github.com/elastic/stack-monitoring/issues/31
-    this.tags(['skipCloud']);
+  describe('node detail - metricbeat and package', function () {
+    ['mb', 'package'].forEach((source) => {
+      describe(`node detail ${source}`, function () {
+        // TODO: https://github.com/elastic/stack-monitoring/issues/31
+        this.tags(['skipCloud']);
 
-    const archive =
-      'x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_mb';
-    const timeRange = {
-      min: '2017-10-05T20:31:48.000Z',
-      max: '2017-10-05T20:35:12.000Z',
-    };
+        const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_${source}`;
+        const timeRange = {
+          min: '2017-10-05T20:31:48.000Z',
+          max: '2017-10-05T20:35:12.000Z',
+        };
 
-    before('load archive', () => {
-      return setup(archive);
-    });
+        before('load archive', () => {
+          return setup(archive);
+        });
 
-    after('unload archive', () => {
-      return tearDown();
-    });
+        after('unload archive', () => {
+          return tearDown(archive);
+        });
 
-    it('should summarize node with metrics', async () => {
-      const { body } = await supertest
-        .post(
-          '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/nodes/jUT5KdxfRbORSCWkb5zjmA'
-        )
-        .set('kbn-xsrf', 'xxx')
-        .send({
-          timeRange,
-          is_advanced: false,
-          showSystemIndices: false,
-        })
-        .expect(200);
+        it('should summarize node with metrics', async () => {
+          const { body } = await supertest
+            .post(
+              '/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/nodes/jUT5KdxfRbORSCWkb5zjmA'
+            )
+            .set('kbn-xsrf', 'xxx')
+            .send({
+              timeRange,
+              is_advanced: false,
+              showSystemIndices: false,
+            })
+            .expect(200);
 
-      expect(body).to.eql(nodeDetailFixture);
+          expect(body).to.eql(nodeDetailFixture);
+        });
+      });
     });
   });
 }

x-pack/test/api_integration/apis/monitoring/elasticsearch/nodes_mb.js

@@ -7,341 +7,43 @@
 
 import expect from '@kbn/expect';
 import nodesListingFixtureGreen from './fixtures/nodes_listing_green.json';
-import nodesListingFixtureRed from './fixtures/nodes_listing_red.json';
-import nodesListingFixtureCgroup from './fixtures/nodes_listing_cgroup.json';
-import nodesListingFixturePagination from './fixtures/nodes_listing_pagination.json';
 import { getLifecycleMethods } from '../data_stream';
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
-  const esArchiver = getService('esArchiver');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('nodes mb', () => {
-    describe('with green platinum cluster', () => {
-      const archive =
-        'x-pack/test/functional/es_archives/monitoring/singlecluster_green_platinum_mb';
-      const timeRange = {
-        min: '2018-02-13T17:04:50.000Z',
-        max: '2018-02-13T17:51:55.000Z',
-      };
-      const pagination = {
-        size: 10,
-        index: 0,
-      };
+  describe('nodes - metricbeat and package', () => {
+    ['mb', 'package'].forEach((source) => {
+      describe(`nodes ${source}`, () => {
+        describe('with green platinum cluster', () => {
+          const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_green_platinum_${source}`;
+          const timeRange = {
+            min: '2018-02-13T17:04:50.000Z',
+            max: '2018-02-13T17:51:55.000Z',
+          };
+          const pagination = {
+            size: 10,
+            index: 0,
+          };
 
-      before('load clusters archive', () => {
-        return setup(archive);
-      });
+          before('load clusters archive', () => {
+            return setup(archive);
+          });
 
-      after('unload clusters archive', () => {
-        return tearDown();
-      });
+          after('unload clusters archive', () => {
+            return tearDown();
+          });
 
-      it('should return data for 2 active nodes', async () => {
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/fHJwISmKTFO8bj57oFBLUQ/elasticsearch/nodes')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, pagination })
-          .expect(200);
-        expect(body).to.eql(nodesListingFixtureGreen);
-      });
-    });
-
-    describe('with red platinum cluster', () => {
-      const archive = 'x-pack/test/functional/es_archives/monitoring/singlecluster_red_platinum';
-      const timeRange = {
-        min: '2017-10-06T19:53:06.000Z',
-        max: '2017-10-06T20:15:30.000Z',
-      };
-      const pagination = {
-        size: 10,
-        index: 0,
-      };
-
-      before('load clusters archive', () => {
-        return esArchiver.load(archive);
-      });
-
-      after('unload clusters archive', () => {
-        return esArchiver.unload(archive);
-      });
-
-      it('should return data for offline nodes', async () => {
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/1LYuyvCCQFS3FAO_h65PQw/elasticsearch/nodes')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, pagination })
-          .expect(200);
-        expect(body).to.eql(nodesListingFixtureRed);
-      });
-    });
-
-    describe('with green trial cluster and node in cpu group', () => {
-      const archive =
-        'x-pack/test/functional/es_archives/monitoring/singlecluster_green_trial_two_nodes_one_cgrouped';
-      const timeRange = {
-        min: '2018-02-13T19:18:02.000Z',
-        max: '2018-02-13T19:26:14.000Z',
-      };
-      const pagination = {
-        size: 10,
-        index: 0,
-      };
-
-      before('load clusters archive', () => {
-        return esArchiver.load(archive);
-      });
-
-      after('unload clusters archive', () => {
-        return esArchiver.unload(archive);
-      });
-
-      it('should return cpu info for cgroup node and cpu info for "regular" node', async () => {
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/Cbo7k85ZRdy--yxmqeykog/elasticsearch/nodes')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, pagination })
-          .expect(200);
-        expect(body).to.eql(nodesListingFixtureCgroup);
-      });
-    });
-
-    describe('with pagination', () => {
-      const archive = 'x-pack/test/functional/es_archives/monitoring/singlecluster_lots_of_nodes';
-      const timeRange = {
-        min: '2019-10-03T19:41:01.402Z',
-        max: '2019-10-03T19:41:13.132Z',
-      };
-
-      before('load clusters archive', () => {
-        return esArchiver.load(archive);
-      });
-
-      after('unload clusters archive', () => {
-        return esArchiver.unload(archive);
-      });
-
-      it('should work with the default pagination', async () => {
-        const pagination = {
-          size: 10,
-          index: 0,
-        };
-
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/Pawu2etQQ9uPwl22Vm_Tug/elasticsearch/nodes')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, pagination })
-          .expect(200);
-
-        const names = body.nodes.map((node) => node.name);
-        expect(body.nodes.length).to.be(pagination.size);
-        expect(names).to.eql(nodesListingFixturePagination.defaultPagination);
-      });
-
-      it('should work with going to page 2', async () => {
-        const pagination = {
-          size: 10,
-          index: 0,
-        };
-
-        const { body: body1 } = await supertest
-          .post('/api/monitoring/v1/clusters/Pawu2etQQ9uPwl22Vm_Tug/elasticsearch/nodes')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, pagination })
-          .expect(200);
-        const names1 = body1.nodes.map((node) => node.name);
-
-        const { body: body2 } = await supertest
-          .post('/api/monitoring/v1/clusters/Pawu2etQQ9uPwl22Vm_Tug/elasticsearch/nodes')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, pagination: { size: 12, index: 1 } })
-          .expect(200);
-        const names2 = body2.nodes.map((node) => node.name);
-
-        for (const name1 of names1) {
-          expect(names2.includes(name1)).to.be(false);
-        }
-        expect(names2.length).to.be(12);
-        expect(names2).to.eql(nodesListingFixturePagination.secondPagePagination);
-      });
-    });
-
-    describe('with sorting', () => {
-      const archive = 'x-pack/test/functional/es_archives/monitoring/singlecluster_lots_of_nodes';
-      const timeRange = {
-        min: '2019-10-03T19:41:01.402Z',
-        max: '2019-10-03T19:41:13.132Z',
-      };
-
-      before('load clusters archive', () => {
-        return esArchiver.load(archive);
-      });
-
-      after('unload clusters archive', () => {
-        return esArchiver.unload(archive);
-      });
-
-      it('should work with the default sorting', async () => {
-        const pagination = {
-          size: 10,
-          index: 0,
-        };
-
-        const sort = {
-          field: 'name',
-          direction: 'asc',
-        };
-
-        const { body: body1 } = await supertest
-          .post('/api/monitoring/v1/clusters/Pawu2etQQ9uPwl22Vm_Tug/elasticsearch/nodes')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, pagination, sort })
-          .expect(200);
-
-        const { body: body2 } = await supertest
-          .post('/api/monitoring/v1/clusters/Pawu2etQQ9uPwl22Vm_Tug/elasticsearch/nodes')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, pagination, sort: { ...sort, direction: 'desc' } })
-          .expect(200);
-
-        const names1 = body1.nodes.map((node) => node.name);
-        const names2 = body2.nodes.map((node) => node.name);
-
-        expect(names1).to.eql(nodesListingFixturePagination.sortByNameAsc);
-        expect(names2).to.eql(nodesListingFixturePagination.sortByNameDesc);
-        for (const name1 of names1) {
-          expect(names2.includes(name1)).to.be(false);
-        }
-      });
-
-      it('should work with sorting by cpu usage', async () => {
-        const pagination = {
-          size: 10,
-          index: 0,
-        };
-
-        const sort = {
-          field: 'node_cpu_utilization',
-          direction: 'asc',
-        };
-
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/Pawu2etQQ9uPwl22Vm_Tug/elasticsearch/nodes')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, pagination, sort })
-          .expect(200);
-
-        const cpuUsage = body.nodes.map((node) => ({
-          name: node.name,
-          cpu_usage: node.node_cpu_utilization.summary,
-        }));
-        expect(cpuUsage).to.eql(nodesListingFixturePagination.sortByCpuUsage);
-      });
-
-      it('should work with sorting by load average', async () => {
-        const pagination = {
-          size: 10,
-          index: 0,
-        };
-
-        const sort = {
-          field: 'node_load_average',
-          direction: 'asc',
-        };
-
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/Pawu2etQQ9uPwl22Vm_Tug/elasticsearch/nodes')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, pagination, sort })
-          .expect(200);
-
-        const loadAverage = body.nodes.map((node) => ({
-          name: node.name,
-          load_average: node.node_load_average.summary,
-        }));
-        expect(loadAverage).to.eql(nodesListingFixturePagination.sortByLoadAverage);
-      });
-
-      it('should work with sorting by jvm memory', async () => {
-        const pagination = {
-          size: 10,
-          index: 0,
-        };
-
-        const sort = {
-          field: 'node_jvm_mem_percent',
-          direction: 'asc',
-        };
-
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/Pawu2etQQ9uPwl22Vm_Tug/elasticsearch/nodes')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, pagination, sort })
-          .expect(200);
-
-        const jvmMemory = body.nodes.map((node) => ({
-          name: node.name,
-          jvm_memory: node.node_jvm_mem_percent.summary,
-        }));
-        expect(jvmMemory).to.eql(nodesListingFixturePagination.sortByJvmMemory);
-      });
-
-      it('should work with sorting by free space', async () => {
-        const pagination = {
-          size: 10,
-          index: 0,
-        };
-
-        const sort = {
-          field: 'node_free_space',
-          direction: 'asc',
-        };
-
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/Pawu2etQQ9uPwl22Vm_Tug/elasticsearch/nodes')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, pagination, sort })
-          .expect(200);
-
-        const freeSpace = body.nodes.map((node) => ({
-          name: node.name,
-          free_space: node.node_free_space.summary,
-        }));
-        expect(freeSpace).to.eql(nodesListingFixturePagination.sortByFreeSpace);
-      });
-    });
-
-    describe('with filtering', () => {
-      const archive = 'x-pack/test/functional/es_archives/monitoring/singlecluster_lots_of_nodes';
-      const timeRange = {
-        min: '2019-10-03T19:41:01.402Z',
-        max: '2019-10-03T19:41:13.132Z',
-      };
-
-      before('load clusters archive', () => {
-        return esArchiver.load(archive);
-      });
-
-      after('unload clusters archive', () => {
-        return esArchiver.unload(archive);
-      });
-
-      it('should work with a simple query', async () => {
-        const pagination = {
-          size: 10,
-          index: 0,
-        };
-        const queryText = '41';
-
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/Pawu2etQQ9uPwl22Vm_Tug/elasticsearch/nodes')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange, pagination, queryText })
-          .expect(200);
-
-        const names = body.nodes.map((node) => node.name);
-        expect(names).to.eql(nodesListingFixturePagination.simpleQuery);
+          it('should return data for 2 active nodes', async () => {
+            const { body } = await supertest
+              .post('/api/monitoring/v1/clusters/fHJwISmKTFO8bj57oFBLUQ/elasticsearch/nodes')
+              .set('kbn-xsrf', 'xxx')
+              .send({ timeRange, pagination })
+              .expect(200);
+            expect(body).to.eql(nodesListingFixtureGreen);
+          });
+        });
       });
     });
   });

x-pack/test/api_integration/apis/monitoring/elasticsearch/overview_mb.js

@@ -8,90 +8,39 @@
 import expect from '@kbn/expect';
 
 import overviewFixtureGreenPlatinum from './fixtures/overview_green_platinum.json';
-import overviewFixtureRedPlatinum from './fixtures/overview_red_platinum.json';
-import overviewFixtureShardsRelocating from './fixtures/overview_shards_relocating.json';
 import { getLifecycleMethods } from '../data_stream';
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
-  const esArchiver = getService('esArchiver');
   const { setup, tearDown } = getLifecycleMethods(getService);
 
-  describe('overview mb', () => {
-    describe('with green platinum cluster', () => {
-      const archive =
-        'x-pack/test/functional/es_archives/monitoring/singlecluster_green_platinum_mb';
-      const timeRange = {
-        min: '2018-02-13T17:04:50.000Z',
-        max: '2018-02-13T17:51:55.000Z',
-      };
+  describe('overview - metricbeat and package', () => {
+    ['mb', 'package'].forEach((source) => {
+      describe(`overview ${source}`, () => {
+        describe('with green platinum cluster', () => {
+          const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_green_platinum_${source}`;
+          const timeRange = {
+            min: '2018-02-13T17:04:50.000Z',
+            max: '2018-02-13T17:51:55.000Z',
+          };
 
-      before('load clusters archive', () => {
-        return setup(archive);
-      });
+          before('load clusters archive', () => {
+            return setup(archive);
+          });
 
-      after('unload clusters archive', async () => {
-        return tearDown();
-      });
+          after('unload clusters archive', async () => {
+            return tearDown(archive);
+          });
 
-      it('should summarize elasticsearch with metrics', async () => {
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/fHJwISmKTFO8bj57oFBLUQ/elasticsearch')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange })
-          .expect(200);
-        expect(body).to.eql(overviewFixtureGreenPlatinum);
-      });
-    });
-
-    describe('with red platinum cluster', () => {
-      const archive = 'x-pack/test/functional/es_archives/monitoring/singlecluster_red_platinum';
-      const timeRange = {
-        min: '2017-10-06T19:53:06.000Z',
-        max: '2017-10-06T20:15:30.000Z',
-      };
-
-      before('load clusters archive', () => {
-        return esArchiver.load(archive);
-      });
-
-      after('unload clusters archive', () => {
-        return esArchiver.unload(archive);
-      });
-
-      it('should summarize elasticsearch with metrics', async () => {
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/1LYuyvCCQFS3FAO_h65PQw/elasticsearch')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange })
-          .expect(200);
-        expect(body).to.eql(overviewFixtureRedPlatinum);
-      });
-    });
-
-    describe('with shards relocating', () => {
-      const archive =
-        'x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation';
-      const timeRange = {
-        min: '2017-10-05T20:31:17.081Z',
-        max: '2017-10-05T20:35:39.428Z',
-      };
-
-      before('load clusters archive', () => {
-        return esArchiver.load(archive);
-      });
-
-      after('unload clusters archive', () => {
-        return esArchiver.unload(archive);
-      });
-
-      it('should summarize elasticsearch with metrics', async () => {
-        const { body } = await supertest
-          .post('/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch')
-          .set('kbn-xsrf', 'xxx')
-          .send({ timeRange })
-          .expect(200);
-        expect(body).to.eql(overviewFixtureShardsRelocating);
+          it('should summarize elasticsearch with metrics', async () => {
+            const { body } = await supertest
+              .post('/api/monitoring/v1/clusters/fHJwISmKTFO8bj57oFBLUQ/elasticsearch')
+              .set('kbn-xsrf', 'xxx')
+              .send({ timeRange })
+              .expect(200);
+            expect(body).to.eql(overviewFixtureGreenPlatinum);
+          });
+        });
       });
     });
   });