[XpackMain] 6.x Backport Add _xpack/usage API (#19232) (#19287)

*  [XpackMain] Add _xpack/usage API (#19232)

* [XpackMain] Add _xpack/usage API

* add xpack usage http api integration test

* comment

* misc test describe fixes

* fix integration test

* fix reply called twice

* enable api test

* enable kibana collection for integration test to work

* throw error comment

* Update config.js

remove whitespace change

x-pack/plugins/monitoring/server/kibana_monitoring/classes/collector_set.js

@@ -136,9 +136,14 @@ export class CollectorSet {
 
     // summarize each type of stat
     return bulk.reduce((accumulatedStats, currentStat) => {
+      /* Suffix removal is a temporary hack: some types have `_stats` suffix
+       * because of how monitoring bulk upload needed to combine types. It can
+       * be removed when bulk upload goes away
+       */
+      const statType = currentStat.type.replace('_stats', '');
       return {
         ...accumulatedStats,
-        [currentStat.type]: currentStat.result,
+        [statType]: currentStat.result,
       };
     }, {});
   }

x-pack/plugins/xpack_main/index.js

@@ -15,6 +15,7 @@ import { replaceInjectedVars } from './server/lib/replace_injected_vars';
 import { setupXPackMain } from './server/lib/setup_xpack_main';
 import {
   xpackInfoRoute,
+  xpackUsageRoute,
   kibanaStatsRoute,
   telemetryRoute,
 } from './server/routes/api/v1';
@@ -110,7 +111,8 @@ export const xpackMain = (kibana) => {
 
       // register routes
       xpackInfoRoute(server);
-      kibanaStatsRoute(server);
+      xpackUsageRoute(server); // To replace kibanaStatsRoute
+      kibanaStatsRoute(server); // Only used internally. Remove in the next major.
       telemetryRoute(server);
     }
   });

x-pack/plugins/xpack_main/server/routes/api/v1/index.js

@@ -6,4 +6,5 @@
 
 export { xpackInfoRoute } from './xpack_info';
 export { kibanaStatsRoute } from './kibana_stats';
-export { telemetryRoute } from './telemetry';
+export { xpackUsageRoute } from './xpack_usage';
+export { telemetryRoute } from './telemetry';

x-pack/plugins/xpack_main/server/routes/api/v1/kibana_stats.js

@@ -4,6 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+/*
+ * TODO: deprecate this API in 7.0
+ */
 import { wrap } from 'boom';
 import { callClusterFactory } from '../../../lib/call_cluster_factory';
 import { getKibanaUsageCollector, getReportingUsageCollector } from '../../../../../monitoring/server/kibana_monitoring';

x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { wrap, serverTimeout as serverUnavailable } from 'boom';
+
+const getClusterUuid = async req => {
+  const { server } = req;
+  const { callWithRequest, } = server.plugins.elasticsearch.getCluster('data');
+  const { cluster_uuid: uuid } = await callWithRequest(req, 'info', { filterPath: 'cluster_uuid', });
+  return uuid;
+};
+
+/*
+ * @return {Object} data from usage stats collectors registered with Monitoring CollectorSet
+ * @throws {Error} if the Monitoring CollectorSet is not ready
+ */
+const getUsage = async req => {
+  const server = req.server;
+  const { collectorSet } = server.plugins.monitoring;
+  if (collectorSet === undefined) {
+    const error = new Error('CollectorSet from Monitoring plugin is not ready for collecting usage'); // moving kibana_monitoring lib to xpack_main will make this unnecessary
+    throw serverUnavailable(error);
+  }
+  return collectorSet.bulkFetchUsage();
+};
+
+export function xpackUsageRoute(server) {
+  server.route({
+    path: '/api/_xpack/usage',
+    method: 'GET',
+    async handler(req, reply) {
+      try {
+        const [ clusterUuid, xpackUsage ] = await Promise.all([
+          getClusterUuid(req),
+          getUsage(req),
+        ]);
+
+        reply({
+          cluster_uuid: clusterUuid,
+          ...xpackUsage
+        });
+      } catch(err) {
+        req.log(['error'], err);
+        if (err.isBoom) {
+          reply(err);
+        } else {
+          reply(wrap(err, err.statusCode, err.message));
+        }
+      }
+    }
+  });
+}

x-pack/test/api_integration/apis/xpack_main/index.js

@@ -7,6 +7,6 @@
 export default function ({ loadTestFile }) {
   describe('xpack_main', () => {
     loadTestFile(require.resolve('./telemetry'));
-    loadTestFile(require.resolve('./stats'));
+    loadTestFile(require.resolve('./usage'));
   });
 }

x-pack/test/api_integration/apis/xpack_main/stats/stats.js

@@ -1,87 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-
-import expect from 'expect.js';
-
-export default function ({ getService }) {
-  const supertest = getService('supertest');
-  const esArchiver = getService('esArchiver');
-
-  describe('stats API', () => {
-    before('load archives', async () => {
-      // Using this archive because it includes reports as well as a range of visualization types.
-      await esArchiver.load('reporting/6_2');
-      // Not really neccessary to have data indexes, but it feels incomplete to leave out, and it is possible that
-      // having data available could potentially interefere with the stats api (unlikely... but possible).
-      await esArchiver.load('logstash_functional');
-    });
-
-    after(async () => {
-      await esArchiver.unload('reporting/6_2');
-      await esArchiver.unload('logstash_functional');
-    });
-
-    describe('from archived data', () => {
-      let stats;
-
-      before('load stats', async () => {
-        const { body } = await supertest
-          .get(`/api/_kibana/v1/stats`)
-          .set('kbn-xsrf', 'xxx')
-          .expect(200);
-
-        stats = body;
-      });
-
-      it('counts 3 dashboards', async () => {
-        expect(stats.kibana.dashboard.total).to.be(3);
-      });
-
-      it('counts 18 visualizations', async () => {
-        expect(stats.kibana.visualization.total).to.be(18);
-      });
-
-      it('counts 1 saved search', async () => {
-        expect(stats.kibana.search.total).to.be(1);
-      });
-
-      it('counts 1 index pattern', async () => {
-        expect(stats.kibana.index_pattern.total).to.be(1);
-      });
-
-      it('counts 0 timelion_sheets', async () => {
-        expect(stats.kibana.timelion_sheet.total).to.be(0);
-      });
-
-      it('counts 0 graph workspaces', async () => {
-        expect(stats.kibana.graph_workspace.total).to.be(0);
-      });
-
-      it('shows reporting as available and enabled', async () => {
-        expect(stats.reporting.available).to.be(true);
-        expect(stats.reporting.enabled).to.be(true);
-      });
-
-      it('is using phantom browser', async () => {
-        expect(stats.reporting.browser_type).to.be('phantom');
-      });
-
-      it('counts 8 total reports', async () => {
-        expect(stats.reporting._all).to.be(8);
-      });
-
-      it('counts 1 csv report', async () => {
-        expect(stats.reporting.csv.available).to.be(true);
-        expect(stats.reporting.csv.total).to.be(1);
-      });
-
-      it('counts 7 printable_pdf reports', async () => {
-        expect(stats.reporting.printable_pdf.available).to.be(true);
-        expect(stats.reporting.printable_pdf.total).to.be(7);
-      });
-    });
-  });
-}

x-pack/test/api_integration/apis/xpack_main/telemetry/index.js

@@ -5,7 +5,7 @@
  */
 
 export default function ({ loadTestFile }) {
-  describe('telemetry', () => {
+  describe('Telemetry', () => {
     loadTestFile(require.resolve('./telemetry'));
   });
 }

x-pack/test/api_integration/apis/xpack_main/telemetry/telemetry.js

@@ -11,7 +11,7 @@ export default function ({ getService }) {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
 
-  describe('clusters/_stats', () => {
+  describe('/api/telemetry/v1/clusters/_stats', () => {
     describe('with trial license clusters', () => {
       const archive = 'monitoring/multicluster';
       const timeRange = {

x-pack/test/api_integration/apis/xpack_main/usage/index.js

@@ -5,7 +5,8 @@
  */
 
 export default function ({ loadTestFile }) {
-  describe('stats', () => {
-    loadTestFile(require.resolve('./stats'));
+  describe('X-Pack Usage', () => {
+    loadTestFile(require.resolve('./usage'));
   });
 }
+

x-pack/test/api_integration/apis/xpack_main/usage/usage.js

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from 'expect.js';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+
+  describe('/api/_xpack/usage', () => {
+    before('load archives', async () => {
+      // Using this archive because it includes reports as well as a range of visualization types.
+      await esArchiver.load('reporting/6_2');
+      // Not really neccessary to have data indexes, but it feels incomplete to leave out, and it is possible that
+      // having data available could potentially interefere with the stats api (unlikely... but possible).
+      await esArchiver.load('logstash_functional');
+    });
+
+    after(async () => {
+      await esArchiver.unload('reporting/6_2');
+      await esArchiver.unload('logstash_functional');
+    });
+
+
+    it('should return xpack usage data', async () => {
+      const { body } = await supertest
+        .get(`/api/_xpack/usage`)
+        .set('kbn-xsrf', 'xxx')
+        .expect(200);
+
+      expect(body.cluster_uuid).to.be.a('string');
+      expect(body.kibana.dashboard.total).to.be(3);
+      expect(body.kibana.visualization.total).to.be(18);
+      expect(body.kibana.search.total).to.be(1);
+      expect(body.kibana.index_pattern.total).to.be(1);
+      expect(body.kibana.timelion_sheet.total).to.be(0);
+      expect(body.kibana.graph_workspace.total).to.be(0);
+      expect(body.reporting.available).to.be(true);
+      expect(body.reporting.enabled).to.be(true);
+      expect(body.reporting.browser_type).to.be('phantom');
+      expect(body.reporting._all).to.be(8);
+      expect(body.reporting.csv.available).to.be(true);
+      expect(body.reporting.csv.total).to.be(1);
+      expect(body.reporting.printable_pdf.available).to.be(true);
+      expect(body.reporting.printable_pdf.total).to.be(7);
+    });
+  });
+}

x-pack/test/functional/config.js

@@ -164,7 +164,6 @@ export default async function ({ readConfigFile }) {
         `--server.uuid=${env.kibana.server.uuid}`,
         `--server.port=${servers.kibana.port}`,
         `--elasticsearch.url=${formatUrl(servers.elasticsearch)}`,
-        '--xpack.monitoring.kibana.collection.enabled=false',
         '--xpack.xpack_main.telemetry.enabled=false',
         '--xpack.security.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"', // server restarts should not invalidate active sessions
       ],