[Usage Collection] Report nodes feature usage (#70108)

* Adds nodes feature usage stats merged into cluster_stats.nodes when usage collection is local

src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js

@@ -19,11 +19,12 @@
 
 import expect from '@kbn/expect';
 import sinon from 'sinon';
+import { merge, omit } from 'lodash';
 
+import { TIMEOUT } from '../constants';
 import { mockGetClusterInfo } from './get_cluster_info';
 import { mockGetClusterStats } from './get_cluster_stats';
 
-import { omit } from 'lodash';
 import { getLocalStats, handleLocalStats } from '../get_local_stats';
 
 const mockUsageCollection = (kibanaUsage = {}) => ({
@@ -51,10 +52,26 @@ const getMockServer = (getCluster = sinon.stub()) => ({
     elasticsearch: { getCluster },
   },
 });
+function mockGetNodesUsage(callCluster, nodesUsage, req) {
+  callCluster
+    .withArgs(
+      req,
+      {
+        method: 'GET',
+        path: '/_nodes/usage',
+        query: {
+          timeout: TIMEOUT,
+        },
+      },
+      'transport.request'
+    )
+    .returns(nodesUsage);
+}
 
-function mockGetLocalStats(callCluster, clusterInfo, clusterStats, req) {
+function mockGetLocalStats(callCluster, clusterInfo, clusterStats, nodesUsage, req) {
   mockGetClusterInfo(callCluster, clusterInfo, req);
   mockGetClusterStats(callCluster, clusterStats, req);
+  mockGetNodesUsage(callCluster, nodesUsage, req);
 }
 
 describe('get_local_stats', () => {
@@ -68,6 +85,28 @@ describe('get_local_stats', () => {
       number: version,
     },
   };
+  const nodesUsage = [
+    {
+      node_id: 'some_node_id',
+      timestamp: 1588617023177,
+      since: 1588616945163,
+      rest_actions: {
+        nodes_usage_action: 1,
+        create_index_action: 1,
+        document_get_action: 1,
+        search_action: 19,
+        nodes_info_action: 36,
+      },
+      aggregations: {
+        terms: {
+          bytes: 2,
+        },
+        scripted_metric: {
+          other: 7,
+        },
+      },
+    },
+  ];
   const clusterStats = {
     _nodes: { failed: 123 },
     cluster_name: 'real-cool',
@@ -75,6 +114,7 @@ describe('get_local_stats', () => {
     nodes: { yup: 'abc' },
     random: 123,
   };
+
   const kibana = {
     kibana: {
       great: 'googlymoogly',
@@ -97,12 +137,16 @@ describe('get_local_stats', () => {
     snow: { chances: 0 },
   };
 
+  const clusterStatsWithNodesUsage = {
+    ...clusterStats,
+    nodes: merge(clusterStats.nodes, { usage: nodesUsage }),
+  };
   const combinedStatsResult = {
     collection: 'local',
     cluster_uuid: clusterUuid,
     cluster_name: clusterName,
     version,
-    cluster_stats: omit(clusterStats, '_nodes', 'cluster_name'),
+    cluster_stats: omit(clusterStatsWithNodesUsage, '_nodes', 'cluster_name'),
     stack_stats: {
       kibana: {
         great: 'googlymoogly',
@@ -135,7 +179,7 @@ describe('get_local_stats', () => {
 
   describe('handleLocalStats', () => {
     it('returns expected object without xpack and kibana data', () => {
-      const result = handleLocalStats(clusterInfo, clusterStats, void 0, context);
+      const result = handleLocalStats(clusterInfo, clusterStatsWithNodesUsage, void 0, context);
       expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
       expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
       expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
@@ -146,7 +190,7 @@ describe('get_local_stats', () => {
     });
 
     it('returns expected object with xpack', () => {
-      const result = handleLocalStats(clusterInfo, clusterStats, void 0, context);
+      const result = handleLocalStats(clusterInfo, clusterStatsWithNodesUsage, void 0, context);
       const { stack_stats: stack, ...cluster } = result;
       expect(cluster.collection).to.be(combinedStatsResult.collection);
       expect(cluster.cluster_uuid).to.be(combinedStatsResult.cluster_uuid);
@@ -167,7 +211,8 @@ describe('get_local_stats', () => {
       mockGetLocalStats(
         callClusterUsageFailed,
         Promise.resolve(clusterInfo),
-        Promise.resolve(clusterStats)
+        Promise.resolve(clusterStats),
+        Promise.resolve(nodesUsage)
       );
       const result = await getLocalStats([], {
         server: getMockServer(),
@@ -177,6 +222,7 @@ describe('get_local_stats', () => {
       expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
       expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
       expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
+      expect(result.cluster_stats.nodes).to.eql(combinedStatsResult.cluster_stats.nodes);
       expect(result.version).to.be('2.3.4');
       expect(result.collection).to.be('local');
 
@@ -188,7 +234,12 @@ describe('get_local_stats', () => {
     it('returns expected object with xpack and kibana data', async () => {
       const callCluster = sinon.stub();
       const usageCollection = mockUsageCollection(kibana);
-      mockGetLocalStats(callCluster, Promise.resolve(clusterInfo), Promise.resolve(clusterStats));
+      mockGetLocalStats(
+        callCluster,
+        Promise.resolve(clusterInfo),
+        Promise.resolve(clusterStats),
+        Promise.resolve(nodesUsage)
+      );
 
       const result = await getLocalStats([], {
         server: getMockServer(callCluster),

src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts

@@ -24,6 +24,7 @@ import {
 import { getClusterInfo, ESClusterInfo } from './get_cluster_info';
 import { getClusterStats } from './get_cluster_stats';
 import { getKibana, handleKibanaStats, KibanaUsageStats } from './get_kibana';
+import { getNodesUsage } from './get_nodes_usage';
 
 /**
  * Handle the separate local calls by combining them into a single object response that looks like the
@@ -67,12 +68,21 @@ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async (
 
   return await Promise.all(
     clustersDetails.map(async (clustersDetail) => {
-      const [clusterInfo, clusterStats, kibana] = await Promise.all([
+      const [clusterInfo, clusterStats, nodesUsage, kibana] = await Promise.all([
         getClusterInfo(callCluster), // cluster info
         getClusterStats(callCluster), // cluster stats (not to be confused with cluster _state_)
+        getNodesUsage(callCluster), // nodes_usage info
         getKibana(usageCollection, callCluster),
       ]);
-      return handleLocalStats(clusterInfo, clusterStats, kibana, context);
+      return handleLocalStats(
+        clusterInfo,
+        {
+          ...clusterStats,
+          nodes: { ...clusterStats.nodes, usage: nodesUsage },
+        },
+        kibana,
+        context
+      );
     })
   );
 };

src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts

@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { getNodesUsage } from './get_nodes_usage';
+import { TIMEOUT } from './constants';
+
+const mockedNodesFetchResponse = {
+  cluster_name: 'test cluster',
+  nodes: {
+    some_node_id: {
+      timestamp: 1588617023177,
+      since: 1588616945163,
+      rest_actions: {
+        nodes_usage_action: 1,
+        create_index_action: 1,
+        document_get_action: 1,
+        search_action: 19,
+        nodes_info_action: 36,
+      },
+      aggregations: {
+        terms: {
+          bytes: 2,
+        },
+        scripted_metric: {
+          other: 7,
+        },
+      },
+    },
+  },
+};
+describe('get_nodes_usage', () => {
+  it('calls fetchNodesUsage', async () => {
+    const callCluster = jest.fn();
+    callCluster.mockResolvedValueOnce(mockedNodesFetchResponse);
+    await getNodesUsage(callCluster);
+    expect(callCluster).toHaveBeenCalledWith('transport.request', {
+      path: '/_nodes/usage',
+      method: 'GET',
+      query: {
+        timeout: TIMEOUT,
+      },
+    });
+  });
+  it('returns a modified array of node usage data', async () => {
+    const callCluster = jest.fn();
+    callCluster.mockResolvedValueOnce(mockedNodesFetchResponse);
+    const result = await getNodesUsage(callCluster);
+    expect(result.nodes).toEqual([
+      {
+        aggregations: { scripted_metric: { other: 7 }, terms: { bytes: 2 } },
+        node_id: 'some_node_id',
+        rest_actions: {
+          create_index_action: 1,
+          document_get_action: 1,
+          nodes_info_action: 36,
+          nodes_usage_action: 1,
+          search_action: 19,
+        },
+        since: 1588616945163,
+        timestamp: 1588617023177,
+      },
+    ]);
+  });
+});

src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts

@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { LegacyAPICaller } from 'kibana/server';
+import { TIMEOUT } from './constants';
+
+export interface NodeAggregation {
+  [key: string]: number;
+}
+
+// we set aggregations as an optional type because it was only added in v7.8.0
+export interface NodeObj {
+  node_id?: string;
+  timestamp: number;
+  since: number;
+  rest_actions: {
+    [key: string]: number;
+  };
+  aggregations?: {
+    [key: string]: NodeAggregation;
+  };
+}
+
+export interface NodesFeatureUsageResponse {
+  cluster_name: string;
+  nodes: {
+    [key: string]: NodeObj;
+  };
+}
+
+export type NodesUsageGetter = (
+  callCluster: LegacyAPICaller
+) => Promise<{ nodes: NodeObj[] | Array<{}> }>;
+/**
+ * Get the nodes usage data from the connected cluster.
+ *
+ * This is the equivalent to GET /_nodes/usage?timeout=30s.
+ *
+ * The Nodes usage API was introduced in v6.0.0
+ */
+export async function fetchNodesUsage(
+  callCluster: LegacyAPICaller
+): Promise<NodesFeatureUsageResponse> {
+  const response = await callCluster('transport.request', {
+    method: 'GET',
+    path: '/_nodes/usage',
+    query: {
+      timeout: TIMEOUT,
+    },
+  });
+  return response;
+}
+
+/**
+ * Get the nodes usage from the connected cluster
+ * @param callCluster APICaller
+ * @returns Object containing array of modified usage information with the node_id nested within the data for that node.
+ */
+export const getNodesUsage: NodesUsageGetter = async (callCluster) => {
+  const result = await fetchNodesUsage(callCluster);
+  const transformedNodes = Object.entries(result?.nodes || {}).map(([key, value]) => ({
+    ...(value as NodeObj),
+    node_id: key,
+  }));
+  return { nodes: transformedNodes };
+};

test/api_integration/apis/telemetry/telemetry_local.js

@@ -113,6 +113,7 @@ export default function ({ getService }) {
         'cluster_stats.nodes.plugins',
         'cluster_stats.nodes.process',
         'cluster_stats.nodes.versions',
+        'cluster_stats.nodes.usage',
         'cluster_stats.status',
         'cluster_stats.timestamp',
         'cluster_uuid',

x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__snapshots__/get_stats_with_xpack.test.ts.snap

@@ -4,7 +4,27 @@ exports[`Telemetry Collection: Get Aggregated Stats OSS-like telemetry (no licen
 Array [
   Object {
     "cluster_name": "test",
-    "cluster_stats": Object {},
+    "cluster_stats": Object {
+      "nodes": Object {
+        "usage": Object {
+          "nodes": Array [
+            Object {
+              "aggregations": Object {
+                "terms": Object {
+                  "bytes": 2,
+                },
+              },
+              "node_id": "some_node_id",
+              "rest_actions": Object {
+                "nodes_usage_action": 1,
+              },
+              "since": 1588616945163,
+              "timestamp": 1588617023177,
+            },
+          ],
+        },
+      },
+    },
     "cluster_uuid": "test",
     "collection": "local",
     "stack_stats": Object {
@@ -62,7 +82,27 @@ exports[`Telemetry Collection: Get Aggregated Stats X-Pack telemetry (license +
 Array [
   Object {
     "cluster_name": "test",
-    "cluster_stats": Object {},
+    "cluster_stats": Object {
+      "nodes": Object {
+        "usage": Object {
+          "nodes": Array [
+            Object {
+              "aggregations": Object {
+                "terms": Object {
+                  "bytes": 2,
+                },
+              },
+              "node_id": "some_node_id",
+              "rest_actions": Object {
+                "nodes_usage_action": 1,
+              },
+              "since": 1588616945163,
+              "timestamp": 1588617023177,
+            },
+          ],
+        },
+      },
+    },
     "cluster_uuid": "test",
     "collection": "local",
     "stack_stats": Object {

x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts

@@ -28,6 +28,20 @@ const kibana = {
   rain: { chances: 2 },
   snow: { chances: 0 },
 };
+const nodesUsage = {
+  some_node_id: {
+    timestamp: 1588617023177,
+    since: 1588616945163,
+    rest_actions: {
+      nodes_usage_action: 1,
+    },
+    aggregations: {
+      terms: {
+        bytes: 2,
+      },
+    },
+  },
+};
 
 const getContext = () => ({
   version: '8675309-snapshot',
@@ -47,6 +61,11 @@ describe('Telemetry Collection: Get Aggregated Stats', () => {
           if (options.path === '/_license' || options.path === '/_xpack/usage') {
             // eslint-disable-next-line no-throw-literal
             throw { statusCode: 404 };
+          } else if (options.path === '/_nodes/usage') {
+            return {
+              cluster_name: 'test cluster',
+              nodes: nodesUsage,
+            };
           }
           return {};
         case 'info':
@@ -81,6 +100,12 @@ describe('Telemetry Collection: Get Aggregated Stats', () => {
           if (options.path === '/_xpack/usage') {
             return {};
           }
+          if (options.path === '/_nodes/usage') {
+            return {
+              cluster_name: 'test cluster',
+              nodes: nodesUsage,
+            };
+          }
         case 'info':
           return { cluster_uuid: 'test', cluster_name: 'test', version: { number: '8.0.0' } };
         default: