[6.x] [ML] adds telemetry to ML (#29121) (#29939)

Adds telemetry to collect the amount of indices created using File Data Visualizer.

x-pack/plugins/ml/common/constants/file_datavisualizer.ts

@@ -4,9 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-
 export const MAX_BYTES = 104857600;
 
 // Value to use in the Elasticsearch index mapping meta data to identify the
 // index as having been created by the ML File Data Visualizer.
-export const INDEX_META_DATA_CREATED_BY = 'ml_file_data_visualizer';
+export const INDEX_META_DATA_CREATED_BY = 'ml-file-data-visualizer';

x-pack/plugins/ml/index.js

@@ -17,6 +17,8 @@ import { jobRoutes } from './server/routes/anomaly_detectors';
 import { dataFeedRoutes } from './server/routes/datafeeds';
 import { indicesRoutes } from './server/routes/indices';
 import { jobValidationRoutes } from './server/routes/job_validation';
+import mappings from './mappings';
+import { makeMlUsageCollector } from './server/lib/ml_telemetry';
 import { notificationRoutes } from './server/routes/notification_settings';
 import { systemRoutes } from './server/routes/system';
 import { dataRecognizer } from './server/routes/modules';
@@ -52,6 +54,12 @@ export const ml = (kibana) => {
       },
       styleSheetPaths: resolve(__dirname, 'public/index.scss'),
       hacks: ['plugins/ml/hacks/toggle_app_link_in_nav'],
+      savedObjectSchemas: {
+        'ml-telemetry': {
+          isNamespaceAgnostic: true
+        }
+      },
+      mappings,
       home: ['plugins/ml/register_feature'],
       injectDefaultVars(server) {
         const config = server.config();
@@ -111,6 +119,7 @@ export const ml = (kibana) => {
       fileDataVisualizerRoutes(server, commonRouteConfig);
 
       initMlServerLog(server);
+      makeMlUsageCollector(server);
     }
 
   });

x-pack/plugins/ml/mappings.json

@@ -0,0 +1,13 @@
+{
+  "ml-telemetry": {
+    "properties": {
+      "file_data_visualizer": {
+        "properties": {
+          "index_creation_count": {
+            "type" : "long"
+          }
+        }
+      }
+    }
+  }
+}

x-pack/plugins/ml/server/client/call_with_internal_user_factory.d.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Server } from 'hapi';
+
+export function callWithInternalUserFactory(server: Server): any;

x-pack/plugins/ml/server/client/call_with_internal_user_factory.test.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { callWithInternalUserFactory } from './call_with_internal_user_factory';
+
+describe('call_with_internal_user_factory', () => {
+  describe('callWithInternalUserFactory', () => {
+    let server: any;
+    let callWithInternalUser: any;
+
+    beforeEach(() => {
+      callWithInternalUser = jest.fn();
+      server = {
+        plugins: {
+          elasticsearch: {
+            getCluster: jest.fn(() => ({ callWithInternalUser })),
+          },
+        },
+      };
+    });
+
+    it('should use internal user "admin"', () => {
+      const callWithInternalUserInstance = callWithInternalUserFactory(server);
+      callWithInternalUserInstance();
+
+      expect(server.plugins.elasticsearch.getCluster).toHaveBeenCalledWith('admin');
+    });
+  });
+});

x-pack/plugins/ml/server/lib/ml_telemetry/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './ml_telemetry';
+export { makeMlUsageCollector } from './make_ml_usage_collector';

x-pack/plugins/ml/server/lib/ml_telemetry/make_ml_usage_collector.ts

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Server } from 'hapi';
+
+import {
+  createMlTelemetry,
+  getSavedObjectsClient,
+  ML_TELEMETRY_DOC_ID,
+  MlTelemetry,
+  MlTelemetrySavedObject,
+} from './ml_telemetry';
+
+// TODO this type should be defined by the platform
+interface KibanaHapiServer extends Server {
+  usage: {
+    collectorSet: {
+      makeUsageCollector: any;
+      register: any;
+    };
+  };
+}
+
+export function makeMlUsageCollector(server: KibanaHapiServer): void {
+  const mlUsageCollector = server.usage.collectorSet.makeUsageCollector({
+    type: 'ml',
+    fetch: async (): Promise<MlTelemetry> => {
+      try {
+        const savedObjectsClient = getSavedObjectsClient(server);
+        const mlTelemetrySavedObject = (await savedObjectsClient.get(
+          'ml-telemetry',
+          ML_TELEMETRY_DOC_ID
+        )) as MlTelemetrySavedObject;
+        return mlTelemetrySavedObject.attributes;
+      } catch (err) {
+        return createMlTelemetry();
+      }
+    },
+  });
+  server.usage.collectorSet.register(mlUsageCollector);
+}

x-pack/plugins/ml/server/lib/ml_telemetry/ml_telemetry.test.ts

@@ -0,0 +1,199 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  createMlTelemetry,
+  getSavedObjectsClient,
+  incrementFileDataVisualizerIndexCreationCount,
+  ML_TELEMETRY_DOC_ID,
+  MlTelemetry,
+  storeMlTelemetry,
+} from './ml_telemetry';
+
+describe('ml_telemetry', () => {
+  describe('createMlTelemetry', () => {
+    it('should create a MlTelemetry object', () => {
+      const mlTelemetry = createMlTelemetry(1);
+      expect(mlTelemetry.file_data_visualizer.index_creation_count).toBe(1);
+    });
+    it('should ignore undefined or unknown values', () => {
+      const mlTelemetry = createMlTelemetry(undefined);
+      expect(mlTelemetry.file_data_visualizer.index_creation_count).toBe(0);
+    });
+  });
+
+  describe('storeMlTelemetry', () => {
+    let server: any;
+    let mlTelemetry: MlTelemetry;
+    let savedObjectsClientInstance: any;
+
+    beforeEach(() => {
+      savedObjectsClientInstance = { create: jest.fn() };
+      const callWithInternalUser = jest.fn();
+      const internalRepository = jest.fn();
+      server = {
+        savedObjects: {
+          SavedObjectsClient: jest.fn(() => savedObjectsClientInstance),
+          getSavedObjectsRepository: jest.fn(() => internalRepository),
+        },
+        plugins: {
+          elasticsearch: {
+            getCluster: jest.fn(() => ({ callWithInternalUser })),
+          },
+        },
+      };
+      mlTelemetry = {
+        file_data_visualizer: {
+          index_creation_count: 1,
+        },
+      };
+    });
+
+    it('should call savedObjectsClient create with the given MlTelemetry object', () => {
+      storeMlTelemetry(server, mlTelemetry);
+      expect(savedObjectsClientInstance.create.mock.calls[0][1]).toBe(mlTelemetry);
+    });
+
+    it('should call savedObjectsClient create with the ml-telemetry document type and ID', () => {
+      storeMlTelemetry(server, mlTelemetry);
+      expect(savedObjectsClientInstance.create.mock.calls[0][0]).toBe('ml-telemetry');
+      expect(savedObjectsClientInstance.create.mock.calls[0][2].id).toBe(ML_TELEMETRY_DOC_ID);
+    });
+
+    it('should call savedObjectsClient create with overwrite: true', () => {
+      storeMlTelemetry(server, mlTelemetry);
+      expect(savedObjectsClientInstance.create.mock.calls[0][2].overwrite).toBe(true);
+    });
+  });
+
+  describe('getSavedObjectsClient', () => {
+    let server: any;
+    let savedObjectsClientInstance: any;
+    let callWithInternalUser: any;
+    let internalRepository: any;
+
+    beforeEach(() => {
+      savedObjectsClientInstance = { create: jest.fn() };
+      callWithInternalUser = jest.fn();
+      internalRepository = jest.fn();
+      server = {
+        savedObjects: {
+          SavedObjectsClient: jest.fn(() => savedObjectsClientInstance),
+          getSavedObjectsRepository: jest.fn(() => internalRepository),
+        },
+        plugins: {
+          elasticsearch: {
+            getCluster: jest.fn(() => ({ callWithInternalUser })),
+          },
+        },
+      };
+    });
+
+    it('should return a SavedObjectsClient initialized with the saved objects internal repository', () => {
+      const result = getSavedObjectsClient(server);
+
+      expect(result).toBe(savedObjectsClientInstance);
+      expect(server.savedObjects.SavedObjectsClient).toHaveBeenCalledWith(internalRepository);
+    });
+  });
+
+  describe('incrementFileDataVisualizerIndexCreationCount', () => {
+    let server: any;
+    let savedObjectsClientInstance: any;
+    let callWithInternalUser: any;
+    let internalRepository: any;
+
+    function createSavedObjectsClientInstance(
+      telemetryEnabled?: boolean,
+      indexCreationCount?: number
+    ) {
+      return {
+        create: jest.fn(),
+        get: jest.fn(obj => {
+          switch (obj) {
+            case 'telemetry':
+              if (telemetryEnabled === undefined) {
+                throw Error;
+              }
+              return {
+                attributes: {
+                  telemetry: {
+                    enabled: telemetryEnabled,
+                  },
+                },
+              };
+            case 'ml-telemetry':
+              // emulate that a non-existing saved object will throw an error
+              if (indexCreationCount === undefined) {
+                throw Error;
+              }
+              return {
+                attributes: {
+                  file_data_visualizer: {
+                    index_creation_count: indexCreationCount,
+                  },
+                },
+              };
+          }
+        }),
+      };
+    }
+
+    function mockInit(telemetryEnabled?: boolean, indexCreationCount?: number): void {
+      savedObjectsClientInstance = createSavedObjectsClientInstance(
+        telemetryEnabled,
+        indexCreationCount
+      );
+      callWithInternalUser = jest.fn();
+      internalRepository = jest.fn();
+      server = {
+        savedObjects: {
+          SavedObjectsClient: jest.fn(() => savedObjectsClientInstance),
+          getSavedObjectsRepository: jest.fn(() => internalRepository),
+        },
+        plugins: {
+          elasticsearch: {
+            getCluster: jest.fn(() => ({ callWithInternalUser })),
+          },
+        },
+      };
+    }
+
+    it('should not increment if telemetry status cannot be determined', async () => {
+      mockInit();
+      await incrementFileDataVisualizerIndexCreationCount(server);
+
+      expect(savedObjectsClientInstance.create.mock.calls).toHaveLength(0);
+    });
+
+    it('should not increment if telemetry status is disabled', async () => {
+      mockInit(false);
+      await incrementFileDataVisualizerIndexCreationCount(server);
+
+      expect(savedObjectsClientInstance.create.mock.calls).toHaveLength(0);
+    });
+
+    it('should initialize index_creation_count with 1', async () => {
+      mockInit(true);
+      await incrementFileDataVisualizerIndexCreationCount(server);
+
+      expect(savedObjectsClientInstance.create.mock.calls[0][0]).toBe('ml-telemetry');
+      expect(savedObjectsClientInstance.create.mock.calls[0][1]).toEqual({
+        file_data_visualizer: { index_creation_count: 1 },
+      });
+    });
+
+    it('should increment index_creation_count to 2', async () => {
+      mockInit(true, 1);
+      await incrementFileDataVisualizerIndexCreationCount(server);
+
+      expect(savedObjectsClientInstance.create.mock.calls[0][0]).toBe('ml-telemetry');
+      expect(savedObjectsClientInstance.create.mock.calls[0][1]).toEqual({
+        file_data_visualizer: { index_creation_count: 2 },
+      });
+    });
+  });
+});

x-pack/plugins/ml/server/lib/ml_telemetry/ml_telemetry.ts

@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Server } from 'hapi';
+import { callWithInternalUserFactory } from '../../client/call_with_internal_user_factory';
+
+export interface MlTelemetry {
+  file_data_visualizer: {
+    index_creation_count: number;
+  };
+}
+
+export interface MlTelemetrySavedObject {
+  attributes: MlTelemetry;
+}
+
+export const ML_TELEMETRY_DOC_ID = 'ml-telemetry';
+
+export function createMlTelemetry(count: number = 0): MlTelemetry {
+  return {
+    file_data_visualizer: {
+      index_creation_count: count,
+    },
+  };
+}
+
+export function storeMlTelemetry(server: Server, mlTelemetry: MlTelemetry): void {
+  const savedObjectsClient = getSavedObjectsClient(server);
+  savedObjectsClient.create('ml-telemetry', mlTelemetry, {
+    id: ML_TELEMETRY_DOC_ID,
+    overwrite: true,
+  });
+}
+
+export function getSavedObjectsClient(server: Server): any {
+  const { SavedObjectsClient, getSavedObjectsRepository } = server.savedObjects;
+  const callWithInternalUser = callWithInternalUserFactory(server);
+  const internalRepository = getSavedObjectsRepository(callWithInternalUser);
+  return new SavedObjectsClient(internalRepository);
+}
+
+export async function incrementFileDataVisualizerIndexCreationCount(server: Server): Promise<void> {
+  const savedObjectsClient = getSavedObjectsClient(server);
+
+  try {
+    const { attributes } = await savedObjectsClient.get('telemetry', 'telemetry');
+    if (attributes.telemetry.enabled === false) {
+      return;
+    }
+  } catch (error) {
+    // if we aren't allowed to get the telemetry document,
+    // we assume we couldn't opt in to telemetry and won't increment the index count.
+    return;
+  }
+
+  let indicesCount = 1;
+
+  try {
+    const { attributes } = (await savedObjectsClient.get(
+      'ml-telemetry',
+      ML_TELEMETRY_DOC_ID
+    )) as MlTelemetrySavedObject;
+    indicesCount = attributes.file_data_visualizer.index_creation_count + 1;
+  } catch (e) {
+    /* silently fail, this will happen if the saved object doesn't exist yet. */
+  }
+
+  const mlTelemetry = createMlTelemetry(indicesCount);
+  storeMlTelemetry(server, mlTelemetry);
+}

x-pack/plugins/ml/server/routes/file_data_visualizer.js

@@ -4,12 +4,13 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-
 import { callWithRequestFactory } from '../client/call_with_request_factory';
 import { wrapError } from '../client/errors';
 import { fileDataVisualizerProvider, importDataProvider } from '../models/file_data_visualizer';
 import { MAX_BYTES } from '../../common/constants/file_datavisualizer';
 
+import { incrementFileDataVisualizerIndexCreationCount } from '../lib/ml_telemetry/ml_telemetry';
+
 function analyzeFiles(callWithRequest, data, overrides) {
   const { analyzeFile } = fileDataVisualizerProvider(callWithRequest);
   return analyzeFile(data, overrides);
@@ -45,6 +46,13 @@ export function fileDataVisualizerRoutes(server, commonRouteConfig) {
       const { id } = request.query;
       const { index, data, settings, mappings, ingestPipeline } = request.payload;
 
+      // `id` being `undefined` tells us that this is a new import due to create a new index.
+      // follow-up import calls to just add additional data will include the `id` of the created
+      // index, we'll ignore those and don't increment the counter.
+      if (id === undefined) {
+        incrementFileDataVisualizerIndexCreationCount(server);
+      }
+
       return importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data)
         .catch(wrapError);
     },

x-pack/test/api_integration/apis/xpack_main/telemetry/telemetry_local.js

@@ -165,6 +165,7 @@ export default function ({ getService }) {
         'stack_stats.kibana.plugins.kql.defaultQueryLanguage',
         'stack_stats.kibana.plugins.kql.optInCount',
         'stack_stats.kibana.plugins.kql.optOutCount',
+        'stack_stats.kibana.plugins.ml.file_data_visualizer.index_creation_count',
         'stack_stats.kibana.plugins.reporting.PNG.available',
         'stack_stats.kibana.plugins.reporting.PNG.total',
         'stack_stats.kibana.plugins.reporting._all',