[6.7] [Telemetry] Localization Stats (#29213) (#31243)

Backports the following commits to 6.7:
 - [Telemetry] Localization Stats  (#29213)

src/server/i18n/index.js

@@ -62,4 +62,5 @@ export async function i18nMixin(kbnServer, server, config) {
     locale,
     ...translations,
   }));
+  server.decorate('server', 'getTranslationsFilePaths', () => translationPaths);
 }

src/server/status/lib/get_kibana_info_for_stats.js

@@ -38,6 +38,7 @@ export function getKibanaInfoForStats(server, kbnServer) {
     name: config.get('server.name'),
     index: config.get('kibana.index'),
     host: config.get('server.host'),
+    locale: config.get('i18n.locale'),
     transport_address: `${config.get('server.host')}:${config.get('server.port')}`,
     version: kbnServer.version.replace(snapshotRegex, ''),
     snapshot: snapshotRegex.test(kbnServer.version),

x-pack/plugins/xpack_main/common/constants.js

@@ -63,3 +63,9 @@ export const LOCALSTORAGE_KEY = 'xpack.data';
  * Link to the Elastic Telemetry privacy statement.
  */
 export const PRIVACY_STATEMENT_URL = `https://www.elastic.co/legal/telemetry-privacy-statement`;
+
+/**
+ * The type name used within the Monitoring index to publish localization stats.
+ * @type {string}
+ */
+export const KIBANA_LOCALIZATION_STATS_TYPE = 'localization';

x-pack/plugins/xpack_main/index.js

@@ -13,6 +13,7 @@ import {
 import { mirrorPluginStatus } from '../../server/lib/mirror_plugin_status';
 import { replaceInjectedVars } from './server/lib/replace_injected_vars';
 import { setupXPackMain } from './server/lib/setup_xpack_main';
+import { getLocalizationUsageCollector } from './server/lib/get_localization_usage_collector';
 import {
   xpackInfoRoute,
   telemetryRoute,
@@ -133,6 +134,7 @@ export const xpackMain = (kibana) => {
       xpackInfoRoute(server);
       telemetryRoute(server);
       settingsRoute(server, this.kbnServer);
+      server.usage.collectorSet.register(getLocalizationUsageCollector(server));
     }
   });
 };

x-pack/plugins/xpack_main/server/lib/file_integrity.test.ts

@@ -0,0 +1,56 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Readable } from 'stream';
+
+jest.mock('fs', () => ({
+  createReadStream(filepath: string): Readable {
+    if (filepath === 'ERROR') {
+      throw new Error('MOCK ERROR - Invalid Path');
+    }
+    const readableStream = new Readable();
+    const streamData = filepath.split('');
+    let cursor = 0;
+
+    readableStream._read = function(size) {
+      const current = streamData[cursor++];
+      if (typeof current === 'undefined') {
+        return this.push(null);
+      }
+      this.push(current);
+    };
+
+    return readableStream;
+  },
+}));
+
+import { getIntegrityHash, getIntegrityHashes } from './file_integrity';
+
+describe('Integrity Hash', () => {
+  it('creates a hash from a file given a file path', async () => {
+    const filePath = 'somepath.json';
+    const expectedHash = '3295d40d2f35ac27145d37fcd5cdc80b';
+    const integrityHash = await getIntegrityHash(filePath);
+    expect(integrityHash).toEqual(expectedHash);
+  });
+
+  it('returns null on error', async () => {
+    const filePath = 'ERROR';
+    const integrityHash = await getIntegrityHash(filePath);
+    expect(integrityHash).toEqual(null);
+  });
+});
+
+describe('Integrity Hashes', () => {
+  it('returns an object with each filename and its hash', async () => {
+    const filePaths = ['somepath1.json', 'somepath2.json'];
+    const integrityHashes = await getIntegrityHashes(filePaths);
+    expect(integrityHashes).toEqual({
+      'somepath1.json': '8cbfe6a9f8174b2d7e77c2111a84f0e6',
+      'somepath2.json': '4177c075ade448d6e69fd94b39d0be15',
+    });
+  });
+});

x-pack/plugins/xpack_main/server/lib/file_integrity.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { createHash } from 'crypto';
+import * as fs from 'fs';
+import { zipObject } from 'lodash';
+import * as stream from 'stream';
+import * as util from 'util';
+
+const pipeline = util.promisify(stream.pipeline);
+
+export type Hash = string;
+
+export interface Integrities {
+  [filePath: string]: Hash;
+}
+
+export async function getIntegrityHashes(filepaths: string[]): Promise<Integrities> {
+  const hashes = await Promise.all(filepaths.map(getIntegrityHash));
+  return zipObject(filepaths, hashes);
+}
+
+export async function getIntegrityHash(filepath: string): Promise<Hash | null> {
+  try {
+    const output = createHash('md5');
+
+    await pipeline(fs.createReadStream(filepath), output);
+    const data = output.read();
+    if (data instanceof Buffer) {
+      return data.toString('hex');
+    }
+    return data;
+  } catch (err) {
+    return null;
+  }
+}

x-pack/plugins/xpack_main/server/lib/get_localization_usage_collector.test.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+interface TranslationsMock {
+  [title: string]: string;
+}
+
+const createI18nLoaderMock = (translations: TranslationsMock) => {
+  return {
+    getTranslationsByLocale() {
+      return {
+        messages: translations,
+      };
+    },
+  };
+};
+
+import { getTranslationCount } from './get_localization_usage_collector';
+
+describe('getTranslationCount', () => {
+  it('returns 0 if no translations registered', async () => {
+    const i18nLoaderMock = createI18nLoaderMock({});
+    const count = await getTranslationCount(i18nLoaderMock, 'en');
+    expect(count).toEqual(0);
+  });
+
+  it('returns number of translations', async () => {
+    const i18nLoaderMock = createI18nLoaderMock({
+      a: '1',
+      b: '2',
+      'b.a': '3',
+    });
+    const count = await getTranslationCount(i18nLoaderMock, 'en');
+    expect(count).toEqual(3);
+  });
+});

x-pack/plugins/xpack_main/server/lib/get_localization_usage_collector.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { i18nLoader } from '@kbn/i18n';
+import { size } from 'lodash';
+// @ts-ignore
+import { KIBANA_LOCALIZATION_STATS_TYPE } from '../../common/constants';
+import { getIntegrityHashes, Integrities } from './file_integrity';
+
+export interface UsageStats {
+  locale: string;
+  integrities: Integrities;
+  labelsCount?: number;
+}
+
+export async function getTranslationCount(loader: any, locale: string): Promise<number> {
+  const translations = await loader.getTranslationsByLocale(locale);
+  return size(translations.messages);
+}
+
+export function createCollectorFetch(server: any) {
+  return async function fetchUsageStats(): Promise<UsageStats> {
+    const config = server.config();
+    const locale: string = config.get('i18n.locale');
+    const translationFilePaths: string[] = server.getTranslationsFilePaths();
+
+    const [labelsCount, integrities] = await Promise.all([
+      getTranslationCount(i18nLoader, locale),
+      getIntegrityHashes(translationFilePaths),
+    ]);
+
+    return {
+      locale,
+      integrities,
+      labelsCount,
+    };
+  };
+}
+
+/*
+ * @param {Object} server
+ * @return {Object} kibana usage stats type collection object
+ */
+export function getLocalizationUsageCollector(server: any) {
+  const { collectorSet } = server.usage;
+  return collectorSet.makeUsageCollector({
+    type: KIBANA_LOCALIZATION_STATS_TYPE,
+    fetch: createCollectorFetch(server),
+  });
+}

x-pack/plugins/xpack_main/server/lib/telemetry/local/__tests__/get_local_stats.js

@@ -64,7 +64,12 @@ describe('get_local_stats', () => {
       os: {
         platform: 'rocky',
         platformRelease: 'iv',
-      }
+      },
+    },
+    localization: {
+      locale: 'en',
+      labelsCount: 0,
+      integrities: {}
     },
     sun: { chances: 5 },
     clouds: { chances: 95 },
@@ -92,6 +97,11 @@ describe('get_local_stats', () => {
         },
         versions: [{ version: '8675309', count: 1 }],
         plugins: {
+          localization: {
+            locale: 'en',
+            labelsCount: 0,
+            integrities: {}
+          },
           sun: { chances: 5 },
           clouds: { chances: 95 },
           rain: { chances: 2 },

x-pack/test/api_integration/apis/xpack_main/telemetry/telemetry_local.js

@@ -165,6 +165,8 @@ export default function ({ getService }) {
         'stack_stats.kibana.plugins.kql.defaultQueryLanguage',
         'stack_stats.kibana.plugins.kql.optInCount',
         'stack_stats.kibana.plugins.kql.optOutCount',
+        "stack_stats.kibana.plugins.localization.labelsCount",
+        "stack_stats.kibana.plugins.localization.locale",
         'stack_stats.kibana.plugins.maps.attributesPerMap.dataSourcesCount.avg',
         'stack_stats.kibana.plugins.maps.attributesPerMap.dataSourcesCount.max',
         'stack_stats.kibana.plugins.maps.attributesPerMap.dataSourcesCount.min',