Implement a build tool for locale files integration (#19826)

Implement a build tool for locale files verification

scripts/i18n_integrate.js

@@ -0,0 +1,21 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+require('../src/setup_node_env');
+require('../src/dev/run_i18n_integrate');

src/dev/i18n/__fixtures__/integrate_locale_files/fr.json

@@ -0,0 +1,63 @@
+{
+  "formats": {
+    "number": {
+      "currency": {
+        "style": "currency"
+      },
+      "percent": {
+        "style": "percent"
+      }
+    },
+    "date": {
+      "short": {
+        "month": "numeric",
+        "day": "numeric",
+        "year": "2-digit"
+      },
+      "medium": {
+        "month": "short",
+        "day": "numeric",
+        "year": "numeric"
+      },
+      "long": {
+        "month": "long",
+        "day": "numeric",
+        "year": "numeric"
+      },
+      "full": {
+        "weekday": "long",
+        "month": "long",
+        "day": "numeric",
+        "year": "numeric"
+      }
+    },
+    "time": {
+      "short": {
+        "hour": "numeric",
+        "minute": "numeric"
+      },
+      "medium": {
+        "hour": "numeric",
+        "minute": "numeric",
+        "second": "numeric"
+      },
+      "long": {
+        "hour": "numeric",
+        "minute": "numeric",
+        "second": "numeric",
+        "timeZoneName": "short"
+      },
+      "full": {
+        "hour": "numeric",
+        "minute": "numeric",
+        "second": "numeric",
+        "timeZoneName": "short"
+      }
+    }
+  },
+  "messages": {
+    "plugin-1.message-id-1": "Translated text 1",
+    "plugin-1.message-id-2": "Translated text 2",
+    "plugin-2.message-id": "Translated text"
+  }
+}

src/dev/i18n/__snapshots__/integrate_locale_files.test.js.snap

@@ -0,0 +1,163 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`dev/i18n/integrate_locale_files integrateLocaleFiles splits locale file by plugins and writes them into the right folders 1`] = `
+Array [
+  "src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_1/translations/fr.json",
+  "{
+  \\"formats\\": {
+    \\"number\\": {
+      \\"currency\\": {
+        \\"style\\": \\"currency\\"
+      },
+      \\"percent\\": {
+        \\"style\\": \\"percent\\"
+      }
+    },
+    \\"date\\": {
+      \\"short\\": {
+        \\"month\\": \\"numeric\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"2-digit\\"
+      },
+      \\"medium\\": {
+        \\"month\\": \\"short\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"numeric\\"
+      },
+      \\"long\\": {
+        \\"month\\": \\"long\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"numeric\\"
+      },
+      \\"full\\": {
+        \\"weekday\\": \\"long\\",
+        \\"month\\": \\"long\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"numeric\\"
+      }
+    },
+    \\"time\\": {
+      \\"short\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\"
+      },
+      \\"medium\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\",
+        \\"second\\": \\"numeric\\"
+      },
+      \\"long\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\",
+        \\"second\\": \\"numeric\\",
+        \\"timeZoneName\\": \\"short\\"
+      },
+      \\"full\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\",
+        \\"second\\": \\"numeric\\",
+        \\"timeZoneName\\": \\"short\\"
+      }
+    }
+  },
+  \\"messages\\": {
+    \\"plugin-1.message-id-1\\": \\"Translated text 1\\",
+    \\"plugin-1.message-id-2\\": \\"Translated text 2\\"
+  }
+}",
+]
+`;
+
+exports[`dev/i18n/integrate_locale_files integrateLocaleFiles splits locale file by plugins and writes them into the right folders 2`] = `
+Array [
+  "src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_2/translations/fr.json",
+  "{
+  \\"formats\\": {
+    \\"number\\": {
+      \\"currency\\": {
+        \\"style\\": \\"currency\\"
+      },
+      \\"percent\\": {
+        \\"style\\": \\"percent\\"
+      }
+    },
+    \\"date\\": {
+      \\"short\\": {
+        \\"month\\": \\"numeric\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"2-digit\\"
+      },
+      \\"medium\\": {
+        \\"month\\": \\"short\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"numeric\\"
+      },
+      \\"long\\": {
+        \\"month\\": \\"long\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"numeric\\"
+      },
+      \\"full\\": {
+        \\"weekday\\": \\"long\\",
+        \\"month\\": \\"long\\",
+        \\"day\\": \\"numeric\\",
+        \\"year\\": \\"numeric\\"
+      }
+    },
+    \\"time\\": {
+      \\"short\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\"
+      },
+      \\"medium\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\",
+        \\"second\\": \\"numeric\\"
+      },
+      \\"long\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\",
+        \\"second\\": \\"numeric\\",
+        \\"timeZoneName\\": \\"short\\"
+      },
+      \\"full\\": {
+        \\"hour\\": \\"numeric\\",
+        \\"minute\\": \\"numeric\\",
+        \\"second\\": \\"numeric\\",
+        \\"timeZoneName\\": \\"short\\"
+      }
+    }
+  },
+  \\"messages\\": {
+    \\"plugin-2.message-id\\": \\"Translated text\\"
+  }
+}",
+]
+`;
+
+exports[`dev/i18n/integrate_locale_files integrateLocaleFiles splits locale file by plugins and writes them into the right folders 3`] = `
+Array [
+  "src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_1/translations",
+  "src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_2/translations",
+]
+`;
+
+exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id and missing id 1`] = `
+"
+Missing translations:
+plugin-1.message-id-2"
+`;
+
+exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id and missing id 2`] = `
+"
+Unused translations:
+plugin-1.message-id-3"
+`;
+
+exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id and missing id 3`] = `
+"
+Unused translations:
+plugin-2.message
+Missing translations:
+plugin-2.message-id"
+`;

src/dev/i18n/__snapshots__/utils.test.js.snap

@@ -9,6 +9,8 @@ exports[`i18n utils should create verbose parser error message 1`] = `
 "
 `;
 
+exports[`i18n utils should normalizePath 1`] = `"src/dev/i18n"`;
+
 exports[`i18n utils should not escape linebreaks 1`] = `
 "Text
  with

src/dev/i18n/extract_default_translations.js

@@ -26,6 +26,7 @@ import {
   extractHandlebarsMessages,
 } from './extractors';
 import { globAsync, readFileAsync, normalizePath } from './utils';
+
 import { createFailError, isFailError } from '../run';
 
 function addMessageToMap(targetMap, key, value, reporter) {
@@ -147,3 +148,13 @@ export async function extractMessagesFromPathToMap(inputPath, targetMap, config,
     })
   );
 }
+
+export async function getDefaultMessagesMap(inputPaths, config, reporter) {
+  const defaultMessagesMap = new Map();
+
+  for (const inputPath of filterPaths(inputPaths, config.paths)) {
+    await extractMessagesFromPathToMap(inputPath, defaultMessagesMap, config, reporter);
+  }
+
+  return defaultMessagesMap;
+}

src/dev/i18n/integrate_locale_files.js

@@ -0,0 +1,112 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import path from 'path';
+
+import {
+  difference,
+  readFileAsync,
+  writeFileAsync,
+  accessAsync,
+  makeDirAsync,
+  normalizePath,
+  ErrorReporter,
+} from './utils';
+import { paths, exclude } from '../../../.i18nrc.json';
+import { getDefaultMessagesMap } from './extract_default_translations';
+import { createFailError } from '../run';
+import { serializeToJson } from './serializers/json';
+
+export function verifyMessages(localizedMessagesMap, defaultMessagesMap) {
+  let errorMessage = '';
+
+  const defaultMessagesIds = [...defaultMessagesMap.keys()];
+  const localizedMessagesIds = [...localizedMessagesMap.keys()];
+
+  const unusedTranslations = difference(localizedMessagesIds, defaultMessagesIds);
+  if (unusedTranslations.length > 0) {
+    errorMessage += `\nUnused translations:\n${unusedTranslations.join(', ')}`;
+  }
+
+  const missingTranslations = difference(defaultMessagesIds, localizedMessagesIds);
+  if (missingTranslations.length > 0) {
+    errorMessage += `\nMissing translations:\n${missingTranslations.join(', ')}`;
+  }
+
+  if (errorMessage) {
+    throw createFailError(errorMessage);
+  }
+}
+
+function groupMessagesByNamespace(localizedMessagesMap) {
+  const localizedMessagesByNamespace = new Map();
+  const knownNamespaces = Object.keys(paths);
+
+  for (const [messageId, messageValue] of localizedMessagesMap) {
+    const namespace = knownNamespaces.find(key => messageId.startsWith(`${key}.`));
+
+    if (!namespace) {
+      throw createFailError(`Unknown namespace in id ${messageId}.`);
+    }
+
+    if (!localizedMessagesByNamespace.has(namespace)) {
+      localizedMessagesByNamespace.set(namespace, []);
+    }
+
+    localizedMessagesByNamespace
+      .get(namespace)
+      .push([messageId, { message: messageValue.text || messageValue }]);
+  }
+
+  return localizedMessagesByNamespace;
+}
+
+async function writeMessages(localizedMessagesByNamespace, fileName, formats, log) {
+  for (const [namespace, messages] of localizedMessagesByNamespace) {
+    const destPath = path.resolve(paths[namespace], 'translations');
+
+    try {
+      await accessAsync(destPath);
+    } catch (_) {
+      await makeDirAsync(destPath);
+    }
+
+    const writePath = path.resolve(destPath, fileName);
+    await writeFileAsync(writePath, serializeToJson(messages, formats));
+    log.success(`Translations have been integrated to ${normalizePath(writePath)}`);
+  }
+}
+
+export async function integrateLocaleFiles(filePath, log) {
+  const reporter = new ErrorReporter();
+  const defaultMessagesMap = await getDefaultMessagesMap(['.'], { paths, exclude }, reporter);
+  const localizedMessages = JSON.parse((await readFileAsync(filePath)).toString());
+
+  if (!localizedMessages.formats) {
+    throw createFailError(`Locale file should contain formats object.`);
+  }
+
+  const localizedMessagesMap = new Map(Object.entries(localizedMessages.messages));
+  verifyMessages(localizedMessagesMap, defaultMessagesMap);
+
+  // use basename of filePath to write the same locale name as the source file has
+  const fileName = path.basename(filePath);
+  const localizedMessagesByNamespace = groupMessagesByNamespace(localizedMessagesMap);
+  await writeMessages(localizedMessagesByNamespace, fileName, localizedMessages.formats, log);
+}

src/dev/i18n/integrate_locale_files.test.js

@@ -0,0 +1,105 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import path from 'path';
+
+import { verifyMessages, integrateLocaleFiles } from './integrate_locale_files';
+import { normalizePath } from './utils';
+
+const localePath = path.resolve(__dirname, '__fixtures__', 'integrate_locale_files', 'fr.json');
+
+const mockDefaultMessagesMap = new Map([
+  ['plugin-1.message-id-1', 'Message text 1'],
+  ['plugin-1.message-id-2', 'Message text 2'],
+  ['plugin-2.message-id', 'Message text'],
+]);
+
+jest.mock('./extract_default_translations.js', () => ({
+  getDefaultMessagesMap: () => mockDefaultMessagesMap,
+}));
+
+jest.mock('../../../.i18nrc.json', () => ({
+  paths: {
+    'plugin-1': 'src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_1',
+    'plugin-2': 'src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_2',
+  },
+  exclude: [],
+}));
+
+const utils = require('./utils');
+utils.writeFileAsync = jest.fn();
+utils.makeDirAsync = jest.fn();
+
+describe('dev/i18n/integrate_locale_files', () => {
+  describe('verifyMessages', () => {
+    test('validates localized messages', () => {
+      const localizedMessagesMap = new Map([
+        ['plugin-1.message-id-1', 'Translated text 1'],
+        ['plugin-1.message-id-2', 'Translated text 2'],
+        ['plugin-2.message-id', 'Translated text'],
+      ]);
+
+      expect(() => verifyMessages(localizedMessagesMap, mockDefaultMessagesMap)).not.toThrow();
+    });
+
+    test('throws an error for unused id and missing id', () => {
+      const localizedMessagesMapWithMissingMessage = new Map([
+        ['plugin-1.message-id-1', 'Translated text 1'],
+        ['plugin-2.message-id', 'Translated text'],
+      ]);
+
+      const localizedMessagesMapWithUnusedMessage = new Map([
+        ['plugin-1.message-id-1', 'Translated text 1'],
+        ['plugin-1.message-id-2', 'Translated text 2'],
+        ['plugin-1.message-id-3', 'Translated text 3'],
+        ['plugin-2.message-id', 'Translated text'],
+      ]);
+
+      const localizedMessagesMapWithIdTypo = new Map([
+        ['plugin-1.message-id-1', 'Message text 1'],
+        ['plugin-1.message-id-2', 'Message text 2'],
+        ['plugin-2.message', 'Message text'],
+      ]);
+
+      expect(() =>
+        verifyMessages(localizedMessagesMapWithMissingMessage, mockDefaultMessagesMap)
+      ).toThrowErrorMatchingSnapshot();
+      expect(() =>
+        verifyMessages(localizedMessagesMapWithUnusedMessage, mockDefaultMessagesMap)
+      ).toThrowErrorMatchingSnapshot();
+      expect(() =>
+        verifyMessages(localizedMessagesMapWithIdTypo, mockDefaultMessagesMap)
+      ).toThrowErrorMatchingSnapshot();
+    });
+  });
+
+  describe('integrateLocaleFiles', () => {
+    test('splits locale file by plugins and writes them into the right folders', async () => {
+      const success = jest.fn();
+      await integrateLocaleFiles(localePath, { success });
+
+      const [[path1, json1], [path2, json2]] = utils.writeFileAsync.mock.calls;
+      const [[dirPath1], [dirPath2]] = utils.makeDirAsync.mock.calls;
+
+      expect([normalizePath(path1), json1]).toMatchSnapshot();
+      expect([normalizePath(path2), json2]).toMatchSnapshot();
+      expect([normalizePath(dirPath1), normalizePath(dirPath2)]).toMatchSnapshot();
+    });
+  });
+});

src/dev/i18n/serializers/json.js

@@ -19,10 +19,10 @@
 
 import { i18n } from '@kbn/i18n';
 
-export function serializeToJson(defaultMessages) {
-  const resultJsonObject = { formats: i18n.formats, messages: {} };
+export function serializeToJson(messages, formats = i18n.formats) {
+  const resultJsonObject = { formats, messages: {} };
 
-  for (const [mapKey, mapValue] of defaultMessages) {
+  for (const [mapKey, mapValue] of messages) {
     if (mapValue.description) {
       resultJsonObject.messages[mapKey] = { text: mapValue.message, comment: mapValue.description };
     } else {

src/dev/i18n/serializers/json.test.js

@@ -21,7 +21,7 @@ import { serializeToJson } from './json';
 
 describe('dev/i18n/serializers/json', () => {
   test('should serialize default messages to JSON', () => {
-    const messages = new Map([
+    const messages = [
       ['plugin1.message.id-1', { message: 'Message text 1 ' }],
       [
         'plugin2.message.id-2',
@@ -30,7 +30,7 @@ describe('dev/i18n/serializers/json', () => {
           description: 'Message description',
         },
       ],
-    ]);
+    ];
 
     expect(serializeToJson(messages)).toMatchSnapshot();
   });

src/dev/i18n/serializers/json5.js

@@ -22,15 +22,15 @@ import { i18n } from '@kbn/i18n';
 
 const ESCAPE_SINGLE_QUOTE_REGEX = /\\([\s\S])|(')/g;
 
-export function serializeToJson5(defaultMessages) {
+export function serializeToJson5(messages, formats = i18n.formats) {
   // .slice(0, -4): remove closing curly braces from json to append messages
   let jsonBuffer = Buffer.from(
-    JSON5.stringify({ formats: i18n.formats, messages: {} }, { quote: `'`, space: 2 })
+    JSON5.stringify({ formats, messages: {} }, { quote: `'`, space: 2 })
       .slice(0, -4)
       .concat('\n')
   );
 
-  for (const [mapKey, mapValue] of defaultMessages) {
+  for (const [mapKey, mapValue] of messages) {
     const formattedMessage = mapValue.message.replace(ESCAPE_SINGLE_QUOTE_REGEX, '\\$1$2');
     const formattedDescription = mapValue.description
       ? mapValue.description.replace(ESCAPE_SINGLE_QUOTE_REGEX, '\\$1$2')

src/dev/i18n/serializers/json5.test.js

@@ -21,7 +21,7 @@ import { serializeToJson5 } from './json5';
 
 describe('dev/i18n/serializers/json5', () => {
   test('should serialize default messages to JSON5', () => {
-    const messages = new Map([
+    const messages = [
       [
         'plugin1.message.id-1',
         {
@@ -35,7 +35,7 @@ describe('dev/i18n/serializers/json5', () => {
           description: 'Message description',
         },
       ],
-    ]);
+    ];
 
     expect(serializeToJson5(messages).toString()).toMatchSnapshot();
   });

src/dev/i18n/utils.js

@@ -31,10 +31,10 @@ import {
 import fs from 'fs';
 import glob from 'glob';
 import { promisify } from 'util';
-import chalk from 'chalk';
-import parser from 'intl-messageformat-parser';
 import normalize from 'normalize-path';
 import path from 'path';
+import chalk from 'chalk';
+import parser from 'intl-messageformat-parser';
 
 import { createFailError } from '../run';
 
@@ -46,6 +46,8 @@ const HTML_KEY_PREFIX = 'html_';
 
 export const readFileAsync = promisify(fs.readFile);
 export const writeFileAsync = promisify(fs.writeFile);
+export const makeDirAsync = promisify(fs.mkdir);
+export const accessAsync = promisify(fs.access);
 export const globAsync = promisify(glob);
 
 export function normalizePath(inputPath) {

src/dev/i18n/utils.test.js

@@ -27,6 +27,7 @@ import {
   formatJSString,
   checkValuesProperty,
   createParserErrorMessage,
+  normalizePath,
   extractMessageValueFromNode,
 } from './utils';
 
@@ -107,6 +108,10 @@ describe('i18n utils', () => {
     }
   });
 
+  test('should normalizePath', () => {
+    expect(normalizePath(__dirname)).toMatchSnapshot();
+  });
+
   test('should validate conformity of "values" and "defaultMessage"', () => {
     const valuesKeys = ['url', 'username', 'password'];
     const defaultMessage = 'Test message with {username}, {password} and [markdown link]({url}).';

src/dev/run_i18n_integrate.js

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import chalk from 'chalk';
+
+import { createFailError, run } from './run';
+import { integrateLocaleFiles } from './i18n/integrate_locale_files';
+
+run(async ({ flags: { path }, log }) => {
+  if (!path || typeof path === 'boolean') {
+    throw createFailError(`${chalk.white.bgRed(' I18N ERROR ')} --path option isn't provided.`);
+  }
+
+  if (Array.isArray(path)) {
+    throw createFailError(
+      `${chalk.white.bgRed(' I18N ERROR ')} --path should be specified only once`
+    );
+  }
+
+  await integrateLocaleFiles(path, log);
+});