[7.x] Add zh-CN.json translations and respective compatibility checks via i18n tools (#30638)

.i18nrc.json

@@ -53,5 +53,8 @@
     "x-pack/plugins/infra/public/utils/loading_state/loading_result.ts",
     "x-pack/plugins/infra/server/graphql/types.ts",
     "x-pack/plugins/infra/server/lib/domains/log_entries_domain/log_entries_domain.ts"
+  ],
+  "translations": [
+    "x-pack/plugins/translations/translations/zh-CN.json"
   ]
 }

package.json

@@ -272,6 +272,7 @@
     "@types/eslint": "^4.16.2",
     "@types/execa": "^0.9.0",
     "@types/fetch-mock": "7.2.1",
+    "@types/json5": "^0.0.30",
     "@types/getopts": "^2.0.0",
     "@types/glob": "^5.0.35",
     "@types/globby": "^8.0.0",

packages/kbn-i18n/GUIDELINE.md

@@ -440,9 +440,10 @@ it('should render normally', async () => {
 
 3. Check functionality of an element (button is clicked, checkbox is checked/unchecked, etc.).
 
-4. Run i18n validation tool and skim through created `en.json`:
-    ```js
-      node scripts/i18n_check --output ./
+4. Run i18n validation/extraction tools and skim through created `en.json`:
+    ```bash
+    $ node scripts/i18n_check --ignore-missing
+    $ node scripts/i18n_extract --output-dir ./
     ```
 
 5. Run linters and type checker as you normally do.

scripts/i18n_extract.js

@@ -17,5 +17,5 @@
  * under the License.
  */
 
-export { serializeToJson } from './json';
-export { serializeToJson5 } from './json5';
+require('../src/setup_node_env');
+require('../src/dev/run_i18n_extract');

src/dev/i18n/README.md

@@ -140,19 +140,20 @@ The `description` is optional, `values` is optional too unless `defaultMessage`
 ### Usage
 
 ```bash
-node scripts/i18n_check --path path/to/plugin --path path/to/another/plugin --output ./translations --output-format json5
+node scripts/i18n_extract --path path/to/plugin --path path/to/another/plugin --output-dir ./translations --output-format json5
 ```
 
 * `path/to/plugin` is an example of path to a directory(-es) where messages searching should start. By default `--path` is `.`, it means that messages from all paths in `.i18nrc.json` will be parsed. Each specified path should start with any path in `.i18nrc.json` or be a part of it.
-* `--output` specifies a path to a directory, where `en.json` will be created, if `--output` is not provided, `en.json` generation will be skipped. It is useful if you want to validate i18n engine usage.\
+* `--output-dir` specifies a path to a directory, where `en.json` will be created.\
 In case of parsing issues, exception with the necessary information will be thrown to console and extraction will be aborted.
-* `--output-format` specifies format of generated `en.json` (if `--output` is provided). By default it is `json`. Use it only if you need a JSON5 file.
+* `--output-format` specifies format of generated `en.json`. By default it is `json`. Use it only if you need a JSON5 file.
+* `--include-config` specifies additional paths to `.i18nrc.json` files (may be useful for 3rd-party plugins)
 
 ### Output
 
 `<output_path>/en.json`
 
-The tool generates a JSON/JSON5 file only if `--output` path is provided. It contains injected `formats` object and `messages` object with `id: message` or `id: {text, comment}` pairs. Messages are sorted by id.
+The generated JSON/JSON5 file contains `formats` object and `messages` object with `id: message` or `id: {text, comment}` pairs. Messages are sorted by id.
 
 **Example**:
 
@@ -169,11 +170,12 @@ The tool generates a JSON/JSON5 file only if `--output` path is provided. It con
 }
 ```
 
-## Locale files verification / integration tool
+## Locale files integration tool
 
 ### Description
 
-The tool is used for verifying locale file, finding unused / missing messages, key duplications, grouping messages by namespaces and creating JSON files in right folders.
+The tool is used for verifying locale file, finding unused / missing messages, key duplications and value references mismatches. If all these
+checks are passing, the tool groups messages by namespaces and creates JSON files in right folders.
 
 ### Notes
 
@@ -182,9 +184,22 @@ The tool throws an exception if `formats` object is missing in locale file.
 ### Usage
 
 ```bash
-node scripts/i18n_integrate --path path/to/locale.json
+node scripts/i18n_integrate --source path/to/locale.json --target x-pack/plugins/translations/translations/locale.json
 ```
 
+* `--source` path to the JSON file with translations that should be integrated.
+* `--target` defines a single path to the JSON file where translations should be integrated to, path mappings from 
+[.i18nrc.json](../../../.i18nrc.json) are ignored in this case. It's currently used for integrating of Kibana built-in
+translations that are located in a single JSON file within `x-pack/translations` plugin.
+* `--dry-run` tells the tool to exit after verification phase and not write translations to the disk.
+* `--ignore-incompatible` specifies whether tool should ignore incompatible translations. It may be useful when the code base you're
+integrating translations to has changed and some default messages switched to ICU structure that is incompatible with the one used in corresponding translation.
+* `--ignore-missing` specifies whether tool should ignore missing translations. It may be useful when the code base you're
+integrating translations to has moved forward since the revision translations were created for.
+* `--ignore-unused` specifies whether tool should ignore unused translations. It may be useful when the code base you're
+integrating translations to has changed and some translations are not needed anymore.
+* `--include-config` specifies additional paths to `.i18nrc.json` files (may be useful for 3rd-party plugins)
+
 ### Output
 
-The tool generates locale files in plugin folders and few other special locations based on namespaces and corresponding mappings defined in [.i18nrc.json](../../../.i18nrc.json).
+Unless `--target` is specified, the tool generates locale files in plugin folders and few other special locations based on namespaces and corresponding mappings defined in [.i18nrc.json](../../../.i18nrc.json).

src/dev/i18n/__snapshots__/integrate_locale_files.test.ts.snap

@@ -142,22 +142,28 @@ Array [
 ]
 `;
 
-exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id and missing id 1`] = `
+exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id, missing id or the incompatible ones 1`] = `
 "
-Missing translations:
+1 missing translation(s):
 plugin-1.message-id-2"
 `;
 
-exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id and missing id 2`] = `
+exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id, missing id or the incompatible ones 2`] = `
 "
-Unused translations:
+1 unused translation(s):
 plugin-1.message-id-3"
 `;
 
-exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id and missing id 3`] = `
+exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id, missing id or the incompatible ones 3`] = `
 "
-Unused translations:
+1 unused translation(s):
 plugin-2.message
-Missing translations:
+1 missing translation(s):
 plugin-2.message-id"
 `;
+
+exports[`dev/i18n/integrate_locale_files verifyMessages throws an error for unused id, missing id or the incompatible ones 4`] = `
+"
+Incompatible translation: some properties are missing in \\"values\\" object (\\"plugin-1.message-id-2\\"): [value].
+"
+`;

src/dev/i18n/__snapshots__/utils.test.js.snap

@@ -20,27 +20,12 @@ exports[`i18n utils should not escape linebreaks 1`] = `
 
 exports[`i18n utils should parse string concatenation 1`] = `"Very long concatenated string"`;
 
-exports[`i18n utils should throw if "values" has a value that is unused in the message 1`] = `
-"\\"values\\" object contains unused properties (\\"namespace.message.id\\"):
-[url]."
-`;
+exports[`i18n utils should throw if "values" has a value that is unused in the message 1`] = `"\\"values\\" object contains unused properties (\\"namespace.message.id\\"): [url]."`;
 
-exports[`i18n utils should throw if "values" property is not provided and defaultMessage requires it 1`] = `
-"some properties are missing in \\"values\\" object (\\"namespace.message.id\\"):
-[username,password,url]."
-`;
+exports[`i18n utils should throw if "values" property is not provided and defaultMessage requires it 1`] = `"some properties are missing in \\"values\\" object (\\"namespace.message.id\\"): [username,password,url]."`;
 
-exports[`i18n utils should throw if "values" property is provided and defaultMessage doesn't include any references 1`] = `
-"\\"values\\" object contains unused properties (\\"namespace.message.id\\"):
-[url,username]."
-`;
+exports[`i18n utils should throw if "values" property is provided and defaultMessage doesn't include any references 1`] = `"\\"values\\" object contains unused properties (\\"namespace.message.id\\"): [url,username]."`;
 
-exports[`i18n utils should throw if some key is missing in "values" 1`] = `
-"some properties are missing in \\"values\\" object (\\"namespace.message.id\\"):
-[password]."
-`;
+exports[`i18n utils should throw if some key is missing in "values" 1`] = `"some properties are missing in \\"values\\" object (\\"namespace.message.id\\"): [password]."`;
 
-exports[`i18n utils should throw on wrong nested ICU message 1`] = `
-"\\"values\\" object contains unused properties (\\"namespace.message.id\\"):
-[third]."
-`;
+exports[`i18n utils should throw on wrong nested ICU message 1`] = `"\\"values\\" object contains unused properties (\\"namespace.message.id\\"): [third]."`;

src/dev/i18n/config.ts

@@ -0,0 +1,93 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { resolve } from 'path';
+
+// @ts-ignore
+import { normalizePath, readFileAsync } from '.';
+// @ts-ignore
+import rootConfig from '../../../.i18nrc.json';
+
+export interface I18nConfig {
+  paths: Record<string, string>;
+  exclude: string[];
+  translations: string[];
+}
+
+/**
+ * Merges root .i18nrc.json config with any other additional configs (e.g. from
+ * third-party plugins).
+ * @param configPaths List of config paths.
+ */
+export async function mergeConfigs(configPaths: string | string[] = []) {
+  const mergedConfig: I18nConfig = { exclude: [], translations: [], ...rootConfig };
+
+  for (const configPath of Array.isArray(configPaths) ? configPaths : [configPaths]) {
+    const additionalConfig: I18nConfig = {
+      paths: {},
+      exclude: [],
+      translations: [],
+      ...JSON.parse(await readFileAsync(resolve(configPath))),
+    };
+
+    for (const [namespace, path] of Object.entries(additionalConfig.paths)) {
+      mergedConfig.paths[namespace] = normalizePath(resolve(configPath, '..', path));
+    }
+
+    for (const exclude of additionalConfig.exclude) {
+      mergedConfig.exclude.push(normalizePath(resolve(configPath, '..', exclude)));
+    }
+
+    for (const translations of additionalConfig.translations) {
+      mergedConfig.translations.push(normalizePath(resolve(configPath, '..', translations)));
+    }
+  }
+
+  return mergedConfig;
+}
+
+/**
+ * Filters out custom paths based on the paths defined in config and that are
+ * known to contain i18n strings.
+ * @param inputPaths List of paths to filter.
+ * @param config I18n config instance.
+ */
+export function filterConfigPaths(inputPaths: string[], config: I18nConfig) {
+  const availablePaths = Object.values(config.paths);
+  const pathsForExtraction = new Set();
+
+  for (const inputPath of inputPaths) {
+    const normalizedPath = normalizePath(inputPath);
+
+    // If input path is the sub path of or equal to any available path, include it.
+    if (
+      availablePaths.some(path => normalizedPath.startsWith(`${path}/`) || path === normalizedPath)
+    ) {
+      pathsForExtraction.add(normalizedPath);
+    } else {
+      // Otherwise go through all available paths and see if any of them is the sub
+      // path of the input path (empty normalized path corresponds to root or above).
+      availablePaths
+        .filter(path => !normalizedPath || path.startsWith(`${normalizedPath}/`))
+        .forEach(ePath => pathsForExtraction.add(ePath));
+    }
+  }
+
+  return [...pathsForExtraction];
+}

src/dev/i18n/extract_default_translations.js

@@ -42,30 +42,6 @@ function addMessageToMap(targetMap, key, value, reporter) {
   }
 }
 
-export function filterPaths(inputPaths, paths) {
-  const availablePaths = Object.values(paths);
-  const pathsForExtraction = new Set();
-
-  for (const inputPath of inputPaths) {
-    const normalizedPath = normalizePath(inputPath);
-
-    // If input path is the sub path of or equal to any available path, include it.
-    if (
-      availablePaths.some(path => normalizedPath.startsWith(`${path}/`) || path === normalizedPath)
-    ) {
-      pathsForExtraction.add(normalizedPath);
-    } else {
-      // Otherwise go through all available paths and see if any of them is the sub
-      // path of the input path (empty normalized path corresponds to root or above).
-      availablePaths
-        .filter(path => !normalizedPath || path.startsWith(`${normalizedPath}/`))
-        .forEach(ePath => pathsForExtraction.add(ePath));
-    }
-  }
-
-  return [...pathsForExtraction];
-}
-
 function filterEntries(entries, exclude) {
   return entries.filter(entry =>
     exclude.every(excludedPath => !normalizePath(entry).startsWith(excludedPath))
@@ -148,13 +124,3 @@ export async function extractMessagesFromPathToMap(inputPath, targetMap, config,
     })
   );
 }
-
-export async function getDefaultMessagesMap(inputPaths, config, reporter) {
-  const defaultMessagesMap = new Map();
-
-  for (const inputPath of filterPaths(inputPaths, config.paths)) {
-    await extractMessagesFromPathToMap(inputPath, defaultMessagesMap, config, reporter);
-  }
-
-  return defaultMessagesMap;
-}

src/dev/i18n/extractors/react.js

@@ -57,7 +57,7 @@ export function extractIntlMessages(node) {
     : undefined;
 
   if (!messageId) {
-    createFailError(`Empty "id" value in intl.formatMessage() is not allowed.`);
+    throw createFailError(`Empty "id" value in intl.formatMessage() is not allowed.`);
   }
 
   const message = messageProperty

src/dev/i18n/index.ts

@@ -17,6 +17,10 @@
  * under the License.
  */
 
-export { filterPaths, extractMessagesFromPathToMap } from './extract_default_translations';
+// @ts-ignore
+export { extractMessagesFromPathToMap } from './extract_default_translations';
+// @ts-ignore
 export { writeFileAsync, readFileAsync, normalizePath, ErrorReporter } from './utils';
 export { serializeToJson, serializeToJson5 } from './serializers';
+export { I18nConfig, filterConfigPaths, mergeConfigs } from './config';
+export { integrateLocaleFiles } from './integrate_locale_files';

src/dev/i18n/integrate_locale_files.js

@@ -1,112 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import path from 'path';
-
-import {
-  difference,
-  readFileAsync,
-  writeFileAsync,
-  accessAsync,
-  makeDirAsync,
-  normalizePath,
-  ErrorReporter,
-} from './utils';
-import { paths, exclude } from '../../../.i18nrc.json';
-import { getDefaultMessagesMap } from './extract_default_translations';
-import { createFailError } from '../run';
-import { serializeToJson } from './serializers/json';
-
-export function verifyMessages(localizedMessagesMap, defaultMessagesMap) {
-  let errorMessage = '';
-
-  const defaultMessagesIds = [...defaultMessagesMap.keys()];
-  const localizedMessagesIds = [...localizedMessagesMap.keys()];
-
-  const unusedTranslations = difference(localizedMessagesIds, defaultMessagesIds);
-  if (unusedTranslations.length > 0) {
-    errorMessage += `\nUnused translations:\n${unusedTranslations.join(', ')}`;
-  }
-
-  const missingTranslations = difference(defaultMessagesIds, localizedMessagesIds);
-  if (missingTranslations.length > 0) {
-    errorMessage += `\nMissing translations:\n${missingTranslations.join(', ')}`;
-  }
-
-  if (errorMessage) {
-    throw createFailError(errorMessage);
-  }
-}
-
-function groupMessagesByNamespace(localizedMessagesMap) {
-  const localizedMessagesByNamespace = new Map();
-  const knownNamespaces = Object.keys(paths);
-
-  for (const [messageId, messageValue] of localizedMessagesMap) {
-    const namespace = knownNamespaces.find(key => messageId.startsWith(`${key}.`));
-
-    if (!namespace) {
-      throw createFailError(`Unknown namespace in id ${messageId}.`);
-    }
-
-    if (!localizedMessagesByNamespace.has(namespace)) {
-      localizedMessagesByNamespace.set(namespace, []);
-    }
-
-    localizedMessagesByNamespace
-      .get(namespace)
-      .push([messageId, { message: messageValue.text || messageValue }]);
-  }
-
-  return localizedMessagesByNamespace;
-}
-
-async function writeMessages(localizedMessagesByNamespace, fileName, formats, log) {
-  for (const [namespace, messages] of localizedMessagesByNamespace) {
-    const destPath = path.resolve(paths[namespace], 'translations');
-
-    try {
-      await accessAsync(destPath);
-    } catch (_) {
-      await makeDirAsync(destPath);
-    }
-
-    const writePath = path.resolve(destPath, fileName);
-    await writeFileAsync(writePath, serializeToJson(messages, formats));
-    log.success(`Translations have been integrated to ${normalizePath(writePath)}`);
-  }
-}
-
-export async function integrateLocaleFiles(filePath, log) {
-  const reporter = new ErrorReporter();
-  const defaultMessagesMap = await getDefaultMessagesMap(['.'], { paths, exclude }, reporter);
-  const localizedMessages = JSON.parse((await readFileAsync(filePath)).toString());
-
-  if (!localizedMessages.formats) {
-    throw createFailError(`Locale file should contain formats object.`);
-  }
-
-  const localizedMessagesMap = new Map(Object.entries(localizedMessages.messages));
-  verifyMessages(localizedMessagesMap, defaultMessagesMap);
-
-  // use basename of filePath to write the same locale name as the source file has
-  const fileName = path.basename(filePath);
-  const localizedMessagesByNamespace = groupMessagesByNamespace(localizedMessagesMap);
-  await writeMessages(localizedMessagesByNamespace, fileName, localizedMessages.formats, log);
-}

src/dev/i18n/integrate_locale_files.test.js

@@ -1,105 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import path from 'path';
-
-import { verifyMessages, integrateLocaleFiles } from './integrate_locale_files';
-import { normalizePath } from './utils';
-
-const localePath = path.resolve(__dirname, '__fixtures__', 'integrate_locale_files', 'fr.json');
-
-const mockDefaultMessagesMap = new Map([
-  ['plugin-1.message-id-1', 'Message text 1'],
-  ['plugin-1.message-id-2', 'Message text 2'],
-  ['plugin-2.message-id', 'Message text'],
-]);
-
-jest.mock('./extract_default_translations.js', () => ({
-  getDefaultMessagesMap: () => mockDefaultMessagesMap,
-}));
-
-jest.mock('../../../.i18nrc.json', () => ({
-  paths: {
-    'plugin-1': 'src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_1',
-    'plugin-2': 'src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_2',
-  },
-  exclude: [],
-}));
-
-const utils = require('./utils');
-utils.writeFileAsync = jest.fn();
-utils.makeDirAsync = jest.fn();
-
-describe('dev/i18n/integrate_locale_files', () => {
-  describe('verifyMessages', () => {
-    test('validates localized messages', () => {
-      const localizedMessagesMap = new Map([
-        ['plugin-1.message-id-1', 'Translated text 1'],
-        ['plugin-1.message-id-2', 'Translated text 2'],
-        ['plugin-2.message-id', 'Translated text'],
-      ]);
-
-      expect(() => verifyMessages(localizedMessagesMap, mockDefaultMessagesMap)).not.toThrow();
-    });
-
-    test('throws an error for unused id and missing id', () => {
-      const localizedMessagesMapWithMissingMessage = new Map([
-        ['plugin-1.message-id-1', 'Translated text 1'],
-        ['plugin-2.message-id', 'Translated text'],
-      ]);
-
-      const localizedMessagesMapWithUnusedMessage = new Map([
-        ['plugin-1.message-id-1', 'Translated text 1'],
-        ['plugin-1.message-id-2', 'Translated text 2'],
-        ['plugin-1.message-id-3', 'Translated text 3'],
-        ['plugin-2.message-id', 'Translated text'],
-      ]);
-
-      const localizedMessagesMapWithIdTypo = new Map([
-        ['plugin-1.message-id-1', 'Message text 1'],
-        ['plugin-1.message-id-2', 'Message text 2'],
-        ['plugin-2.message', 'Message text'],
-      ]);
-
-      expect(() =>
-        verifyMessages(localizedMessagesMapWithMissingMessage, mockDefaultMessagesMap)
-      ).toThrowErrorMatchingSnapshot();
-      expect(() =>
-        verifyMessages(localizedMessagesMapWithUnusedMessage, mockDefaultMessagesMap)
-      ).toThrowErrorMatchingSnapshot();
-      expect(() =>
-        verifyMessages(localizedMessagesMapWithIdTypo, mockDefaultMessagesMap)
-      ).toThrowErrorMatchingSnapshot();
-    });
-  });
-
-  describe('integrateLocaleFiles', () => {
-    test('splits locale file by plugins and writes them into the right folders', async () => {
-      const success = jest.fn();
-      await integrateLocaleFiles(localePath, { success });
-
-      const [[path1, json1], [path2, json2]] = utils.writeFileAsync.mock.calls;
-      const [[dirPath1], [dirPath2]] = utils.makeDirAsync.mock.calls;
-
-      expect([normalizePath(path1), json1]).toMatchSnapshot();
-      expect([normalizePath(path2), json2]).toMatchSnapshot();
-      expect([normalizePath(dirPath1), normalizePath(dirPath2)]).toMatchSnapshot();
-    });
-  });
-});

src/dev/i18n/integrate_locale_files.test.ts

@@ -0,0 +1,184 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+const mockWriteFileAsync = jest.fn();
+const mockMakeDirAsync = jest.fn();
+jest.mock('./utils', () => ({
+  // Jest typings don't define `requireActual` for some reason.
+  ...(jest as any).requireActual('./utils'),
+  writeFileAsync: mockWriteFileAsync,
+  makeDirAsync: mockMakeDirAsync,
+}));
+
+import path from 'path';
+import { integrateLocaleFiles, verifyMessages } from './integrate_locale_files';
+// @ts-ignore
+import { normalizePath } from './utils';
+
+const localePath = path.resolve(__dirname, '__fixtures__', 'integrate_locale_files', 'fr.json');
+
+const mockDefaultMessagesMap = new Map([
+  ['plugin-1.message-id-1', { message: 'Message text 1' }],
+  ['plugin-1.message-id-2', { message: 'Message text 2' }],
+  ['plugin-2.message-id', { message: 'Message text' }],
+]);
+
+const defaultIntegrateOptions = {
+  sourceFileName: localePath,
+  dryRun: false,
+  ignoreIncompatible: false,
+  ignoreMissing: false,
+  ignoreUnused: false,
+  config: {
+    paths: {
+      'plugin-1': 'src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_1',
+      'plugin-2': 'src/dev/i18n/__fixtures__/integrate_locale_files/test_plugin_2',
+    },
+    exclude: [],
+    translations: [],
+  },
+  log: { success: jest.fn(), warning: jest.fn() } as any,
+};
+
+describe('dev/i18n/integrate_locale_files', () => {
+  describe('verifyMessages', () => {
+    test('validates localized messages', () => {
+      const localizedMessagesMap = new Map([
+        ['plugin-1.message-id-1', 'Translated text 1'],
+        ['plugin-1.message-id-2', 'Translated text 2'],
+        ['plugin-2.message-id', 'Translated text'],
+      ]);
+
+      expect(() =>
+        verifyMessages(localizedMessagesMap, mockDefaultMessagesMap, defaultIntegrateOptions)
+      ).not.toThrow();
+    });
+
+    test('throws an error for unused id, missing id or the incompatible ones', () => {
+      const localizedMessagesMapWithMissingMessage = new Map([
+        ['plugin-1.message-id-1', 'Translated text 1'],
+        ['plugin-2.message-id', 'Translated text'],
+      ]);
+
+      const localizedMessagesMapWithUnusedMessage = new Map([
+        ['plugin-1.message-id-1', 'Translated text 1'],
+        ['plugin-1.message-id-2', 'Translated text 2'],
+        ['plugin-1.message-id-3', 'Translated text 3'],
+        ['plugin-2.message-id', 'Translated text'],
+      ]);
+
+      const localizedMessagesMapWithIdTypo = new Map([
+        ['plugin-1.message-id-1', 'Message text 1'],
+        ['plugin-1.message-id-2', 'Message text 2'],
+        ['plugin-2.message', 'Message text'],
+      ]);
+
+      const localizedMessagesMapWithUnknownValues = new Map([
+        ['plugin-1.message-id-1', 'Translated text 1'],
+        ['plugin-1.message-id-2', 'Translated text 2 with some unknown {value}'],
+        ['plugin-2.message-id', 'Translated text'],
+      ]);
+
+      expect(() =>
+        verifyMessages(
+          localizedMessagesMapWithMissingMessage,
+          mockDefaultMessagesMap,
+          defaultIntegrateOptions
+        )
+      ).toThrowErrorMatchingSnapshot();
+      expect(() =>
+        verifyMessages(
+          localizedMessagesMapWithUnusedMessage,
+          mockDefaultMessagesMap,
+          defaultIntegrateOptions
+        )
+      ).toThrowErrorMatchingSnapshot();
+      expect(() =>
+        verifyMessages(
+          localizedMessagesMapWithIdTypo,
+          mockDefaultMessagesMap,
+          defaultIntegrateOptions
+        )
+      ).toThrowErrorMatchingSnapshot();
+      expect(() =>
+        verifyMessages(
+          localizedMessagesMapWithUnknownValues,
+          mockDefaultMessagesMap,
+          defaultIntegrateOptions
+        )
+      ).toThrowErrorMatchingSnapshot();
+    });
+
+    test('removes unused ids if `ignoreUnused` is set', () => {
+      const localizedMessagesMapWithUnusedMessage = new Map([
+        ['plugin-1.message-id-1', 'Translated text 1'],
+        ['plugin-1.message-id-2', 'Translated text 2'],
+        ['plugin-1.message-id-3', 'Some old translated text 3'],
+        ['plugin-2.message-id', 'Translated text'],
+        ['plugin-2.message', 'Some old translated text'],
+      ]);
+
+      verifyMessages(localizedMessagesMapWithUnusedMessage, mockDefaultMessagesMap, {
+        ...defaultIntegrateOptions,
+        ignoreUnused: true,
+      });
+
+      expect(localizedMessagesMapWithUnusedMessage).toMatchInlineSnapshot(`
+Map {
+  "plugin-1.message-id-1" => "Translated text 1",
+  "plugin-1.message-id-2" => "Translated text 2",
+  "plugin-2.message-id" => "Translated text",
+}
+`);
+    });
+
+    test('removes ids with incompatible ICU structure if `ignoreIncompatible` is set', () => {
+      const localizedMessagesMapWithIncompatibleMessage = new Map([
+        ['plugin-1.message-id-1', 'Translated text 1'],
+        ['plugin-1.message-id-2', 'Translated text 2 with some unknown {value}'],
+        ['plugin-2.message-id', 'Translated text'],
+      ]);
+
+      verifyMessages(localizedMessagesMapWithIncompatibleMessage, mockDefaultMessagesMap, {
+        ...defaultIntegrateOptions,
+        ignoreIncompatible: true,
+      });
+
+      expect(localizedMessagesMapWithIncompatibleMessage).toMatchInlineSnapshot(`
+Map {
+  "plugin-1.message-id-1" => "Translated text 1",
+  "plugin-2.message-id" => "Translated text",
+}
+`);
+    });
+  });
+
+  describe('integrateLocaleFiles', () => {
+    test('splits locale file by plugins and writes them into the right folders', async () => {
+      await integrateLocaleFiles(mockDefaultMessagesMap, defaultIntegrateOptions);
+
+      const [[path1, json1], [path2, json2]] = mockWriteFileAsync.mock.calls;
+      const [[dirPath1], [dirPath2]] = mockMakeDirAsync.mock.calls;
+
+      expect([normalizePath(path1), json1]).toMatchSnapshot();
+      expect([normalizePath(path2), json2]).toMatchSnapshot();
+      expect([normalizePath(dirPath1), normalizePath(dirPath2)]).toMatchSnapshot();
+    });
+  });
+});

src/dev/i18n/integrate_locale_files.ts

@@ -0,0 +1,200 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { ToolingLog } from '@kbn/dev-utils';
+import { i18n } from '@kbn/i18n';
+import path from 'path';
+
+import {
+  accessAsync,
+  checkValuesProperty,
+  difference,
+  extractValueReferencesFromMessage,
+  makeDirAsync,
+  normalizePath,
+  readFileAsync,
+  writeFileAsync,
+  // @ts-ignore
+} from './utils';
+
+import { createFailError } from '../run';
+import { I18nConfig } from './config';
+import { serializeToJson } from './serializers';
+
+interface IntegrateOptions {
+  sourceFileName: string;
+  targetFileName?: string;
+  dryRun: boolean;
+  ignoreIncompatible: boolean;
+  ignoreUnused: boolean;
+  ignoreMissing: boolean;
+  config: I18nConfig;
+  log: ToolingLog;
+}
+
+type MessageMap = Map<string, { message: string }>;
+type GroupedMessageMap = Map<string, Array<[string, { message: string }]>>;
+type LocalizedMessageMap = Map<string, string | { text: string }>;
+
+export function verifyMessages(
+  localizedMessagesMap: LocalizedMessageMap,
+  defaultMessagesMap: MessageMap,
+  options: IntegrateOptions
+) {
+  let errorMessage = '';
+
+  const defaultMessagesIds = [...defaultMessagesMap.keys()];
+  const localizedMessagesIds = [...localizedMessagesMap.keys()];
+
+  const unusedTranslations = difference(localizedMessagesIds, defaultMessagesIds);
+  if (unusedTranslations.length > 0) {
+    if (!options.ignoreUnused) {
+      errorMessage += `\n${
+        unusedTranslations.length
+      } unused translation(s):\n${unusedTranslations.join(', ')}`;
+    } else {
+      for (const unusedTranslationId of unusedTranslations) {
+        localizedMessagesMap.delete(unusedTranslationId);
+      }
+    }
+  }
+
+  if (!options.ignoreMissing) {
+    const missingTranslations = difference(defaultMessagesIds, localizedMessagesIds);
+    if (missingTranslations.length > 0) {
+      errorMessage += `\n${
+        missingTranslations.length
+      } missing translation(s):\n${missingTranslations.join(', ')}`;
+    }
+  }
+
+  for (const messageId of localizedMessagesIds) {
+    const defaultMessage = defaultMessagesMap.get(messageId);
+    if (defaultMessage) {
+      try {
+        const message = localizedMessagesMap.get(messageId)!;
+        checkValuesProperty(
+          extractValueReferencesFromMessage(defaultMessage.message, messageId),
+          typeof message === 'string' ? message : message.text,
+          messageId
+        );
+      } catch (err) {
+        if (options.ignoreIncompatible) {
+          localizedMessagesMap.delete(messageId);
+          options.log.warning(`Incompatible translation ignored: ${err.message}`);
+        } else {
+          errorMessage += `\nIncompatible translation: ${err.message}\n`;
+        }
+      }
+    }
+  }
+
+  if (errorMessage) {
+    throw createFailError(errorMessage);
+  }
+}
+
+function groupMessagesByNamespace(
+  localizedMessagesMap: LocalizedMessageMap,
+  knownNamespaces: string[]
+) {
+  const localizedMessagesByNamespace = new Map();
+  for (const [messageId, messageValue] of localizedMessagesMap) {
+    const namespace = knownNamespaces.find(key => messageId.startsWith(`${key}.`));
+    if (!namespace) {
+      throw createFailError(`Unknown namespace in id ${messageId}.`);
+    }
+
+    if (!localizedMessagesByNamespace.has(namespace)) {
+      localizedMessagesByNamespace.set(namespace, []);
+    }
+
+    localizedMessagesByNamespace
+      .get(namespace)
+      .push([
+        messageId,
+        { message: typeof messageValue === 'string' ? messageValue : messageValue.text },
+      ]);
+  }
+
+  return localizedMessagesByNamespace;
+}
+
+async function writeMessages(
+  localizedMessagesByNamespace: GroupedMessageMap,
+  formats: typeof i18n.formats,
+  options: IntegrateOptions
+) {
+  // If target file name is specified we need to write all the translations into one file,
+  // irrespective to the namespace.
+  if (options.targetFileName) {
+    await writeFileAsync(
+      options.targetFileName,
+      serializeToJson(
+        [...localizedMessagesByNamespace.values()].reduce((acc, val) => acc.concat(val), []),
+        formats
+      )
+    );
+
+    return options.log.success(
+      `Translations have been integrated to ${normalizePath(options.targetFileName)}`
+    );
+  }
+
+  // Use basename of source file name to write the same locale name as the source file has.
+  const fileName = path.basename(options.sourceFileName);
+  for (const [namespace, messages] of localizedMessagesByNamespace) {
+    const destPath = path.resolve(options.config.paths[namespace], 'translations');
+
+    try {
+      await accessAsync(destPath);
+    } catch (_) {
+      await makeDirAsync(destPath);
+    }
+
+    const writePath = path.resolve(destPath, fileName);
+    await writeFileAsync(writePath, serializeToJson(messages, formats));
+    options.log.success(`Translations have been integrated to ${normalizePath(writePath)}`);
+  }
+}
+
+export async function integrateLocaleFiles(
+  defaultMessagesMap: MessageMap,
+  options: IntegrateOptions
+) {
+  const localizedMessages = JSON.parse((await readFileAsync(options.sourceFileName)).toString());
+  if (!localizedMessages.formats) {
+    throw createFailError(`Locale file should contain formats object.`);
+  }
+
+  const localizedMessagesMap: LocalizedMessageMap = new Map(
+    Object.entries(localizedMessages.messages)
+  );
+  verifyMessages(localizedMessagesMap, defaultMessagesMap, options);
+
+  const knownNamespaces = Object.keys(options.config.paths);
+  const groupedLocalizedMessagesMap = groupMessagesByNamespace(
+    localizedMessagesMap,
+    knownNamespaces
+  );
+
+  if (!options.dryRun) {
+    await writeMessages(groupedLocalizedMessagesMap, localizedMessages.formats, options);
+  }
+}

src/dev/i18n/serializers/index.ts

@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { i18n } from '@kbn/i18n';
+
+export { serializeToJson } from './json';
+export { serializeToJson5 } from './json5';
+
+export type Serializer = (
+  messages: Array<[string, { message: string; description?: string }]>,
+  formats?: typeof i18n.formats
+) => string;

src/dev/i18n/serializers/json.test.ts

@@ -21,7 +21,7 @@ import { serializeToJson } from './json';
 
 describe('dev/i18n/serializers/json', () => {
   test('should serialize default messages to JSON', () => {
-    const messages = [
+    const messages: Array<[string, { message: string; description?: string }]> = [
       ['plugin1.message.id-1', { message: 'Message text 1 ' }],
       [
         'plugin2.message.id-2',

src/dev/i18n/serializers/json.ts

@@ -18,9 +18,13 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import { Serializer } from '.';
 
-export function serializeToJson(messages, formats = i18n.formats) {
-  const resultJsonObject = { formats, messages: {} };
+export const serializeToJson: Serializer = (messages, formats = i18n.formats) => {
+  const resultJsonObject = {
+    formats,
+    messages: {} as Record<string, string | { text: string; comment: string }>,
+  };
 
   for (const [mapKey, mapValue] of messages) {
     if (mapValue.description) {
@@ -31,4 +35,4 @@ export function serializeToJson(messages, formats = i18n.formats) {
   }
 
   return JSON.stringify(resultJsonObject, undefined, 2);
-}
+};

src/dev/i18n/serializers/json5.test.ts

@@ -21,7 +21,7 @@ import { serializeToJson5 } from './json5';
 
 describe('dev/i18n/serializers/json5', () => {
   test('should serialize default messages to JSON5', () => {
-    const messages = [
+    const messages: Array<[string, { message: string; description?: string }]> = [
       [
         'plugin1.message.id-1',
         {

src/dev/i18n/serializers/json5.ts

@@ -17,12 +17,13 @@
  * under the License.
  */
 
-import JSON5 from 'json5';
 import { i18n } from '@kbn/i18n';
+import JSON5 from 'json5';
+import { Serializer } from '.';
 
 const ESCAPE_SINGLE_QUOTE_REGEX = /\\([\s\S])|(')/g;
 
-export function serializeToJson5(messages, formats = i18n.formats) {
+export const serializeToJson5: Serializer = (messages, formats = i18n.formats) => {
   // .slice(0, -4): remove closing curly braces from json to append messages
   let jsonBuffer = Buffer.from(
     JSON5.stringify({ formats, messages: {} }, { quote: `'`, space: 2 })
@@ -46,5 +47,5 @@ export function serializeToJson5(messages, formats = i18n.formats) {
   // append previously removed closing curly braces
   jsonBuffer = Buffer.concat([jsonBuffer, Buffer.from('  },\n}\n')]);
 
-  return jsonBuffer;
-}
+  return jsonBuffer.toString();
+};

src/dev/i18n/tasks/extract_default_translations.ts

@@ -0,0 +1,74 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import chalk from 'chalk';
+import Listr from 'listr';
+
+import { ErrorReporter, extractMessagesFromPathToMap, filterConfigPaths, I18nConfig } from '..';
+import { createFailError } from '../../run';
+
+export async function extractDefaultMessages({
+  path,
+  config,
+}: {
+  path?: string | string[];
+  config: I18nConfig;
+}) {
+  const filteredPaths = filterConfigPaths(Array.isArray(path) ? path : [path || './'], config);
+  if (filteredPaths.length === 0) {
+    throw createFailError(
+      `${chalk.white.bgRed(
+        ' I18N ERROR '
+      )} None of input paths is covered by the mappings in .i18nrc.json.`
+    );
+  }
+
+  const reporter = new ErrorReporter();
+
+  const list = new Listr(
+    filteredPaths.map(filteredPath => ({
+      task: async (messages: Map<string, unknown>) => {
+        const initialErrorsNumber = reporter.errors.length;
+
+        // Return result if no new errors were reported for this path.
+        const result = await extractMessagesFromPathToMap(filteredPath, messages, config, reporter);
+        if (reporter.errors.length === initialErrorsNumber) {
+          return result;
+        }
+
+        // Throw an empty error to make Listr mark the task as failed without any message.
+        throw new Error('');
+      },
+      title: filteredPath,
+    })),
+    {
+      exitOnError: false,
+    }
+  );
+
+  try {
+    return await list.run(new Map());
+  } catch (error) {
+    if (error.name === 'ListrError' && reporter.errors.length) {
+      throw createFailError(reporter.errors.join('\n\n'));
+    }
+
+    throw error;
+  }
+}

src/dev/i18n/tasks/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { extractDefaultMessages } from './extract_default_translations';

src/dev/i18n/utils.js

@@ -177,8 +177,10 @@ function extractValueReferencesFromIcuAst(node, keys = new Set()) {
  * @throws if "values" and "defaultMessage" don't correspond to each other
  */
 export function checkValuesProperty(prefixedValuesKeys, defaultMessage, messageId) {
-  // skip validation if defaultMessage doesn't use ICU and values prop has no keys
-  if (!prefixedValuesKeys.length && !defaultMessage.includes('{')) {
+  // Skip validation if `defaultMessage` doesn't include any ICU values and
+  // `values` prop has no keys.
+  const defaultMessageValueReferences = extractValueReferencesFromMessage(defaultMessage, messageId);
+  if (!prefixedValuesKeys.length && defaultMessageValueReferences.length === 0) {
     return;
   }
 
@@ -186,13 +188,39 @@ export function checkValuesProperty(prefixedValuesKeys, defaultMessage, messageI
     key.startsWith(HTML_KEY_PREFIX) ? key.slice(HTML_KEY_PREFIX.length) : key
   );
 
-  let defaultMessageAst;
+  const missingValuesKeys = difference(defaultMessageValueReferences, valuesKeys);
+  if (missingValuesKeys.length) {
+    throw createFailError(
+      `some properties are missing in "values" object ("${messageId}"): [${missingValuesKeys}].`
+    );
+  }
 
+  const unusedValuesKeys = difference(valuesKeys, defaultMessageValueReferences);
+  if (unusedValuesKeys.length) {
+    throw createFailError(
+      `"values" object contains unused properties ("${messageId}"): [${unusedValuesKeys}].`
+    );
+  }
+}
+
+/**
+ * Extracts value references from the ICU message.
+ * @param message ICU message.
+ * @param messageId ICU message id
+ * @returns {string[]}
+ */
+export function extractValueReferencesFromMessage(message, messageId) {
+  // Skip validation if message doesn't use ICU.
+  if (!message.includes('{')) {
+    return [];
+  }
+
+  let messageAST;
   try {
-    defaultMessageAst = parser.parse(defaultMessage);
+    messageAST = parser.parse(message);
   } catch (error) {
     if (error.name === 'SyntaxError') {
-      const errorWithContext = createParserErrorMessage(defaultMessage, {
+      const errorWithContext = createParserErrorMessage(message, {
         loc: {
           line: error.location.start.line,
           column: error.location.start.column - 1,
@@ -208,26 +236,12 @@ export function checkValuesProperty(prefixedValuesKeys, defaultMessage, messageI
     throw error;
   }
 
-  // skip validation if intl-messageformat-parser didn't return an AST with nonempty elements array
-  if (!defaultMessageAst || !defaultMessageAst.elements || !defaultMessageAst.elements.length) {
-    return;
+  // Skip extraction if intl-messageformat-parser didn't return an AST with nonempty elements array.
+  if (!messageAST || !messageAST.elements || !messageAST.elements.length) {
+    return [];
   }
 
-  const defaultMessageValueReferences = extractValueReferencesFromIcuAst(defaultMessageAst);
-
-  const missingValuesKeys = difference(defaultMessageValueReferences, valuesKeys);
-  if (missingValuesKeys.length) {
-    throw createFailError(
-      `some properties are missing in "values" object ("${messageId}"):\n[${missingValuesKeys}].`
-    );
-  }
-
-  const unusedValuesKeys = difference(valuesKeys, defaultMessageValueReferences);
-  if (unusedValuesKeys.length) {
-    throw createFailError(
-      `"values" object contains unused properties ("${messageId}"):\n[${unusedValuesKeys}].`
-    );
-  }
+  return extractValueReferencesFromIcuAst(messageAST);
 }
 
 export function extractMessageIdFromNode(node) {

src/dev/run/index.d.ts

@@ -17,4 +17,9 @@
  * under the License.
  */
 
+import { ToolingLog } from '@kbn/dev-utils';
+
 export function createFailError(msg: string, exitCode?: number): Error;
+export function run(
+  body: (args: { flags: Record<string, any>; log: ToolingLog }) => void
+): Promise<void>;

src/dev/run_i18n_check.js

@@ -1,110 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import chalk from 'chalk';
-import Listr from 'listr';
-import { resolve } from 'path';
-
-import { run, createFailError } from './run';
-import config from '../../.i18nrc.json';
-import {
-  filterPaths,
-  extractMessagesFromPathToMap,
-  writeFileAsync,
-  readFileAsync,
-  serializeToJson,
-  serializeToJson5,
-  ErrorReporter,
-  normalizePath,
-} from './i18n/';
-
-run(async ({ flags: { path, output, 'output-format': outputFormat, include = [] } }) => {
-  const paths = Array.isArray(path) ? path : [path || './'];
-  const additionalI18nConfigPaths = Array.isArray(include) ? include : [include];
-  const mergedConfig = { exclude: [], ...config };
-
-  for (const configPath of additionalI18nConfigPaths) {
-    const additionalConfig = JSON.parse(await readFileAsync(resolve(configPath)));
-
-    for (const [pathNamespace, pathValue] of Object.entries(additionalConfig.paths)) {
-      mergedConfig.paths[pathNamespace] = normalizePath(resolve(configPath, '..', pathValue));
-    }
-
-    for (const exclude of additionalConfig.exclude || []) {
-      mergedConfig.exclude.push(normalizePath(resolve(configPath, '..', exclude)));
-    }
-  }
-
-  const filteredPaths = filterPaths(paths, mergedConfig.paths);
-
-  if (filteredPaths.length === 0) {
-    throw createFailError(
-      `${chalk.white.bgRed(' I18N ERROR ')} \
-None of input paths is available for extraction or validation. See .i18nrc.json.`
-    );
-  }
-
-  const reporter = new ErrorReporter();
-
-  const list = new Listr(
-    filteredPaths.map(filteredPath => ({
-      task: async messages => {
-        const initialErrorsNumber = reporter.errors.length;
-
-        // Return result if no new errors were reported for this path.
-        const result = await extractMessagesFromPathToMap(
-          filteredPath,
-          messages,
-          mergedConfig,
-          reporter
-        );
-        if (reporter.errors.length === initialErrorsNumber) {
-          return result;
-        }
-
-        // throw an empty error to make listr mark the task as failed without any message
-        throw new Error('');
-      },
-      title: filteredPath,
-    })),
-    {
-      exitOnError: false,
-    }
-  );
-
-  try {
-    // messages shouldn't be extracted to a file if output is not supplied
-    const messages = await list.run(new Map());
-    if (!output || !messages.size) {
-      return;
-    }
-
-    const sortedMessages = [...messages].sort(([key1], [key2]) => key1.localeCompare(key2));
-    await writeFileAsync(
-      resolve(output, 'en.json'),
-      outputFormat === 'json5' ? serializeToJson5(sortedMessages) : serializeToJson(sortedMessages)
-    );
-  } catch (error) {
-    if (error.name === 'ListrError' && reporter.errors.length) {
-      throw createFailError(reporter.errors.join('\n\n'));
-    }
-
-    throw error;
-  }
-});

src/dev/run_i18n_check.ts

@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import chalk from 'chalk';
+import Listr from 'listr';
+
+import { integrateLocaleFiles, mergeConfigs } from './i18n';
+import { extractDefaultMessages } from './i18n/tasks';
+import { createFailError, run } from './run';
+
+run(
+  async ({
+    flags: {
+      'ignore-incompatible': ignoreIncompatible,
+      'ignore-missing': ignoreMissing,
+      'ignore-unused': ignoreUnused,
+      'include-config': includeConfig,
+      fix = false,
+      path,
+    },
+    log,
+  }) => {
+    if (
+      fix &&
+      (ignoreIncompatible !== undefined ||
+        ignoreUnused !== undefined ||
+        ignoreMissing !== undefined)
+    ) {
+      throw createFailError(
+        `${chalk.white.bgRed(
+          ' I18N ERROR '
+        )} none of the --ignore-incompatible, --ignore-unused or --ignore-missing is allowed when --fix is set.`
+      );
+    }
+
+    const config = await mergeConfigs(includeConfig);
+    const defaultMessages = await extractDefaultMessages({ path, config });
+
+    if (config.translations.length === 0) {
+      return;
+    }
+
+    const list = new Listr(
+      config.translations.map(translationsPath => ({
+        task: async () => {
+          // If `--fix` is set we should try apply all possible fixes and override translations file.
+          await integrateLocaleFiles(defaultMessages, {
+            sourceFileName: translationsPath,
+            targetFileName: fix ? translationsPath : undefined,
+            dryRun: !fix,
+            ignoreIncompatible: fix || !!ignoreIncompatible,
+            ignoreUnused: fix || !!ignoreUnused,
+            ignoreMissing: fix || !!ignoreMissing,
+            config,
+            log,
+          });
+        },
+        title: `Compatibility check with ${translationsPath}`,
+      })),
+      {
+        concurrent: true,
+        exitOnError: false,
+      }
+    );
+
+    try {
+      await list.run();
+    } catch (error) {
+      process.exitCode = 1;
+
+      if (!error.errors) {
+        log.error('Unhandled exception!');
+        log.error(error);
+        process.exit();
+      }
+
+      for (const e of error.errors) {
+        log.error(e);
+      }
+    }
+  }
+);

src/dev/run_i18n_extract.ts

@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import chalk from 'chalk';
+import { resolve } from 'path';
+
+import { mergeConfigs, serializeToJson, serializeToJson5, writeFileAsync } from './i18n';
+import { extractDefaultMessages } from './i18n/tasks';
+import { createFailError, run } from './run';
+
+run(
+  async ({
+    flags: {
+      path,
+      'output-dir': outputDir,
+      'output-format': outputFormat,
+      'include-config': includeConfig,
+    },
+  }) => {
+    if (!outputDir || typeof outputDir !== 'string') {
+      throw createFailError(
+        `${chalk.white.bgRed(' I18N ERROR ')} --output-dir option should be specified.`
+      );
+    }
+
+    const config = await mergeConfigs(includeConfig);
+    const defaultMessages = await extractDefaultMessages({ path, config });
+
+    // Messages shouldn't be written to a file if output is not supplied.
+    if (!outputDir || !defaultMessages.size) {
+      return;
+    }
+
+    const sortedMessages = [...defaultMessages].sort(([key1], [key2]) => key1.localeCompare(key2));
+    await writeFileAsync(
+      resolve(outputDir, 'en.json'),
+      outputFormat === 'json5' ? serializeToJson5(sortedMessages) : serializeToJson(sortedMessages)
+    );
+  }
+);

src/dev/run_i18n_integrate.js

@@ -1,37 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import chalk from 'chalk';
-
-import { createFailError, run } from './run';
-import { integrateLocaleFiles } from './i18n/integrate_locale_files';
-
-run(async ({ flags: { path }, log }) => {
-  if (!path || typeof path === 'boolean') {
-    throw createFailError(`${chalk.white.bgRed(' I18N ERROR ')} --path option isn't provided.`);
-  }
-
-  if (Array.isArray(path)) {
-    throw createFailError(
-      `${chalk.white.bgRed(' I18N ERROR ')} --path should be specified only once`
-    );
-  }
-
-  await integrateLocaleFiles(path, log);
-});

src/dev/run_i18n_integrate.ts

@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import chalk from 'chalk';
+
+import { integrateLocaleFiles, mergeConfigs } from './i18n';
+import { extractDefaultMessages } from './i18n/tasks';
+import { createFailError, run } from './run';
+
+run(
+  async ({
+    flags: {
+      'dry-run': dryRun = false,
+      'ignore-incompatible': ignoreIncompatible = false,
+      'ignore-missing': ignoreMissing = false,
+      'ignore-unused': ignoreUnused = false,
+      'include-config': includeConfig,
+      path,
+      source,
+      target,
+    },
+    log,
+  }) => {
+    if (!source || typeof source === 'boolean') {
+      throw createFailError(`${chalk.white.bgRed(' I18N ERROR ')} --source option isn't provided.`);
+    }
+
+    if (Array.isArray(source)) {
+      throw createFailError(
+        `${chalk.white.bgRed(' I18N ERROR ')} --source should be specified only once.`
+      );
+    }
+
+    if (Array.isArray(target)) {
+      throw createFailError(
+        `${chalk.white.bgRed(' I18N ERROR ')} --target should be specified only once.`
+      );
+    }
+
+    const config = await mergeConfigs(includeConfig);
+    const defaultMessages = await extractDefaultMessages({ path, config });
+
+    await integrateLocaleFiles(defaultMessages, {
+      sourceFileName: source,
+      targetFileName: target,
+      dryRun,
+      ignoreIncompatible,
+      ignoreUnused,
+      ignoreMissing,
+      config,
+      log,
+    });
+  }
+);

tasks/config/run.js

@@ -115,6 +115,7 @@ module.exports = function (grunt) {
       cmd: process.execPath,
       args: [
         require.resolve('../../scripts/i18n_check'),
+        '--ignore-missing',
       ]
     },
 

yarn.lock

@@ -1565,6 +1565,11 @@
   resolved "https://registry.yarnpkg.com/@types/json-stable-stringify/-/json-stable-stringify-1.0.32.tgz#121f6917c4389db3923640b2e68de5fa64dda88e"
   integrity sha512-q9Q6+eUEGwQkv4Sbst3J4PNgDOvpuVuKj79Hl/qnmBMEIPzB5QoFRUtjcgcg2xNUZyYUGXBk5wYIBKHt0A+Mxw==
 
+"@types/json5@^0.0.30":
+  version "0.0.30"
+  resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.30.tgz#44cb52f32a809734ca562e685c6473b5754a7818"
+  integrity sha512-sqm9g7mHlPY/43fcSNrCYfOeX9zkTTK+euO5E6+CVijSMm5tTjkVdwdqRkY3ljjIAf8679vps5jKUoJBCLsMDA==
+
 "@types/jsonwebtoken@^7.2.7":
   version "7.2.8"
   resolved "https://registry.yarnpkg.com/@types/jsonwebtoken/-/jsonwebtoken-7.2.8.tgz#8d199dab4ddb5bba3234f8311b804d2027af2b3a"