Add ability to skip file writing for messages extraction tool (#21588) (#22247)

* Add ability to skip 'en.json' file writing for default messages extraction tool

* Update default messages extractor

* Add namespace validation

* Update tests

* Fix i18n config file

* Add ability to choose output JSON format

* Update output json properties keys
This commit is contained in:
Leanid Shutau 2018-08-22 16:48:15 +03:00 committed by GitHub
parent 704bd1ffed
commit c6309b91e8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 282 additions and 160 deletions

14
.i18nrc.json Normal file
View file

@ -0,0 +1,14 @@
{
"paths": {
"kbn": "src/core_plugins/kibana",
"common.server": "src/server",
"common.ui": "src/ui",
"xpack.idxMgmt": "xpack/plugins/index_management"
},
"exclude": [
"src/ui/ui_render/bootstrap/app_bootstrap.js",
"src/ui/ui_render/ui_render_mixin.js",
"x-pack/plugins/monitoring/public/components/cluster/overview/alerts_panel.js",
"x-pack/plugins/monitoring/public/directives/alerts/index.js"
]
}

View file

@ -327,6 +327,7 @@
"mutation-observer": "^1.0.3",
"nock": "8.0.0",
"node-sass": "^4.9.0",
"normalize-path": "^3.0.0",
"pixelmatch": "4.0.2",
"postcss": "^7.0.2",
"prettier": "^1.14.0",

View file

@ -1 +1 @@
<p>{{ 'test-plugin.message-id' | i18n: { defaultMessage: 'Message text' } }}</p>
<p>{{ 'plugin_2.message-id' | i18n: { defaultMessage: 'Message text' } }}</p>

View file

@ -2,132 +2,135 @@
exports[`dev/i18n/extract_default_translations extracts messages to en.json 1`] = `
"{
formats: {
number: {
currency: {
style: 'currency',
},
percent: {
style: 'percent',
\\"formats\\": {
\\"number\\": {
\\"currency\\": {
\\"style\\": \\"currency\\"
},
\\"percent\\": {
\\"style\\": \\"percent\\"
}
},
date: {
short: {
month: 'numeric',
day: 'numeric',
year: '2-digit',
\\"date\\": {
\\"short\\": {
\\"month\\": \\"numeric\\",
\\"day\\": \\"numeric\\",
\\"year\\": \\"2-digit\\"
},
medium: {
month: 'short',
day: 'numeric',
year: 'numeric',
\\"medium\\": {
\\"month\\": \\"short\\",
\\"day\\": \\"numeric\\",
\\"year\\": \\"numeric\\"
},
long: {
month: 'long',
day: 'numeric',
year: 'numeric',
},
full: {
weekday: 'long',
month: 'long',
day: 'numeric',
year: 'numeric',
\\"long\\": {
\\"month\\": \\"long\\",
\\"day\\": \\"numeric\\",
\\"year\\": \\"numeric\\"
},
\\"full\\": {
\\"weekday\\": \\"long\\",
\\"month\\": \\"long\\",
\\"day\\": \\"numeric\\",
\\"year\\": \\"numeric\\"
}
},
time: {
short: {
hour: 'numeric',
minute: 'numeric',
\\"time\\": {
\\"short\\": {
\\"hour\\": \\"numeric\\",
\\"minute\\": \\"numeric\\"
},
medium: {
hour: 'numeric',
minute: 'numeric',
second: 'numeric',
\\"medium\\": {
\\"hour\\": \\"numeric\\",
\\"minute\\": \\"numeric\\",
\\"second\\": \\"numeric\\"
},
long: {
hour: 'numeric',
minute: 'numeric',
second: 'numeric',
timeZoneName: 'short',
\\"long\\": {
\\"hour\\": \\"numeric\\",
\\"minute\\": \\"numeric\\",
\\"second\\": \\"numeric\\",
\\"timeZoneName\\": \\"short\\"
},
full: {
hour: 'numeric',
minute: 'numeric',
second: 'numeric',
timeZoneName: 'short',
},
},
\\"full\\": {
\\"hour\\": \\"numeric\\",
\\"minute\\": \\"numeric\\",
\\"second\\": \\"numeric\\",
\\"timeZoneName\\": \\"short\\"
}
}
},
'plugin_1.id_1': 'Message 1',
'plugin_1.id_2': 'Message 2', // Message context
'plugin_1.id_3': 'Message 3',
'plugin_1.id_4': 'Message 4',
'plugin_1.id_5': 'Message 5',
'plugin_1.id_6': 'Message 6',
'plugin_1.id_7': 'Message 7',
}
"
\\"plugin_1.id_1\\": \\"Message 1\\",
\\"plugin_1.id_2\\": {
\\"text\\": \\"Message 2\\",
\\"comment\\": \\"Message context\\"
},
\\"plugin_1.id_3\\": \\"Message 3\\",
\\"plugin_1.id_4\\": \\"Message 4\\",
\\"plugin_1.id_5\\": \\"Message 5\\",
\\"plugin_1.id_6\\": \\"Message 6\\",
\\"plugin_1.id_7\\": \\"Message 7\\"
}"
`;
exports[`dev/i18n/extract_default_translations injects default formats into en.json 1`] = `
"{
formats: {
number: {
currency: {
style: 'currency',
},
percent: {
style: 'percent',
\\"formats\\": {
\\"number\\": {
\\"currency\\": {
\\"style\\": \\"currency\\"
},
\\"percent\\": {
\\"style\\": \\"percent\\"
}
},
date: {
short: {
month: 'numeric',
day: 'numeric',
year: '2-digit',
\\"date\\": {
\\"short\\": {
\\"month\\": \\"numeric\\",
\\"day\\": \\"numeric\\",
\\"year\\": \\"2-digit\\"
},
medium: {
month: 'short',
day: 'numeric',
year: 'numeric',
\\"medium\\": {
\\"month\\": \\"short\\",
\\"day\\": \\"numeric\\",
\\"year\\": \\"numeric\\"
},
long: {
month: 'long',
day: 'numeric',
year: 'numeric',
},
full: {
weekday: 'long',
month: 'long',
day: 'numeric',
year: 'numeric',
\\"long\\": {
\\"month\\": \\"long\\",
\\"day\\": \\"numeric\\",
\\"year\\": \\"numeric\\"
},
\\"full\\": {
\\"weekday\\": \\"long\\",
\\"month\\": \\"long\\",
\\"day\\": \\"numeric\\",
\\"year\\": \\"numeric\\"
}
},
time: {
short: {
hour: 'numeric',
minute: 'numeric',
\\"time\\": {
\\"short\\": {
\\"hour\\": \\"numeric\\",
\\"minute\\": \\"numeric\\"
},
medium: {
hour: 'numeric',
minute: 'numeric',
second: 'numeric',
\\"medium\\": {
\\"hour\\": \\"numeric\\",
\\"minute\\": \\"numeric\\",
\\"second\\": \\"numeric\\"
},
long: {
hour: 'numeric',
minute: 'numeric',
second: 'numeric',
timeZoneName: 'short',
\\"long\\": {
\\"hour\\": \\"numeric\\",
\\"minute\\": \\"numeric\\",
\\"second\\": \\"numeric\\",
\\"timeZoneName\\": \\"short\\"
},
full: {
hour: 'numeric',
minute: 'numeric',
second: 'numeric',
timeZoneName: 'short',
},
},
\\"full\\": {
\\"hour\\": \\"numeric\\",
\\"minute\\": \\"numeric\\",
\\"second\\": \\"numeric\\",
\\"timeZoneName\\": \\"short\\"
}
}
},
'test-plugin.message-id': 'Message text',
}
"
\\"plugin_2.message-id\\": \\"Message text\\"
}"
`;
exports[`dev/i18n/extract_default_translations throws on wrong message namespace 1`] = `"Expected \\"wrong_plugin_namespace.message-id\\" id to have \\"plugin_2\\" namespace. See i18nrc.json for the list of supported namespaces."`;

View file

@ -17,15 +17,19 @@
* under the License.
*/
import { resolve } from 'path';
import path from 'path';
import { i18n } from '@kbn/i18n';
import JSON5 from 'json5';
import normalize from 'normalize-path';
import { extractHtmlMessages } from './extract_html_messages';
import { extractCodeMessages } from './extract_code_messages';
import { extractPugMessages } from './extract_pug_messages';
import { extractHandlebarsMessages } from './extract_handlebars_messages';
import { globAsync, makeDirAsync, accessAsync, readFileAsync, writeFileAsync } from './utils';
import { globAsync, readFileAsync, writeFileAsync } from './utils';
import { paths, exclude } from '../../../.i18nrc.json';
const ESCAPE_SINGLE_QUOTE_REGEX = /\\([\s\S])|(')/g;
function addMessageToMap(targetMap, key, value) {
const existingValue = targetMap.get(key);
@ -38,7 +42,48 @@ function addMessageToMap(targetMap, key, value) {
targetMap.set(key, value);
}
export async function extractDefaultTranslations(inputPath) {
function normalizePath(inputPath) {
return normalize(path.relative('.', inputPath));
}
function filterPaths(inputPaths) {
const availablePaths = Object.values(paths);
const pathsForExtraction = new Set();
for (const inputPath of inputPaths) {
const normalizedPath = normalizePath(inputPath);
// If input path is the sub path of or equal to any available path, include it.
if (
availablePaths.some(path => normalizedPath.startsWith(`${path}/`) || path === normalizedPath)
) {
pathsForExtraction.add(normalizedPath);
} else {
// Otherwise go through all available paths and see if any of them is the sub
// path of the input path (empty normalized path corresponds to root or above).
availablePaths
.filter(path => !normalizedPath || path.startsWith(`${normalizedPath}/`))
.forEach(ePath => pathsForExtraction.add(ePath));
}
}
return [...pathsForExtraction];
}
export function validateMessageNamespace(id, filePath) {
const normalizedPath = normalizePath(filePath);
const [expectedNamespace] = Object.entries(paths).find(([, pluginPath]) =>
normalizedPath.startsWith(`${pluginPath}/`)
);
if (!id.startsWith(`${expectedNamespace}.`)) {
throw new Error(`Expected "${id}" id to have "${expectedNamespace}" namespace. \
See i18nrc.json for the list of supported namespaces.`);
}
}
export async function extractMessagesFromPathToMap(inputPath, targetMap) {
const entries = await globAsync('*.{js,jsx,pug,ts,tsx,html,hbs,handlebars}', {
cwd: inputPath,
matchBase: true,
@ -46,7 +91,7 @@ export async function extractDefaultTranslations(inputPath) {
const { htmlEntries, codeEntries, pugEntries, hbsEntries } = entries.reduce(
(paths, entry) => {
const resolvedPath = resolve(inputPath, entry);
const resolvedPath = path.resolve(inputPath, entry);
if (resolvedPath.endsWith('.html')) {
paths.htmlEntries.push(resolvedPath);
@ -63,8 +108,6 @@ export async function extractDefaultTranslations(inputPath) {
{ htmlEntries: [], codeEntries: [], pugEntries: [], hbsEntries: [] }
);
const defaultMessagesMap = new Map();
await Promise.all(
[
[htmlEntries, extractHtmlMessages],
@ -73,7 +116,7 @@ export async function extractDefaultTranslations(inputPath) {
[hbsEntries, extractHandlebarsMessages],
].map(async ([entries, extractFunction]) => {
const files = await Promise.all(
entries.map(async entry => {
entries.filter(entry => !exclude.includes(normalizePath(entry))).map(async entry => {
return {
name: entry,
content: await readFileAsync(entry),
@ -84,7 +127,8 @@ export async function extractDefaultTranslations(inputPath) {
for (const { name, content } of files) {
try {
for (const [id, value] of extractFunction(content)) {
addMessageToMap(defaultMessagesMap, id, value);
validateMessageNamespace(id, name);
addMessageToMap(targetMap, id, value);
}
} catch (error) {
throw new Error(`Error in ${name}\n${error.message || error}`);
@ -92,32 +136,65 @@ export async function extractDefaultTranslations(inputPath) {
}
})
);
}
function serializeToJson5(defaultMessages) {
// .slice(0, -1): remove closing curly brace from json to append messages
let jsonBuffer = Buffer.from(
JSON5.stringify({ formats: i18n.formats }, { quote: `'`, space: 2 }).slice(0, -1)
);
const defaultMessages = [...defaultMessagesMap].sort(([key1], [key2]) => {
return key1 < key2 ? -1 : 1;
});
for (const [mapKey, mapValue] of defaultMessages) {
const formattedMessage = mapValue.message.replace(ESCAPE_SINGLE_QUOTE_REGEX, '\\$1$2');
const formattedContext = mapValue.context
? mapValue.context.replace(ESCAPE_SINGLE_QUOTE_REGEX, '\\$1$2')
: '';
jsonBuffer = Buffer.concat([
jsonBuffer,
Buffer.from(` '${mapKey}': '${mapValue.message}',`),
Buffer.from(mapValue.context ? ` // ${mapValue.context}\n` : '\n'),
Buffer.from(` '${mapKey}': '${formattedMessage}',`),
Buffer.from(formattedContext ? ` // ${formattedContext}\n` : '\n'),
]);
}
// append previously removed closing curly brace
jsonBuffer = Buffer.concat([jsonBuffer, Buffer.from('}\n')]);
try {
await accessAsync(resolve(inputPath, 'translations'));
} catch (_) {
await makeDirAsync(resolve(inputPath, 'translations'));
return jsonBuffer;
}
function serializeToJson(defaultMessages) {
const resultJsonObject = { formats: i18n.formats };
for (const [mapKey, mapValue] of defaultMessages) {
if (mapValue.context) {
resultJsonObject[mapKey] = { text: mapValue.message, comment: mapValue.context };
} else {
resultJsonObject[mapKey] = mapValue.message;
}
}
await writeFileAsync(resolve(inputPath, 'translations', 'en.json'), jsonBuffer);
return JSON.stringify(resultJsonObject, undefined, 2);
}
export async function extractDefaultTranslations({ paths, output, outputFormat }) {
const defaultMessagesMap = new Map();
for (const inputPath of filterPaths(paths)) {
await extractMessagesFromPathToMap(inputPath, defaultMessagesMap);
}
// messages shouldn't be extracted to a file if output is not supplied
if (!output || !defaultMessagesMap.size) {
return;
}
const defaultMessages = [...defaultMessagesMap].sort(([key1], [key2]) =>
key1.localeCompare(key2)
);
await writeFileAsync(
path.resolve(output, 'en.json'),
outputFormat === 'json5' ? serializeToJson5(defaultMessages) : serializeToJson(defaultMessages)
);
}

View file

@ -18,14 +18,11 @@
*/
import path from 'path';
import fs from 'fs';
import { promisify } from 'util';
import { extractDefaultTranslations } from './extract_default_translations';
const readFileAsync = promisify(fs.readFile);
const removeDirAsync = promisify(fs.rmdir);
const unlinkAsync = promisify(fs.unlink);
import {
extractDefaultTranslations,
validateMessageNamespace,
} from './extract_default_translations';
const fixturesPath = path.resolve(__dirname, '__fixtures__', 'extract_default_translations');
const pluginsPaths = [
@ -34,40 +31,72 @@ const pluginsPaths = [
path.join(fixturesPath, 'test_plugin_3'),
];
jest.mock('../../../.i18nrc.json', () => ({
paths: {
plugin_1: 'src/dev/i18n/__fixtures__/extract_default_translations/test_plugin_1',
plugin_2: 'src/dev/i18n/__fixtures__/extract_default_translations/test_plugin_2',
plugin_3: 'src/dev/i18n/__fixtures__/extract_default_translations/test_plugin_3',
},
exclude: [],
}));
const utils = require('./utils');
utils.writeFileAsync = jest.fn();
describe('dev/i18n/extract_default_translations', () => {
test('extracts messages to en.json', async () => {
const [pluginPath] = pluginsPaths;
await extractDefaultTranslations(pluginPath);
const extractedJSONBuffer = await readFileAsync(
path.join(pluginPath, 'translations', 'en.json')
);
utils.writeFileAsync.mockClear();
await extractDefaultTranslations({
paths: [pluginPath],
output: pluginPath,
});
await unlinkAsync(path.join(pluginPath, 'translations', 'en.json'));
await removeDirAsync(path.join(pluginPath, 'translations'));
const [[, json]] = utils.writeFileAsync.mock.calls;
expect(extractedJSONBuffer.toString()).toMatchSnapshot();
expect(json.toString()).toMatchSnapshot();
});
test('injects default formats into en.json', async () => {
const [, pluginPath] = pluginsPaths;
await extractDefaultTranslations(pluginPath);
const extractedJSONBuffer = await readFileAsync(
path.join(pluginPath, 'translations', 'en.json')
);
utils.writeFileAsync.mockClear();
await extractDefaultTranslations({
paths: [pluginPath],
output: pluginPath,
});
await unlinkAsync(path.join(pluginPath, 'translations', 'en.json'));
await removeDirAsync(path.join(pluginPath, 'translations'));
const [[, json]] = utils.writeFileAsync.mock.calls;
expect(extractedJSONBuffer.toString()).toMatchSnapshot();
expect(json.toString()).toMatchSnapshot();
});
test('throws on id collision', async () => {
const [, , pluginPath] = pluginsPaths;
await expect(extractDefaultTranslations(pluginPath)).rejects.toMatchObject({
await expect(
extractDefaultTranslations({ paths: [pluginPath], output: pluginPath })
).rejects.toMatchObject({
message: `Error in ${path.join(pluginPath, 'test_file.jsx')}
There is more than one default message for the same id "plugin_3.duplicate_id": "Message 1" and "Message 2"`,
});
});
test('validates message namespace', () => {
const id = 'plugin_2.message-id';
const filePath = path.resolve(
__dirname,
'__fixtures__/extract_default_translations/test_plugin_2/test_file.html'
);
expect(() => validateMessageNamespace(id, filePath)).not.toThrow();
});
test('throws on wrong message namespace', () => {
const id = 'wrong_plugin_namespace.message-id';
const filePath = path.resolve(
__dirname,
'__fixtures__/extract_default_translations/test_plugin_2/test_file.html'
);
expect(() => validateMessageNamespace(id, filePath)).toThrowErrorMatchingSnapshot();
});
});

View file

@ -29,15 +29,12 @@ import glob from 'glob';
import { promisify } from 'util';
const ESCAPE_LINE_BREAK_REGEX = /(?<!\\)\\\n/g;
const ESCAPE_SINGLE_QUOTE_REGEX = /\\([\s\S])|(')/g;
const HTML_LINE_BREAK_REGEX = /[\s]*\n[\s]*/g;
const LINE_BREAK_REGEX = /\n/g;
export const readFileAsync = promisify(fs.readFile);
export const writeFileAsync = promisify(fs.writeFile);
export const globAsync = promisify(glob);
export const makeDirAsync = promisify(fs.mkdir);
export const accessAsync = promisify(fs.access);
export function isPropertyWithKey(property, identifierName) {
return isObjectProperty(property) && isIdentifier(property.key, { name: identifierName });
@ -61,16 +58,11 @@ export function isI18nTranslateFunction(node) {
}
export function formatJSString(string) {
return (string || '')
.replace(ESCAPE_LINE_BREAK_REGEX, '')
.replace(ESCAPE_SINGLE_QUOTE_REGEX, '\\$1$2')
.replace(LINE_BREAK_REGEX, '\\n');
return (string || '').replace(ESCAPE_LINE_BREAK_REGEX, '').replace(LINE_BREAK_REGEX, '\\n');
}
export function formatHTMLString(string) {
return (string || '')
.replace(ESCAPE_SINGLE_QUOTE_REGEX, '\\$1$2')
.replace(HTML_LINE_BREAK_REGEX, ' ');
return (string || '').replace(HTML_LINE_BREAK_REGEX, ' ');
}
/**

View file

@ -20,8 +20,10 @@
import { run } from './run';
import { extractDefaultTranslations } from './i18n/extract_default_translations';
run(async () => {
for (const inputPath of process.argv.slice(2)) {
await extractDefaultTranslations(inputPath);
}
run(async ({ flags: { path, output, 'output-format': outputFormat } }) => {
await extractDefaultTranslations({
paths: Array.isArray(path) ? path : [path || './'],
output,
outputFormat,
});
});

View file

@ -9342,6 +9342,10 @@ normalize-path@^2.0.0, normalize-path@^2.0.1, normalize-path@^2.1.1:
dependencies:
remove-trailing-separator "^1.0.1"
normalize-path@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
normalize-range@^0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/normalize-range/-/normalize-range-0.1.2.tgz#2d10c06bdfd312ea9777695a4d28439456b75942"