1
0
mirror of https://github.com/laurent22/joplin.git synced 2024-12-12 08:54:00 +02:00
joplin/Tools/build-translation.js

327 lines
10 KiB
JavaScript
Raw Normal View History

'use strict';
2019-06-05 19:07:11 +02:00
// Dependencies:
//
// sudo apt install gettext
// sudo apt install translate-toolkit
2019-09-19 23:51:18 +02:00
require('app-module-path').addPath(`${__dirname}/../ReactNativeClient`);
2019-09-19 23:51:18 +02:00
const rootDir = `${__dirname}/..`;
2019-09-19 23:51:18 +02:00
const { filename, fileExtension } = require(`${rootDir}/ReactNativeClient/lib/path-utils.js`);
const fs = require('fs-extra');
const gettextParser = require('gettext-parser');
2019-09-19 23:51:18 +02:00
const cliDir = `${rootDir}/CliClient`;
const cliLocalesDir = `${cliDir}/locales`;
const rnDir = `${rootDir}/ReactNativeClient`;
const electronDir = `${rootDir}/ElectronClient`;
2017-07-18 20:04:47 +02:00
2019-07-18 19:36:29 +02:00
const { execCommand, isMac, insertContentIntoFile } = require('./tool-utils.js');
2018-02-01 22:21:54 +02:00
const { countryDisplayName, countryCodeOnly } = require('lib/locale.js');
function parsePoFile(filePath) {
const content = fs.readFileSync(filePath);
return gettextParser.po.parse(content);
}
function serializeTranslation(translation) {
const output = {};
const translations = translation.translations[''];
for (const n in translations) {
if (!translations.hasOwnProperty(n)) continue;
if (n == '') continue;
2017-07-19 00:14:20 +02:00
const t = translations[n];
let translated = '';
2017-07-19 00:14:20 +02:00
if (t.comments && t.comments.flag && t.comments.flag.indexOf('fuzzy') >= 0) {
// Don't include fuzzy translations
} else {
translated = t['msgstr'][0];
2017-07-19 00:14:20 +02:00
}
if (translated) output[n] = translated;
}
return JSON.stringify(output);
}
function saveToFile(filePath, data) {
fs.writeFileSync(filePath, data);
}
function buildLocale(inputFile, outputFile) {
const r = parsePoFile(inputFile);
const translation = serializeTranslation(r);
saveToFile(outputFile, translation);
}
function executablePath(file) {
const potentialPaths = [
'/usr/local/opt/gettext/bin/',
'/opt/local/bin/',
'/usr/local/bin/',
];
for (const path of potentialPaths) {
const pathFile = path + file;
if (fs.existsSync(pathFile)) {
return pathFile;
}
}
2019-09-19 23:51:18 +02:00
throw new Error(`${file} could not be found. Please install via brew or MacPorts.\n`);
}
2019-01-10 20:53:18 +02:00
async function removePoHeaderDate(filePath) {
2019-01-10 20:53:18 +02:00
let sedPrefix = 'sed -i';
if (isMac()) sedPrefix += ' ""'; // Note: on macOS it has to be 'sed -i ""' (BSD quirk)
2019-09-19 23:51:18 +02:00
await execCommand(`${sedPrefix} -e'/POT-Creation-Date:/d' "${filePath}"`);
await execCommand(`${sedPrefix} -e'/PO-Revision-Date:/d' "${filePath}"`);
2017-07-19 22:39:48 +02:00
}
2017-07-18 20:04:47 +02:00
async function createPotFile(potFilePath, sources) {
const baseArgs = [];
2017-07-18 20:04:47 +02:00
baseArgs.push('--from-code=utf-8');
2019-09-19 23:51:18 +02:00
baseArgs.push(`--output="${potFilePath}"`);
2017-07-18 20:04:47 +02:00
baseArgs.push('--language=JavaScript');
baseArgs.push('--copyright-holder="Laurent Cozic"');
baseArgs.push('--package-name=Joplin-CLI');
baseArgs.push('--package-version=1.0.0');
// baseArgs.push('--no-location');
baseArgs.push('--keyword=_n:1,2');
2017-07-18 20:04:47 +02:00
for (let i = 0; i < sources.length; i++) {
const args = baseArgs.slice();
2017-07-18 20:04:47 +02:00
if (i > 0) args.push('--join-existing');
args.push(sources[i]);
2019-01-10 20:53:18 +02:00
let xgettextPath = 'xgettext';
if (isMac()) xgettextPath = executablePath('xgettext'); // Needs to have been installed with `brew install gettext`
const cmd = `${xgettextPath} ${args.join(' ')}`;
const result = await execCommand(cmd);
2017-07-18 20:04:47 +02:00
if (result) console.error(result);
2017-07-19 22:39:48 +02:00
await removePoHeaderDate(potFilePath);
2017-07-18 20:04:47 +02:00
}
}
async function mergePotToPo(potFilePath, poFilePath) {
2019-02-09 21:28:19 +02:00
let msgmergePath = 'msgmerge';
if (isMac()) msgmergePath = executablePath('msgmerge'); // Needs to have been installed with `brew install gettext`
2019-02-09 21:28:19 +02:00
2019-09-19 23:51:18 +02:00
const command = `${msgmergePath} -U "${poFilePath}" "${potFilePath}"`;
2017-07-18 20:04:47 +02:00
const result = await execCommand(command);
if (result) console.error(result);
2017-07-19 22:39:48 +02:00
await removePoHeaderDate(poFilePath);
2017-07-18 20:04:47 +02:00
}
function buildIndex(locales, stats) {
const output = [];
2017-07-19 23:26:30 +02:00
output.push('var locales = {};');
output.push('var stats = {};');
2017-07-19 23:26:30 +02:00
for (let i = 0; i < locales.length; i++) {
const locale = locales[i];
2019-09-19 23:51:18 +02:00
output.push(`locales['${locale}'] = require('./${locale}.json');`);
2017-07-19 23:26:30 +02:00
}
for (let i = 0; i < stats.length; i++) {
const stat = Object.assign({}, stats[i]);
const locale = stat.locale;
delete stat.locale;
delete stat.translatorName;
delete stat.languageName;
delete stat.untranslatedCount;
2019-09-19 23:51:18 +02:00
output.push(`stats['${locale}'] = ${JSON.stringify(stat)};`);
}
output.push('module.exports = { locales: locales, stats: stats };');
return output.join('\n');
2017-07-19 23:26:30 +02:00
}
2017-11-30 20:29:10 +02:00
function availableLocales(defaultLocale) {
const output = [defaultLocale];
fs.readdirSync(cliLocalesDir).forEach((path) => {
if (fileExtension(path) !== 'po') return;
const locale = filename(path);
if (locale === defaultLocale) return;
output.push(locale);
});
return output;
}
2018-02-10 15:03:01 +02:00
function extractTranslator(regex, poContent) {
const translatorMatch = poContent.match(regex);
let translatorName = '';
2018-02-10 15:03:01 +02:00
if (translatorMatch && translatorMatch.length >= 1) {
translatorName = translatorMatch[1];
translatorName = translatorName.replace(/["\s]+$/, '');
translatorName = translatorName.replace(/\\n$/, '');
translatorName = translatorName.replace(/^\s*/, '');
}
if (translatorName.indexOf('FULL NAME') >= 0) return '';
if (translatorName.indexOf('LL@li.org') >= 0) return '';
return translatorName;
}
async function translationStatus(isDefault, poFile) {
// "apt install translate-toolkit" to have pocount
let pocountPath = 'pocount';
if (isMac()) pocountPath = executablePath('pocount');
2019-09-19 23:51:18 +02:00
const command = `${pocountPath} "${poFile}"`;
const result = await execCommand(command);
const matches = result.match(/Translated:\s*?(\d+)\s*\((.+?)%\)/);
2019-09-19 23:51:18 +02:00
if (!matches || matches.length < 3) throw new Error(`Cannot extract status: ${command}:\n${result}`);
const percentDone = Number(matches[2]);
2019-09-19 23:51:18 +02:00
if (isNaN(percentDone)) throw new Error(`Cannot extract percent translated: ${command}:\n${result}`);
const untranslatedMatches = result.match(/Untranslated:\s*?(\d+)/);
if (!untranslatedMatches) throw new Error(`Cannot extract untranslated: ${command}:\n${result}`);
const untranslatedCount = Number(untranslatedMatches[1]);
let translatorName = '';
const content = await fs.readFile(poFile, 'utf-8');
2018-02-10 15:03:01 +02:00
translatorName = extractTranslator(/Last-Translator:\s*?(.*)/, content);
if (!translatorName) {
translatorName = extractTranslator(/Language-Team:\s*?(.*)/, content);
}
// Remove <> around email otherwise it's converted to HTML with (apparently) non-deterministic
// encoding, so it changes on every update.
2018-05-09 18:06:02 +02:00
translatorName = translatorName.replace(/ </, ' (');
translatorName = translatorName.replace(/>/, ')');
const isAlways100 = poFile.endsWith('en_US.po');
return {
2019-04-26 19:58:40 +02:00
percentDone: isDefault || isAlways100 ? 100 : percentDone,
translatorName: translatorName,
untranslatedCount: untranslatedCount,
};
}
2018-02-10 14:52:57 +02:00
function flagImageUrl(locale) {
2019-04-18 15:59:17 +02:00
const baseUrl = 'https://joplinapp.org/images/flags';
2019-09-19 23:51:18 +02:00
if (locale === 'ar') return `${baseUrl}/country-4x3/arableague.png`;
if (locale === 'eu') return `${baseUrl}/es/basque_country.png`;
if (locale === 'gl_ES') return `${baseUrl}/es/galicia.png`;
if (locale === 'ca') return `${baseUrl}/es/catalonia.png`;
if (locale === 'ko') return `${baseUrl}/country-4x3/kr.png`;
if (locale === 'sv') return `${baseUrl}/country-4x3/se.png`;
if (locale === 'nb_NO') return `${baseUrl}/country-4x3/no.png`;
if (locale === 'ro') return `${baseUrl}/country-4x3/ro.png`;
if (locale === 'fa') return `${baseUrl}/country-4x3/ir.png`;
2019-11-18 10:55:23 +02:00
if (locale === 'eo') return `${baseUrl}/esperanto.png`;
2019-09-19 23:51:18 +02:00
return `${baseUrl}/country-4x3/${countryCodeOnly(locale).toLowerCase()}.png`;
2018-02-10 14:52:57 +02:00
}
function poFileUrl(locale) {
2019-09-19 23:51:18 +02:00
return `https://github.com/laurent22/joplin/blob/master/CliClient/locales/${locale}.po`;
}
function translationStatusToMdTable(status) {
const output = [];
output.push(['&nbsp;', 'Language', 'Po File', 'Last translator', 'Percent done'].join(' | '));
2018-02-01 22:21:54 +02:00
output.push(['---', '---', '---', '---', '---'].join('|'));
for (let i = 0; i < status.length; i++) {
const stat = status[i];
const flagUrl = flagImageUrl(stat.locale);
2019-09-19 23:51:18 +02:00
output.push([`![](${flagUrl})`, stat.languageName, `[${stat.locale}](${poFileUrl(stat.locale)})`, stat.translatorName, `${stat.percentDone}%`].join(' | '));
}
return output.join('\n');
}
async function updateReadmeWithStats(stats) {
2019-07-18 19:36:29 +02:00
await insertContentIntoFile(
2019-09-19 23:51:18 +02:00
`${rootDir}/README.md`,
2019-07-18 19:36:29 +02:00
'<!-- LOCALE-TABLE-AUTO-GENERATED -->\n',
'\n<!-- LOCALE-TABLE-AUTO-GENERATED -->',
translationStatusToMdTable(stats)
);
}
2017-07-18 20:04:47 +02:00
async function main() {
const argv = require('yargs').argv;
const potFilePath = `${cliLocalesDir}/joplin.pot`;
const jsonLocalesDir = `${cliDir}/build/locales`;
2017-07-18 20:04:47 +02:00
const defaultLocale = 'en_GB';
const oldPotStatus = await translationStatus(false, potFilePath);
2017-07-18 20:04:47 +02:00
await createPotFile(potFilePath, [
2019-09-19 23:51:18 +02:00
`${cliDir}/app/*.js`,
`${cliDir}/app/gui/*.js`,
`${electronDir}/*.js`,
`${electronDir}/gui/*.js`,
`${electronDir}/gui/utils/*.js`,
`${electronDir}/plugins/*.js`,
`${rnDir}/lib/*.js`,
`${rnDir}/lib/models/*.js`,
`${rnDir}/lib/services/*.js`,
`${rnDir}/lib/components/*.js`,
`${rnDir}/lib/components/shared/*.js`,
`${rnDir}/lib/components/screens/*.js`,
2017-07-18 20:04:47 +02:00
]);
const newPotStatus = await translationStatus(false, potFilePath);
console.info(`Updated pot file. Total strings: ${oldPotStatus.untranslatedCount} => ${newPotStatus.untranslatedCount}`);
const deletedCount = oldPotStatus.untranslatedCount - newPotStatus.untranslatedCount;
if (deletedCount >= 10) {
if (argv['skip-missing-strings-check']) {
console.info(`${deletedCount} strings have been deleted, but proceeding anyway due to --skip-missing-strings-check flag`);
} else {
throw new Error(`${deletedCount} strings have been deleted - aborting as it could be a bug. To override, use the --skip-missing-strings-check flag.`);
}
}
2019-09-19 23:51:18 +02:00
await execCommand(`cp "${potFilePath}" ` + `"${cliLocalesDir}/${defaultLocale}.po"`);
2017-07-18 20:04:47 +02:00
fs.mkdirpSync(jsonLocalesDir, 0o755);
const stats = [];
const locales = availableLocales(defaultLocale);
2017-07-18 20:04:47 +02:00
for (let i = 0; i < locales.length; i++) {
const locale = locales[i];
2018-12-06 00:30:30 +02:00
2019-09-19 23:51:18 +02:00
console.info(`Building ${locale}...`);
2018-12-06 00:30:30 +02:00
2019-09-19 23:51:18 +02:00
const poFilePäth = `${cliLocalesDir}/${locale}.po`;
const jsonFilePath = `${jsonLocalesDir}/${locale}.json`;
2017-07-18 20:04:47 +02:00
if (locale != defaultLocale) await mergePotToPo(potFilePath, poFilePäth);
buildLocale(poFilePäth, jsonFilePath);
const stat = await translationStatus(defaultLocale === locale, poFilePäth);
stat.locale = locale;
stat.languageName = countryDisplayName(locale);
stats.push(stat);
2017-07-18 20:04:47 +02:00
}
stats.sort((a, b) => a.languageName < b.languageName ? -1 : +1);
2019-09-19 23:51:18 +02:00
saveToFile(`${jsonLocalesDir}/index.js`, buildIndex(locales, stats));
2017-07-19 23:26:30 +02:00
const destDirs = [
`${rnDir}/locales`,
`${electronDir}/locales`,
`${cliDir}/locales-build`,
];
2017-11-04 13:46:06 +02:00
for (const destDir of destDirs) {
await execCommand(`rsync -a "${jsonLocalesDir}/" "${destDir}/"`);
}
await updateReadmeWithStats(stats);
2017-07-18 20:04:47 +02:00
}
main().catch((error) => {
console.error(error);
process.exit(1);
});