strong-globalize-cli
Version:
StrongLoop Globalize - CLI
415 lines • 16.5 kB
JavaScript
// Copyright IBM Corp. 2018,2020. All Rights Reserved.
// Node module: strong-globalize-cli
// This file is licensed under the Artistic License 2.0.
// License text available at https://opensource.org/licenses/Artistic-2.0
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.translateResource = exports.getCredentials = exports.reverseAdjustLangFromGPB = exports.adjustLangForGPB = exports.setTranslationUnit = exports.GPB_MAX_NUMBER_OF_KEYS = void 0;
const _ = require("lodash");
const assert = require("assert");
const dbg = require("debug");
const debug = dbg('strong-globalize-cli');
const fs = require("fs");
const gpb = require('g11n-pipeline');
const util_1 = require("util");
const SG = require("strong-globalize");
const { helper } = SG;
const lint = require("./lint");
const os = require("os");
const path = require("path");
const mkdirp = require("mkdirp");
const mktmpdir = require('mktmpdir');
const wc = require('word-count');
const createTmpDir = util_1.promisify(mktmpdir);
let INTL_DIR = helper.myIntlDir();
// GPB service limits
exports.GPB_MAX_NUMBER_OF_KEYS = 500; // per messages.json
let MY_TRANSLATION_UNIT = exports.GPB_MAX_NUMBER_OF_KEYS;
function bound(n, lowerBound, upperBound) {
n = typeof n === 'number' ? Math.round(n) : upperBound;
n = Math.max(n, lowerBound);
n = Math.min(n, upperBound);
return n;
}
function setTranslationUnit(unit) {
MY_TRANSLATION_UNIT = bound(unit, 1, exports.GPB_MAX_NUMBER_OF_KEYS);
return MY_TRANSLATION_UNIT;
}
exports.setTranslationUnit = setTranslationUnit;
function adjustLangForGPB(lang) {
if (lang === 'pt')
return 'pt-BR';
return lang;
}
exports.adjustLangForGPB = adjustLangForGPB;
function reverseAdjustLangFromGPB(lang) {
if (lang === 'pt-BR')
return 'pt';
return lang;
}
exports.reverseAdjustLangFromGPB = reverseAdjustLangFromGPB;
const msgStore = {};
function storeMsg() {
fs.writeFileSync(path.join(INTL_DIR, 'MSG.json'), JSON.stringify(helper.sortMsges(msgStore), null, 2) + '\n');
}
function writeToMsg(lang, key, value) {
assert(typeof value === 'string', 'Message value type is not <string>: ' + typeof value);
if (!msgStore[lang])
msgStore[lang] = {};
msgStore[lang][key] = value;
}
function writeAllToMsg(lang, json) {
assert(helper.isSupportedLanguage(lang), 'Unsupported language key: ' + lang);
Object.keys(json)
.sort()
.forEach(function (key) {
writeToMsg(lang, key, json[key]);
});
}
function getCredentials() {
let LC;
try {
LC = require(path.join(os.homedir(), '.strong-globalize/ibm-cloud-credentials.json'));
}
catch (e) {
// Ignore error
}
const BLUEMIX_URL = process.env.BLUEMIX_URL || 'url';
const BLUEMIX_USER = process.env.BLUEMIX_USER || 'user';
const BLUEMIX_PASSWORD = process.env.BLUEMIX_PASSWORD || 'password';
const BLUEMIX_INSTANCE = process.env.BLUEMIX_INSTANCE || 'instanceid';
if (!LC || !LC.credentials)
LC = {
credentials: {},
};
LC.credentials.url = LC.credentials.url || BLUEMIX_URL;
LC.credentials.userId = LC.credentials.userId || BLUEMIX_USER;
LC.credentials.password = LC.credentials.password || BLUEMIX_PASSWORD;
LC.credentials.instanceId = LC.credentials.instanceId || BLUEMIX_INSTANCE;
return LC;
}
exports.getCredentials = getCredentials;
/**
* translateResource
*
* @param {Function} Optional callback function. If not provided, a promise
* will be returned.
*/
// tslint:disable-next-line:no-any
function translateResource(cb) {
if (!cb) {
return _translateResource();
}
else {
_translateResourceWithCallback(cb);
return Promise.resolve();
}
}
exports.translateResource = translateResource;
/**
* translateResource
*
* @param {Function} function(err)
*/
function _translateResource() {
return __awaiter(this, void 0, void 0, function* () {
INTL_DIR = path.join(helper.getRootDir(), 'intl');
const myTargetLangs = [];
helper.enumerateLanguageSync((lang) => {
if (lang === helper.ENGLISH)
return false;
myTargetLangs.push(adjustLangForGPB(lang));
return false;
});
const credentials = getCredentials();
const gpClient = gpb.getClient(credentials);
let supportedLangs;
let err;
try {
const supportedTranslations = util_1.promisify(gpClient.supportedTranslations.bind(gpClient));
supportedLangs = yield supportedTranslations({});
}
catch (e) {
err = e;
}
if (err || !(supportedLangs && supportedLangs.en)) {
const e = helper.MSG_GPB_UNAVAILABLE;
console.error(e);
throw e;
}
const langs = [];
myTargetLangs.forEach(function (targetLang) {
if (supportedLangs.en.indexOf(targetLang) >= 0)
langs.push(targetLang);
});
const tempDir = yield createTmpDir();
return yield translateResourcePriv(gpClient, langs, tempDir);
});
}
// tslint:disable-next-line:no-any
function _translateResourceWithCallback(cb) {
const promise = _translateResource();
promise
.then(() => {
cb();
})
.catch((err) => {
cb(err);
});
}
function reduceMsgFiles(intlDir, tempDir) {
const langDirs = fs.readdirSync(tempDir);
if (!langDirs)
return;
// console.log('======= langDirs:', langDirs);
langDirs.forEach(function (lang) {
if (!helper.isSupportedLanguage(lang))
return;
const tempLangDir = path.join(tempDir, lang);
const tempMsgFiles = fs.readdirSync(tempLangDir);
const jsonData = {};
tempMsgFiles.forEach(function (tempMsgFile) {
if (helper.getTrailerAfterDot(tempMsgFile) !== 'json')
return;
const matched = tempMsgFile.match(/^(.+)_[0-9]*\.json$/);
if (!matched)
return;
const base = matched[1];
if (!(base in jsonData))
jsonData[base] = {};
_.merge(jsonData[base], require(path.join(tempLangDir, tempMsgFile)));
});
const bases = Object.keys(jsonData);
bases.forEach(function (base) {
fs.writeFileSync(path.join(intlDir, lang, base) + '.json', JSON.stringify(helper.sortMsges(jsonData[base]), null, 2));
});
});
}
function translateResourcePriv(gpClient, langs, tempDir) {
return __awaiter(this, void 0, void 0, function* () {
const packageName = helper.getPackageName();
if (!packageName)
throw new Error('Package.json not found.');
if (!helper.initIntlDirs())
throw new Error('English resource does not exist.');
const malformed = lint.lintMessageFiles(true);
if (malformed) {
throw new Error('English resource is malformed.');
}
const enDirPath = helper.intlDir('en');
const msgFiles = fs.readdirSync(enDirPath);
let msgCount = 0;
let wordCount = 0;
let characterCount = 0;
for (const msgFile of msgFiles) {
const trailer = helper.getTrailerAfterDot(msgFile);
if (trailer !== 'json' && trailer !== 'txt') {
continue;
}
const source = helper.readToJson(enDirPath, msgFile, 'en');
if (source) {
const srcKeys = Object.keys(source);
const unit = MY_TRANSLATION_UNIT;
const useTempDir = srcKeys.length > unit;
let unitIx = 0;
const outputDir = useTempDir ? tempDir : INTL_DIR;
while (unit * unitIx < srcKeys.length) {
let unitStr = unitIx.toString();
while (unitStr.length < 6) {
unitStr = '0' + unitStr;
}
const msgFileX = useTempDir
? msgFile.replace('.json', '_' + unitStr + '.json')
: msgFile;
const sourceX = {};
const startIx = unit * unitIx;
const endIx = Math.min(startIx + unit, srcKeys.length);
for (let ix = startIx; ix < endIx; ix++) {
const key = srcKeys[ix];
sourceX[key] = source[key];
}
try {
yield translate(gpClient, sourceX, INTL_DIR, outputDir, langs, msgFileX);
for (const key in sourceX) {
msgCount++;
wordCount += wc(sourceX[key]);
characterCount += sourceX[key].length;
}
}
catch (err) {
console.log('*** translation failed: %s', msgFileX);
langs.forEach(function (lang) {
lang = reverseAdjustLangFromGPB(lang);
const msgFilePath = path.join(tempDir, lang, msgFileX);
try {
fs.unlinkSync(msgFilePath);
console.log('*** removed the residual file: %s%s%s', lang, path.sep, msgFileX);
}
catch (e) { }
});
}
unitIx++;
}
}
}
console.log('--- translated', msgCount, 'messages,', wordCount, 'words,', characterCount, 'characters');
storeMsg();
reduceMsgFiles(INTL_DIR, tempDir);
lint.lintMessageFiles(false);
});
}
function translate(gpClient, source, intlDir, outputDir, targetLangs, msgFile) {
return __awaiter(this, void 0, void 0, function* () {
const sourceCount = Object.keys(source).length;
const bundleName = helper.getPackageName() + '_' + msgFile;
const myBundle = gpClient.bundle(bundleName);
console.log('--- translating %s', bundleName);
debug('*** 1 *** GPB.create');
try {
const create = util_1.promisify(myBundle.create.bind(myBundle));
yield create({
sourceLanguage: 'en',
targetLanguages: targetLangs,
});
}
catch (err) {
if (err.obj &&
err.obj.message &&
err.obj.message.indexOf('DuplicatedResourceException') >= 0) {
err = null;
// If it exists, ignore error and use it.
}
else {
console.error('*** GPB.create error: %j', err);
}
if (err)
throw err;
}
debug('*** 2 *** GPB.uploadStrings');
writeAllToMsg('en', source);
try {
yield myBundle.uploadStrings({
languageId: 'en',
strings: source,
});
}
catch (err) {
if (err)
console.error('*** GPB.uploadStrings error: %j', err);
throw err;
}
debug('*** 3 *** GPB.getStrings');
function writeToTxt(jsonObj, targetLang) {
const keys = Object.keys(jsonObj);
keys.forEach(function (key) {
if (helper.getTrailerAfterDot(key) === 'txt') {
let content = JSON.stringify(jsonObj[key]).slice(1, -1);
delete jsonObj[key];
content = content.replace(/\\.?/g, function (esc) {
if (esc === '\\n')
return os.EOL;
if (esc === '\\t')
return '\x09';
if (esc === '\\"')
return '"';
if (esc === "\\'")
return "'";
return esc;
});
// fs.writeFileSync(path.join(intlDir, targetLang, key), content);
fs.writeFileSync(path.join(intlDir, targetLang, msgFile), content);
}
});
}
function storeTranslatedStrings(targetLang, result) {
const translatedJson = result;
targetLang = reverseAdjustLangFromGPB(targetLang);
// writeAllToMsg(targetLang, translatedJson);
writeToTxt(translatedJson, targetLang);
if (Object.keys(translatedJson).length > 0) {
const targetLandPath = path.join(outputDir, targetLang);
mkdirp.sync(targetLandPath);
fs.writeFileSync(path.join(targetLandPath, msgFile), JSON.stringify(helper.sortMsges(translatedJson), null, 2) + '\n');
}
console.log('--- translated to %s', targetLang);
}
function sleep(ms) {
return __awaiter(this, void 0, void 0, function* () {
return new Promise((resolve) => {
setTimeout(() => resolve(), ms);
});
});
}
for (const lang of targetLangs) {
const intervalMsec = 500;
const maxTry = 10;
let tryCount = 0;
const opts = {
languageId: lang,
resourceKey: Object.keys(source)[0],
};
let data;
let retry = true;
while (retry) {
try {
data = yield myBundle.getStrings(opts);
}
catch (err) {
if (err.obj &&
err.obj.message &&
/Language [^ ]+ does not exist./.test(err.obj.message)) {
console.error('*** GPB.getStrings error: %j', err.obj.message);
retry = false;
}
else {
if (++tryCount >= maxTry) {
console.error('*** translation to %s failed and skipped.', lang);
retry = false;
}
else {
process.stdout.write('.');
yield sleep(intervalMsec);
// retry
}
}
continue;
}
const resultCount = Object.keys(data.resourceStrings).length;
if (resultCount === sourceCount) {
storeTranslatedStrings(lang, data.resourceStrings);
break;
}
try {
data = yield myBundle.getEntryInfo(opts);
if (data && data.resourceEntry.translationStatus === 'FAILED') {
// ['SOURCE_LANGUAGE', 'TRALSLATED', 'IN_PROGRESS', 'FAILED']
console.error('*** translation to %s was incomplete.\n' +
'Try to delete the bundle %s from the GPB dashboard and ' +
'"slt-translate -t" again.', lang, bundleName);
}
}
catch (err) {
if (++tryCount >= maxTry) {
console.error('*** GPB.getEntryInfo error: %j', err);
retry = false;
}
else {
process.stdout.write('.');
yield sleep(intervalMsec);
}
continue;
}
}
}
});
}
//# sourceMappingURL=translate.js.map