i18n-ai-translate
Version:
Use LLMs to translate your i18n JSON to any language.
290 lines • 12.2 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.translate = translate;
exports.translateDiff = translateDiff;
const constants_1 = require("./constants");
const fastest_levenshtein_1 = require("fastest-levenshtein");
const flat_1 = require("flat");
const utils_1 = require("./utils");
const chat_factory_1 = __importDefault(require("./chats/chat_factory"));
const generate_1 = __importDefault(require("./generate_json/generate"));
const prompt_mode_1 = __importDefault(require("./enums/prompt_mode"));
const rate_limiter_1 = __importDefault(require("./rate_limiter"));
const generate_2 = __importDefault(require("./generate_csv/generate"));
function getChats(options) {
const rateLimiter = new rate_limiter_1.default(options.rateLimitMs, options.verbose);
return {
generateTranslationChat: chat_factory_1.default.newChat(options.engine, options.model, rateLimiter, options.apiKey, options.host, options.chatParams),
verifyStylingChat: chat_factory_1.default.newChat(options.engine, options.model, rateLimiter, options.apiKey, options.host, options.chatParams),
verifyTranslationChat: chat_factory_1.default.newChat(options.engine, options.model, rateLimiter, options.apiKey, options.host, options.chatParams),
};
}
function replaceNewlinesWithPlaceholder(templatedStringPrefix, templatedStringSuffix, flatInput) {
for (const key in flatInput) {
if (Object.prototype.hasOwnProperty.call(flatInput, key)) {
flatInput[key] = flatInput[key].replaceAll("\n", `${templatedStringPrefix}NEWLINE${templatedStringSuffix}`);
}
}
}
function replacePlaceholderWithNewLines(templatedStringPrefix, templatedStringSuffix, sortedOutput) {
for (const key in sortedOutput) {
if (Object.prototype.hasOwnProperty.call(sortedOutput, key)) {
sortedOutput[key] = sortedOutput[key].replaceAll(`${templatedStringPrefix}NEWLINE${templatedStringSuffix}`, "\n");
}
}
}
function groupSimilarValues(flatInput) {
const groups = [];
for (const key in flatInput) {
if (Object.prototype.hasOwnProperty.call(flatInput, key)) {
const val = flatInput[key];
const existingGroup = groups.find((group) => Object.values(group).some((entry) => {
const distPercent = (0, fastest_levenshtein_1.distance)(val, entry) /
Math.max(val.length, entry.length);
return distPercent < 0.3;
}));
if (existingGroup) {
existingGroup[key] = val;
}
else {
groups.push({ [key]: val });
}
}
}
for (let i = groups.length - 1; i > 0; i--) {
const j = Math.floor(Math.random() * (i + 1));
[groups[i], groups[j]] = [groups[j], groups[i]];
}
flatInput = {};
for (const groupObj of groups) {
for (const [k, v] of Object.entries(groupObj)) {
flatInput[k] = v;
}
}
return flatInput;
}
function startTranslationStatsItem() {
return {
batchStartTime: 0,
enqueuedItems: 0,
processedItems: 0,
processedTokens: 0,
totalItems: 0,
totalTokens: 0,
};
}
function startTranslationStats() {
return {
translate: startTranslationStatsItem(),
verify: startTranslationStatsItem(),
};
}
async function getTranslation(flatInput, options, chats, translationStats) {
if (options.verbose) {
(0, utils_1.printInfo)(`Translation prompting mode: ${options.promptMode}\n`);
}
switch (options.promptMode) {
case prompt_mode_1.default.JSON: {
const generateTranslationJSON = new generate_1.default(options);
return generateTranslationJSON.translateJSON(flatInput, options, chats, translationStats);
}
case prompt_mode_1.default.CSV:
return (0, generate_2.default)(flatInput, options, chats, translationStats.translate);
default:
throw new Error("Prompt mode is not set");
}
}
function setDefaults(options) {
if (!options.templatedStringPrefix)
options.templatedStringPrefix = constants_1.DEFAULT_TEMPLATED_STRING_PREFIX;
if (!options.templatedStringSuffix)
options.templatedStringSuffix = constants_1.DEFAULT_TEMPLATED_STRING_SUFFIX;
if (!options.batchMaxTokens)
options.batchMaxTokens = constants_1.DEFAULT_REQUEST_TOKENS;
if (!options.batchSize)
options.batchSize = constants_1.DEFAULT_BATCH_SIZE;
if (!options.verbose)
options.verbose = false;
if (!options.ensureChangedTranslation)
options.ensureChangedTranslation = false;
if (!options.skipTranslationVerification)
options.skipTranslationVerification = false;
if (!options.skipStylingVerification)
options.skipStylingVerification = false;
}
/**
* Translate the input JSON to the given language
* @param options - The options for the translation
*/
async function translate(options) {
setDefaults(options);
// Validate the input and output languages are valid
if (!(0, utils_1.isValidLanguageCode)(options.inputLanguage)) {
throw new Error(`Invalid input language code: ${options.inputLanguage}`);
}
if (!(0, utils_1.isValidLanguageCode)(options.outputLanguage)) {
throw new Error(`Invalid output language code: ${options.outputLanguage}`);
}
if (options.verbose) {
(0, utils_1.printInfo)(`Translating from ${options.inputLanguage} to ${options.outputLanguage}...`);
}
const chats = getChats(options);
let flatInput = (0, flat_1.flatten)(options.inputJSON, {
delimiter: constants_1.FLATTEN_DELIMITER,
});
replaceNewlinesWithPlaceholder(options.templatedStringPrefix, options.templatedStringSuffix, flatInput);
const canonicalToDupes = {};
const valueBuckets = {};
for (const [k, v] of Object.entries(flatInput)) {
(valueBuckets[v] ??= []).push(k);
}
for (const keys of Object.values(valueBuckets)) {
if (keys.length > 1) {
const [canonical, ...dupes] = keys;
canonicalToDupes[canonical] = dupes;
for (const k of dupes) {
delete flatInput[k];
}
}
}
if (options.verbose) {
for (const [canonical, dupes] of Object.entries(canonicalToDupes)) {
(0, utils_1.printInfo)(`De-duplicating ${canonical}\n=>\n${dupes.join("\n")}\n\n`);
}
}
flatInput = groupSimilarValues(flatInput);
const translationStats = startTranslationStats();
const output = await getTranslation(flatInput, options, chats, translationStats);
for (const [canonical, dupes] of Object.entries(canonicalToDupes)) {
const translated = output[canonical];
for (const k of dupes) {
output[k] = translated;
}
}
const sortedOutput = {};
for (const key of Object.keys(output).sort()) {
sortedOutput[key] = output[key];
}
replacePlaceholderWithNewLines(options.templatedStringPrefix, options.templatedStringSuffix, sortedOutput);
const unflattenedOutput = (0, flat_1.unflatten)(sortedOutput, {
delimiter: constants_1.FLATTEN_DELIMITER,
});
if (options.verbose) {
(0, utils_1.printExecutionTime)(translationStats.translate.batchStartTime, "Total execution time: ");
}
return unflattenedOutput;
}
/**
* Translate the difference of an input JSON to the given languages
* @param options - The options for the translation
*/
async function translateDiff(options) {
const flatInputBefore = (0, flat_1.flatten)(options.inputJSONBefore, {
delimiter: constants_1.FLATTEN_DELIMITER,
});
const flatInputAfter = (0, flat_1.flatten)(options.inputJSONAfter, {
delimiter: constants_1.FLATTEN_DELIMITER,
});
const flatToUpdateJSONs = {};
for (const lang in options.toUpdateJSONs) {
if (Object.prototype.hasOwnProperty.call(options.toUpdateJSONs, lang)) {
const flatToUpdateJSON = (0, flat_1.flatten)(options.toUpdateJSONs[lang], {
delimiter: constants_1.FLATTEN_DELIMITER,
});
flatToUpdateJSONs[lang] = flatToUpdateJSON;
}
}
const addedKeys = [];
const modifiedKeys = [];
const deletedKeys = [];
for (const key in flatInputBefore) {
if (flatInputBefore[key] !== flatInputAfter[key]) {
if (flatInputAfter[key] === undefined) {
deletedKeys.push(key);
}
else {
modifiedKeys.push(key);
}
}
}
for (const key in flatInputAfter) {
if (flatInputBefore[key] === undefined) {
addedKeys.push(key);
}
}
if (options.verbose) {
(0, utils_1.printInfo)(`Added keys: ${addedKeys.join("\n")}\n`);
(0, utils_1.printInfo)(`Modified keys: ${modifiedKeys.join("\n")}\n`);
(0, utils_1.printInfo)(`Deleted keys: ${deletedKeys.join("\n")}\n`);
}
for (const key of deletedKeys) {
for (const lang in flatToUpdateJSONs) {
if (Object.prototype.hasOwnProperty.call(flatToUpdateJSONs, lang)) {
delete flatToUpdateJSONs[lang][key];
}
}
}
const translatedJSONs = {};
for (const languageCode in flatToUpdateJSONs) {
if (Object.prototype.hasOwnProperty.call(flatToUpdateJSONs, languageCode)) {
translatedJSONs[languageCode] = {};
const addedAndModifiedTranslations = {};
for (const key of addedKeys) {
addedAndModifiedTranslations[key] = flatInputAfter[key];
}
for (const key of modifiedKeys) {
addedAndModifiedTranslations[key] = flatInputAfter[key];
}
// eslint-disable-next-line no-await-in-loop
const translated = await translate({
apiKey: options.apiKey,
batchMaxTokens: options.batchMaxTokens,
batchSize: options.batchSize,
chatParams: options.chatParams,
engine: options.engine,
ensureChangedTranslation: options.ensureChangedTranslation,
host: options.host,
inputJSON: addedAndModifiedTranslations,
inputLanguage: options.inputLanguage,
model: options.model,
outputLanguage: languageCode,
overridePrompt: options.overridePrompt,
promptMode: options.promptMode,
rateLimitMs: options.rateLimitMs,
skipStylingVerification: options.skipStylingVerification,
skipTranslationVerification: options.skipTranslationVerification,
templatedStringPrefix: options.templatedStringPrefix,
templatedStringSuffix: options.templatedStringSuffix,
verbose: options.verbose,
});
const flatTranslated = (0, flat_1.flatten)(translated, {
delimiter: constants_1.FLATTEN_DELIMITER,
});
for (const key in flatTranslated) {
if (Object.prototype.hasOwnProperty.call(flatTranslated, key)) {
translatedJSONs[languageCode][key] = flatTranslated[key];
}
}
// Sort the keys
translatedJSONs[languageCode] = Object.keys(translatedJSONs[languageCode])
.sort()
.reduce((obj, key) => {
obj[key] = translatedJSONs[languageCode][key];
return obj;
}, {});
}
}
const unflatToUpdateJSONs = {};
for (const lang in translatedJSONs) {
if (Object.prototype.hasOwnProperty.call(translatedJSONs, lang)) {
unflatToUpdateJSONs[lang] = (0, flat_1.unflatten)(translatedJSONs[lang], {
delimiter: constants_1.FLATTEN_DELIMITER,
});
}
}
return unflatToUpdateJSONs;
}
//# sourceMappingURL=translate.js.map