i18n-llm-translate
Version:
Automatically translates namespace-based JSON translation files across multiple languages from any source language
229 lines (228 loc) • 13.1 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.translate = translate;
const cache_1 = require("./cache");
const cleaner_1 = require("./cleaner");
const namespace_1 = require("./namespace");
const util_1 = require("./util");
const logger_1 = require("./logger");
const cache_2 = require("./engines/cache");
const validation_1 = require("./validation");
const break_silent_error_1 = require("./break-silent-error");
function translate(engine, options) {
return __awaiter(this, void 0, void 0, function* () {
var _a, _b;
if (!process.env.TEST_ENGINES && engine.type !== 'llm' && engine.type !== 'ml') {
throw new Error('Wrong translate engine');
}
const startTime = Date.now();
// Setup logger
const logger = options.logger || logger_1.defaultLogger;
if (options.debug !== undefined)
(_a = logger.setDebug) === null || _a === void 0 ? void 0 : _a.call(logger, options.debug);
if (options.verbose !== undefined)
(_b = logger.setVerbose) === null || _b === void 0 ? void 0 : _b.call(logger, options.verbose);
// Validate configuration before filtering
(0, validation_1.validateTranslateOptions)(options);
options.baseLanguageCode = options.baseLanguageCode.toLowerCase();
options.targetLanguageCodes = options.targetLanguageCodes.map(languageCode => languageCode.toLowerCase());
// We filter out base language, as it is used only as reference
options.targetLanguageCodes = options.targetLanguageCodes.filter(languageCode => languageCode !== options.baseLanguageCode);
// If no target languages remain after filtering, return early
if (options.targetLanguageCodes.length === 0) {
logger.info('No target languages to translate after filtering out base language');
return;
}
// We operate only on json cache
if (!options.namesMapping)
options.namesMapping = {};
if (!options.namesMapping.jsonCache)
options.namesMapping.jsonCache = '.translations-cache';
if (!options.namesMapping.jsonCache.endsWith(".json"))
options.namesMapping.jsonCache = `${options.namesMapping.jsonCache}.json`;
// Fix application context. Each entry is treated as separate sentence
for (let i = 0; i < options.applicationContextEntries.length; i++) {
let entry = options.applicationContextEntries[i].trim();
if (!entry.endsWith("."))
entry = `${entry}.`;
options.applicationContextEntries[i] = entry;
}
if (options.maxRetriesOnEngineValidationError === undefined)
options.maxRetriesOnEngineValidationError = 3;
if (!options.retryOnEngineValidationErrorFor)
options.retryOnEngineValidationErrorFor = 'llm-only';
try {
yield runTranslatePipeline(engine, options, startTime);
}
catch (e) {
if ((0, break_silent_error_1.isBreakSilentError)(e)) {
logger.error(e.message);
if (options.verboseEngineErrors) {
throw e;
}
return;
}
throw e;
}
});
}
function runTranslatePipeline(engine, options, startTime) {
return __awaiter(this, void 0, void 0, function* () {
var _a;
const logger = options.logger || logger_1.defaultLogger;
let namespaces = yield (0, namespace_1.readTranslationsNamespaces)(options);
const cache = yield (0, cache_1.readTranslationsCache)(options);
if (options.cleanup) {
// Remove files that are neither language directories nor cache file
const dirtyNamespaces = yield (0, cleaner_1.cleanLanguagesDirectory)(options);
if (dirtyNamespaces)
namespaces = yield (0, namespace_1.readTranslationsNamespaces)(options);
// Clean each language directory by removing files that are not namespace files from the base language directory
yield (0, cleaner_1.cleanNamespaces)(options, namespaces);
}
const dirtyCache = cache.cleanCache(namespaces);
// Fix cache structure so it matches all paths with base namespaces
cache.syncCacheWithNamespaces(namespaces, false);
logger.info(`Using engine: "${engine.name}"`);
if (engine.initializeEstimate) {
const estimateInitialization = yield engine.initializeEstimate(options);
if (estimateInitialization.message) {
const message = `Estimate engine: ${estimateInitialization.message}`;
if (estimateInitialization.ok) {
logger.info(message);
}
else {
logger.warn(message);
}
}
}
let dirty = false;
let totalBaseDifferencesCount = 0;
let totalBaseDifferencesTranslationCount = 0;
let totalMissingTranslationCount = 0;
let totalCacheLoadedCount = 0;
let totalCharactersTranslated = 0;
let totalSourceKeysTranslated = 0;
for (let namespace of namespaces) {
let baseDifferences = cache.getBaseLanguageTranslationDifferences(namespace);
if (baseDifferences) {
baseDifferences = (0, namespace_1.stripEmptyStringLeavesFromDiff)(baseDifferences, namespace);
if (namespace.targetLanguages.some(tl => tl.dirty)) {
dirty = true;
}
}
if (baseDifferences) {
totalBaseDifferencesCount += (0, util_1.countKeysInObject)(baseDifferences);
dirty = true;
const baseDifferencesSchema = (0, validation_1.generateTranslationsZodSchema)(baseDifferences);
const engineResultSchema = (0, validation_1.generateLanguagesTranslateReturnZodSchema)(options.targetLanguageCodes, baseDifferencesSchema);
logger.info(`Translating base differences for namespace: "${namespace.jsonFileName}"`);
// Count characters and keys from source text being sent for translation
totalCharactersTranslated += (0, util_1.countTranslatedCharacters)(baseDifferences) * options.targetLanguageCodes.length;
totalSourceKeysTranslated += (0, util_1.countKeysInObject)(baseDifferences);
const translationsResults = yield engine.translate(baseDifferences, options);
const engineCheck = engineResultSchema.safeParse(translationsResults);
// TODO: RETRIES
if (!engineCheck.success) {
logger.error(`Engine does not returned proper translation structure!`);
logger.debug(`Base differences:`, baseDifferences);
logger.error(`Validation error:`, engineCheck.error.issues);
return;
}
totalBaseDifferencesTranslationCount += (0, util_1.countTranslatedKeys)(translationsResults);
(0, namespace_1.applyEngineTranslations)(namespace, translationsResults);
}
}
// Find missing translations
for (let namespace of namespaces) {
const cacheEngine = (0, cache_2.createCacheTranslateEngine)(cache, namespace.jsonFileName);
let missed = namespace.getMissingTranslations();
if (missed) {
dirty = true;
const translationsResults = yield cacheEngine.translateMissed(missed, options);
// Cache results can be nullish, we clear them so they are not written back to cache
const cleanedResult = (0, util_1.clearNullsFromResult)(translationsResults);
// Count how many translations were loaded from cache
totalCacheLoadedCount += (0, util_1.countTranslatedKeys)(cleanedResult);
(0, namespace_1.applyEngineTranslations)(namespace, cleanedResult);
}
missed = namespace.getMissingTranslations();
if (missed) {
dirty = true;
// "Missed" translations are already structured according to their respective languages.
// Different languages may have varying missing translations,
// unlike differential translations based on the primary language.
const engineResultSchema = (0, validation_1.generateTranslationsZodSchema)(missed.targetLanguageTranslationsKeys);
logger.info(`Translating missed translations for namespace: "${namespace.jsonFileName}"`);
logger.debug(`Missed translations structure prepared`);
// Count characters and keys from source text being sent for translation
// For missing translations, we need to count characters multiplied by target languages that need them
totalCharactersTranslated += (0, util_1.countMissingTranslationCharacters)(missed.baseLanguageTranslations, missed.targetLanguageTranslationsKeys);
totalSourceKeysTranslated += (0, util_1.countKeysInObject)(missed.baseLanguageTranslations);
// `baseLanguageTranslations` contains merged missing translations, regardless of the language,
// while avoiding duplicates. This reduces the required context, leading to lower token consumption.
const translationsResults = yield engine.translateMissed(missed, options);
const engineCheck = engineResultSchema.safeParse(translationsResults);
// TODO: RETRIES
if (!engineCheck.success) {
logger.error(`Engine does not returned proper translation structure!`);
logger.error(`Validation error:`, engineCheck.error.issues);
return;
}
totalMissingTranslationCount += (0, util_1.countTranslatedKeys)(translationsResults);
(0, namespace_1.applyEngineTranslations)(namespace, translationsResults);
}
}
if (totalCacheLoadedCount > 0) {
logger.info(`Total translations loaded from cache: ${totalCacheLoadedCount}`);
}
if (dirty || dirtyCache) {
// Overwrite cache with namespace values
if (dirty)
cache.syncCacheWithNamespaces(namespaces, true);
yield cache.write();
}
if (dirty) {
for (let namespace of namespaces) {
yield namespace.write();
}
const duration = Date.now() - startTime;
logger.info(`Total source keys sent for translation: ${totalSourceKeysTranslated}`);
logger.info(`Total translations processed: ${totalBaseDifferencesTranslationCount + totalMissingTranslationCount}`);
if (totalMissingTranslationCount) {
logger.info(` of which ${totalMissingTranslationCount} were missing`);
}
logger.info(`Total characters sent for translation: ${totalCharactersTranslated}`);
const usage = ((_a = engine.getUsage) === null || _a === void 0 ? void 0 : _a.call(engine)) || { charactersCount: totalCharactersTranslated };
if ('inputTokens' in usage) {
logger.info(`Total tokens sent for translation: input ${usage.inputTokens}, output ${usage.outputTokens}`);
}
if (engine.estimatePrice && ('charactersCount' in usage ? usage.charactersCount > 0 : true)) {
const priceEstimate = engine.estimatePrice(usage);
if (typeof priceEstimate === 'string') {
logger.info(`Estimated cost: ${priceEstimate}`);
}
else if (priceEstimate) {
logger.info(`Estimated cost: ${priceEstimate.formatted}`);
if (!priceEstimate.available && priceEstimate.message) {
logger.warn(priceEstimate.message);
}
}
}
logger.success(`Translated and saved successfully in ${(0, util_1.formatDuration)(duration)}`);
}
else {
const duration = Date.now() - startTime;
logger.success(`No changes detected (completed in ${(0, util_1.formatDuration)(duration)})`);
}
});
}