i18n-llm-translate
Version:
Automatically translates namespace-based JSON translation files across multiple languages from any source language
309 lines (308 loc) • 16 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createOpenAITranslateEngine = createOpenAITranslateEngine;
const util_1 = require("../../util");
const logger_1 = require("../../logger");
const openai_1 = require("openai");
const zod_1 = require("openai/helpers/zod");
const zod_2 = require("zod");
const iso_639_1_1 = __importDefault(require("iso-639-1"));
function toLanguagesContext(baseLanguageCode, languages) {
const getLanguageName = (code) => {
// Try full code first
let name = iso_639_1_1.default.getName(code);
// If not found and code contains region (e.g., en-gb), try just the language part
if (!name && code.includes('-')) {
const languagePart = code.split('-')[0];
name = iso_639_1_1.default.getName(languagePart);
// If found, add region info
if (name) {
const regionPart = code.split('-')[1].toUpperCase();
return `${name} (${regionPart})`;
}
}
return name || code; // Fallback to code if name not found
};
const baseLanguageName = getLanguageName(baseLanguageCode);
const targetLanguagesWithNames = languages.map(code => `${code} (${getLanguageName(code)})`);
return `You are translating from language with code "${baseLanguageCode}" (${baseLanguageName}) to the following language codes: "${targetLanguagesWithNames.join(', ')}".`;
}
const ABSOLUTE_CONTEXT = [
'You are a professional translator.',
'When translating a value, consider the key name to better understand the context.',
'Variables enclosed in {} are coded and their names should remain unchanged.',
'IMPORTANT: Ensure that each translation is in the correct target language. Double-check that you are translating to the specific language requested for each language code.',
];
// | 'gpt-3.5-turbo' // Does not support structured output???
const DEFAULT_MODEL = 'gpt-4o-mini';
const DEBUG_CHUNKS = false;
function createOpenAITranslateEngine(config) {
if (!config.apiKey) {
throw new Error('OpenAI > Missing apiKey');
}
const model = config.model || DEFAULT_MODEL;
const MAX_CHUNK_SIZE = Math.min(100, Math.max(5, config.chunkSize || 50));
const TIMEOUT_MS = (config.timeoutSeconds || 25) * 1000;
const MAX_RETRIES = config.maxRetries || 10;
const RATE_LIMIT_MULTIPLIER = config.rateLimitRetryMultiplier || 4;
const RATE_LIMIT_EXTRA_DELAY = config.rateLimitRetryExtraDelay || 1200;
const openai = new openai_1.OpenAI({
apiKey: config.apiKey,
timeout: TIMEOUT_MS
});
// Global timeout synchronization
let globalTimeoutPromise = null;
function sleep(ms) {
return __awaiter(this, void 0, void 0, function* () {
return new Promise(resolve => setTimeout(resolve, ms));
});
}
function withRetry(operation, operationName, options) {
return __awaiter(this, void 0, void 0, function* () {
var _a;
const logger = options.logger || logger_1.defaultLogger;
let lastError;
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
try {
// Wait for any ongoing global timeout before attempting
if (globalTimeoutPromise) {
const startTime = Date.now();
yield globalTimeoutPromise;
const waitTime = Date.now() - startTime;
if (waitTime > 0) {
logger.engineDebug('OpenAI', `Waited ${waitTime}ms for global timeout synchronization`);
}
}
return yield operation();
}
catch (error) {
lastError = error;
const isTimeout = error instanceof Error && (error.message.includes('timeout') ||
error.message.includes('ETIMEDOUT') ||
error.name === 'TimeoutError' ||
error.name === 'APIConnectionTimeoutError' ||
error.constructor.name === 'APIConnectionTimeoutError');
// Check if it's a rate limit error (429)
const isRateLimit = (error === null || error === void 0 ? void 0 : error.status) === 429;
if ((isTimeout || isRateLimit) && attempt < MAX_RETRIES) {
if (isRateLimit) {
// Get retry-after-ms from headers and multiply by configured multiplier
const retryAfterMs = (_a = error === null || error === void 0 ? void 0 : error.headers) === null || _a === void 0 ? void 0 : _a['retry-after-ms'];
const waitTime = retryAfterMs ? parseInt(retryAfterMs) * RATE_LIMIT_MULTIPLIER + RATE_LIMIT_EXTRA_DELAY : 1000; // Default 1s if no header
logger.engineDebug('OpenAI', `Rate limit hit on attempt ${attempt}/${MAX_RETRIES}, waiting ${waitTime}ms before retry...`);
// Set global timeout promise so other chunks wait
const sleepPromise = sleep(waitTime);
globalTimeoutPromise = sleepPromise;
const startTime = Date.now();
yield sleepPromise;
const actualWaitTime = Date.now() - startTime;
logger.engineDebug('OpenAI', `Waited ${actualWaitTime}ms for rate limit retry`);
globalTimeoutPromise = null;
}
else if (isTimeout) {
const timeoutWaitTime = 2000; // 2 second delay for timeout
logger.engineDebug('OpenAI', `Timeout on attempt ${attempt}/${MAX_RETRIES}, waiting ${timeoutWaitTime}ms before retry...`);
// Set global timeout promise so other chunks wait
const sleepPromise = sleep(timeoutWaitTime);
globalTimeoutPromise = sleepPromise;
const startTime = Date.now();
yield sleepPromise;
const actualWaitTime = Date.now() - startTime;
logger.engineDebug('OpenAI', `Waited ${actualWaitTime}ms for timeout retry`);
globalTimeoutPromise = null;
}
continue;
}
// If it's not a retryable error or we've exhausted retries, throw the error
throw error;
}
}
throw lastError;
});
}
/**
* OpenAI imposes limitations on structured outputs:
* - A schema can have up to 100 object properties in total.
* - Nesting is limited to a maximum depth of 5 levels.
*
* Reference: https://platform.openai.com/docs/guides/structured-outputs/objects-have-limitations-on-nesting-depth-and-size
*
* To comply with these constraints:
* - We flatten the structure before sending requests to the API, ensuring each chunk contains at most 100 properties.
* - Once all chunks receive a response, we reconstruct (unflatten) the structure and merge the results.
*
* Schema below counts as 5 properties:
* const schema = z.object({
* pl: z.object({ // 1
* foo: z.string(), // 2
* bar: z.string(), // 3
* }),
* jp: z.object({ // 4
* xyz: z.string(), // 5
* })
* });
*/
function prepareTranslationsAndChunkThem(flatBaseTranslations, languagesTranslations) {
const chunks = [];
let currentChunkSize = 0;
let currentChunkBaseTranslations = {};
let currentChunkSchema = {};
function pushChunk() {
chunks.push({
// We use an unflattened object to save tokens.
baseTranslations: (0, util_1.unflattenObject)(currentChunkBaseTranslations),
schema: DEBUG_CHUNKS ? currentChunkSchema : zod_2.z.object(currentChunkSchema)
});
currentChunkSize = 0;
currentChunkBaseTranslations = {};
currentChunkSchema = {};
}
for (let languageCode in languagesTranslations) {
const flat = Object.entries((0, util_1.flattenObject)(languagesTranslations[languageCode]));
let languageObject = {};
do {
currentChunkSize++; // language code property
const rest = Math.min(MAX_CHUNK_SIZE - currentChunkSize, flat.length);
const chunkPart = flat.splice(0, rest);
for (const [key] of chunkPart) {
currentChunkBaseTranslations[key] = flatBaseTranslations[key];
languageObject[key] = DEBUG_CHUNKS ? '' : zod_2.z.string();
}
currentChunkSchema[languageCode] = DEBUG_CHUNKS ? languageObject : zod_2.z.object(languageObject);
currentChunkSize += rest;
if (currentChunkSize == MAX_CHUNK_SIZE) {
pushChunk();
languageObject = {};
}
} while (flat.length);
/**
* We want to avoid pushing chunks that only contain a language code, e.g.:
*
* MAX_CHUNK_SIZE = 3
*
* Example of a chunk structure that we want to avoid:
* {
* pl: { // 1
* 'order.navigation.title': '...' // 2
* },
* zh: {} // 3
* }
*
* In this example, the chunk for the 'zh' language code is empty, which is undesirable.
* We want to ensure that only meaningful chunks with actual content are pushed.
*/
if (currentChunkSize >= MAX_CHUNK_SIZE - 1) {
pushChunk();
}
}
if (currentChunkSize)
pushChunk();
return chunks;
}
function fetchTranslations(chunks, options) {
return __awaiter(this, void 0, void 0, function* () {
const logger = options.logger || logger_1.defaultLogger;
const systemContext = [
...ABSOLUTE_CONTEXT,
toLanguagesContext(options.baseLanguageCode, options.targetLanguageCodes),
...options.applicationContextEntries
].join(' ');
function fetchChunk(chunk) {
return __awaiter(this, void 0, void 0, function* () {
return yield withRetry(() => __awaiter(this, void 0, void 0, function* () {
const response = yield openai.beta.chat.completions.parse({
model,
response_format: (0, zod_1.zodResponseFormat)(chunk.schema, "language_translations"),
messages: [
{ role: 'system', content: systemContext },
{ role: 'user', content: JSON.stringify(chunk.baseTranslations) }
]
});
const translatedChunk = response.choices[0].message.parsed;
if (!translatedChunk) {
throw new Error("Received null translatedChunk from API.");
}
return translatedChunk;
}), 'translate', options);
});
}
const mergedFlat = {};
let fetchedCount = 0;
const translatedChunks = yield Promise.all(chunks.map((chunk, i) => __awaiter(this, void 0, void 0, function* () {
if (chunks.length > 1) {
logger.engineDebug('OpenAI', `Fetching chunk ${i + 1}/${chunks.length}`);
}
let result = yield fetchChunk(chunk);
fetchedCount++;
logger.engineDebug('OpenAI', `Fetched chunk ${i + 1} (${fetchedCount}/${chunks.length})`);
return result;
})));
logger.engineDebug('OpenAI', `Finished fetching all chunks`);
for (const translatedChunk of translatedChunks) {
for (const languageCode in translatedChunk) {
let mergedLanguageCode = mergedFlat[languageCode];
if (!mergedLanguageCode) {
mergedLanguageCode = {};
mergedFlat[languageCode] = mergedLanguageCode;
}
for (const path in translatedChunk[languageCode]) {
mergedLanguageCode[path] = translatedChunk[languageCode][path];
}
}
}
// for (let i = 0; i < chunks.length; i++) {
// if (options.debug && chunks.length > 1) console.log(`OpenAI translate > Fetching chunk ${i + 1} of ${chunks.length}...`);
// const translatedChunk = await fetchChunk(chunks[i]);
//
// for (let languageCode in translatedChunk) {
// let mergedLanguageCode = mergedFlat[languageCode];
// if (!mergedLanguageCode) {
// mergedLanguageCode = {};
// mergedFlat[languageCode] = mergedLanguageCode
// }
//
// for (let path in translatedChunk[languageCode]) {
// mergedLanguageCode[path] = translatedChunk[languageCode][path];
// }
// }
// }
for (let languageCode in mergedFlat) {
mergedFlat[languageCode] = (0, util_1.unflattenObject)(mergedFlat[languageCode]);
}
return (0, util_1.unflattenObject)(mergedFlat);
});
}
return {
name: `OpenAI (${model})`,
type: 'llm',
canBeTrustedWithVariablesTranslation: true,
translate(translations, options) {
return __awaiter(this, void 0, void 0, function* () {
const languagesTranslations = {};
for (let targetLanguageCode of options.targetLanguageCodes) {
languagesTranslations[targetLanguageCode] = translations;
}
const chunks = prepareTranslationsAndChunkThem((0, util_1.flattenObject)(translations), languagesTranslations);
return yield fetchTranslations(chunks, options);
});
},
translateMissed(missingTranslations, options) {
return __awaiter(this, void 0, void 0, function* () {
const chunks = prepareTranslationsAndChunkThem((0, util_1.flattenObject)(missingTranslations.baseLanguageTranslations), missingTranslations.targetLanguageTranslationsKeys);
return yield fetchTranslations(chunks, options);
});
}
};
}