i18n-llm-translate
Version:
Automatically translates namespace-based JSON translation files across multiple languages from any source language
448 lines (447 loc) • 21.9 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.isOpenAIInsufficientQuotaError = isOpenAIInsufficientQuotaError;
exports.isOpenAIEngineNonRetryableError = isOpenAIEngineNonRetryableError;
exports.createOpenAITranslateEngine = createOpenAITranslateEngine;
const util_1 = require("../../util");
const logger_1 = require("../../logger");
const openai_1 = __importStar(require("openai"));
const break_silent_error_1 = require("../../break-silent-error");
const zod_1 = require("openai/helpers/zod");
const zod_2 = require("zod");
const iso_639_1_1 = __importDefault(require("iso-639-1"));
const openrouter_1 = require("../../engines/estimate/openrouter");
function rateLimitApiErrorFromChain(error) {
let e = error;
const seen = new Set();
while (e instanceof Error && !seen.has(e)) {
seen.add(e);
if (e instanceof openai_1.RateLimitError)
return e;
if (e instanceof openai_1.APIError && e.status === 429)
return e;
if (e instanceof openai_1.APIError && (e.code === 'insufficient_quota' || e.code === 'rate_limit_exceeded')) {
return e;
}
e = e.cause;
}
return null;
}
/** Billing / monthly quota — retrying will not help. */
function isOpenAIInsufficientQuotaError(error) {
var _a;
const api = rateLimitApiErrorFromChain(error);
const nested = api === null || api === void 0 ? void 0 : api.error;
const code = (_a = api === null || api === void 0 ? void 0 : api.code) !== null && _a !== void 0 ? _a : nested === null || nested === void 0 ? void 0 : nested.code;
if (code === 'insufficient_quota')
return true;
const msg = error instanceof Error ? error.message : '';
if (/exceeded your current quota|check your plan and billing/i.test(msg))
return true;
return false;
}
/** @deprecated Use {@link isBreakSilentError} */
function isOpenAIEngineNonRetryableError(error) {
return (0, break_silent_error_1.isBreakSilentError)(error);
}
function isTransientOpenAIRateLimit(error) {
var _a;
if (isOpenAIInsufficientQuotaError(error))
return false;
if (((_a = rateLimitApiErrorFromChain(error)) === null || _a === void 0 ? void 0 : _a.code) === 'rate_limit_exceeded')
return true;
const api = rateLimitApiErrorFromChain(error);
if (api && api.status === 429)
return true;
if (error instanceof Error && /^429(\s|$)/.test(error.message))
return true;
return false;
}
function headerValue(headers, name) {
var _a;
if (headers == null)
return undefined;
const h = headers;
if (typeof h.get === 'function') {
return (_a = h.get(name)) !== null && _a !== void 0 ? _a : undefined;
}
const found = Object.keys(headers).find(k => k.toLowerCase() === name.toLowerCase());
return found ? String(headers[found]) : undefined;
}
function toLanguagesContext(baseLanguageCode, languages) {
const getLanguageName = (code) => {
// Try full code first
let name = iso_639_1_1.default.getName(code);
// If not found and code contains region (e.g., en-gb), try just the language part
if (!name && code.includes('-')) {
const languagePart = code.split('-')[0];
name = iso_639_1_1.default.getName(languagePart);
// If found, add region info
if (name) {
const regionPart = code.split('-')[1].toUpperCase();
return `${name} (${regionPart})`;
}
}
return name || code; // Fallback to code if name not found
};
const baseLanguageName = getLanguageName(baseLanguageCode);
const targetLanguagesWithNames = languages.map(code => `${code} (${getLanguageName(code)})`);
return `You are translating from language with code "${baseLanguageCode}" (${baseLanguageName}) to the following language codes: "${targetLanguagesWithNames.join(', ')}".`;
}
const ABSOLUTE_CONTEXT = [
'You are a professional translator.',
'When translating a value, consider the key name to better understand the context.',
'Variables enclosed in {} are coded and their names should remain unchanged.',
'IMPORTANT: Ensure that each translation is in the correct target language. Double-check that you are translating to the specific language requested for each language code.',
];
// | 'gpt-3.5-turbo' // Does not support structured output???
const DEFAULT_MODEL = 'gpt-4o-mini';
const DEBUG_CHUNKS = false;
function createOpenAITranslateEngine(config) {
if (!config.apiKey) {
throw new Error('OpenAI > Missing apiKey');
}
const model = config.model || DEFAULT_MODEL;
const estimateEngine = (0, openrouter_1.createOpenRouterEstimateEngine)({
provider: 'openai',
model: config.estimateModel || model
});
const tokenUsage = {
inputTokens: 0,
outputTokens: 0
};
const MAX_CHUNK_SIZE = Math.min(100, Math.max(5, config.chunkSize || 50));
const TIMEOUT_MS = (config.timeoutSeconds || 25) * 1000;
const MAX_RETRIES = config.maxRetries || 10;
const RATE_LIMIT_MULTIPLIER = config.rateLimitRetryMultiplier || 4;
const RATE_LIMIT_EXTRA_DELAY = config.rateLimitRetryExtraDelay || 1200;
const openai = new openai_1.default({
apiKey: config.apiKey,
timeout: TIMEOUT_MS
});
// Global timeout synchronization
let globalTimeoutPromise = null;
function sleep(ms) {
return __awaiter(this, void 0, void 0, function* () {
return new Promise(resolve => setTimeout(resolve, ms));
});
}
function computeRateLimitWaitMs(error) {
var _a;
const apiErr = rateLimitApiErrorFromChain(error);
const headers = (_a = apiErr === null || apiErr === void 0 ? void 0 : apiErr.headers) !== null && _a !== void 0 ? _a : error === null || error === void 0 ? void 0 : error.headers;
const retryAfterMsRaw = headerValue(headers, 'retry-after-ms');
if (retryAfterMsRaw) {
const n = parseInt(retryAfterMsRaw, 10);
if (!Number.isNaN(n))
return n * RATE_LIMIT_MULTIPLIER + RATE_LIMIT_EXTRA_DELAY;
}
const retryAfterSecRaw = headerValue(headers, 'retry-after');
if (retryAfterSecRaw) {
const n = parseInt(retryAfterSecRaw, 10);
if (!Number.isNaN(n))
return n * 1000 * RATE_LIMIT_MULTIPLIER + RATE_LIMIT_EXTRA_DELAY;
}
return 1000 + RATE_LIMIT_EXTRA_DELAY;
}
function withRetry(operation, operationName, options) {
return __awaiter(this, void 0, void 0, function* () {
const logger = options.logger || logger_1.defaultLogger;
let lastError;
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
try {
// Wait for any ongoing global timeout before attempting
if (globalTimeoutPromise) {
const startTime = Date.now();
yield globalTimeoutPromise;
const waitTime = Date.now() - startTime;
if (waitTime > 0) {
logger.engineDebug('OpenAI', `Waited ${waitTime}ms for global timeout synchronization`);
}
}
return yield operation();
}
catch (error) {
lastError = error;
if (isOpenAIInsufficientQuotaError(error)) {
throw new break_silent_error_1.BreakSilentError('OpenAI: insufficient quota / billing — not retrying. (or deprecated API Token)', error);
}
const isRateLimit = isTransientOpenAIRateLimit(error);
const isTimeout = !isRateLimit && error instanceof Error && (error instanceof openai_1.APIConnectionTimeoutError ||
error.message.includes('timeout') ||
error.message.includes('ETIMEDOUT') ||
error.name === 'TimeoutError' ||
error.name === 'APIConnectionTimeoutError' ||
error.constructor.name === 'APIConnectionTimeoutError');
if ((isTimeout || isRateLimit) && attempt < MAX_RETRIES) {
if (isRateLimit) {
const waitTime = computeRateLimitWaitMs(error);
logger.engineDebug('OpenAI', `Rate limit hit on attempt ${attempt}/${MAX_RETRIES}, waiting ${waitTime}ms before retry...`);
// Set global timeout promise so other chunks wait
const sleepPromise = sleep(waitTime);
globalTimeoutPromise = sleepPromise;
const startTime = Date.now();
yield sleepPromise;
const actualWaitTime = Date.now() - startTime;
logger.engineDebug('OpenAI', `Waited ${actualWaitTime}ms for rate limit retry`);
globalTimeoutPromise = null;
}
else if (isTimeout) {
const timeoutWaitTime = 2000; // 2 second delay for timeout
logger.engineDebug('OpenAI', `Timeout on attempt ${attempt}/${MAX_RETRIES}, waiting ${timeoutWaitTime}ms before retry...`);
// Set global timeout promise so other chunks wait
const sleepPromise = sleep(timeoutWaitTime);
globalTimeoutPromise = sleepPromise;
const startTime = Date.now();
yield sleepPromise;
const actualWaitTime = Date.now() - startTime;
logger.engineDebug('OpenAI', `Waited ${actualWaitTime}ms for timeout retry`);
globalTimeoutPromise = null;
}
continue;
}
// If it's not a retryable error or we've exhausted retries, throw the error
throw error;
}
}
throw lastError;
});
}
/**
* OpenAI imposes limitations on structured outputs:
* - A schema can have up to 100 object properties in total.
* - Nesting is limited to a maximum depth of 5 levels.
*
* Reference: https://platform.openai.com/docs/guides/structured-outputs/objects-have-limitations-on-nesting-depth-and-size
*
* To comply with these constraints:
* - We flatten the structure before sending requests to the API, ensuring each chunk contains at most 100 properties.
* - Once all chunks receive a response, we reconstruct (unflatten) the structure and merge the results.
*
* Schema below counts as 5 properties:
* const schema = z.object({
* pl: z.object({ // 1
* foo: z.string(), // 2
* bar: z.string(), // 3
* }),
* jp: z.object({ // 4
* xyz: z.string(), // 5
* })
* });
*/
function prepareTranslationsAndChunkThem(flatBaseTranslations, languagesTranslations) {
const chunks = [];
let currentChunkSize = 0;
let currentChunkBaseTranslations = {};
let currentChunkSchema = {};
function pushChunk() {
chunks.push({
// We use an unflattened object to save tokens.
baseTranslations: (0, util_1.unflattenObject)(currentChunkBaseTranslations),
schema: DEBUG_CHUNKS ? currentChunkSchema : zod_2.z.object(currentChunkSchema)
});
currentChunkSize = 0;
currentChunkBaseTranslations = {};
currentChunkSchema = {};
}
for (let languageCode in languagesTranslations) {
const flat = Object.entries((0, util_1.flattenObject)(languagesTranslations[languageCode]));
let languageObject = {};
do {
currentChunkSize++; // language code property
const rest = Math.min(MAX_CHUNK_SIZE - currentChunkSize, flat.length);
const chunkPart = flat.splice(0, rest);
for (const [key] of chunkPart) {
currentChunkBaseTranslations[key] = flatBaseTranslations[key];
languageObject[key] = DEBUG_CHUNKS ? '' : zod_2.z.string();
}
currentChunkSchema[languageCode] = DEBUG_CHUNKS ? languageObject : zod_2.z.object(languageObject);
currentChunkSize += rest;
if (currentChunkSize == MAX_CHUNK_SIZE) {
pushChunk();
languageObject = {};
}
} while (flat.length);
/**
* We want to avoid pushing chunks that only contain a language code, e.g.:
*
* MAX_CHUNK_SIZE = 3
*
* Example of a chunk structure that we want to avoid:
* {
* pl: { // 1
* 'order.navigation.title': '...' // 2
* },
* zh: {} // 3
* }
*
* In this example, the chunk for the 'zh' language code is empty, which is undesirable.
* We want to ensure that only meaningful chunks with actual content are pushed.
*/
if (currentChunkSize >= MAX_CHUNK_SIZE - 1) {
pushChunk();
}
}
if (currentChunkSize)
pushChunk();
return chunks;
}
function fetchTranslations(chunks, options) {
return __awaiter(this, void 0, void 0, function* () {
const logger = options.logger || logger_1.defaultLogger;
const systemContext = [
...ABSOLUTE_CONTEXT,
toLanguagesContext(options.baseLanguageCode, options.targetLanguageCodes),
...options.applicationContextEntries
].join(' ');
function fetchChunk(chunk) {
return __awaiter(this, void 0, void 0, function* () {
return yield withRetry(() => __awaiter(this, void 0, void 0, function* () {
var _a, _b;
const response = yield openai.beta.chat.completions.parse({
model,
response_format: (0, zod_1.zodResponseFormat)(chunk.schema, "language_translations"),
messages: [
{ role: 'system', content: systemContext },
{ role: 'user', content: JSON.stringify(chunk.baseTranslations) }
]
});
tokenUsage.inputTokens += ((_a = response.usage) === null || _a === void 0 ? void 0 : _a.prompt_tokens) || 0;
tokenUsage.outputTokens += ((_b = response.usage) === null || _b === void 0 ? void 0 : _b.completion_tokens) || 0;
const translatedChunk = response.choices[0].message.parsed;
if (!translatedChunk) {
throw new Error("Received null translatedChunk from API.");
}
return translatedChunk;
}), 'translate', options);
});
}
const mergedFlat = {};
let fetchedCount = 0;
const translatedChunks = yield Promise.all(chunks.map((chunk, i) => __awaiter(this, void 0, void 0, function* () {
if (chunks.length > 1) {
logger.engineDebug('OpenAI', `Fetching chunk ${i + 1}/${chunks.length}`);
}
let result = yield fetchChunk(chunk);
fetchedCount++;
logger.engineDebug('OpenAI', `Fetched chunk ${i + 1} (${fetchedCount}/${chunks.length})`);
return result;
})));
logger.engineDebug('OpenAI', `Finished fetching all chunks`);
for (const translatedChunk of translatedChunks) {
for (const languageCode in translatedChunk) {
let mergedLanguageCode = mergedFlat[languageCode];
if (!mergedLanguageCode) {
mergedLanguageCode = {};
mergedFlat[languageCode] = mergedLanguageCode;
}
for (const path in translatedChunk[languageCode]) {
mergedLanguageCode[path] = translatedChunk[languageCode][path];
}
}
}
// for (let i = 0; i < chunks.length; i++) {
// if (options.debug && chunks.length > 1) console.log(`OpenAI translate > Fetching chunk ${i + 1} of ${chunks.length}...`);
// const translatedChunk = await fetchChunk(chunks[i]);
//
// for (let languageCode in translatedChunk) {
// let mergedLanguageCode = mergedFlat[languageCode];
// if (!mergedLanguageCode) {
// mergedLanguageCode = {};
// mergedFlat[languageCode] = mergedLanguageCode
// }
//
// for (let path in translatedChunk[languageCode]) {
// mergedLanguageCode[path] = translatedChunk[languageCode][path];
// }
// }
// }
for (let languageCode in mergedFlat) {
mergedFlat[languageCode] = (0, util_1.unflattenObject)(mergedFlat[languageCode]);
}
return (0, util_1.unflattenObject)(mergedFlat);
});
}
return {
name: `OpenAI (${model})`,
type: 'llm',
canBeTrustedWithVariablesTranslation: true,
initializeEstimate(options) {
return __awaiter(this, void 0, void 0, function* () {
tokenUsage.inputTokens = 0;
tokenUsage.outputTokens = 0;
const result = yield estimateEngine.initialize();
// const logger = options.logger || defaultLogger;
// logger.engineDebug('OpenAI', `${estimateEngine.name} ${result.message}`);
return result;
});
},
estimatePrice(usage) {
return estimateEngine.estimatePrice(usage);
},
getUsage() {
return Object.assign({}, tokenUsage);
},
translate(translations, options) {
return __awaiter(this, void 0, void 0, function* () {
const languagesTranslations = {};
for (let targetLanguageCode of options.targetLanguageCodes) {
languagesTranslations[targetLanguageCode] = translations;
}
const chunks = prepareTranslationsAndChunkThem((0, util_1.flattenObject)(translations), languagesTranslations);
return yield fetchTranslations(chunks, options);
});
},
translateMissed(missingTranslations, options) {
return __awaiter(this, void 0, void 0, function* () {
const chunks = prepareTranslationsAndChunkThem((0, util_1.flattenObject)(missingTranslations.baseLanguageTranslations), missingTranslations.targetLanguageTranslationsKeys);
return yield fetchTranslations(chunks, options);
});
}
};
}