i18n-llm-translate
Version:
Automatically translates namespace-based JSON translation files across multiple languages from any source language
207 lines (206 loc) • 10 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createOpenAITranslateEngine = createOpenAITranslateEngine;
const util_1 = require("../../util");
const openai_1 = require("openai");
const zod_1 = require("openai/helpers/zod");
const zod_2 = require("zod");
function toLanguagesContext(baseLanguageCode, languages) {
return `You are translating from language with code "${baseLanguageCode}" to the following language codes: "${languages.join(', ')}".`;
}
const ABSOLUTE_CONTEXT = [
'You are a professional translator.',
'When translating a value, consider the key name to better understand the context.',
'Variables enclosed in {} are coded and their names should remain unchanged.',
];
// | 'gpt-3.5-turbo' // Does not support structured output???
const DEFAULT_MODEL = 'gpt-4o-mini';
const DEBUG_CHUNKS = false;
function createOpenAITranslateEngine(config) {
if (!config.apiKey) {
throw new Error('OpenAI > Missing apiKey');
}
const model = config.model || DEFAULT_MODEL;
const MAX_CHUNK_SIZE = Math.min(100, Math.max(5, config.chunkSize || 50));
const openai = new openai_1.OpenAI({ apiKey: config.apiKey });
/**
* OpenAI imposes limitations on structured outputs:
* - A schema can have up to 100 object properties in total.
* - Nesting is limited to a maximum depth of 5 levels.
*
* Reference: https://platform.openai.com/docs/guides/structured-outputs/objects-have-limitations-on-nesting-depth-and-size
*
* To comply with these constraints:
* - We flatten the structure before sending requests to the API, ensuring each chunk contains at most 100 properties.
* - Once all chunks receive a response, we reconstruct (unflatten) the structure and merge the results.
*
* Schema below counts as 5 properties:
* const schema = z.object({
* pl: z.object({ // 1
* foo: z.string(), // 2
* bar: z.string(), // 3
* }),
* jp: z.object({ // 4
* xyz: z.string(), // 5
* })
* });
*/
function prepareTranslationsAndChunkThem(flatBaseTranslations, languagesTranslations) {
const chunks = [];
let currentChunkSize = 0;
let currentChunkBaseTranslations = {};
let currentChunkSchema = {};
function pushChunk() {
chunks.push({
// We use an unflattened object to save tokens.
baseTranslations: (0, util_1.unflattenObject)(currentChunkBaseTranslations),
schema: DEBUG_CHUNKS ? currentChunkSchema : zod_2.z.object(currentChunkSchema)
});
currentChunkSize = 0;
currentChunkBaseTranslations = {};
currentChunkSchema = {};
}
for (let languageCode in languagesTranslations) {
const flat = Object.entries((0, util_1.flattenObject)(languagesTranslations[languageCode]));
let languageObject = {};
do {
currentChunkSize++; // language code property
const rest = Math.min(MAX_CHUNK_SIZE - currentChunkSize, flat.length);
const chunkPart = flat.splice(0, rest);
for (const [key] of chunkPart) {
currentChunkBaseTranslations[key] = flatBaseTranslations[key];
languageObject[key] = DEBUG_CHUNKS ? '' : zod_2.z.string();
}
currentChunkSchema[languageCode] = DEBUG_CHUNKS ? languageObject : zod_2.z.object(languageObject);
currentChunkSize += rest;
if (currentChunkSize == MAX_CHUNK_SIZE) {
pushChunk();
languageObject = {};
}
} while (flat.length);
/**
* We want to avoid pushing chunks that only contain a language code, e.g.:
*
* MAX_CHUNK_SIZE = 3
*
* Example of a chunk structure that we want to avoid:
* {
* pl: { // 1
* 'order.navigation.title': '...' // 2
* },
* zh: {} // 3
* }
*
* In this example, the chunk for the 'zh' language code is empty, which is undesirable.
* We want to ensure that only meaningful chunks with actual content are pushed.
*/
if (currentChunkSize >= MAX_CHUNK_SIZE - 1) {
pushChunk();
}
}
if (currentChunkSize)
pushChunk();
return chunks;
}
function fetchTranslations(chunks, options) {
return __awaiter(this, void 0, void 0, function* () {
const systemContext = [
...ABSOLUTE_CONTEXT,
toLanguagesContext(options.baseLanguageCode, options.targetLanguageCodes),
...options.applicationContextEntries
].join(' ');
function fetchChunk(chunk) {
return __awaiter(this, void 0, void 0, function* () {
const response = yield openai.beta.chat.completions.parse({
model,
response_format: (0, zod_1.zodResponseFormat)(chunk.schema, "language_translations"),
messages: [
{ role: 'system', content: systemContext },
{ role: 'user', content: JSON.stringify(chunk.baseTranslations) }
]
});
const translatedChunk = response.choices[0].message.parsed;
if (!translatedChunk) {
throw new Error("Received null translatedChunk from API.");
}
return translatedChunk;
});
}
const mergedFlat = {};
let fetchedCount = 0;
const translatedChunks = yield Promise.all(chunks.map((chunk, i) => __awaiter(this, void 0, void 0, function* () {
if (options.debug && chunks.length > 1) {
console.log(`OpenAI translate > Fetching chunk ${i + 1}...`);
}
let result = yield fetchChunk(chunk);
fetchedCount++;
if (options.debug) {
console.log(`OpenAI translate > Fetched chunk ${i + 1} (${fetchedCount}/${chunks.length})`);
}
return result;
})));
if (options.debug)
console.log(`OpenAI translate > Finished fetching all chunks.`);
for (const translatedChunk of translatedChunks) {
for (const languageCode in translatedChunk) {
let mergedLanguageCode = mergedFlat[languageCode];
if (!mergedLanguageCode) {
mergedLanguageCode = {};
mergedFlat[languageCode] = mergedLanguageCode;
}
for (const path in translatedChunk[languageCode]) {
mergedLanguageCode[path] = translatedChunk[languageCode][path];
}
}
}
// for (let i = 0; i < chunks.length; i++) {
// if (options.debug && chunks.length > 1) console.log(`OpenAI translate > Fetching chunk ${i + 1} of ${chunks.length}...`);
// const translatedChunk = await fetchChunk(chunks[i]);
//
// for (let languageCode in translatedChunk) {
// let mergedLanguageCode = mergedFlat[languageCode];
// if (!mergedLanguageCode) {
// mergedLanguageCode = {};
// mergedFlat[languageCode] = mergedLanguageCode
// }
//
// for (let path in translatedChunk[languageCode]) {
// mergedLanguageCode[path] = translatedChunk[languageCode][path];
// }
// }
// }
for (let languageCode in mergedFlat) {
mergedFlat[languageCode] = (0, util_1.unflattenObject)(mergedFlat[languageCode]);
}
return (0, util_1.unflattenObject)(mergedFlat);
});
}
return {
name: `OpenAI (${model})`,
translate(translations, options) {
return __awaiter(this, void 0, void 0, function* () {
const languagesTranslations = {};
for (let targetLanguageCode of options.targetLanguageCodes) {
languagesTranslations[targetLanguageCode] = translations;
}
const chunks = prepareTranslationsAndChunkThem((0, util_1.flattenObject)(translations), languagesTranslations);
return yield fetchTranslations(chunks, options);
});
},
translateMissed(missingTranslations, options) {
return __awaiter(this, void 0, void 0, function* () {
const chunks = prepareTranslationsAndChunkThem((0, util_1.flattenObject)(missingTranslations.baseLanguageTranslations), missingTranslations.targetLanguageTranslationsKeys);
return yield fetchTranslations(chunks, options);
});
}
};
}