wuchale/dist/ai/index.js

Protobuf-like i18n from plain code

// $$ cd .. && npm run test
// $$ node %f
import PO from 'pofile';
import { color } from '../log.js';
// implements a queue for a sequential translation useful for vite's transform during dev
// as vite can do async transform
export default class AIQueue {
    batches = [];
    nextBatchId = 0;
    running = null;
    sourceLang;
    targetLang;
    ai;
    instruction;
    onComplete;
    log;
    constructor(sourceLang, targetLang, ai, onComplete, log) {
        this.sourceLang = sourceLang;
        this.targetLang = targetLang;
        this.ai = ai;
        this.instruction = `
            You will be given the contents of a gettext .po file for a web app.
            Translate each of the items from ${this.sourceLang} to ${this.targetLang}.
            You can read all of the information for the items including contexts,
            comments and references to get the appropriate context about each item.
            Provide the same content with the only difference being that the
            empty msgstr quotes should be filled with the appropriate translations,
            preserving all placeholders.
            The placeholder format is like the following examples:
                - {0}: means arbitrary values.
                - <0>something</0>: means something enclosed in some tags, like HTML tags
                - <0/>: means a self closing tag, like in HTML
            In all of the examples, 0 is an example for any integer.
        `;
        this.onComplete = onComplete;
        this.log = log;
    }
    #requestName = (id) => `${color.cyan(this.ai.name)}: ${this.targetLang} [${id}]`;
    translate = async (batch) => {
        const logStart = this.#requestName(batch.id);
        let translated;
        try {
            const po = new PO();
            po.items = batch.messages;
            const translatedstr = await this.ai.translate(po.toString(), this.instruction);
            translated = PO.parse(translatedstr).items;
        }
        catch (err) {
            this.log.error(`${logStart}: ${color.red(`error: ${err}`)}`);
            return;
        }
        let unTranslated = batch.messages.slice(translated.length);
        for (const [i, item] of translated.entries()) {
            if (item.msgid !== batch.messages[i]?.msgid) {
                unTranslated.push(item);
                continue;
            }
            if (item.msgstr[0]) {
                batch.messages[i].msgstr = item.msgstr;
            }
            else {
                unTranslated.push(item);
            }
        }
        if (unTranslated.length) {
            this.log.warn(`${logStart}: ${unTranslated.length} ${color.yellow('messages not translated. Retrying...')}`);
            await this.translate({ id: batch.id, messages: unTranslated });
        }
        else {
            this.log.info(`${logStart}: ${color.green('translated')}`);
        }
    };
    run = async () => {
        while (this.batches.length > 0) {
            const allBatches = [];
            while (this.batches.length > 0 && allBatches.length < this.ai.parallel) {
                allBatches.push(this.batches.pop());
            }
            await Promise.all(allBatches.map(this.translate));
        }
        await this.onComplete();
        this.running = null;
    };
    add = (messages) => {
        if (!this.ai) {
            return;
        }
        const opInfo = [];
        const lastBatch = this.batches.at(-1);
        if (lastBatch && lastBatch.messages.length < this.ai.batchSize) {
            const lastBatchFree = this.ai.batchSize - lastBatch.messages.length;
            const msgs = messages.slice(0, lastBatchFree);
            opInfo.push(['(add)', lastBatch.id, msgs.length]);
            lastBatch.messages.push(...msgs);
            messages = messages.slice(lastBatchFree);
        }
        if (messages.length > 0) {
            opInfo.push([color.yellow('(new)'), this.nextBatchId, messages.length]);
            this.batches.push({ id: this.nextBatchId, messages });
            this.nextBatchId++;
        }
        for (const [opType, batchId, msgsLen] of opInfo) {
            this.log.info(`${this.#requestName(batchId)}: ${opType} translate ${color.cyan(msgsLen)} messages`);
        }
        if (!this.running) {
            this.running = this.run();
        }
    };
}