i18n-translate-agent/src/index.ts

An intelligent i18n translation agent powered by OpenAI, supporting automatic translation of JSON files with caching and progress tracking

import path from "path";
import fs from "fs";
import colors from "ansi-colors";
import { OpenAI, ClientOptions } from "openai";
import cliProgress from "cli-progress";
import {
  ICwalletTranslateParams,
  IJson,
  IOutputLanguageFile,
  ISingleTranslate,
  ITranslateChat,
  ITranslateChatResponse,
  SupportLanguageType,
} from "./types";
import {
  chunkArray,
  getRandomNumber,
  notExistsToCreateFile,
  readFileOfDirSync,
  readJsonFileSync,
  flattenJson,
  unflattenJson,
} from "./lib/utils.js";
import {
  getCacheFileSync,
  registerLanguageCacheFile,
  translateJSONDiffToJson,
} from "./lib/cache/index.js";
import { logErrorToFile } from "./lib/log/index.js";
import { SUPPORT_LANGUAGE_MAP } from "./lib/support.js";
import { ChatCompletionCreateParams } from "openai/resources";

export { generateCache, deleteBatchCache } from "./lib/cache/index.js";

const DEFAULT_OPENAI_CONFIG: ClientOptions = {};

export class CwalletTranslate {
  /** open ai api key  */
  /** */
  CACHE_ROOT_PATH: string;
  ENTRY_ROOT_PATH: string;
  /** default en */
  SOURCE_LANGUAGE: SupportLanguageType;
  OUTPUT_ROOT_PATH: string | undefined;
  languages: SupportLanguageType[];
  client: OpenAI | null = null;
  /** default model gpt-4o */
  openaiClientConfig: ClientOptions;
  fineTune: string[];
  chatCompletionCreateParams: Partial<ChatCompletionCreateParams>;

  constructor(params: ICwalletTranslateParams) {
    this.CACHE_ROOT_PATH = params.cacheFileRootPath;
    this.ENTRY_ROOT_PATH = params.fileRootPath;
    this.openaiClientConfig =
      params.openaiClientConfig ?? DEFAULT_OPENAI_CONFIG;
    this.chatCompletionCreateParams =
      params.chatCompletionCreateParams ??
      ({
        model: "gpt-4o",
      } as Partial<ChatCompletionCreateParams>);
    this.SOURCE_LANGUAGE = params.sourceLanguage ?? "en";
    this.OUTPUT_ROOT_PATH = params.outputRootPath;
    this.fineTune = params.fineTune;
    this.languages = params.languages ?? [];
    this.createOpenAIClient();
  }

  get supportLanguages() {
    return Object.entries(SUPPORT_LANGUAGE_MAP)
      .map(([key, val]) => val)
      .filter(
        ({ code }) =>
          this.languages.includes(code) || code === this.SOURCE_LANGUAGE
      );
  }

  get outputPath() {
    return this.OUTPUT_ROOT_PATH ?? this.ENTRY_ROOT_PATH;
  }

  searchLanguage(code: SupportLanguageType) {
    return this.supportLanguages.find((item) => item.code === code);
  }

  createOpenAIClient = () => {
    /** Initialize OpenAI */
    const client = new OpenAI(this.openaiClientConfig);

    this.client = client;
  };
  /**
   * Translate all supported language folders and files in the entry file
   */
  translate = async () => {
    console.log("🚀 Starting translation");
    console.log(
      `🚀 Model being used: ${this.chatCompletionCreateParams.model} 🚀`
    );
    console.log(`🚀 Fine-tuning: ${this.fineTune} 🚀`);

    const translateFolderPath = path.join(
      this.ENTRY_ROOT_PATH,
      this.SOURCE_LANGUAGE
    );
    console.log("🚀 ~ translateFolderPath:", translateFolderPath);
    // Translate all json files under the source language folder
    const translateFolders = await readFileOfDirSync(translateFolderPath);
    console.log("🚀 ~ Files to be translated:", translateFolders);
    // Create progress bar
    const multiBar = new cliProgress.MultiBar(
      {
        clearOnComplete: false,
        hideCursor: true,
        format:
          colors.cyan("{bar}") +
          "| {percentage}% || {filename} {value}/{total} ",
      },
      cliProgress.Presets.legacy
    );

    let promises = [];
    const arr: (() => Promise<void>)[] = [];

    for (const item of this.supportLanguages) {
      // Source language does not need translation
      if (item.code === this.SOURCE_LANGUAGE) continue;
      for (const fileName of translateFolders) {
        // 直接读取源语言文件
        const sourceFilePath = path.join(
          this.ENTRY_ROOT_PATH,
          this.SOURCE_LANGUAGE,
          fileName
        );

        if (!fs.existsSync(sourceFilePath)) {
          console.log(`Source file not found: ${sourceFilePath}`);
          continue;
        }

        const sourceContent = await readJsonFileSync(sourceFilePath);
        if (!sourceContent || Object.keys(sourceContent).length === 0) {
          console.log(`${item.code}:${fileName} has no content to translate`);
          continue;
        }

        arr.push(() =>
          this.singleTranslate({
            language: item.code,
            fileName,
            multiBar,
            translateJson: sourceContent,
          })
        );
      }
    }

    promises = chunkArray(arr, 8);

    for (const chunk of promises) {
      await Promise.all(chunk.map((fn) => fn()));
    }

    multiBar.stop();
    console.log("🚀 Translation completed");
  };
  /**
   * Translate a single file
   * @param params
   * @returns
   */
  singleTranslate = async (params: ISingleTranslate) => {
    const {
      /** Language to be translated */
      language,
      /** File name to be translated */
      fileName,
      translateJson,
      multiBar,
      callback,
    } = params;

    try {
      // 扁平化嵌套 JSON 对象
      const flattenedJson = flattenJson(translateJson);

      // Array waiting for translation
      const jsonMap: IJson = {};

      // 获取缓存对象以区分新翻译和缓存命中的内容
      const cacheFilePath = path.join(this.CACHE_ROOT_PATH, language, fileName);
      const cacheObject = await getCacheFileSync(cacheFilePath);
      const flattenedCacheObject = flattenJson(cacheObject);

      // 分离需要新翻译的内容和缓存命中的内容
      const needsTranslation: IJson = {};
      const cachedContent: IJson = {};

      Object.entries(flattenedJson).forEach(([key, value]) => {
        if (flattenedCacheObject.hasOwnProperty(key)) {
          // 缓存命中，使用缓存中的译文
          cachedContent[key] = flattenedCacheObject[key];
        } else {
          // 需要新翻译
          needsTranslation[key] = value;
        }
      });

      // 处理需要API翻译的内容
      if (Object.keys(needsTranslation).length > 0) {
        console.log(
          `🔄 Translating ${Object.keys(needsTranslation).length} new items`
        );

        const promiseList = Object.entries(needsTranslation).map(
          ([key, value], index) =>
            () =>
              this.translateChat({
                key,
                value,
                language,
                index,
                fileName,
              })
        );

        const progressBar = multiBar.create(promiseList.length, 0);

        for (const fn of promiseList) {
          const result = await fn();
          jsonMap[result.key] = result.value;
          progressBar.update(result.index + 1, {
            filename: `${language}:${fileName}`,
          });
        }
      }

      // 添加缓存命中的内容（使用缓存中的译文）
      if (Object.keys(cachedContent).length > 0) {
        console.log(
          `📋 Using ${Object.keys(cachedContent).length} cached translations`
        );
        // 直接使用缓存中的译文，而不是源文本
        Object.assign(jsonMap, cachedContent);
      }

      // 由于存在路径冲突，直接使用扁平化结构，不进行unflattenJson转换

      // 只有新翻译的内容需要写入缓存（也保持扁平化）
      const newTranslationsForCache =
        Object.keys(needsTranslation).length > 0
          ? Object.fromEntries(
              Object.entries(jsonMap).filter(([key]) =>
                needsTranslation.hasOwnProperty(key)
              )
            )
          : {};

      this.outputLanguageFile({
        jsonMap: jsonMap, // 直接使用扁平化的jsonMap
        newTranslations: newTranslationsForCache, // 只有新翻译的内容
        folderName: language,
        fileName,
      });
      callback && callback();
    } catch (error) {
      logErrorToFile({
        error: error as Error,
        language,
        fileName,
        key: "",
      });
      return;
    }
  };

  /**
   * Use OpenAI for translation
   * @param {string} key
   * @param {string} value
   * @param {OpenAI} client
   * @param {string} language
   * @returns
   */
  translateChat = (params: ITranslateChat): Promise<ITranslateChatResponse> => {
    return new Promise((resolve) => {
      const { key, value, language, index, fileName } = params;
      try {
        if (!this.client) throw new Error("Connection failed");
        const targetLanguage = this.searchLanguage(language);
        const originLanguage = this.searchLanguage(this.SOURCE_LANGUAGE);

        if (!targetLanguage) {
          throw new Error(`Unsupported language: ${language}`);
        }

        if (!originLanguage) {
          throw new Error(`Unsupported language: ${this.SOURCE_LANGUAGE}`);
        }

        setTimeout(async () => {
          const chatCompletion = await this.client!.chat.completions.create({
            model: "gpt-4o",
            ...this.chatCompletionCreateParams,
            stream: false,
            messages: [
              ...this.fineTune.map(
                (val) =>
                  ({
                    role: "system",
                    content: val,
                  } as OpenAI.Chat.Completions.ChatCompletionMessageParam)
              ),
              {
                role: "system",
                content: `Please translate ${originLanguage!.name} to ${
                  targetLanguage!.name
                }`,
              },
              {
                role: "system",
                content: `After translation is complete, directly output the corresponding meaning without any irrelevant content`,
              },
              {
                role: "user",
                content: value,
              },
            ],
          });
          resolve({
            key,
            value: chatCompletion?.choices[0]?.message.content ?? value,
            index,
          });
        }, getRandomNumber(200, 300));
      } catch (error) {
        logErrorToFile({ error: error as Error, key, fileName, language });
        resolve({
          key,
          value,
          index,
          error: error as Error,
        });
      }
    });
  };

  /**
   * Output language file
   * @param {Object} jsonMap
   */
  outputLanguageFile = async (params: IOutputLanguageFile) => {
    const { folderName, fileName, jsonMap, newTranslations } = params;
    const outputFilePath = path.join(this.outputPath, folderName, fileName);

    // Create output folder
    notExistsToCreateFile(this.outputPath);
    // Create output language folder
    notExistsToCreateFile(`${this.outputPath}/${folderName}`);
    let oldJsonData: string = "";
    // Check if file exists
    if (!fs.existsSync(outputFilePath)) {
      oldJsonData = await fs.readFileSync(
        path.join(this.ENTRY_ROOT_PATH, this.SOURCE_LANGUAGE, fileName),
        "utf8"
      );
    } else {
      oldJsonData = await fs.readFileSync(outputFilePath, "utf8");
    }
    const oldJsonMap: IJson = JSON.parse(oldJsonData);
    const newJsonMap: IJson = Object.assign(oldJsonMap, jsonMap);

    await fs.writeFileSync(
      path.resolve(outputFilePath),
      JSON.stringify(newJsonMap, null, 2),
      "utf8"
    );

    // 只有新翻译的内容才写入缓存
    if (newTranslations && Object.keys(newTranslations).length > 0) {
      console.log(
        `💾 Caching ${Object.keys(newTranslations).length} new translations`
      );
      registerLanguageCacheFile({
        sourceFilePath: path.join(
          this.ENTRY_ROOT_PATH,
          this.SOURCE_LANGUAGE,
          fileName
        ),
        jsonMap: newTranslations,
        fileName,
        language: folderName,
        folderName: this.CACHE_ROOT_PATH,
      });
    } else {
      console.log(`📋 No new translations to cache (all from cache)`);
    }
  };
}