next-intl-scanner/dist/utils/extractTranslations.js

A tool to extract and manage translations from Next.js projects using next-intl

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const glob_1 = require("glob");
const logger_1 = __importDefault(require("./logger.js"));
const translate_1 = require("./translate.js");
const extractWithBabelParser_1 = require("./extractWithBabelParser.js");
const extractTranslations = async (config, options) => {
    if (!config) {
        logger_1.default.error("Could not load configuration file");
        return;
    }
    if (options.watch) {
        logger_1.default.info("Starting watch mode...");
        await startWatchMode(config, options);
        return;
    }
    logger_1.default.info("Extracting translations...");
    await performExtraction(config, options);
};
// Helpers
function getAllKeys(obj, prefix = "") {
    const keys = [];
    for (const [key, value] of Object.entries(obj)) {
        const fullKey = prefix ? `${prefix}.${key}` : key;
        if (typeof value === "object" && value !== null && !Array.isArray(value)) {
            keys.push(...getAllKeys(value, fullKey));
        }
        else {
            keys.push(fullKey);
        }
    }
    return keys;
}
function removeKeyFromObject(obj, keyPath) {
    const keys = keyPath.split(".");
    let current = obj;
    for (let i = 0; i < keys.length - 1; i++) {
        if (current[keys[i]] && typeof current[keys[i]] === "object") {
            current = current[keys[i]];
        }
        else {
            return;
        }
    }
    delete current[keys[keys.length - 1]];
}
function createNestedObject(namespace, messageKey, value) {
    const parts = namespace.split(".");
    const result = {};
    let current = result;
    // Create nested structure for all parts
    for (let i = 0; i < parts.length; i++) {
        if (i === parts.length - 1) {
            // Last part: add the message key
            current[parts[i]] = { [messageKey]: value };
        }
        else {
            // Not the last part: create nested object
            current[parts[i]] = {};
            current = current[parts[i]];
        }
    }
    return result;
}
function mergeNestedObjects(target, source) {
    for (const [key, value] of Object.entries(source)) {
        if (typeof value === "object" && value !== null && !Array.isArray(value)) {
            if (!target[key]) {
                target[key] = {};
            }
            mergeNestedObjects(target[key], value);
        }
        else {
            target[key] = value;
        }
    }
}
function getNestedValue(obj, keyPath) {
    const keys = keyPath.split(".");
    let current = obj;
    for (const key of keys) {
        if (current && typeof current === "object" && key in current) {
            current = current[key];
        }
        else {
            return undefined;
        }
    }
    return current;
}
async function startWatchMode(config, options) {
    const basepath = path_1.default.resolve(process.cwd(), config.sourceDirectory);
    if (!fs_1.default.existsSync(basepath)) {
        logger_1.default.error(`Source directory ${basepath} does not exist`);
        return;
    }
    // Initial extraction
    await performExtraction(config, options);
    // Get all files to watch
    const allFiles = [];
    for (const page of config.pages) {
        const files = await new Promise((resolve, reject) => {
            (0, glob_1.glob)(page.match, { cwd: basepath, ignore: page.ignore.concat(config.ignore) }, (err, matches) => (err ? reject(err) : resolve(matches)));
        });
        allFiles.push(...files.map((file) => path_1.default.resolve(basepath, file)));
    }
    // Watch directories for changes
    const watchedDirs = new Set();
    for (const file of allFiles) {
        const dir = path_1.default.dirname(file);
        if (!watchedDirs.has(dir)) {
            watchedDirs.add(dir);
            fs_1.default.watch(dir, { recursive: true }, async (eventType, filename) => {
                if (filename && shouldProcessFile(filename, config, basepath)) {
                    logger_1.default.info(`File changed: ${filename}`);
                    await performExtraction(config, options);
                }
            });
        }
    }
    logger_1.default.success(`Watching ${watchedDirs.size} directories for changes...`);
    logger_1.default.info("Press Ctrl+C to stop watching");
    // Keep the process alive
    process.on("SIGINT", () => {
        logger_1.default.info("Stopping watch mode...");
        process.exit(0);
    });
}
function shouldProcessFile(filename, config, basepath) {
    const filePath = path_1.default.resolve(basepath, filename);
    // Check if file matches any of the page patterns
    for (const page of config.pages) {
        const matches = glob_1.glob.sync(page.match, { cwd: basepath });
        if (matches.includes(filename)) {
            // Check if file is not ignored
            for (const ignorePattern of page.ignore.concat(config.ignore)) {
                if (glob_1.glob.sync(ignorePattern, { cwd: basepath }).includes(filename)) {
                    return false;
                }
            }
            return true;
        }
    }
    return false;
}
async function performExtraction(config, options) {
    var _a, _b;
    const basepath = path_1.default.resolve(process.cwd(), config.sourceDirectory);
    if (!fs_1.default.existsSync(basepath)) {
        logger_1.default.error(`Source directory ${basepath} does not exist`);
        return;
    }
    const BATCH_SIZE = 50;
    const allFiles = [];
    for (const page of config.pages) {
        const files = await new Promise((resolve, reject) => {
            (0, glob_1.glob)(page.match, { cwd: basepath, ignore: page.ignore.concat(config.ignore) }, (err, matches) => (err ? reject(err) : resolve(matches)));
        });
        allFiles.push(...files.map((file) => path_1.default.resolve(basepath, file)));
    }
    if (!fs_1.default.existsSync(config.outputDirectory)) {
        fs_1.default.mkdirSync(config.outputDirectory, { recursive: true });
    }
    for (const locale of config.locales) {
        const localeFile = path_1.default.resolve(config.outputDirectory, `${locale}.json`);
        if (!fs_1.default.existsSync(localeFile)) {
            fs_1.default.writeFileSync(localeFile, "{}");
        }
    }
    const duplicateKeyMap = new Map();
    const allExtractedKeys = new Set();
    for (let i = 0; i < allFiles.length; i += BATCH_SIZE) {
        const batch = allFiles.slice(i, i + BATCH_SIZE);
        const batchTranslations = [];
        for (const file of batch) {
            const source = fs_1.default.readFileSync(file, "utf-8");
            if (!source) {
                logger_1.default.error(`Could not read file: ${file}`);
                continue;
            }
            const extracted = (0, extractWithBabelParser_1.extractTranslationsFromSource)(source, file, config);
            for (const translation of extracted) {
                // Allow dots in message keys - they will be handled by replacing dots with underscores in values
                // checkForDots(translation.messageKey); // Removed to allow dots in keys
                batchTranslations.push(translation);
                const key = translation.nameSpace
                    ? `${translation.nameSpace}.${translation.messageKey}`
                    : translation.messageKey;
                allExtractedKeys.add(key);
                const mapKey = key;
                const existing = duplicateKeyMap.get(mapKey);
                if (existing) {
                    existing.files.add(translation.file || "unknown");
                    if (existing.value !== translation.string) {
                        existing.duplicateValues.add(translation.string);
                    }
                }
                else {
                    duplicateKeyMap.set(mapKey, {
                        value: translation.string,
                        files: new Set([translation.file || "unknown"]),
                        duplicateValues: new Set(),
                    });
                }
            }
        }
        if ((_a = config.locales) === null || _a === void 0 ? void 0 : _a.length) {
            for (const locale of config.locales) {
                const localeFile = path_1.default.resolve(config.outputDirectory, `${locale}.json`);
                let localeTranslations = {};
                if (fs_1.default.existsSync(localeFile)) {
                    try {
                        localeTranslations = JSON.parse(fs_1.default.readFileSync(localeFile, "utf-8"));
                        for (const { nameSpace, string, messageKey } of batchTranslations) {
                            if (nameSpace) {
                                // Create nested structure for namespaces with dots
                                if (nameSpace.includes(".")) {
                                    const nestedObject = createNestedObject(nameSpace, messageKey, string);
                                    mergeNestedObjects(localeTranslations, nestedObject);
                                }
                                else {
                                    // Handle simple namespaces
                                    localeTranslations[nameSpace] =
                                        localeTranslations[nameSpace] || {};
                                    if (options.overwrite ||
                                        !localeTranslations[nameSpace][messageKey]) {
                                        localeTranslations[nameSpace][messageKey] = string;
                                    }
                                }
                            }
                            else {
                                if (options.overwrite || !localeTranslations[messageKey]) {
                                    localeTranslations[messageKey] = string;
                                }
                            }
                        }
                    }
                    catch (error) {
                        logger_1.default.error(`Error parsing JSON for locale ${locale}: ${error}`);
                    }
                }
                fs_1.default.writeFileSync(localeFile, JSON.stringify(localeTranslations, null, 2));
                logger_1.default.info(`Translations written for locale ${locale}`);
            }
        }
    }
    if (options.clean && ((_b = config.locales) === null || _b === void 0 ? void 0 : _b.length)) {
        for (const locale of config.locales) {
            const localeFile = path_1.default.resolve(config.outputDirectory, `${locale}.json`);
            if (fs_1.default.existsSync(localeFile)) {
                try {
                    const localeTranslations = JSON.parse(fs_1.default.readFileSync(localeFile, "utf-8"));
                    const existingKeys = getAllKeys(localeTranslations);
                    const keysToRemove = existingKeys.filter((key) => !allExtractedKeys.has(key));
                    for (const key of keysToRemove) {
                        removeKeyFromObject(localeTranslations, key);
                    }
                    if (keysToRemove.length) {
                        fs_1.default.writeFileSync(localeFile, JSON.stringify(localeTranslations, null, 2));
                        logger_1.default.info(`Cleaned ${keysToRemove.length} unused keys from ${locale}.json`);
                    }
                }
                catch (error) {
                    logger_1.default.error(`Error cleaning locale ${locale}: ${error}`);
                }
            }
        }
    }
    if (options.autoTranslate) {
        const defaultLocaleFile = path_1.default.resolve(config.outputDirectory, `${config.defaultLocale}.json`);
        const defaultLocaleTranslations = JSON.parse(fs_1.default.readFileSync(defaultLocaleFile, "utf-8"));
        for (const locale of config.locales) {
            const localeFile = path_1.default.resolve(config.outputDirectory, `${locale}.json`);
            const localeTranslations = JSON.parse(fs_1.default.readFileSync(localeFile, "utf-8"));
            // Get all keys from both objects for comparison
            const defaultKeys = getAllKeys(defaultLocaleTranslations);
            const localeKeys = getAllKeys(localeTranslations);
            const untranslated = {};
            for (const key of defaultKeys) {
                if (!localeKeys.includes(key) ||
                    getNestedValue(defaultLocaleTranslations, key) ===
                        getNestedValue(localeTranslations, key)) {
                    untranslated[key] = getNestedValue(defaultLocaleTranslations, key);
                }
            }
            const translated = await (0, translate_1.translateBatch)(untranslated, config.defaultLocale, locale);
            // Merge translated keys back into the nested structure
            for (const [key, value] of Object.entries(translated)) {
                const parts = key.split(".");
                let current = localeTranslations;
                // Navigate to the correct nested location
                for (let i = 0; i < parts.length - 1; i++) {
                    if (!current[parts[i]]) {
                        current[parts[i]] = {};
                    }
                    current = current[parts[i]];
                }
                // Set the translated value
                current[parts[parts.length - 1]] = value;
            }
            fs_1.default.writeFileSync(localeFile, JSON.stringify(localeTranslations, null, 2));
        }
    }
    if (Array.from(duplicateKeyMap.values()).some((v) => v.duplicateValues.size)) {
        logger_1.default.warn("\nSummary of duplicate keys with conflicting values:");
        for (const [key, { value, duplicateValues }] of duplicateKeyMap.entries()) {
            if (duplicateValues.size) {
                logger_1.default.warn(`- ${key}: original='${value}', others=[${Array.from(duplicateValues).join(", ")}])`);
            }
        }
    }
    logger_1.default.success("Translations extracted successfully");
}
exports.default = extractTranslations;