@akson/cortex-shopify-translations/translations/scripts/translate-file.mjs

Unified Shopify translations management client with product extraction, translation sync, and CLI tools

#!/usr/bin/env node
/**
 * Main translation script that processes category files using AI
 * Handles batching, progress tracking, and status updates
 */
import fs from 'fs/promises';
import path from 'path';
import AITranslator from './ai-translator.mjs';
import chalk from 'chalk';

class TranslationManager {
  constructor() {
    this.translator = new AITranslator();
    this.batchSize = 5; // Process 5 translations at once (reduced for rate limiting)
    this.pauseMs = 5000; // Pause 5 seconds between batches (increased for rate limiting)
    this.contentDir = path.join(process.cwd(), 'translations', 'content');
    this.statusFile = path.join(process.cwd(), 'translations', 'status.json');
  }

  async run(category = null, language = null) {
    console.log(chalk.cyan('🧠 MyArmy Translation System'));
    console.log(chalk.gray('═'.repeat(50)));

    // Load translator config
    await this.translator.loadConfig();

    // Get files to process
    const files = await this.getFilesToProcess(category);
    const languages = language ? [language] : ['de', 'it', 'en'];

    console.log(chalk.yellow(`📁 Processing files: ${files.join(', ')}`));
    console.log(chalk.yellow(`🌍 Target languages: ${languages.join(', ')}`));
    console.log();

    let totalProcessed = 0;
    let totalCost = 0;

    for (const file of files) {
      const result = await this.processFile(file, languages);
      totalProcessed += result.processed;
      totalCost += result.cost;
    }

    // Update global status
    await this.updateGlobalStatus();

    console.log();
    console.log(chalk.green('✅ Translation complete!'));
    console.log(chalk.white(`📊 Total processed: ${totalProcessed}`));
    console.log(chalk.white(`💰 Estimated cost: $${totalCost.toFixed(4)}`));
  }

  async getFilesToProcess(category) {
    if (category) {
      const filePath = path.join(this.contentDir, `${category}.json`);
      const exists = await fs.access(filePath).then(() => true).catch(() => false);
      if (!exists) {
        throw new Error(`Category file not found: ${category}.json`);
      }
      return [`${category}.json`];
    }

    // Get all category files
    const files = await fs.readdir(this.contentDir);
    return files.filter(f => f.endsWith('.json'));
  }

  async processFile(fileName, languages) {
    console.log(chalk.blue(`\n📂 Processing: ${fileName}`));
    console.log(chalk.gray('─'.repeat(40)));

    const filePath = path.join(this.contentDir, fileName);
    const data = JSON.parse(await fs.readFile(filePath, 'utf-8'));

    let processed = 0;
    let totalCost = 0;

    for (const lang of languages) {
      console.log(chalk.yellow(`\n🌍 Translating to ${lang.toUpperCase()}...`));

      // Get pending translations for this language
      const pending = data.translations.filter(t =>
        t.status[lang] === 'pending' || t.status[lang] === 'failed'
      );

      if (pending.length === 0) {
        console.log(chalk.gray(`  ✓ No pending translations for ${lang}`));
        continue;
      }

      console.log(chalk.white(`  📝 ${pending.length} translations to process`));

      // Process in batches
      for (let i = 0; i < pending.length; i += this.batchSize) {
        const batch = pending.slice(i, i + this.batchSize);
        const batchNum = Math.floor(i / this.batchSize) + 1;
        const totalBatches = Math.ceil(pending.length / this.batchSize);

        console.log(chalk.gray(`  🔄 Batch ${batchNum}/${totalBatches}`));

        try {
          // Mark as in_progress
          batch.forEach(item => {
            const original = data.translations.find(t => t.key === item.key);
            original.status[lang] = 'in_progress';
          });

          // Prepare batch for translation
          const textsToTranslate = batch.map(item => ({
            text: item.fr,
            context: item.key,
            key: item.key
          }));

          // Translate batch
          const result = await this.translator.translateBatch(textsToTranslate, lang);

          // Update translations with results
          result.translations.forEach((trans, idx) => {
            const original = data.translations.find(t => t.key === batch[idx].key);

            if (trans.validation.valid && trans.confidence >= 0.7) {
              original[lang] = trans.translated;
              original.status[lang] = 'completed';
              processed++;
              console.log(chalk.green(`    ✓ ${batch[idx].key.substring(0, 50)}...`));
            } else {
              original.status[lang] = 'failed';
              original[`${lang}_error`] = `Low confidence: ${trans.confidence.toFixed(2)}`;
              console.log(chalk.red(`    ✗ ${batch[idx].key.substring(0, 50)}... (confidence: ${trans.confidence.toFixed(2)})`));
            }

            // Store validation info
            if (!original.validation) original.validation = {};
            original.validation[lang] = trans.validation;
          });

          totalCost += result.cost || 0;

          // Save after each batch
          await fs.writeFile(filePath, JSON.stringify(data, null, 2));

        } catch (error) {
          console.error(chalk.red(`    ❌ Batch failed: ${error.message}`));

          // Mark batch as failed
          batch.forEach(item => {
            const original = data.translations.find(t => t.key === item.key);
            original.status[lang] = 'failed';
            original[`${lang}_error`] = error.message;
          });

          await fs.writeFile(filePath, JSON.stringify(data, null, 2));
        }

        // Pause between batches
        if (i + this.batchSize < pending.length) {
          console.log(chalk.gray(`    ⏸  Pausing ${this.pauseMs}ms...`));
          await new Promise(resolve => setTimeout(resolve, this.pauseMs));
        }
      }
    }

    // Update file metadata
    data.metadata.last_updated = new Date().toISOString();

    // Recalculate stats
    const stats = {
      total: data.translations.length,
      de: { pending: 0, in_progress: 0, completed: 0, failed: 0, reviewed: 0 },
      it: { pending: 0, in_progress: 0, completed: 0, failed: 0, reviewed: 0 },
      en: { pending: 0, in_progress: 0, completed: 0, failed: 0, reviewed: 0 }
    };

    data.translations.forEach(t => {
      ['de', 'it', 'en'].forEach(lang => {
        const status = t.status[lang] || 'pending';
        if (!stats[lang][status]) stats[lang][status] = 0;
        stats[lang][status]++;
      });
    });

    data.metadata.stats = stats;
    await fs.writeFile(filePath, JSON.stringify(data, null, 2));

    return { processed, cost: totalCost };
  }

  async updateGlobalStatus() {
    const files = await fs.readdir(this.contentDir);
    const categoryFiles = files.filter(f => f.endsWith('.json'));

    const globalStats = {
      summary: {
        total_keys: 0,
        last_update: new Date().toISOString(),
        source_language: 'fr',
        target_languages: ['de', 'it', 'en']
      },
      progress: {
        de: { pending: 0, in_progress: 0, completed: 0, failed: 0, reviewed: 0 },
        it: { pending: 0, in_progress: 0, completed: 0, failed: 0, reviewed: 0 },
        en: { pending: 0, in_progress: 0, completed: 0, failed: 0, reviewed: 0 }
      },
      files: {}
    };

    for (const fileName of categoryFiles) {
      const filePath = path.join(this.contentDir, fileName);
      const data = JSON.parse(await fs.readFile(filePath, 'utf-8'));

      globalStats.summary.total_keys += data.metadata.stats.total;
      globalStats.files[fileName.replace('.json', '')] = data.metadata.stats;

      ['de', 'it', 'en'].forEach(lang => {
        Object.keys(globalStats.progress[lang]).forEach(status => {
          globalStats.progress[lang][status] += data.metadata.stats[lang][status] || 0;
        });
      });
    }

    // Calculate percentages
    globalStats.progress_percentage = {};
    ['de', 'it', 'en'].forEach(lang => {
      const total = globalStats.summary.total_keys;
      const completed = globalStats.progress[lang].completed + globalStats.progress[lang].reviewed;
      globalStats.progress_percentage[lang] = Math.round((completed / total) * 100);
    });

    await fs.writeFile(this.statusFile, JSON.stringify(globalStats, null, 2));
  }
}

// CLI handling
async function main() {
  const args = process.argv.slice(2);
  let category = null;
  let language = null;

  // Parse arguments
  args.forEach(arg => {
    if (arg.startsWith('--category=')) {
      category = arg.split('=')[1];
    } else if (arg.startsWith('--lang=')) {
      language = arg.split('=')[1];
    }
  });

  // Validate language
  if (language && !['de', 'it', 'en'].includes(language)) {
    console.error(chalk.red('❌ Invalid language. Use: de, it, or en'));
    process.exit(1);
  }

  try {
    const manager = new TranslationManager();
    await manager.run(category, language);
  } catch (error) {
    console.error(chalk.red(`\n❌ Error: ${error.message}`));
    process.exit(1);
  }
}

// Check if we have chalk installed
try {
  await import('chalk');
} catch {
  console.log('📦 Installing chalk for colored output...');
  const { execSync } = await import('child_process');
  execSync('npm install chalk', { stdio: 'inherit' });
}

main();