@mediafly/translate/src/translate.js

Translate strings extracted from apps using Google Translate

var _ = require('lodash')
var crypto = require('crypto')
var path = require('path')
var chalk = require('chalk')
var fs = require('fs')
var highland = require('highland')
var flatten = require('flat')
var path = require('path')
var config = require(path.join(process.cwd(), 'mediafly-translate.config'))
var languages = config.languages
var format = config.format || 'json'
var apiKey = config.apiKey
var readFile = require('./read-file').read
var retry = require('retry')
var fileFormat = require('./format')
var en = readFile('locale-en')
var { requireOrDefault, getLanguageName, getTranslationsDir } = require('./util')
var googleTranslate = require('./google-translate')(apiKey)

const TRANSLATIONS_DIRECTORY = getTranslationsDir()

var magicWord = '777'

function replaceBindingsWithMagicCharacter(text) {
	if (typeof text !== 'string') {
		return text
	}
	var regExp = config.placeholderRegularExpression || '(\{\{.+?\}\})'
	return text.replace(new RegExp(regExp, 'gi'), magicWord)
}

function replaceMagicCharsWithBindings(result) {
	var keyValues = flatten(en)
	var values = highland.values(keyValues)

	var originalText = values
		.find(v => replaceBindingsWithMagicCharacter(v) === result.originalText)
		.map(v => {
			var regExp = config.placeholderRegularExpression || '(\{\{.+?\}\})'
			var replacements = v.match(new RegExp(regExp, 'gi'))
			return result.translatedText.replace(new RegExp(magicWord, 'g'), () => replacements.shift() || '')
		})

	return originalText
}

function isString(str) {
	return typeof str === 'string'
}

function updateOverridesFile() {

	const overridesFile = flatten(requireOrDefault(`${TRANSLATIONS_DIRECTORY}/locale.overrides`))
	const flattenedEnglishFile = flatten(en)

	const newStrings = _.pickBy(flattenedEnglishFile, (v, k) => {
		var hashFromEnglishFile = crypto.createHash('md5').update(v).digest('hex')
		var overrideHash = overridesFile[k]
		return hashFromEnglishFile !== overrideHash
	})
	_.forOwn(newStrings, (_value, key) => {
		overridesFile[key] = crypto.createHash('md5').update(flattenedEnglishFile[key]).digest('hex')
	})

	Object.keys(overridesFile, (key) => {
		if (!flattenedEnglishFile[key]) {
			delete overridesFile[key]
		}
	})

	const outputStream = fs.createWriteStream(`${TRANSLATIONS_DIRECTORY}/locale.overrides.json`)
	const overridesStream = highland(Object.entries(overridesFile))
		.filter(([key]) => !!flattenedEnglishFile[key])
		.reduce((x, y) => highland.set(y[0], y[1], x), {})

	outputStream.on('error', (err) => {
		console.error(`Error writing to the file: ${TRANSLATIONS_DIRECTORY}/locale.overrides.json`, err)
	})

	outputStream.on('finish', () => {
		console.log(`Writing to file ${TRANSLATIONS_DIRECTORY}/locale.overrides.json finished successfully.`)
	})

	overridesStream
		.map((obj) => {
			const unflattenedObj = flatten.unflatten(obj)
			return JSON.stringify(unflattenedObj, null, 4)
		})
		.pipe(outputStream)

	return overridesFile
}

async function translate(argv) {
	console.log(`Translating using Google Translate...`)

	const overridesFile = flatten(requireOrDefault(`${TRANSLATIONS_DIRECTORY}/locale.overrides`))

	var streams = languages.map(language => {
		if (language === 'ke') {
			//special handling for the "keys" language
			return require('./keys-language').translate()
		}

		var flattenedEnglishFile = flatten(en)

		//only take keys if they still exist in the English file
		const previousTranslation = (config.format === 'js')
			? readFile(`locale-${language}`).default
			: readFile(`locale-${language}`)

		var flatPreviousTranslation = _.pickBy(
			flatten(previousTranslation), (v, k) => !!flattenedEnglishFile[k]
		)

		var flattenedOverridesFile = flatten(overridesFile)

		var newStrings = _.pickBy(flattenedEnglishFile, (v, k) => {
			var hashFromEnglishFile = crypto.createHash('md5').update(v).digest('hex')
			var overrideHash = flattenedOverridesFile[k]
			return hashFromEnglishFile !== overrideHash
		})

		if (Object.keys(newStrings).length > 0) {
			console.log(`Translating ${getLanguageName(language)}`)
			console.log(`The following keys are new in the English file. They will be translated using Google Translate.`)
			Object.keys(newStrings).forEach(key => {
				console.log(`Key: ${key}`)
			})
		} else if (Object.keys(newStrings).length === 0) {
			console.log(`No new keys in ${getLanguageName(language)}`)
		}

		var englishKeys = highland.keys(newStrings)

		var arrayOfEnglishText = highland.values(newStrings)

		var languageCodeWithoutRegion = language.split('-')[0]
		var translateToLanguage = (s, cb) => {

			var operation = retry.operation({ retries: 3, minTimeout: 1000 })
			operation.attempt(function() {

				if (languageCodeWithoutRegion !== 'en') {
					googleTranslate.translate(s, 'en', languageCodeWithoutRegion, (err, translations) => {
						if (operation.retry(err)) {
							console.error(err)
							console.error('Failed to translate ', s, '. Retrying...')
							return
						}

						cb(err ? operation.mainError() : null, translations)
					})
				} else {
					cb(null, s.map(each => ({ originalText: each, translatedText: each })))
				}
			})
		}

		var translate = highland
			.wrapCallback(translateToLanguage)

		var translations = arrayOfEnglishText
			.filter(isString)
			.map(replaceBindingsWithMagicCharacter)
			.batch(10)
			.map(translate)
			.flatten()
			.flatMap(replaceMagicCharsWithBindings)

		var writeFileOptions

		if (format === 'strings') {
			writeFileOptions = { encoding: 'ucs2' }
		}
		return englishKeys
			.zip(translations)
			.reduce((x, y) => highland.set(y[0], y[1], x), {})
			.map(obj => {
				//sort by key
				return Object.keys(obj)
					.sort()
					.reduce((previous, current) => {
						return highland.set(current, obj[current], previous)
					}, {})
			})
			.map(o => Object.assign({}, flatPreviousTranslation, o))
			.map(fileFormat.transform)
			.pipe(
				fs.createWriteStream(`${TRANSLATIONS_DIRECTORY}/locale-${language}.${format}`, writeFileOptions)
			)
	})

	var promises = streams
		.filter(s => !!s)
		.map(writeStream => new Promise((resolve) => {
			writeStream.on('finish', resolve)
		}))


	//map the languages array to a single promise
	return Promise.all(promises)
		.then(() => console.log(chalk.green('All languages translated!')))
		.then(() => updateOverridesFile())
}

module.exports = {
	handler: translate
}