tc-source-content-updater
Version:
Module that updates source content for the desktop application translationCore.
379 lines (322 loc) • 16.9 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.processTranslationWordsTSV = exports.twlTsvToGroupData = undefined;
var _extends = Object.assign || function (target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i]; for (var key in source) { if (Object.prototype.hasOwnProperty.call(source, key)) { target[key] = source[key]; } } } return target; };
/**
* process the TSV file into index files
* @param {string} tsvPath
* @param {object} project
* @param {string} resourcesPath
* @param {string} originalBiblePath
* @param {string} outputPath
*/
let twlTsvToGroupData = exports.twlTsvToGroupData = (() => {
var _ref = _asyncToGenerator(function* (tsvPath, project, resourcesPath, originalBiblePath, outputPath) {
const bookId = project.identifier;
let groupData;
const {
tsvItems,
error
} = yield (0, _tsvGroupdataParser.tsvToObjects)(tsvPath, project);
if (error) {
throw error;
}
const tsvItems_ = [];
for (const tsvItem of tsvItems) {
const cleanedRef = (0, _tsvGroupdataParser.parseReference)(tsvItem.Reference);
const tsvItem_ = _extends({}, tsvItem, cleanedRef);
tsvItems_.push(tsvItem_);
}
try {
groupData = tsvObjectsToGroupData(tsvItems_, originalBiblePath, resourcesPath, bookId, project.languageId, 'translationWords', { categorized: true });
(0, _tnArticleHelpers.convertEllipsisToAmpersand)(groupData, tsvPath);
yield (0, _tsvGroupdataParser.formatAndSaveGroupData)(groupData, outputPath, bookId);
} catch (e) {
console.error(`twArticleHelpers.twlTsvToGroupData() - error processing filepath: ${tsvPath}`, e);
throw e;
}
return groupData;
});
return function twlTsvToGroupData(_x, _x2, _x3, _x4, _x5) {
return _ref.apply(this, arguments);
};
})();
/**
* @description Processes the extracted files for translationWord to create the folder
* structure and produce the index.js file for the language with the title of each article.
* @param {Object} resource - Resource object
* @param {String} sourcePath - Path to the extracted files that came from the zip file from the catalog
* @param {String} outputPath - Path to place the processed resource files WIHTOUT the version in the path
* @param {string} resourcesPath
* @param {Array} downloadErrors - parsed list of download errors with details such as if the download completed (vs. parsing error), error, and url
* @param {object} config - configuration object
* @return {Boolean} true if success
*/
let processTranslationWordsTSV = exports.processTranslationWordsTSV = (() => {
var _ref2 = _asyncToGenerator(function* (resource, sourcePath, outputPath, resourcesPath, downloadErrors, config = {}) {
try {
if (!resource || !(0, _util.isObject)(resource) || !resource.languageId || !resource.resourceId) throw Error(resourcesHelpers.formatError(resource, errors.RESOURCE_NOT_GIVEN));
if (!sourcePath) throw Error(resourcesHelpers.formatError(resource, errors.SOURCE_PATH_NOT_GIVEN));
if (!_fsExtra2.default.pathExistsSync(sourcePath)) throw Error(resourcesHelpers.formatError(resource, errors.SOURCE_PATH_NOT_EXIST));
if (!outputPath) throw Error(resourcesHelpers.formatError(resource, errors.OUTPUT_PATH_NOT_GIVEN));
if (_fsExtra2.default.pathExistsSync(outputPath)) _fsExtra2.default.removeSync(outputPath);
const { otQuery, ntQuery } = yield (0, _tnArticleHelpers.getMissingResources)(sourcePath, resourcesPath, _tnArticleHelpers.getMissingOriginalResource, downloadErrors, resource.languageId, resource.owner, false, config);
// make sure tW is already installed
const twPath = _pathExtra2.default.join(resourcesPath, resource.languageId, 'translationHelps/translationWords');
const twVersionPath = resourcesHelpers.getLatestVersionInPath(twPath, resource.owner);
if (_fsExtra2.default.existsSync(twVersionPath)) {
(0, _unzipFileHelpers.makeSureResourceUnzipped)(twVersionPath);
} else {
const resource_ = `${resource.owner}/${resource.languageId}_tw`;
throw new Error(`processTranslationWordsTSV() - cannot find '${resource_}' at ${twPath} for ${resource.owner}`);
}
const manifest = (0, _resourcesHelpers.getResourceManifest)(sourcePath);
if (!(manifest && Array.isArray(manifest.projects))) {
throw new Error(`processTranslationWordsTSV() - no projects in manifest at ${sourcePath} for ${resource.owner}`);
}
const twlErrors = [];
for (const project of manifest.projects) {
const tsvPath = _pathExtra2.default.join(sourcePath, project.path);
try {
const bookId = project.identifier;
const isNewTestament = _bible.BIBLE_BOOKS.newTestament[project.identifier];
const originalLanguageId = isNewTestament ? _bible.NT_ORIG_LANG : _bible.OT_ORIG_LANG;
const originalLanguageBibleId = isNewTestament ? _bible.NT_ORIG_LANG_BIBLE : _bible.OT_ORIG_LANG_BIBLE;
const version = isNewTestament && ntQuery ? 'v' + ntQuery : otQuery ? 'v' + otQuery : null;
if (!version) {
console.warn('processTranslationWordsTSV() - There was a missing version for book ' + bookId + ' of resource ' + originalLanguageBibleId + ' from ' + resource.downloadUrl);
continue;
}
const originalLanguageOwner = (0, _apiHelpers.getOwnerForOriginalLanguage)(resource.owner);
const originalBiblePath = _pathExtra2.default.join(resourcesPath, originalLanguageId, 'bibles', originalLanguageBibleId, `${version}_${originalLanguageOwner}`);
if (!_fsExtra2.default.existsSync(tsvPath)) {
const message = `processTranslationWordsTSV() - cannot find '${tsvPath}' from manifest projects - skipping`;
console.warn(message);
} else if (_fsExtra2.default.existsSync(originalBiblePath)) {
const groupData = yield twlTsvToGroupData(tsvPath, project, resourcesPath, originalBiblePath, outputPath);
(0, _tnArticleHelpers.convertEllipsisToAmpersand)(groupData, tsvPath);
yield (0, _tsvGroupdataParser.formatAndSaveGroupData)(groupData, outputPath, bookId);
} else {
const resource = `${originalLanguageOwner}/${originalLanguageId}_${originalLanguageBibleId}`;
const message = `processTranslationWordsTSV() - cannot find '${resource}' at ${originalBiblePath}:`;
console.error(message);
twlErrors.push(message);
}
} catch (e) {
const message = `processTranslationWordsTSV() - error processing ${tsvPath}:`;
console.error(message, e);
twlErrors.push(message + e.toString());
}
}
yield (0, _utils.delay)(200);
if (twlErrors.length) {
// report errors
const message = `processTranslationWordsTSV() - error processing ${sourcePath}`;
console.error(message);
throw new Error(`${message}:\n${twlErrors.join('\n')}`);
}
} catch (error) {
console.error('processTranslationWordsTSV() - error:', error);
throw error;
}
});
return function processTranslationWordsTSV(_x6, _x7, _x8, _x9, _x10) {
return _ref2.apply(this, arguments);
};
})();
/**
* @description - Generates the groups index for the tw articles (both kt, other and names).
* @param {String} filesPath - Path to all tw markdown artciles.
* @param {String} twOutputPath Path to the resource location in the static folder.
* @param {String} folderName article type. ex. kt or other.
*/
exports.processTranslationWords = processTranslationWords;
var _fsExtra = require('fs-extra');
var _fsExtra2 = _interopRequireDefault(_fsExtra);
var _pathExtra = require('path-extra');
var _pathExtra2 = _interopRequireDefault(_pathExtra);
var _util = require('util');
var _tsvGroupdataParser = require('tsv-groupdata-parser');
var _resourcesHelpers = require('../resourcesHelpers');
var resourcesHelpers = _interopRequireWildcard(_resourcesHelpers);
var _errors = require('../../resources/errors');
var errors = _interopRequireWildcard(_errors);
var _apiHelpers = require('../apiHelpers');
var _unzipFileHelpers = require('../unzipFileHelpers');
var _bible = require('../../resources/bible');
var _tnArticleHelpers = require('./tnArticleHelpers');
var _utils = require('../utils');
function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; } /* eslint-disable curly */
// helpers
// eslint-disable-next-line no-duplicate-imports
// constants
/**
* @description Processes the extracted files for translationWord to create the folder
* structure and produce the index.js file for the language with the title of each article.
* @param {Object} resource - Resource object
* @param {String} sourcePath - Path to the extracted files that came from the zip file from the catalog
* @param {String} outputPath - Path to place the processed resource files WIHTOUT the version in the path
* @return {Boolean} true if success
*/
function processTranslationWords(resource, sourcePath, outputPath) {
if (!resource || !(0, _util.isObject)(resource) || !resource.languageId || !resource.resourceId) throw Error(resourcesHelpers.formatError(resource, errors.RESOURCE_NOT_GIVEN));
if (!sourcePath) throw Error(resourcesHelpers.formatError(resource, errors.SOURCE_PATH_NOT_GIVEN));
if (!_fsExtra2.default.pathExistsSync(sourcePath)) throw Error(resourcesHelpers.formatError(resource, errors.SOURCE_PATH_NOT_EXIST));
if (!outputPath) throw Error(resourcesHelpers.formatError(resource, errors.OUTPUT_PATH_NOT_GIVEN));
if (_fsExtra2.default.pathExistsSync(outputPath)) _fsExtra2.default.removeSync(outputPath);
const typesPath = _pathExtra2.default.join(sourcePath, 'bible');
const isDirectory = item => _fsExtra2.default.lstatSync(_pathExtra2.default.join(typesPath, item)).isDirectory();
let typeDirs = [];
if (_fsExtra2.default.existsSync(typesPath)) {
typeDirs = _fsExtra2.default.readdirSync(typesPath).filter(isDirectory);
}
typeDirs.forEach(typeDir => {
const typePath = _pathExtra2.default.join(typesPath, typeDir);
const files = _fsExtra2.default.readdirSync(typePath).filter(filename => _pathExtra2.default.extname(filename) === '.md');
generateGroupsIndex(typePath, outputPath, typeDir);
files.forEach(fileName => {
const sourcePath = _pathExtra2.default.join(typePath, fileName);
const destinationPath = _pathExtra2.default.join(outputPath, typeDir, 'articles', fileName);
_fsExtra2.default.copySync(sourcePath, destinationPath);
});
});
return true;
}
/**
* organize group data by catalogories and groupID
* @param {array} groupData
* @return {Object}
*/
const categorizeGroupData = groupData => {
const categorizedGroupData = {
kt: {},
names: {},
other: {}
};
Object.keys(groupData).forEach(category_ => {
let category = category_;
const items = groupData[category];
if (!categorizedGroupData[category]) {
category = 'other'; // unknown categories go in other
}
for (const item of items) {
const groupId = item.contextId && item.contextId.groupId;
if (groupId) {
if (!categorizedGroupData[category][groupId]) {
categorizedGroupData[category][groupId] = [];
}
categorizedGroupData[category][groupId].push(item);
}
}
});
return categorizedGroupData;
};
/**
* Parses list of tsv items and returns an object holding the lists of group ids.
* @param {Array} tsvItems - list of items to process
* @param {string} originalBiblePath path to original bible.
* e.g. /resources/el-x-koine/bibles/ugnt/v0.11
* @param {string} resourcesPath path to the resources dir
* e.g. /User/john/translationCore/resources
* @param {string} bookId
* @param {string} langId
* @param {string} toolName tC tool name.
* @param {object} params When it includes { categorized: true }
* then it returns the object organized by tn article category.
* @return {Object} - groupData
*/
function tsvObjectsToGroupData(tsvItems, originalBiblePath, resourcesPath, bookId, langId, toolName, params) {
const groupData = {};
const twLinkMatch = /^rc:\/\/\*\/tw\/dict\/bible\/(\w+)\/([\w\d]+)/;
const twLinkRE = new RegExp(twLinkMatch);
bookId = bookId.toLowerCase();
try {
const resourceApi = new _tsvGroupdataParser.ManageResource(originalBiblePath, bookId);
for (const tsvItem of tsvItems) {
if (tsvItem.Reference && tsvItem.ID && tsvItem.OrigWords && tsvItem.Occurrence && tsvItem.TWLink) {
// const tags = cleanGroupId(tsvItem.Tags) || 'other';
const twLink = tsvItem.TWLink.match(twLinkRE);
if (!twLink) {
console.warn(`tsvObjectsToGroupData() - invalid TWLink: ${tsvItem.TWLink}`);
continue;
}
const chapter = tsvItem.Chapter;
const verse = tsvItem.Verse;
tsvItem.Book = bookId;
tsvItem.Chapter = chapter;
tsvItem.Verse = verse;
tsvItem.OrigQuote = tsvItem.OrigWords;
tsvItem.Catagory = twLink[1];
tsvItem.SupportReference = twLink[2];
let verseString = null;
if (!tsvItem.Catagory || !tsvItem.SupportReference) {
console.warn(`tsvObjectsToGroupData() - invalid TWLink: ${tsvItem.TWLink}`);
continue;
}
try {
verseString = resourceApi.getVerseStringFromRef(tsvItem.Reference);
} catch (e) {
if (parseInt(chapter, 10) && parseInt(verse, 10)) {
console.warn(`tsvObjectsToGroupData() - error getting verse string: chapter ${chapter}, verse ${verse} from ${JSON.stringify(tsvItem)}`, e);
}
}
if (verseString) {
const key = tsvItem.Catagory;
const groupDataItem = (0, _tsvGroupdataParser.generateGroupDataItem)(tsvItem, toolName, verseString);
if (groupData[key]) {
groupData[key].push(groupDataItem);
} else {
groupData[key] = [groupDataItem];
}
}
} else {
console.warn('tsvObjectsToGroupData() - error processing item:', JSON.stringify(tsvItem));
}
}
return params && params.categorized ? categorizeGroupData(groupData) : groupData;
} catch (e) {
console.error(`tsvObjectsToGroupData() - error processing TSVs`, e);
throw e;
}
}function generateGroupsIndex(filesPath, twOutputPath, folderName) {
const groupsIndex = [];
const groupIds = _fsExtra2.default.readdirSync(filesPath).filter(filename => {
return filename.split('.').pop() === 'md';
});
groupIds.forEach(fileName => {
const groupObject = {};
const filePath = _pathExtra2.default.join(filesPath, fileName);
const articleFile = _fsExtra2.default.readFileSync(filePath, 'utf8');
const groupId = fileName.replace('.md', '');
// get the article's first line and remove #'s and spaces from beginning/end
const groupName = articleFile.split('\n')[0].replace(/(^\s*#\s*|\s*#\s*$)/gi, '');
groupObject.id = groupId;
groupObject.name = groupName;
groupsIndex.push(groupObject);
});
groupsIndex.sort(compareByFirstUniqueWord);
const groupsIndexOutputPath = _pathExtra2.default.join(twOutputPath, folderName, 'index.json');
_fsExtra2.default.outputJsonSync(groupsIndexOutputPath, groupsIndex, { spaces: 2 });
}
/**
* Splits the string into words delimited by commas and compares the first unique word
* @param {String} a first string to be compared
* @param {String} b second string to be compared
* @return {int} comparison result
*/
function compareByFirstUniqueWord(a, b) {
const aWords = a.name.toUpperCase().split(',');
const bWords = b.name.toUpperCase().split(',');
while (aWords.length || bWords.length) {
if (!aWords.length) return -1;
if (!bWords.length) return 1;
const aWord = aWords.shift().trim();
const bWord = bWords.shift().trim();
if (aWord !== bWord) return aWord < bWord ? -1 : 1;
}
return 0; // both lists are the same
}