tc-source-content-updater
Version:
Module that updates source content for the desktop application translationCore.
452 lines (399 loc) • 22.2 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.processTranslationNotes = exports.getMissingResources = undefined;
/**
* search to see if we need to get any missing resources needed for tN processing
* @param {String} sourcePath - Path to the extracted files that came from the zip file from the catalog
* e.g. /Users/mannycolon/translationCore/resources/imports/en_tn_v16/en_tn
* @param {String} resourcesPath Path to resources folder
* @param {Function} getMissingOriginalResource - function called to fetch missing resources
* @param {Array} downloadErrors - parsed list of download errors with details such as if the download completed (vs. parsing error), error, and url
* @param {String} languageId - language ID for tA
* @param {String} ownerStr
* @param {boolean} needTa - set to false if resource does not depend on TA
* @param {object} config - configuration object
* @return {Promise<{otQuery: string, ntQuery: string}>}
*/
let getMissingResources = exports.getMissingResources = (() => {
var _ref = _asyncToGenerator(function* (sourcePath, resourcesPath, getMissingOriginalResource, downloadErrors, languageId, ownerStr, needTa = true, config = {}) {
const tsvManifest = resourcesHelpers.getResourceManifestFromYaml(sourcePath);
// array of related resources used to generated the tsv.
const tsvRelations = tsvManifest.dublin_core.relation;
const OT_ORIG_LANG_QUERY = (0, _utils.getQueryStringForBibleId)(tsvRelations, _bible.OT_ORIG_LANG);
const otQuery = (0, _utils.getQueryVariable)(OT_ORIG_LANG_QUERY, 'v');
const NT_ORIG_LANG_QUERY = (0, _utils.getQueryStringForBibleId)(tsvRelations, _bible.NT_ORIG_LANG);
const ntQuery = (0, _utils.getQueryVariable)(NT_ORIG_LANG_QUERY, 'v');
for (const isNewTestament of [false, true]) {
const query = isNewTestament ? ntQuery : otQuery;
if (query) {
const origLangVersion = (0, _apiHelpers.formatVersionWithV)(query);
const origLangId = isNewTestament ? _bible.NT_ORIG_LANG : _bible.OT_ORIG_LANG;
const origLangBibleId = isNewTestament ? _bible.NT_ORIG_LANG_BIBLE : _bible.OT_ORIG_LANG_BIBLE;
const originalLanguageOwner = (0, _apiHelpers.getOwnerForOriginalLanguage)(ownerStr);
yield getMissingOriginalResource(resourcesPath, origLangId, origLangBibleId, origLangVersion, downloadErrors, originalLanguageOwner, config);
const originalBiblePath = _pathExtra2.default.join(resourcesPath, origLangId, 'bibles', origLangBibleId, resourcesHelpers.addOwnerToKey(origLangVersion, originalLanguageOwner));
(0, _unzipFileHelpers.makeSureResourceUnzipped)(originalBiblePath);
}
}
yield (0, _utils.delay)(500);
if (needTa) {
// make sure tA is unzipped
const tAPath = _pathExtra2.default.join(resourcesPath, languageId, 'translationHelps/translationAcademy');
const taVersionPath = resourcesHelpers.getLatestVersionInPath(tAPath, ownerStr, true);
if (taVersionPath) {
(0, _unzipFileHelpers.makeSureResourceUnzipped)(taVersionPath);
} else {
const resource = `${ownerStr}/${languageId}_ta`;
throw new Error(`tnArticleHelpers.getMissingResources() - cannot find '${resource}', at ${tAPath} for ${ownerStr}`);
}
}
return { otQuery, ntQuery };
});
return function getMissingResources(_x, _x2, _x3, _x4, _x5, _x6) {
return _ref.apply(this, arguments);
};
})();
/**
* iterate through group data converting older format ellipsis breaks to ampersand
* @param {object} groupData
* @param {string} filepath
*/
/**
* @description Processes the extracted files for translationNotes to separate the folder
* structure and produce the index.json file for the language with the title of each article.
* @param {Object} resource - Resource object
* @param {String} sourcePath - Path to the extracted files that came from the zip file from the catalog
* e.g. /Users/mannycolon/translationCore/resources/imports/en_tn_v16/en_tn
* @param {String} outputPath - Path to place the processed resource files WITHOUT the version in the path
* @param {String} resourcesPath Path to resources folder
* @param {Array} downloadErrors - parsed list of download errors with details such as if the download completed (vs. parsing error), error, and url
* @param {object} config - configuration object
*/
let processTranslationNotes = exports.processTranslationNotes = (() => {
var _ref2 = _asyncToGenerator(function* (resource, sourcePath, outputPath, resourcesPath, downloadErrors, config = {}) {
try {
if (!resource || !(0, _util.isObject)(resource) || !resource.languageId || !resource.resourceId) {
throw Error(resourcesHelpers.formatError(resource, errors.RESOURCE_NOT_GIVEN));
}
if (!sourcePath) {
throw Error(resourcesHelpers.formatError(resource, errors.SOURCE_PATH_NOT_GIVEN));
}
if (!_fsExtra2.default.pathExistsSync(sourcePath)) {
throw Error(resourcesHelpers.formatError(resource, errors.SOURCE_PATH_NOT_EXIST));
}
if (!outputPath) {
throw Error(resourcesHelpers.formatError(resource, errors.OUTPUT_PATH_NOT_GIVEN));
}
if (_fsExtra2.default.pathExistsSync(outputPath)) {
_fsExtra2.default.removeSync(outputPath);
}
const translationAcademyPath = _pathExtra2.default.join(resourcesPath, resource.languageId, 'translationHelps', 'translationAcademy');
let taCategoriesPath = resourcesHelpers.getLatestVersionInPath(translationAcademyPath, resource.owner);
if (!taCategoriesPath) {
console.log(`tnArticleHelpers.processTranslationNotes() - download missing tA resource`);
yield getMissingHelpsResource(resourcesPath, resource, 'ta', 'Translation_Academy', downloadErrors, config);
console.log(`tnArticleHelpers.processTranslationNotes() - have tA resource`);
taCategoriesPath = resourcesHelpers.getLatestVersionInPath(translationAcademyPath, resource.owner);
}
const originalLanguageOwner = (0, _apiHelpers.getOwnerForOriginalLanguage)(resource.owner);
const { otQuery, ntQuery } = yield getMissingResources(sourcePath, resourcesPath, getMissingOriginalResource, downloadErrors, resource.languageId, resource.owner, true, config);
console.log(`tnArticleHelpers.processTranslationNotes() - have needed original bibles for ${sourcePath}, starting processing`);
const manifest = resourcesHelpers.getResourceManifest(sourcePath);
if (!(manifest && Array.isArray(manifest.projects))) {
throw new Error(`tnArticleHelpers.processTranslationNotes() - no projects in manifest at ${sourcePath} for ${resource_}`);
}
const tnErrors = [];
let bookCount = 0;
for (const project of manifest.projects) {
const filepath = _pathExtra2.default.join(sourcePath, project.path);
const filename = _pathExtra2.default.basename(filepath);
try {
const isSevenCol = filename.toLowerCase().indexOf('tn_') === 0; // file names are as tn_2JN.tsv
const splitter = isSevenCol ? '_' : '-';
const bookId = filename.split(splitter)[1].toLowerCase().replace('.tsv', '');
if (!_bible.BOOK_CHAPTER_VERSES[bookId]) console.error(`tnArticleHelpers.processTranslationNotes() - ${bookId} is not a valid book id.`);
let isNewTestament = true;
if (isSevenCol) {
isNewTestament = _bible.BIBLE_LIST_NT.find(function (bookNumberAndId) {
return bookNumberAndId.split('-')[1].toLowerCase() === bookId;
});
} else {
const bookNumberAndIdMatch = filename.match(/(\d{2}-\w{3})/ig) || [];
const bookNumberAndId = bookNumberAndIdMatch[0];
isNewTestament = _bible.BIBLE_LIST_NT.includes(bookNumberAndId);
}
const originalLanguageId = isNewTestament ? _bible.NT_ORIG_LANG : _bible.OT_ORIG_LANG;
const originalLanguageBibleId = isNewTestament ? _bible.NT_ORIG_LANG_BIBLE : _bible.OT_ORIG_LANG_BIBLE;
const version = isNewTestament && ntQuery ? 'v' + ntQuery : otQuery ? 'v' + otQuery : null;
if (!version) {
console.warn('tnArticleHelpers.processTranslationNotes() - There was a missing original language version for book ' + bookId + ' of resource ' + originalLanguageBibleId + ' from ' + resource.downloadUrl);
continue;
}
const originalBiblePath = _pathExtra2.default.join(resourcesPath, originalLanguageId, 'bibles', originalLanguageBibleId, `${version}_${originalLanguageOwner}`);
if (!_fsExtra2.default.existsSync(filepath)) {
const message = `tnArticleHelpers.processTranslationNote() - cannot find '${filepath}' from manifest projects - skipping`;
console.warn(message);
} else if (_fsExtra2.default.existsSync(originalBiblePath)) {
let groupData;
const params = { categorized: true };
const toolName = 'translationNotes';
if (isSevenCol) {
groupData = yield (0, _tsvGroupdataParser.tsvToGroupData7Cols)(filepath, bookId, resourcesPath, resource.languageId, toolName, originalBiblePath, params);
} else {
groupData = yield (0, _tsvGroupdataParser.tsvToGroupData)(filepath, toolName, params, originalBiblePath, resourcesPath, resource.languageId);
}
convertEllipsisToAmpersand(groupData, filepath);
yield (0, _tsvGroupdataParser.formatAndSaveGroupData)(groupData, outputPath, bookId);
bookCount += 1;
} else {
const resource = `${originalLanguageOwner}/${originalLanguageId}_${originalLanguageBibleId}`;
const message = `tnArticleHelpers.processTranslationNotes() - cannot find '${resource}' at ${originalBiblePath}:`;
console.error(message);
tnErrors.push(message);
}
} catch (e) {
const message = `tnArticleHelpers.processTranslationNotes() - error processing ${filename}:`;
console.error(message, e);
tnErrors.push(message + e.toString());
}
}
yield (0, _utils.delay)(200);
if (tnErrors.length) {
// report errors
const message = `tnArticleHelpers.processTranslationNotes() - error processing ${sourcePath}`;
console.error(message);
throw new Error(`${message}:\n${tnErrors.join('\n')}`);
}
if (!bookCount) {
const message = `tnArticleHelpers.processTranslationNotes() - no books could be processed in ${sourcePath}`;
console.error(message);
throw new Error(`${message}:\n${tnErrors.join('\n')}`);
}
// Generate groupsIndex using tN groupData & tA articles.
(0, _unzipFileHelpers.makeSureResourceUnzipped)(taCategoriesPath);
const categorizedGroupsIndex = (0, _tsvGroupdataParser.generateGroupsIndex)(outputPath, taCategoriesPath);
(0, _tsvGroupdataParser.saveGroupsIndex)(categorizedGroupsIndex, outputPath);
} catch (error) {
console.error('tnArticleHelpers.processTranslationNotes() - error:', error);
throw error;
}
});
return function processTranslationNotes(_x7, _x8, _x9, _x10, _x11) {
return _ref2.apply(this, arguments);
};
})();
/**
* Get missing original language resource
* @param {String} resourcesPath - resources Path
* @param {String} originalLanguageId - original language Id
* @param {String} originalLanguageBibleId - original language bible Id
* @param {String} version - version number
* @param {Array} downloadErrors - parsed list of download errors with details such as if the download completed (vs. parsing error), error, and url
* @param {String} ownerStr
* @param {object} config - configuration object
* @return {Promise}
*/
exports.convertEllipsisToAmpersand = convertEllipsisToAmpersand;
exports.getMissingOriginalResource = getMissingOriginalResource;
exports.getMissingHelpsResource = getMissingHelpsResource;
exports.getOtherTnsOLVersions = getOtherTnsOLVersions;
var _fsExtra = require('fs-extra');
var _fsExtra2 = _interopRequireDefault(_fsExtra);
var _pathExtra = require('path-extra');
var _pathExtra2 = _interopRequireDefault(_pathExtra);
var _util = require('util');
var _tsvGroupdataParser = require('tsv-groupdata-parser');
var _resourcesHelpers = require('../resourcesHelpers');
var resourcesHelpers = _interopRequireWildcard(_resourcesHelpers);
var _resourcesDownloadHelpers = require('../resourcesDownloadHelpers');
var _utils = require('../utils');
var _errors = require('../../resources/errors');
var errors = _interopRequireWildcard(_errors);
var _bible = require('../../resources/bible');
var _unzipFileHelpers = require('../unzipFileHelpers');
var _apiHelpers = require('../apiHelpers');
var _constants = require('tsv-groupdata-parser/lib/utils/constants');
function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
// helpers
// constants
const ELLIPSIS_WITH_SPACES = ` ${_constants.ELLIPSIS} `;function convertEllipsisToAmpersand(groupData, filepath) {
if (groupData) {
const categoriesKeys = Object.keys(groupData);
for (const categoryKey of categoriesKeys) {
const categoryData = groupData[categoryKey];
const checkKeys = Object.keys(categoryData);
for (const checkKey of checkKeys) {
const checks = categoryData[checkKey];
for (const check of checks) {
const contextId = check && check.contextId && check.contextId;
if (contextId) {
let quote = contextId.quote;
let quoteString = contextId.quoteString;
const foundEllipsis = quoteString && quoteString.includes(_constants.ELLIPSIS);
if (foundEllipsis) {
// console.log(`convertEllipsisToAmpersand(${filepath}) - found ellipsis in `, JSON.stringify(contextId));
quoteString = quoteString.replaceAll(ELLIPSIS_WITH_SPACES, ' & ');
quoteString = quoteString.replaceAll(_constants.ELLIPSIS, ' & ');
if (Array.isArray(quote) && quote.length) {
quote = quote.map(item => item.word === _constants.ELLIPSIS ? { word: '&' } : item);
contextId.quote = quote;
contextId.quoteString = quoteString;
} else {
console.log(`convertEllipsisToAmpersand(${filepath}) - missing quote array in `, JSON.stringify(contextId));
}
}
}
}
}
}
}
}function getMissingOriginalResource(resourcesPath, originalLanguageId, originalLanguageBibleId, version, downloadErrors, ownerStr, config = {}) {
return new Promise((() => {
var _ref3 = _asyncToGenerator(function* (resolve, reject) {
try {
const version_ = (0, _apiHelpers.formatVersionWithV)(version);
const originalBiblePath = _pathExtra2.default.join(resourcesPath, originalLanguageId, 'bibles', originalLanguageBibleId, `${version_}_${ownerStr}`);
// If version needed is not in the resources download it.
if (!_fsExtra2.default.existsSync(originalBiblePath)) {
const resourceName = `${originalLanguageId}_${originalLanguageBibleId}`;
let downloadUrl;
let origOwner = ownerStr;
const baseUrl = config.DCS_BASE_URL || _apiHelpers.DCS_BASE_URL;
if (ownerStr === _apiHelpers.DOOR43_CATALOG) {
// Download orig. lang. resource from UW org, but use as Door43-Catalog
downloadUrl = `${baseUrl}/${_apiHelpers.CN_CATALOG}/${resourceName}/archive/${version_}.zip`;
} else {
// otherwise we read from uW org
origOwner = 'unfoldingWord'; // change owner of resource
downloadUrl = `${baseUrl}/${origOwner}/${resourceName}/archive/${version_}.zip`;
}
console.log(`tnArticleHelpers.getMissingOriginalResource() - downloading missing original bible: ${downloadUrl}`);
const resource = {
languageId: originalLanguageId,
resourceId: originalLanguageBibleId,
remoteModifiedTime: '0001-01-01T00:00:00+00:00',
downloadUrl,
name: resourceName,
version: (0, _apiHelpers.formatVersionWithoutV)(version),
subject: 'Bible',
owner: origOwner,
catalogEntry: {
subject: {},
resource: {},
format: {}
}
};
// Delay to try to avoid Socket timeout
yield (0, _utils.delay)(1000);
yield (0, _resourcesDownloadHelpers.downloadAndProcessResource)(resource, resourcesPath, downloadErrors, config);
resolve();
} else {
resolve();
}
} catch (error) {
reject(error);
}
});
return function (_x12, _x13) {
return _ref3.apply(this, arguments);
};
})());
}
/**
* download a missing resource that matches parentResource, but has fetchResourceId
* @param {String} resourcesPath - resources Path
* @param {object} parentResource - resource of object loading this as a dependency
* @param {String} fetchResourceId - id of resource to fetch, such as 'ta'
* @param {String} fetchSubject - subject string of resource to fetch, such as 'Translation_Academy'
* @param {Array} downloadErrors - parsed list of download errors with details such as if the download completed (vs. parsing error), error, and url
* @param {object} config - configuration object
* @return {Promise}
*/
function getMissingHelpsResource(resourcesPath, parentResource, fetchResourceId, fetchSubject, downloadErrors, config = {}) {
return new Promise((() => {
var _ref4 = _asyncToGenerator(function* (resolve, reject) {
try {
const resourceName = `${parentResource.languageId}_${fetchResourceId}`;
// get latest version
const latest = yield (0, _apiHelpers.getLatestRelease)({
owner: parentResource.owner,
repo: resourceName,
stage: config.stage,
baseUrl: config.DCS_BASE_URL || _apiHelpers.DCS_BASE_URL
});
if (!latest) {
console.warn('tnArticleHelpers.getMissingHelpsResource() - no release found');
throw 'no release found';
}
const release = latest && latest.release;
const version = release && release.tag_name || 'master';
const version_ = (0, _apiHelpers.formatVersionWithV)(version);
const baseUrl = config.DCS_BASE_URL || _apiHelpers.DCS_BASE_URL;
const downloadUrl = `${baseUrl}/${parentResource.owner}/${resourceName}/archive/${version_}.zip`;
console.log(`tnArticleHelpers.getMissingHelpsResource() - downloading missing helps: ${downloadUrl}`);
const remoteModifiedTime = latest && latest.released || release && release.published_at;
const resource = {
languageId: latest.language,
resourceId: fetchResourceId,
remoteModifiedTime,
downloadUrl,
name: resourceName,
owner: parentResource.owner,
version: (0, _apiHelpers.formatVersionWithoutV)(version),
subject: latest.subject
};
// Delay to try to avoid Socket timeout
yield (0, _utils.delay)(1000);
yield (0, _resourcesDownloadHelpers.downloadAndProcessResource)(resource, resourcesPath, downloadErrors, config);
resolve();
} catch (error) {
reject(error);
}
});
return function (_x14, _x15) {
return _ref4.apply(this, arguments);
};
})());
}
/**
* Get the version of the other Tns orginal language.
* @param {String} resourcesPath - resources Path
* @param {string} originalLanguageId - original Language Id.
* @return {array}
*/
function getOtherTnsOLVersions(resourcesPath, originalLanguageId) {
const languageIds = _fsExtra2.default.readdirSync(resourcesPath).filter(filename => resourcesHelpers.isDirectory(resourcesPath, filename));
const versionsToNotDelete = [];
languageIds.forEach(languageId => {
const tnHelpsPath = _pathExtra2.default.join(resourcesPath, languageId, 'translationHelps', 'translationNotes');
if (_fsExtra2.default.existsSync(tnHelpsPath)) {
const owners = resourcesHelpers.getLatestVersionsAndOwners(tnHelpsPath) || {};
for (const owner of Object.keys(owners)) {
const tnHelpsVersionPath = owners[owner];
if (tnHelpsVersionPath) {
const tnManifestPath = _pathExtra2.default.join(tnHelpsVersionPath, 'manifest.json');
if (_fsExtra2.default.existsSync(tnManifestPath)) {
const manifest = _fsExtra2.default.readJsonSync(tnManifestPath);
const { relation } = manifest.dublin_core || {};
const query = (0, _utils.getQueryStringForBibleId)(relation, originalLanguageId);
if (query) {
let requiredVersion = (0, _utils.getQueryVariable)(query, 'v');
if (requiredVersion) {
const version = 'v' + requiredVersion;
// console.log(`tnArticleHelpers.getOtherTnsOLVersions() - for ${languageId}, found dependency: ${query}`);
versionsToNotDelete.push(version);
} else {
console.log(`getOtherTnsOLVersions() - could not find version for ${query} in ${tnHelpsPath}`);
}
}
}
}
}
}
});
return versionsToNotDelete;
}