tc-source-content-updater
Version:
Module that updates source content for the desktop application translationCore.
586 lines (525 loc) • 22.7 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.sortDownloableResources = exports.downloadResources = exports.findMatchingResource = exports.downloadAndProcessResourceWithCatch = exports.downloadAndProcessResource = undefined;
exports.addDownloadError = addDownloadError;
exports.removeUnusedResources = removeUnusedResources;
exports.getVersionFolder = getVersionFolder;
exports.setResourcesPreReleaseStateFromStage = setResourcesPreReleaseStateFromStage;
exports.getMainManifestPath = getMainManifestPath;
exports.setResourcesContainPreReleaseData = setResourcesContainPreReleaseData;
exports.doResourcesContainPreReleaseData = doResourcesContainPreReleaseData;
exports.isStagePreRelease = isStagePreRelease;
exports.doingPreReleaseFetching = doingPreReleaseFetching;
exports.doingPreRelease = doingPreRelease;
exports.showOnlineStatus = showOnlineStatus;
exports.resourceSort = resourceSort;
var _fsExtra = require('fs-extra');
var _fsExtra2 = _interopRequireDefault(_fsExtra);
var _pathExtra = require('path-extra');
var _pathExtra2 = _interopRequireDefault(_pathExtra);
var _rimraf = require('rimraf');
var _rimraf2 = _interopRequireDefault(_rimraf);
var _promiseParallelThrottle = require('promise-parallel-throttle');
var Throttle = _interopRequireWildcard(_promiseParallelThrottle);
var _resourcesHelpers = require('./resourcesHelpers');
var _parseHelpers = require('./parseHelpers');
var parseHelpers = _interopRequireWildcard(_parseHelpers);
var _downloadHelpers = require('./downloadHelpers');
var downloadHelpers = _interopRequireWildcard(_downloadHelpers);
var _moveResourcesHelpers = require('./moveResourcesHelpers');
var moveResourcesHelpers = _interopRequireWildcard(_moveResourcesHelpers);
var _tnArticleHelpers = require('./translationHelps/tnArticleHelpers');
var _errors = require('../resources/errors');
var errors = _interopRequireWildcard(_errors);
var _bible = require('../resources/bible');
var Bible = _interopRequireWildcard(_bible);
var _apiHelpers = require('./apiHelpers');
var _utils = require('./utils');
var _index = require('../index');
function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
// helpers
// constants
/**
* add download error keeping track of error message, download url, and if parse error type (if not parse error, then download error)
* @param {Array} downloadErrors - list of download errors
* @param {Boolean} parseError
* @param {String} errorMessage
* @param {String} downloadUrl
*/
function addDownloadError(downloadErrors, parseError, errorMessage, downloadUrl) {
downloadErrors.push({
parseError,
errorMessage,
downloadUrl
});
}
/**
* if not original language resource, removes all but latest. If original language resource, only removes unused old original language resources
* @param {string} resourcesPath - path to all resources
* @param {string} currentResourcePath - path for current download resource version
* @param {string} originalLanguageId
* @param {string} version
* @param {boolean} isGreekOrHebrew - true if original language resource
* @param {string} owner
*/
function removeUnusedResources(resourcesPath, currentResourcePath, originalLanguageId, version, isGreekOrHebrew, owner) {
let versionsToNotDelete = [];
const resourceVersionsPath = _pathExtra2.default.dirname(currentResourcePath); // get folder for all the versions
// Get the version numbers of the original language used by other tNs so that needed versions are not deleted.
if (isGreekOrHebrew) {
versionsToNotDelete = (0, _tnArticleHelpers.getOtherTnsOLVersions)(resourcesPath, originalLanguageId);
}
// Make sure that the resource currently being downloaded is not deleted
versionsToNotDelete.push('v' + version);
(0, _resourcesHelpers.removeAllButLatestVersion)(resourceVersionsPath, versionsToNotDelete, owner);
}
/**
*
* @param {object} resource
* @return {string}
*/
function getVersionFolder(resource) {
let versionDir = 'v' + resource.version;
if (resource.owner) {
const ownerStr = (0, _resourcesHelpers.encodeOwnerStr)(resource.owner);
versionDir += `${_apiHelpers.OWNER_SEPARATOR}${ownerStr}`;
}
return versionDir;
}
/**
* save flag that resources contain pre-release data if stage is preprod
* @param {String} resourcesPath
* @param {String} stage
*/
function setResourcesPreReleaseStateFromStage(resourcesPath, stage) {
if (isStagePreRelease(stage)) {
setResourcesContainPreReleaseData(resourcesPath, true);
}
}
/**
* get path for main manifest
* @param {String} resourcesPath
* @return {*}
*/
function getMainManifestPath(resourcesPath) {
const mainManifestPath = _pathExtra2.default.join(resourcesPath, 'source-content-updater-manifest.json');
return mainManifestPath;
}
/**
* save flag that resources contain pre-release data. Flag is truthy
* @param {String} resourcesPath
* @param {boolean} flag
*/
function setResourcesContainPreReleaseData(resourcesPath, flag) {
const mainManifestPath = getMainManifestPath(resourcesPath);
let manifest = (0, _apiHelpers.readJsonFile)(mainManifestPath);
if (!manifest) {
manifest = {};
}
manifest.havePreReleaseData = !!flag;
try {
_fsExtra2.default.outputJsonSync(mainManifestPath, manifest);
} catch (e) {
console.error(`setResourcesContainPreReleaseData(): could not save ${mainManifestPath}`, e);
}
}
/**
* checks flag to see if we have pre-release data
* @param {String} resourcesPath
* @return {boolean} true if we have downloaded pre-release data
*/
function doResourcesContainPreReleaseData(resourcesPath) {
const mainManifestPath = getMainManifestPath(resourcesPath);
const manifest = (0, _apiHelpers.readJsonFile)(mainManifestPath);
if (manifest) {
return manifest.havePreReleaseData;
}
return false;
}
/**
* returns true is stage is pre-release
* @param {string} stage
* @return {boolean} true if pre-release
*/
function isStagePreRelease(stage) {
return stage === _index.STAGE.PRE_PROD;
}
/**
* checks flag to see if we have pre-release data
* @param {object} config
* @return {boolean} true if we are downloading pre-release data
*/
function doingPreReleaseFetching(config = {}) {
return isStagePreRelease(config.stage);
}
/**
* check if we are doing pre-release checking or have prerelease downloads
* @param {String} resourcesPath
* @param {object} config
* @return {boolean} true if we are doing pre-release data
*/
function doingPreRelease(resourcesPath, config = {}) {
const doingPrerelease = doingPreReleaseFetching(config) || doResourcesContainPreReleaseData(resourcesPath);
return doingPrerelease;
}
/**
* @description Downloads the resources that need to be updated for a given language using the DCS API
* @param {Object.<{
* languageId: String,
* resourceId: String,
* localModifiedTime: String,
* remoteModifiedTime: String,
* downloadUrl: String,
* version: String,
* subject: String,
* catalogEntry: {langResource, bookResource, format}
* }>} resource - resource to download
* @param {String} resourcesPath Path to the user resources directory
* @param {Array} downloadErrors - parsed list of download errors with details such as if the download completed (vs. parsing error), error, and url
* @param {object} config - configuration object
* @return {Promise} Download promise
*/
const downloadAndProcessResource = exports.downloadAndProcessResource = (() => {
var _ref = _asyncToGenerator(function* (resource, resourcesPath, downloadErrors, config = {}) {
if (!resource) {
throw Error(errors.RESOURCE_NOT_GIVEN);
} else if (!resourcesPath) {
throw Error((0, _resourcesHelpers.formatError)(resource, errors.RESOURCES_PATH_NOT_GIVEN));
}
if (config.getCancelState && config.getCancelState()) {
console.warn(`downloadAndProcessResource() download of ${resource.downloadUrl} cancelled`);
return; // if user cancelled then skip
}
const doingPreRelease_ = doingPreRelease(resourcesPath, config);
const resourceData = resource.catalogEntry ? resource.catalogEntry.resource : resource;
if (!resource.version || resource.version === 'master') {
if (!(0, _utils.areWeOnline)()) {
const message = `Download ${resource.downloadUrl} error, disconnected from internet`;
console.log(message);
throw message;
}
const owner = resourceData.owner || resource.owner;
const tag = resource.version || 'master';
const repo = resourceData.name || `${resource.languageId}_${resource.resourceId}`;
const latest = yield (0, _apiHelpers.getReleaseMetaData)({ owner, repo, tag, baseUrl: config.DCS_BASE_URL });
const release = latest && latest.release;
let version = release && release.tag_name;
resource.downloadUrl = release && release.zipball_url || latest.zipball_url;
if (!version) {
// if no release found, then get version from manifest data
const manifest = yield (0, _apiHelpers.downloadManifestData)({ owner, repo, tag, baseUrl: config.DCS_BASE_URL });
version = manifest && manifest.dublin_core && manifest.dublin_core.version;
}
if (version) {
resource.version = version;
}
}
// Resource is the Greek UGNT or Hebrew UHB
const isGreekOrHebrew = resource.languageId === Bible.NT_ORIG_LANG && resource.resourceId === Bible.NT_ORIG_LANG_BIBLE || resource.languageId === Bible.OT_ORIG_LANG && resource.resourceId === Bible.OT_ORIG_LANG_BIBLE;
_fsExtra2.default.ensureDirSync(resourcesPath);
const importsPath = _pathExtra2.default.join(resourcesPath, 'imports');
_fsExtra2.default.ensureDirSync(importsPath);
let importPath = null;
let zipFilePath = null;
let downloadComplete = false;
try {
try {
const zipFileName = `${resource.languageId}_${resource.resourceId}_v${resource.version}_${encodeURIComponent(resource.owner)}.zip`;
zipFilePath = _pathExtra2.default.join(importsPath, zipFileName);
console.log('Downloading: ' + resource.downloadUrl);
if (!(0, _utils.areWeOnline)()) {
const message = `Download ${resource.downloadUrl} error, disconnected from internet`;
console.log(message);
throw message;
}
const results = yield downloadHelpers.download(resource.downloadUrl, zipFilePath);
if (results.status === 200) {
downloadComplete = true;
} else {
const message = `Download ${resource.downloadUrl} error, status: ${results.status}`;
console.log(message);
throw message;
}
} catch (err) {
throw Error((0, _resourcesHelpers.appendError)(errors.UNABLE_TO_DOWNLOAD_RESOURCES, err));
}
try {
if (config.getCancelState && config.getCancelState()) {
console.warn(`downloadAndProcessResource() unzipping of ${resource.downloadUrl} cancelled`);
return; // if user cancelled then skip
}
console.log('Unzipping: ' + resource.downloadUrl);
importPath = yield (0, _resourcesHelpers.unzipResource)(resource, zipFilePath, resourcesPath);
} catch (err) {
throw Error((0, _resourcesHelpers.appendError)(errors.UNABLE_TO_UNZIP_RESOURCES, err));
}
if (config.getCancelState && config.getCancelState()) {
console.warn(`downloadAndProcessResource() processing of ${resource.downloadUrl} cancelled`);
return; // if user cancelled then skip
}
console.log('Processing: ' + resource.downloadUrl);
const importSubdirPath = (0, _resourcesHelpers.getSubdirOfUnzippedResource)(importPath);
const processedFilesPath = yield (0, _resourcesHelpers.processResource)(resource, importSubdirPath, resourcesPath, downloadErrors, config);
if (processedFilesPath) {
const currentResourcePath = (0, _resourcesHelpers.getActualResourcePath)(resource, resourcesPath);
try {
yield moveResourcesHelpers.moveResources(processedFilesPath, currentResourcePath);
} catch (err) {
throw Error((0, _resourcesHelpers.appendError)(errors.UNABLE_TO_MOVE_RESOURCE_INTO_RESOURCES, err));
}
if (!doingPreRelease_) {
removeUnusedResources(resourcesPath, currentResourcePath, resource.languageId, resource.version, isGreekOrHebrew);
}
} else {
throw Error(errors.FAILED_TO_PROCESS_RESOURCE);
}
} catch (err) {
const errorMessage = (0, _resourcesHelpers.getErrorMessage)(err);
console.error('Error getting ' + resource.downloadUrl + ': ' + errorMessage);
addDownloadError(downloadErrors, downloadComplete, errorMessage, resource.downloadUrl);
throw Error((0, _resourcesHelpers.formatError)(resource, errorMessage));
} finally {
if (zipFilePath) {
_rimraf2.default.sync(zipFilePath, _fsExtra2.default);
}
if (importPath) {
_rimraf2.default.sync(importPath, _fsExtra2.default);
}
}
console.log('Processed: ' + resource.downloadUrl);
return resource;
});
return function downloadAndProcessResource(_x, _x2, _x3) {
return _ref.apply(this, arguments);
};
})();
/**
* downloads and processes the resource catching and saving errors
* @param {Object} resource being downloaded
* @param {String} resourcesPath - path to save resources
* @param {Array} errorList - keeps track of errors
* @param {Array} downloadErrors - parsed list of download errors with details such as if the download completed (vs. parsing error), error, and url
* @param {object} config - configuration object
* @return {Promise} promise
*/
const downloadAndProcessResourceWithCatch = exports.downloadAndProcessResourceWithCatch = (() => {
var _ref2 = _asyncToGenerator(function* (resource, resourcesPath, errorList, downloadErrors, config = {}) {
let result = null;
try {
result = yield downloadAndProcessResource(resource, resourcesPath, downloadErrors, config);
console.log('Update Success: ' + resource.downloadUrl);
} catch (e) {
console.log('Update Error:');
console.error(e);
errorList.push(e);
}
return result;
});
return function downloadAndProcessResourceWithCatch(_x4, _x5, _x6, _x7) {
return _ref2.apply(this, arguments);
};
})();
/**
* searches resources to find item with matching languageId and resourceId
* @param {Array} resources
* @param {String} languageId
* @param {String} resourceId
* @return {Object}
*/
const findMatchingResource = exports.findMatchingResource = (resources, languageId, resourceId) => {
const found = resources.find(item => item.languageId === languageId && item.resourceId === resourceId);
return found;
};
/**
* orders resources - ta,tw,tn then other resources
* @param {object} resource
* @return {number}
*/
function helpsOrder(resource) {
if (resource && resource.resourceId) {
if (resource.resourceId) {
switch (resource.resourceId.toLowerCase()) {
case 'ta':
return 0;
case 'tw':
return 1;
case 'tn':
return 2;
default:
return 100;
}
}
}
return 100000;
}
/**
* move helps resources to first in list - ta,tw,tn then other resources
* @param {object} a
* @param {object} b
* @return {number}
*/
function sortHelps(a, b) {
const order = helpsOrder(a) > helpsOrder(b) ? 1 : -1;
return order;
}
/**
* log the online status
*/
function showOnlineStatus() {
if (!global.navigator) {
console.log('showOnlineStatus - navigator is not defined, so we will try anyway since we may be running as a script');
} else {
const online = (0, _utils.areWeOnline)();
console.log(`showOnlineStatus - navigator is defined, and online status is ${online}`);
}
}
/**
* @description Downloads the resources that need to be updated for the given languages using the DCS API
* @param {Array} languageList - Array of languages to download the resources for
* @param {String} resourcesPath - Path to the resources directory where each resource will be placed
* @param {Array.<{
* languageId: String,
* resourceId: String,
* localModifiedTime: String,
* remoteModifiedTime: String,
* downloadUrl: String,
* version: String,
* subject: String,
* catalogEntry: {langResource, bookResource, format}
* }>} resources - resources that will be downloaded if the lanugage IDs match
* @param {Array} downloadErrors - parsed list of download errors with details such as if the download completed (vs. parsing error), error, and url
* @param {Boolean} allAlignedBibles - if true all aligned Bibles from all languages are updated also
* @param {object} config - configuration object
* @return {Promise} Promise that returns a list of all the resources updated, rejects if
* any fail
*/
const downloadResources = exports.downloadResources = (languageList, resourcesPath, resources, downloadErrors, allAlignedBibles = false, config = {}) => {
return new Promise((resolve, reject) => {
if (!allAlignedBibles && (!languageList || !languageList.length)) {
reject(errors.LANGUAGE_LIST_EMPTY);
return;
}
if (!resourcesPath) {
reject(errors.RESOURCES_PATH_NOT_GIVEN);
return;
}
showOnlineStatus();
_fsExtra2.default.ensureDirSync(resourcesPath);
const importsDir = _pathExtra2.default.join(resourcesPath, 'imports');
let downloadableResources = [];
languageList.forEach(languageId => {
let resourcesForLanguage = parseHelpers.getResourcesForLanguage(resources, languageId);
resourcesForLanguage = resourcesForLanguage.sort(sortHelps); // fetch helps first
resourcesForLanguage.forEach(resource => {
const loadAfter = resource.catalogEntry && resource.catalogEntry.resource && resource.catalogEntry.resource.loadAfter;
if (loadAfter) {
loadAfter.forEach(afterResource => {
resourcesForLanguage.push(afterResource);
});
}
});
downloadableResources = downloadableResources.concat(resourcesForLanguage);
});
if (allAlignedBibles) {
console.log('downloadResources() - updating all aligned bibles');
resources.forEach(resource => {
if (resource.subject === 'Aligned_Bible') {
const duplicate = findMatchingResource(downloadableResources, resource.languageId, resource.resourceId);
if (!duplicate) {
downloadableResources.push(resource);
}
}
});
}
if (!downloadableResources || !downloadableResources.length) {
resolve([]);
return;
}
const errorList = [];
downloadableResources = sortDownloableResources(downloadableResources.filter(resource => resource));
const queue = downloadableResources.map(resource => () => downloadAndProcessResourceWithCatch(resource, resourcesPath, errorList, downloadErrors, config));
Throttle.all(queue, { maxInProgress: 2 }).then(result => {
_rimraf2.default.sync(importsDir, _fsExtra2.default);
if (!errorList.length) {
resolve(result);
} else {
const errorMessages = errorList.map(e => e.message || e);
const returnErrorMessage = errorMessages.join('\n');
reject(new Error(returnErrorMessage));
}
}, err => {
_rimraf2.default.sync(importsDir, _fsExtra2.default);
reject(err);
});
});
};
/**
* find order for resourceId
* @param {Array.<string>} resourcePrecedence
* @param {string} resourceId
* @return {*}
*/
function getResourcePrecidence(resourcePrecedence, resourceId) {
let index = resourcePrecedence.indexOf(resourceId);
if (index < 0) {
index = 1000000;
}
return index;
}
/**
* Return book code with highest precedence to sort method
* @param {*} a - First book code of 2
* @param {*} b - second book code
* @return {Number}
*/
function resourceSort(a, b) {
const resourcePrecedence = ['ta', 'ult', 'glt', 'tw', 'twl', 'tn']; // we should download these resources in this order, others will just be alphabetical
const indexA = getResourcePrecidence(resourcePrecedence, a);
const indexB = getResourcePrecidence(resourcePrecedence, b);
let diff = 0;
if (indexA === indexB) {
// same resource types or both not in list
diff = a < b ? -1 : a > b ? 1 : 0;
} else {
diff = indexA - indexB; // this plays off the fact other resources will be high index value
}
return diff;
}
/**
* Sorts the list of downloadable resources. Sorts by language, then owner and moves tA
* to the front of the array in order to be downloaded before tN
* since tN will use tA articles to generate the groupsIndex files.
* @param {array} downloadableResources list of downloadable resources.
* @return {array} sorted list of downloadable resources.
*/
const sortDownloableResources = exports.sortDownloableResources = downloadableResources => {
return downloadableResources.sort((resourceA, resourceB) => {
const langA = resourceA.languageId;
const langB = resourceB.languageId;
if (langB > langA) {
return -1;
} else if (langB === langA) {
const ownerA = resourceA.owner;
const ownerB = resourceB.owner;
if (ownerB > ownerA) {
return -1;
} else if (ownerB === ownerA) {
const idA = resourceA.resourceId.toLowerCase();
const idB = resourceB.resourceId.toLowerCase();
const compareResult = resourceSort(idA, idB);
// console.log(compareResult);
return compareResult;
} else {
// ownerB < ownerA
return 1;
}
} else {
// langB < langA
return 1;
}
});
};