tc-source-content-updater/lib/helpers/resourcesHelpers.js

Module that updates source content for the desktop application translationCore.

'use strict';

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.isDirectory = exports.processResource = exports.unzipResource = exports.cleanReaddirSync = exports.TRANSLATION_HELPS_INDEX = undefined;

/**
 * @description Processes a resource in the imports directory as needed
 * @param {Object} resource Resource object
 * @param {String} sourcePath Path to the source dictory of the resource
 * @param {String} resourcesPath Path to user resources folder
 * @return {String} Path to the directory of the processed files
 * @param {Array} downloadErrors - parsed list of download errors with details such as if the download completed (vs. parsing error), error, and url
 * @param {object} config - configuration object
 */
let processResource = exports.processResource = (() => {
  var _ref2 = _asyncToGenerator(function* (resource, sourcePath, resourcesPath, downloadErrors, config = {}) {
    try {
      if (!resource || !(0, _util.isObject)(resource) || !resource.languageId || !resource.resourceId) {
        throw Error(formatError(resource, errors.RESOURCE_NOT_GIVEN));
      }
      if (!sourcePath) {
        throw Error(formatError(resource, errors.SOURCE_PATH_NOT_GIVEN));
      }
      if (!_fsExtra2.default.pathExistsSync(sourcePath)) {
        throw Error(formatError(resource, errors.SOURCE_PATH_NOT_EXIST));
      }

      const processedFilesPath = sourcePath + '_processed';
      _fsExtra2.default.ensureDirSync(processedFilesPath);

      const subject = resource.subject && resource.subject.replaceAll(' ', '_'); // replace spaces with underscores;
      switch (subject) {
        case 'Translation_Words':
          twArticleHelpers.processTranslationWords(resource, sourcePath, processedFilesPath);
          break;
        case 'TSV_Translation_Words_Links':
          yield twArticleHelpers.processTranslationWordsTSV(resource, sourcePath, processedFilesPath, resourcesPath, downloadErrors, config);
          break;
        case 'TSV_Translation_Notes':
          yield tnArticleHelpers.processTranslationNotes(resource, sourcePath, processedFilesPath, resourcesPath, downloadErrors, config);
          break;
        case 'Translation_Academy':
          taArticleHelpers.processTranslationAcademy(resource, sourcePath, processedFilesPath);
          break;
        case 'Bible':
        case 'Aligned_Bible':
        case 'Greek_New_Testament':
        case 'Hebrew_Old_Testament':
          packageParseHelpers.parseBiblePackage(resource, sourcePath, processedFilesPath, config);
          break;
        default:
          _fsExtra2.default.copySync(sourcePath, processedFilesPath);
      }

      let manifest = getResourceManifest(processedFilesPath);
      if (!manifest) {
        // if manifest not found, create
        manifest = getResourceManifest(sourcePath);
      }

      const stage = resource.catalogEntry && resource.catalogEntry.resource && resource.catalogEntry.resource.stage;

      if (manifest) {
        if (resource.version) {
          manifest.version = resource.version;
        }
        manifest.modifiedTime = manifest.catalog_modified_time = resource.remoteModifiedTime || resource.modified;
        if (stage) {
          manifest.stage = stage;
        }
        _fsExtra2.default.outputJsonSync(_pathExtra2.default.join(processedFilesPath, 'manifest.json'), manifest, { spaces: 2 });
      }

      if (stage) {
        (0, _resourcesDownloadHelpers.setResourcesPreReleaseStateFromStage)(resourcesPath, stage);
      }

      return processedFilesPath;
    } catch (error) {
      console.error(error);
      throw Error(appendError(errors.FAILED_TO_PROCESS_RESOURCE, error));
    }
  });

  return function processResource(_x4, _x5, _x6, _x7) {
    return _ref2.apply(this, arguments);
  };
})();

/**
 * encode owner to string without characters unsupported by file system
 * @param {string} owner
 * @return {string}
 */


exports.getVersionFromManifest = getVersionFromManifest;
exports.getResourceManifest = getResourceManifest;
exports.getResourceManifestFromJson = getResourceManifestFromJson;
exports.getResourceManifestFromYaml = getResourceManifestFromYaml;
exports.getVersionsInPath = getVersionsInPath;
exports.getVersionAndOwnerFromPath = getVersionAndOwnerFromPath;
exports.splitVersionAndOwner = splitVersionAndOwner;
exports.getLatestVersionsAndOwners = getLatestVersionsAndOwners;
exports.sortVersionsByNumber = sortVersionsByNumber;
exports.listVersions = listVersions;
exports.compareVersions = compareVersions;
exports.sortVersions = sortVersions;
exports.getLatestVersionInPath = getLatestVersionInPath;
exports.getLatestVersionFromList = getLatestVersionFromList;
exports.getSubdirOfUnzippedResource = getSubdirOfUnzippedResource;
exports.encodeOwnerStr = encodeOwnerStr;
exports.addOwnerToKey = addOwnerToKey;
exports.getActualResourcePath = getActualResourcePath;
exports.makeTwGroupDataResource = makeTwGroupDataResource;
exports.removeAllButLatestVersion = removeAllButLatestVersion;
exports.formatError = formatError;
exports.getErrorMessage = getErrorMessage;
exports.appendError = appendError;

var _fsExtra = require('fs-extra');

var _fsExtra2 = _interopRequireDefault(_fsExtra);

var _pathExtra = require('path-extra');

var _pathExtra2 = _interopRequireDefault(_pathExtra);

var _yamljs = require('yamljs');

var _yamljs2 = _interopRequireDefault(_yamljs);

var _util = require('util');

var _semver = require('semver');

var _semver2 = _interopRequireDefault(_semver);

var _zipFileHelpers = require('./zipFileHelpers');

var zipFileHelpers = _interopRequireWildcard(_zipFileHelpers);

var _twArticleHelpers = require('./translationHelps/twArticleHelpers');

var twArticleHelpers = _interopRequireWildcard(_twArticleHelpers);

var _tnArticleHelpers = require('./translationHelps/tnArticleHelpers');

var tnArticleHelpers = _interopRequireWildcard(_tnArticleHelpers);

var _taArticleHelpers = require('./translationHelps/taArticleHelpers');

var taArticleHelpers = _interopRequireWildcard(_taArticleHelpers);

var _twGroupDataHelpers = require('./translationHelps/twGroupDataHelpers');

var twGroupDataHelpers = _interopRequireWildcard(_twGroupDataHelpers);

var _packageParseHelpers = require('./packageParseHelpers');

var packageParseHelpers = _interopRequireWildcard(_packageParseHelpers);

var _errors = require('../resources/errors');

var errors = _interopRequireWildcard(_errors);

var _bible = require('../resources/bible');

var Bible = _interopRequireWildcard(_bible);

var _apiHelpers = require('./apiHelpers');

var _resourcesDownloadHelpers = require('./resourcesDownloadHelpers');

function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } else { var newObj = {}; if (obj != null) { for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) newObj[key] = obj[key]; } } newObj.default = obj; return newObj; } }

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; } /* eslint-disable no-console,max-len,camelcase */

// helpers

// constants


const TRANSLATION_HELPS_INDEX = exports.TRANSLATION_HELPS_INDEX = {
  ta: 'translationAcademy',
  tn: 'translationNotes',
  tw: 'translationWords',
  twl: 'translationWordsLinks',
  tq: 'translationQuestions'
};

/**
 * @description - Gets the version from the manifest
 * @param {String} resourcePath - folder for manifest.json or yaml
 * @return {String} version
 */
function getVersionFromManifest(resourcePath) {
  const manifest = getResourceManifest(resourcePath);
  if (!manifest || !manifest.dublin_core || !manifest.dublin_core.version) {
    return null;
  }
  return manifest.dublin_core.version;
}

/**
 * @description Helper function to get manifest file from the resources folder. First
 * it will try manifest.json, then manifest.yaml.
 * @param {String} resourcePath - path to a resource folder which contains the manifest file in its root.
 * @return {Object} manifest
 */
function getResourceManifest(resourcePath) {
  const manifest = getResourceManifestFromJson(resourcePath);
  if (!manifest) {
    return getResourceManifestFromYaml(resourcePath);
  }
  return manifest;
}

/**
 * @description - Turns a manifest.json file into an object and returns it, null if doesn't exist
 * @param {String} resourcePath - folder for manifest.json
 * @return {Object} manifest
 */
function getResourceManifestFromJson(resourcePath) {
  const fileName = 'manifest.json';
  const manifestPath = _pathExtra2.default.join(resourcePath, fileName);
  let manifest = null;
  if (_fsExtra2.default.existsSync(manifestPath)) {
    manifest = _fsExtra2.default.readJsonSync(manifestPath);
  }
  return manifest;
}

/**
 * @description - Turns a manifest.yaml file into an object and returns it, null if doesn't exist
 * @param {String} resourcePath - folder for manifest.yaml
 * @return {Object} manifest
 */
function getResourceManifestFromYaml(resourcePath) {
  const fileName = 'manifest.yaml';
  const manifestPath = _pathExtra2.default.join(resourcePath, fileName);
  let manifest = null;
  if (_fsExtra2.default.existsSync(manifestPath)) {
    let yamlManifest = _fsExtra2.default.readFileSync(manifestPath, 'utf8').replace(/^\uFEFF/, '');
    try {
      manifest = _yamljs2.default.parse(yamlManifest);
    } catch (e) {
      console.error(`getResourceManifestFromYaml - manifest.yaml invalid ${manifestPath}`, e);
      yamlManifest = yamlManifest.replace('---', '').trimStart();
      try {
        manifest = YAML.parse(yamlManifest);
        // copy some data for more convenient access
        console.log(`getResourceManifestFromYaml - manifest.yaml cleaned yaml worked`);
      } catch (e) {
        console.error(`getResourceManifestFromYaml - manifest.yaml invalid even without --- ${manifestPath}`, e);
        manifest = null;
        throw e;
      }
    }
  }
  return manifest;
}

/**
 * Returns an array of versions found in the path that start with [vV]\d
 * @param {String} resourcePath - base path to search for versions
 * @param {string} ownerStr - optional owner to filter by
 * @param {boolean} fallbackToDefaultOwner
 * @return {Array} - array of versions, e.g. ['v1', 'v10', 'v1.1']
 */
function getVersionsInPath(resourcePath, ownerStr = _apiHelpers.DEFAULT_OWNER, fallbackToDefaultOwner = false) {
  if (!resourcePath || !_fsExtra2.default.pathExistsSync(resourcePath)) {
    return null;
  }

  ownerStr = encodeOwnerStr(ownerStr);

  if (ownerStr[0] !== _apiHelpers.OWNER_SEPARATOR) {
    // prefix the separator character if missing
    ownerStr = _apiHelpers.OWNER_SEPARATOR + ownerStr;
  }

  const isVersionDirectory = name => {
    const fullPath = _pathExtra2.default.join(resourcePath, name);
    let isMatch = _fsExtra2.default.lstatSync(fullPath).isDirectory() && name.match(/^v\d/i);
    if (isMatch && ownerStr) {
      // if we need to filter by owner
      isMatch = name.endsWith(ownerStr);
    }
    return isMatch;
  };
  let versions = sortVersions(_fsExtra2.default.readdirSync(resourcePath).filter(isVersionDirectory));
  if ((!versions || !versions.length) && fallbackToDefaultOwner) {
    versions = getVersionsInPath(resourcePath, _apiHelpers.DEFAULT_OWNER);
  }
  return versions;
}

/**
 * reads files names from folder and remove system files
 * @param {string} path
 * @return {*[]}
 */
const cleanReaddirSync = exports.cleanReaddirSync = path => {
  let cleanDirectories = [];

  if (_fsExtra2.default.existsSync(path)) {
    cleanDirectories = _fsExtra2.default.readdirSync(path).filter(file => file !== '.DS_Store');
  }

  return cleanDirectories;
};

/**
 * takes path that has version and owner such as `~/resources/bible/v1.1_unfoldingWord` and extracts the version and owner.
 * @param {string} versionPath
 * @return {{owner: string, version: string}}
 */
function getVersionAndOwnerFromPath(versionPath) {
  if (versionPath) {
    const versionAndOwner = _pathExtra2.default.base(versionPath, true);
    return splitVersionAndOwner(versionAndOwner);
  }
  return {};
}

/**
 * takes string that has version and owner such as `v1.1_unfoldingWord` and splits into version and owner.
 * @param {string} versionAndOwner
 * @return {{owner: string, version: string}}
 */
function splitVersionAndOwner(versionAndOwner) {
  let version = versionAndOwner;
  let owner = '';
  const pos = versionAndOwner && versionAndOwner.indexOf(_apiHelpers.OWNER_SEPARATOR);
  if (pos >= 0) {
    owner = decodeURIComponent(versionAndOwner.substr(pos + 1));
    version = versionAndOwner.substr(0, pos);
  }
  return { version, owner };
}

/**
 * Returns the versioned folder within the directory with the highest value.
 * e.g. `v10` is greater than `v9`
 * @param {string} dir - the directory to read
 * @return {string} the full path to the latest version directory.
 */
function getLatestVersionsAndOwners(dir) {
  const versionAndOwners = listVersions(dir, true);
  const orgs = {};

  for (const versionAndOwner of versionAndOwners) {
    const { owner } = splitVersionAndOwner(versionAndOwner);
    if (!orgs[owner]) {
      orgs[owner] = [];
    }
    orgs[owner].push(versionAndOwner);
  }

  const orgsKeys = Object.keys(orgs);
  for (const org of orgsKeys) {
    const versions = orgs[org];
    const latest = _pathExtra2.default.join(dir, versions[0]);
    orgs[org] = latest;
  }

  if (orgsKeys.length > 0) {
    return orgs;
  } else {
    return null;
  }
}

/**
 * sorts array by version number
 * @param {Array} versionedDirs
 * @param {boolean} inverted - if true then do reverse sort
 * @return {*}
 */
function sortVersionsByNumber(versionedDirs, inverted = false) {
  if (inverted) {
    return versionedDirs.sort((a, b) => -compareVersions(a, b) // do inverted sort
    );
  }
  return versionedDirs.sort((a, b) => compareVersions(a, b) // do regular sort
  );
}

/**
 * Returns an array of paths found in the directory filtered and sorted by version
 * @param {string} dir
 * @param {boolean} inverted - if true then do reverse sort
 * @return {string[]}
 */
function listVersions(dir, inverted = false) {
  if (_fsExtra2.default.pathExistsSync(dir)) {
    const versionedDirs = _fsExtra2.default.readdirSync(dir).filter(file => _fsExtra2.default.lstatSync(_pathExtra2.default.join(dir, file)).isDirectory() && file.match(/^v\d/i));
    return sortVersionsByNumber(versionedDirs, inverted);
  }
  return [];
}

/**
 * compares version numbers, if a > b returns 1; if a < b return -1; else are equal and return 0
 * @param {String} a
 * @param {String} b
 * @return {number}
 */
function compareVersions(a, b) {
  const cleanA = _semver2.default.coerce(a);
  const cleanB = _semver2.default.coerce(b);

  if (_semver2.default.gt(cleanA, cleanB)) {
    return 1;
  } else if (_semver2.default.lt(cleanA, cleanB)) {
    return -1;
  } else {
    return 0;
  }
}

/**
 * Returns a sorted an array of versions so that numeric parts are properly ordered (e.g. v10a < v100)
 * @param {Array} versions - array of versions unsorted: ['v05.5.2', 'v5.5.1', 'V6.21.0', 'v4.22.0', 'v6.1.0', 'v6.1a.0', 'v5.1.0', 'V4.5.0']
 * @return {Array} - array of versions sorted:  ["V4.5.0", "v4.22.0", "v5.1.0", "v5.5.1", "v05.5.2", "v6.1.0", "v6.1a.0", "V6.21.0"]
 */
function sortVersions(versions) {
  // Don't sort if null, empty or not an array
  if (!versions || !Array.isArray(versions)) {
    return versions;
  }
  // Only sort if all items are strings
  for (let i = 0; i < versions.length; ++i) {
    if (typeof versions[i] !== 'string') {
      return versions;
    }
  }
  versions = sortVersionsByNumber(versions); // sort by standard version order
  return versions;
}

/**
 * Return the full path to the highest version folder in resource path
 * @param {String} resourcePath - base path to search for versions
 * @param {string} ownerStr - optional owner to filter by
 * @param {boolean} fallbackToDefaultOwner - if version not found for owner, then fall back to using default owner
 * @return {String} - path to highest version
 */
function getLatestVersionInPath(resourcePath, ownerStr = _apiHelpers.DEFAULT_OWNER, fallbackToDefaultOwner = false) {
  const versions = getVersionsInPath(resourcePath, ownerStr, fallbackToDefaultOwner);
  if (versions && versions.length) {
    // versions = sortVersions(versions);
    return _pathExtra2.default.join(resourcePath, versions[versions.length - 1]);
  }
  return null; // return illegal path
}

/**
 * returns the highest version for array that matches owner
 * @param {Array} versions - base path to search for versions
 * @param {string} ownerStr - optional owner to filter by, default to DOOR43_CATALOG
 * @return {String} - highest version
 */
function getLatestVersionFromList(versions, ownerStr = _apiHelpers.DEFAULT_OWNER) {
  if (Array.isArray(versions)) {
    if (versions.length) {
      ownerStr = encodeOwnerStr(ownerStr);

      if (ownerStr[0] !== _apiHelpers.OWNER_SEPARATOR) {
        // prefix the separator character if missing
        ownerStr = _apiHelpers.OWNER_SEPARATOR + ownerStr;
      }

      versions = versions.filter(version => version.endsWith(ownerStr));
      versions = sortVersions(versions); // sort by standard version order
      return versions[versions.length - 1];
    }
  }
  return null;
}

/**
 * @description Unzips a resource's zip file to an imports directory for processing
 * @param {Object} resource Resource object containing resourceId and languageId
 * @param {String} zipFilePath Path to the zip file
 * @param {string} resourcesPath Path to the resources directory
 * @return {String} Path to the resource's import directory
 */
const unzipResource = exports.unzipResource = (() => {
  var _ref = _asyncToGenerator(function* (resource, zipFilePath, resourcesPath) {
    const importsPath = _pathExtra2.default.join(resourcesPath, 'imports');
    _fsExtra2.default.ensureDirSync(importsPath);
    const importPath = zipFilePath.split('.').slice(0, -1).join('.');
    yield zipFileHelpers.extractZipFile(zipFilePath, importPath);
    return importPath;
  });

  return function unzipResource(_x, _x2, _x3) {
    return _ref.apply(this, arguments);
  };
})();

/**
 * Gets the single subdirectory of an extracted zip file path
 * @param {String} extractedFilesPath Extracted files path
 * @return {String} The subdir in the extracted path
 */
function getSubdirOfUnzippedResource(extractedFilesPath) {
  const subdirs = _fsExtra2.default.readdirSync(extractedFilesPath);
  if (subdirs.length === 1 && _fsExtra2.default.lstatSync(_pathExtra2.default.join(extractedFilesPath, subdirs[0])).isDirectory()) {
    return _pathExtra2.default.join(extractedFilesPath, subdirs[0]);
  }
  return extractedFilesPath;
}function encodeOwnerStr(owner) {
  const ownerStr = encodeURIComponent(owner || '');
  return ownerStr;
}

/**
 * combine owner with key
 * @param {string} key
 * @param {string} owner
 * @return {string|*}
 */
function addOwnerToKey(key, owner) {
  const ownerStr = encodeOwnerStr(owner);
  const versionDir = ownerStr ? `${key}${_apiHelpers.OWNER_SEPARATOR}${ownerStr}` : key;
  return versionDir;
}

/**
 * @description Gets the actual path to a resource based on the resource object
 * @param {Object} resource The resource object
 * @param {String} resourcesPath The path to the resources directory
 * @return {String} The resource path
 */
function getActualResourcePath(resource, resourcesPath) {
  const languageId = resource.languageId;
  let resourceName = resource.resourceId;
  let type = 'bibles';
  if (TRANSLATION_HELPS_INDEX[resourceName]) {
    resourceName = TRANSLATION_HELPS_INDEX[resourceName];
    type = 'translationHelps';
  }
  const version = 'v' + resource.version;
  const versionDir = addOwnerToKey(version, resource.owner);
  const actualResourcePath = _pathExtra2.default.join(resourcesPath, languageId, type, resourceName, versionDir);
  _fsExtra2.default.ensureDirSync(actualResourcePath);
  return actualResourcePath;
}

/**
 * @description Downloads the resources that need to be updated for a given language using the DCS API
 * @param {Object.<{
 *             languageId: String,
 *             resourceId: String,
 *             localModifiedTime: String,
 *             remoteModifiedTime: String,
 *             downloadUrl: String,
 *             version: String,
 *             subject: String,
 *             catalogEntry: {langResource, bookResource, format}
 *           }>} resource - resource to download
 * @param {String} sourcePath Path to the Bible directory
 * @return {String} Path to the processed tw Group Data files
 */
function makeTwGroupDataResource(resource, sourcePath) {
  if (!resource) {
    throw Error(formatError(resource, errors.RESOURCE_NOT_GIVEN));
  }
  if (!_fsExtra2.default.pathExistsSync(sourcePath)) {
    throw Error(formatError(resource, errors.SOURCE_PATH_NOT_EXIST));
  }
  if (resource.languageId === Bible.NT_ORIG_LANG && resource.resourceId === Bible.NT_ORIG_LANG_BIBLE || resource.languageId === Bible.OT_ORIG_LANG && resource.resourceId === Bible.OT_ORIG_LANG_BIBLE) {
    const twGroupDataPath = _pathExtra2.default.join(sourcePath + '_tw_group_data_' + resource.languageId + '_v' + resource.version);
    const result = twGroupDataHelpers.generateTwGroupDataFromAlignedBible(resource, sourcePath, twGroupDataPath);
    if (result) {
      return twGroupDataPath;
    }
  }
}

/**
 * Removes all version directories except the latest
 * @param {String} resourcePath Path to the resource directory that has subdirs of versions
 * @param {array} versionsToNotDelete List of versions not to be deleted.
 * @param {string} ownerStr - optional owner to filter by
 * @return {Boolean} True if versions were deleted, false if nothing was touched
 */
function removeAllButLatestVersion(resourcePath, versionsToNotDelete = [], ownerStr = '') {
  // Remove the previous version(s)
  const versionDirs = getVersionsInPath(resourcePath, ownerStr);
  if (versionDirs && versionDirs.length > 1) {
    const lastVersion = versionDirs[versionDirs.length - 1];
    versionDirs.forEach(versionDir => {
      if (versionDir !== lastVersion && !versionsToNotDelete.includes(versionDir)) {
        const versionPath = _pathExtra2.default.join(resourcePath, versionDir);
        console.log(`removeAllButLatestVersion() - removing old resource: ${versionPath}`);
        _fsExtra2.default.removeSync(versionPath);
      }
    });
    return true;
  }
  return false;
}

/**
 * @description Formats an error for all resources to have a standard format
 * @param {Object} resource Resource object
 * @param {String} errMessage Error message
 * @return {String} The formatted error message
 */
function formatError(resource, errMessage) {
  if (!resource || !(0, _util.isObject)(resource) || !resource.languageId || !resource.resourceId) {
    resource = {
      languageId: 'unknown',
      resourceId: 'unknown'
    };
  }
  return resource.owner + '/' + resource.languageId + '_' + resource.resourceId + ': ' + errMessage;
}

/**
 *  converts error to string
 * @param {Error|String} error - error to append
 * @return {string} concatenated message
 */
function getErrorMessage(error) {
  return error && error.message || error || 'UNDEFINED';
}

/**
 * appends error message to string
 * @param {string} str - string to use as prefix
 * @param {Error|String} err - error to append
 * @return {string} concatenated message
 */
function appendError(str, err) {
  return str + ': ' + getErrorMessage(err);
}

/**
 * Determines if the rootpath plus a filename is a directory.
 * @param {string} rootPath
 * @param {string} filename
 * @return {bool} - Whether the path is a directory or not.
 */
const isDirectory = exports.isDirectory = (rootPath, filename) => {
  const fullPath = _pathExtra2.default.join(rootPath, filename);
  return _fsExtra2.default.lstatSync(fullPath).isDirectory();
};