page-loader-ivan
Version:
Web page loader
145 lines (119 loc) • 13.1 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", {
value: true
});
var _debug = require('debug');
var _debug2 = _interopRequireDefault(_debug);
var _listr = require('listr');
var _listr2 = _interopRequireDefault(_listr);
var _fs = require('mz/fs');
var _fs2 = _interopRequireDefault(_fs);
var _url = require('url');
var _url2 = _interopRequireDefault(_url);
var _path = require('path');
var _path2 = _interopRequireDefault(_path);
var _fs3 = require('./fs');
var _loader = require('./loader');
var _names = require('./names');
var _parser = require('./parser');
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
var debug = (0, _debug2.default)('page-loader');
var chackParameters = function chackParameters(dir, filepath, dirFiles) {
return (0, _fs3.checkExistDir)(dir).then(function () {
return (0, _fs3.checkCreation)(filepath);
}).then(function () {
return (0, _fs3.checkCreation)(dirFiles);
});
};
var handlePage = function handlePage(page, html, dir, simpleDirFiles) {
var _url$parse = _url2.default.parse(page),
protocol = _url$parse.protocol,
hostname = _url$parse.hostname;
var urlData = { protocol: protocol, hostname: hostname };
var linksToLoad = (0, _names.exchangeToNativLinks)((0, _parser.localLinks)(html), simpleDirFiles).map(function (exch) {
var linkToLoad = {
originalLink: exch.link,
link: (0, _names.fullLink)(exch.link, urlData),
nativeLink: exch.nativeLink,
path: _path2.default.join(dir, exch.nativeLink)
};
return linkToLoad;
});
var nativeHtml = linksToLoad.reduce(function (str, linkToLoad) {
return str.replace(new RegExp(linkToLoad.originalLink, 'g'), linkToLoad.nativeLink);
}, html);
return { nativeHtml: nativeHtml, linksToLoad: linksToLoad };
};
var createFolder = function createFolder(linksToLoad, dirFiles) {
if (linksToLoad.length === 0) {
return null;
}
return _fs2.default.mkdir(dirFiles);
};
var savePage = function savePage(nativeHtml, filepath, linksToLoad) {
return new _listr2.default([{
title: 'save main html',
task: function task() {
return _fs2.default.writeFile(filepath, nativeHtml).then(function () {
return debug('saved a main html %s', filepath);
});
}
}, {
title: 'load local links',
task: function task() {
return new _listr2.default(linksToLoad.map(function (loadLink) {
var task = {
title: 'load link ' + loadLink.link,
task: function task() {
return (0, _loader.loadFile)(loadLink.link, loadLink.path);
}
};
return task;
}), { concurrent: true });
}
}]);
};
exports.default = function (page, dir) {
debug('start load a page %s', page);
var names = (0, _names.basicNames)(page, dir);
var runSpinner = function runSpinner() {
var tasks = new _listr2.default([{
title: 'check parameters for loading',
task: function task() {
return chackParameters(dir, names.filepath, names.dirFiles);
}
}, {
title: 'read a page ' + page,
task: function task(ctx) {
return (0, _loader.getHtml)(page).then(function (html) {
ctx.html = html;
});
}
}, {
title: 'handle a page ' + page,
task: function task(ctx) {
var data = handlePage(page, ctx.html, dir, names.simpleDirFiles);
ctx.nativeHtml = data.nativeHtml;
ctx.linksToLoad = data.linksToLoad;
}
}, {
title: 'create a folder for local links',
task: function task(ctx) {
return createFolder(ctx.linksToLoad, names.dirFiles);
}
}, {
title: 'save page',
task: function task(ctx) {
return savePage(ctx.nativeHtml, names.filepath, ctx.linksToLoad);
}
}]);
return tasks.run();
};
return runSpinner().then(function () {
return debug('end load a page %s', page);
}).catch(function (err) {
debug('page ' + page + ' not loaded due to reason: ' + err.message);
throw err;
});
};
//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIi4uL3NyYy9pbmRleC5qcyJdLCJuYW1lcyI6WyJkZWJ1ZyIsImNoYWNrUGFyYW1ldGVycyIsImRpciIsImZpbGVwYXRoIiwiZGlyRmlsZXMiLCJ0aGVuIiwiaGFuZGxlUGFnZSIsInBhZ2UiLCJodG1sIiwic2ltcGxlRGlyRmlsZXMiLCJ1cmwiLCJwYXJzZSIsInByb3RvY29sIiwiaG9zdG5hbWUiLCJ1cmxEYXRhIiwibGlua3NUb0xvYWQiLCJtYXAiLCJleGNoIiwibGlua1RvTG9hZCIsIm9yaWdpbmFsTGluayIsImxpbmsiLCJuYXRpdmVMaW5rIiwicGF0aCIsImpvaW4iLCJuYXRpdmVIdG1sIiwicmVkdWNlIiwic3RyIiwicmVwbGFjZSIsIlJlZ0V4cCIsImNyZWF0ZUZvbGRlciIsImxlbmd0aCIsImZzIiwibWtkaXIiLCJzYXZlUGFnZSIsIkxpc3RyIiwidGl0bGUiLCJ0YXNrIiwid3JpdGVGaWxlIiwibG9hZExpbmsiLCJjb25jdXJyZW50IiwibmFtZXMiLCJydW5TcGlubmVyIiwidGFza3MiLCJjdHgiLCJkYXRhIiwicnVuIiwiY2F0Y2giLCJlcnIiLCJtZXNzYWdlIl0sIm1hcHBpbmdzIjoiOzs7Ozs7QUFBQTs7OztBQUNBOzs7O0FBQ0E7Ozs7QUFDQTs7OztBQUNBOzs7O0FBQ0E7O0FBQ0E7O0FBQ0E7O0FBQ0E7Ozs7QUFFQSxJQUFNQSxRQUFRLHFCQUFZLGFBQVosQ0FBZDs7QUFFQSxJQUFNQyxrQkFBa0IsU0FBbEJBLGVBQWtCLENBQUNDLEdBQUQsRUFBTUMsUUFBTixFQUFnQkMsUUFBaEI7QUFBQSxTQUN0Qix3QkFBY0YsR0FBZCxFQUNHRyxJQURILENBQ1E7QUFBQSxXQUFNLHdCQUFjRixRQUFkLENBQU47QUFBQSxHQURSLEVBRUdFLElBRkgsQ0FFUTtBQUFBLFdBQU0sd0JBQWNELFFBQWQsQ0FBTjtBQUFBLEdBRlIsQ0FEc0I7QUFBQSxDQUF4Qjs7QUFLQSxJQUFNRSxhQUFhLFNBQWJBLFVBQWEsQ0FBQ0MsSUFBRCxFQUFPQyxJQUFQLEVBQWFOLEdBQWIsRUFBa0JPLGNBQWxCLEVBQXFDO0FBQUEsbUJBQ3ZCQyxjQUFJQyxLQUFKLENBQVVKLElBQVYsQ0FEdUI7QUFBQSxNQUM5Q0ssUUFEOEMsY0FDOUNBLFFBRDhDO0FBQUEsTUFDcENDLFFBRG9DLGNBQ3BDQSxRQURvQzs7QUFFdEQsTUFBTUMsVUFBVSxFQUFFRixrQkFBRixFQUFZQyxrQkFBWixFQUFoQjtBQUNBLE1BQU1FLGNBQWMsaUNBQXFCLHdCQUFXUCxJQUFYLENBQXJCLEVBQXVDQyxjQUF2QyxFQUNqQk8sR0FEaUIsQ0FDYixVQUFDQyxJQUFELEVBQVU7QUFDYixRQUFNQyxhQUFhO0FBQ2pCQyxvQkFBY0YsS0FBS0csSUFERjtBQUVqQkEsWUFBTSxxQkFBU0gsS0FBS0csSUFBZCxFQUFvQk4sT0FBcEIsQ0FGVztBQUdqQk8sa0JBQVlKLEtBQUtJLFVBSEE7QUFJakJDLFlBQU1BLGVBQUtDLElBQUwsQ0FBVXJCLEdBQVYsRUFBZWUsS0FBS0ksVUFBcEI7QUFKVyxLQUFuQjtBQU1BLFdBQU9ILFVBQVA7QUFDRCxHQVRpQixDQUFwQjtBQVVBLE1BQU1NLGFBQWFULFlBQVlVLE1BQVosQ0FBbUIsVUFBQ0MsR0FBRCxFQUFNUixVQUFOO0FBQUEsV0FDcENRLElBQUlDLE9BQUosQ0FBWSxJQUFJQyxNQUFKLENBQVdWLFdBQVdDLFlBQXRCLEVBQW9DLEdBQXBDLENBQVosRUFBc0RELFdBQVdHLFVBQWpFLENBRG9DO0FBQUEsR0FBbkIsRUFDNkRiLElBRDdELENBQW5CO0FBRUEsU0FBTyxFQUFFZ0Isc0JBQUYsRUFBY1Qsd0JBQWQsRUFBUDtBQUNELENBaEJEOztBQWtCQSxJQUFNYyxlQUFlLFNBQWZBLFlBQWUsQ0FBQ2QsV0FBRCxFQUFjWCxRQUFkLEVBQTJCO0FBQzlDLE1BQUlXLFlBQVllLE1BQVosS0FBdUIsQ0FBM0IsRUFBOEI7QUFDNUIsV0FBTyxJQUFQO0FBQ0Q7QUFDRCxTQUFPQyxhQUFHQyxLQUFILENBQVM1QixRQUFULENBQVA7QUFDRCxDQUxEOztBQU9BLElBQU02QixXQUFXLFNBQVhBLFFBQVcsQ0FBQ1QsVUFBRCxFQUFhckIsUUFBYixFQUF1QlksV0FBdkI7QUFBQSxTQUNmLElBQUltQixlQUFKLENBQVUsQ0FBQztBQUNUQyxXQUFPLGdCQURFO0FBRVRDLFVBQU07QUFBQSxhQUNKTCxhQUFHTSxTQUFILENBQWFsQyxRQUFiLEVBQXVCcUIsVUFBdkIsRUFDR25CLElBREgsQ0FDUTtBQUFBLGVBQU1MLE1BQU0sc0JBQU4sRUFBOEJHLFFBQTlCLENBQU47QUFBQSxPQURSLENBREk7QUFBQTtBQUZHLEdBQUQsRUFNVjtBQUNFZ0MsV0FBTyxrQkFEVDtBQUVFQyxVQUFNO0FBQUEsYUFBTSxJQUFJRixlQUFKLENBQVVuQixZQUFZQyxHQUFaLENBQWdCLFVBQUNzQixRQUFELEVBQWM7QUFDbEQsWUFBTUYsT0FBTztBQUNYRCxnQ0FBb0JHLFNBQVNsQixJQURsQjtBQUVYZ0IsZ0JBQU07QUFBQSxtQkFBTSxzQkFBU0UsU0FBU2xCLElBQWxCLEVBQXdCa0IsU0FBU2hCLElBQWpDLENBQU47QUFBQTtBQUZLLFNBQWI7QUFJQSxlQUFPYyxJQUFQO0FBQ0QsT0FOcUIsQ0FBVixFQU1SLEVBQUVHLFlBQVksSUFBZCxFQU5RLENBQU47QUFBQTtBQUZSLEdBTlUsQ0FBVixDQURlO0FBQUEsQ0FBakI7O2tCQWtCZSxVQUFDaEMsSUFBRCxFQUFPTCxHQUFQLEVBQWU7QUFDNUJGLFFBQU0sc0JBQU4sRUFBOEJPLElBQTlCOztBQUVBLE1BQU1pQyxRQUFRLHVCQUFXakMsSUFBWCxFQUFpQkwsR0FBakIsQ0FBZDs7QUFFQSxNQUFNdUMsYUFBYSxTQUFiQSxVQUFhLEdBQU07QUFDdkIsUUFBTUMsUUFBUSxJQUFJUixlQUFKLENBQVUsQ0FBQztBQUN2QkMsYUFBTyw4QkFEZ0I7QUFFdkJDLFlBQU07QUFBQSxlQUFNbkMsZ0JBQWdCQyxHQUFoQixFQUFxQnNDLE1BQU1yQyxRQUEzQixFQUFxQ3FDLE1BQU1wQyxRQUEzQyxDQUFOO0FBQUE7QUFGaUIsS0FBRCxFQUl4QjtBQUNFK0IsOEJBQXNCNUIsSUFEeEI7QUFFRTZCLFlBQU07QUFBQSxlQUFPLHFCQUFRN0IsSUFBUixFQUFjRixJQUFkLENBQW1CLFVBQUNHLElBQUQsRUFBVTtBQUFFbUMsY0FBSW5DLElBQUosR0FBV0EsSUFBWDtBQUFrQixTQUFqRCxDQUFQO0FBQUE7QUFGUixLQUp3QixFQVF4QjtBQUNFMkIsZ0NBQXdCNUIsSUFEMUI7QUFFRTZCLFlBQU0sY0FBQ08sR0FBRCxFQUFTO0FBQ2IsWUFBTUMsT0FBT3RDLFdBQVdDLElBQVgsRUFBaUJvQyxJQUFJbkMsSUFBckIsRUFBMkJOLEdBQTNCLEVBQWdDc0MsTUFBTS9CLGNBQXRDLENBQWI7QUFDQWtDLFlBQUluQixVQUFKLEdBQWlCb0IsS0FBS3BCLFVBQXRCO0FBQ0FtQixZQUFJNUIsV0FBSixHQUFrQjZCLEtBQUs3QixXQUF2QjtBQUNEO0FBTkgsS0FSd0IsRUFnQnhCO0FBQ0VvQixhQUFPLGlDQURUO0FBRUVDLFlBQU07QUFBQSxlQUFPUCxhQUFhYyxJQUFJNUIsV0FBakIsRUFBOEJ5QixNQUFNcEMsUUFBcEMsQ0FBUDtBQUFBO0FBRlIsS0FoQndCLEVBb0J4QjtBQUNFK0IsYUFBTyxXQURUO0FBRUVDLFlBQU07QUFBQSxlQUFPSCxTQUFTVSxJQUFJbkIsVUFBYixFQUF5QmdCLE1BQU1yQyxRQUEvQixFQUF5Q3dDLElBQUk1QixXQUE3QyxDQUFQO0FBQUE7QUFGUixLQXBCd0IsQ0FBVixDQUFkO0FBd0JBLFdBQU8yQixNQUFNRyxHQUFOLEVBQVA7QUFDRCxHQTFCRDs7QUE0QkEsU0FBT0osYUFDSnBDLElBREksQ0FDQztBQUFBLFdBQU1MLE1BQU0sb0JBQU4sRUFBNEJPLElBQTVCLENBQU47QUFBQSxHQURELEVBRUp1QyxLQUZJLENBRUUsVUFBQ0MsR0FBRCxFQUFTO0FBQ2QvQyxvQkFBY08sSUFBZCxtQ0FBZ0R3QyxJQUFJQyxPQUFwRDtBQUNBLFVBQU1ELEdBQU47QUFDRCxHQUxJLENBQVA7QUFNRCxDIiwiZmlsZSI6ImluZGV4LmpzIiwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IGNyZWF0ZURlYnVnIGZyb20gJ2RlYnVnJztcbmltcG9ydCBMaXN0ciBmcm9tICdsaXN0cic7XG5pbXBvcnQgZnMgZnJvbSAnbXovZnMnO1xuaW1wb3J0IHVybCBmcm9tICd1cmwnO1xuaW1wb3J0IHBhdGggZnJvbSAncGF0aCc7XG5pbXBvcnQgeyBjaGVja0V4aXN0RGlyLCBjaGVja0NyZWF0aW9uIH0gZnJvbSAnLi9mcyc7XG5pbXBvcnQgeyBnZXRIdG1sLCBsb2FkRmlsZSB9IGZyb20gJy4vbG9hZGVyJztcbmltcG9ydCB7IGJhc2ljTmFtZXMsIGV4Y2hhbmdlVG9OYXRpdkxpbmtzLCBmdWxsTGluayB9IGZyb20gJy4vbmFtZXMnO1xuaW1wb3J0IHsgbG9jYWxMaW5rcyB9IGZyb20gJy4vcGFyc2VyJztcblxuY29uc3QgZGVidWcgPSBjcmVhdGVEZWJ1ZygncGFnZS1sb2FkZXInKTtcblxuY29uc3QgY2hhY2tQYXJhbWV0ZXJzID0gKGRpciwgZmlsZXBhdGgsIGRpckZpbGVzKSA9PlxuICBjaGVja0V4aXN0RGlyKGRpcilcbiAgICAudGhlbigoKSA9PiBjaGVja0NyZWF0aW9uKGZpbGVwYXRoKSlcbiAgICAudGhlbigoKSA9PiBjaGVja0NyZWF0aW9uKGRpckZpbGVzKSk7XG5cbmNvbnN0IGhhbmRsZVBhZ2UgPSAocGFnZSwgaHRtbCwgZGlyLCBzaW1wbGVEaXJGaWxlcykgPT4ge1xuICBjb25zdCB7IHByb3RvY29sLCBob3N0bmFtZSB9ID0gdXJsLnBhcnNlKHBhZ2UpO1xuICBjb25zdCB1cmxEYXRhID0geyBwcm90b2NvbCwgaG9zdG5hbWUgfTtcbiAgY29uc3QgbGlua3NUb0xvYWQgPSBleGNoYW5nZVRvTmF0aXZMaW5rcyhsb2NhbExpbmtzKGh0bWwpLCBzaW1wbGVEaXJGaWxlcylcbiAgICAubWFwKChleGNoKSA9PiB7XG4gICAgICBjb25zdCBsaW5rVG9Mb2FkID0ge1xuICAgICAgICBvcmlnaW5hbExpbms6IGV4Y2gubGluayxcbiAgICAgICAgbGluazogZnVsbExpbmsoZXhjaC5saW5rLCB1cmxEYXRhKSxcbiAgICAgICAgbmF0aXZlTGluazogZXhjaC5uYXRpdmVMaW5rLFxuICAgICAgICBwYXRoOiBwYXRoLmpvaW4oZGlyLCBleGNoLm5hdGl2ZUxpbmspLFxuICAgICAgfTtcbiAgICAgIHJldHVybiBsaW5rVG9Mb2FkO1xuICAgIH0pO1xuICBjb25zdCBuYXRpdmVIdG1sID0gbGlua3NUb0xvYWQucmVkdWNlKChzdHIsIGxpbmtUb0xvYWQpID0+XG4gICAgc3RyLnJlcGxhY2UobmV3IFJlZ0V4cChsaW5rVG9Mb2FkLm9yaWdpbmFsTGluaywgJ2cnKSwgbGlua1RvTG9hZC5uYXRpdmVMaW5rKSwgaHRtbCk7XG4gIHJldHVybiB7IG5hdGl2ZUh0bWwsIGxpbmtzVG9Mb2FkIH07XG59O1xuXG5jb25zdCBjcmVhdGVGb2xkZXIgPSAobGlua3NUb0xvYWQsIGRpckZpbGVzKSA9PiB7XG4gIGlmIChsaW5rc1RvTG9hZC5sZW5ndGggPT09IDApIHtcbiAgICByZXR1cm4gbnVsbDtcbiAgfVxuICByZXR1cm4gZnMubWtkaXIoZGlyRmlsZXMpO1xufTtcblxuY29uc3Qgc2F2ZVBhZ2UgPSAobmF0aXZlSHRtbCwgZmlsZXBhdGgsIGxpbmtzVG9Mb2FkKSA9PlxuICBuZXcgTGlzdHIoW3tcbiAgICB0aXRsZTogJ3NhdmUgbWFpbiBodG1sJyxcbiAgICB0YXNrOiAoKSA9PlxuICAgICAgZnMud3JpdGVGaWxlKGZpbGVwYXRoLCBuYXRpdmVIdG1sKVxuICAgICAgICAudGhlbigoKSA9PiBkZWJ1Zygnc2F2ZWQgYSBtYWluIGh0bWwgJXMnLCBmaWxlcGF0aCkpLFxuICB9LFxuICB7XG4gICAgdGl0bGU6ICdsb2FkIGxvY2FsIGxpbmtzJyxcbiAgICB0YXNrOiAoKSA9PiBuZXcgTGlzdHIobGlua3NUb0xvYWQubWFwKChsb2FkTGluaykgPT4ge1xuICAgICAgY29uc3QgdGFzayA9IHtcbiAgICAgICAgdGl0bGU6IGBsb2FkIGxpbmsgJHtsb2FkTGluay5saW5rfWAsXG4gICAgICAgIHRhc2s6ICgpID0+IGxvYWRGaWxlKGxvYWRMaW5rLmxpbmssIGxvYWRMaW5rLnBhdGgpLFxuICAgICAgfTtcbiAgICAgIHJldHVybiB0YXNrO1xuICAgIH0pLCB7IGNvbmN1cnJlbnQ6IHRydWUgfSksXG4gIH1dKTtcblxuZXhwb3J0IGRlZmF1bHQgKHBhZ2UsIGRpcikgPT4ge1xuICBkZWJ1Zygnc3RhcnQgbG9hZCBhIHBhZ2UgJXMnLCBwYWdlKTtcblxuICBjb25zdCBuYW1lcyA9IGJhc2ljTmFtZXMocGFnZSwgZGlyKTtcblxuICBjb25zdCBydW5TcGlubmVyID0gKCkgPT4ge1xuICAgIGNvbnN0IHRhc2tzID0gbmV3IExpc3RyKFt7XG4gICAgICB0aXRsZTogJ2NoZWNrIHBhcmFtZXRlcnMgZm9yIGxvYWRpbmcnLFxuICAgICAgdGFzazogKCkgPT4gY2hhY2tQYXJhbWV0ZXJzKGRpciwgbmFtZXMuZmlsZXBhdGgsIG5hbWVzLmRpckZpbGVzKSxcbiAgICB9LFxuICAgIHtcbiAgICAgIHRpdGxlOiBgcmVhZCBhIHBhZ2UgJHtwYWdlfWAsXG4gICAgICB0YXNrOiBjdHggPT4gZ2V0SHRtbChwYWdlKS50aGVuKChodG1sKSA9PiB7IGN0eC5odG1sID0gaHRtbDsgfSksXG4gICAgfSxcbiAgICB7XG4gICAgICB0aXRsZTogYGhhbmRsZSBhIHBhZ2UgJHtwYWdlfWAsXG4gICAgICB0YXNrOiAoY3R4KSA9PiB7XG4gICAgICAgIGNvbnN0IGRhdGEgPSBoYW5kbGVQYWdlKHBhZ2UsIGN0eC5odG1sLCBkaXIsIG5hbWVzLnNpbXBsZURpckZpbGVzKTtcbiAgICAgICAgY3R4Lm5hdGl2ZUh0bWwgPSBkYXRhLm5hdGl2ZUh0bWw7XG4gICAgICAgIGN0eC5saW5rc1RvTG9hZCA9IGRhdGEubGlua3NUb0xvYWQ7XG4gICAgICB9LFxuICAgIH0sXG4gICAge1xuICAgICAgdGl0bGU6ICdjcmVhdGUgYSBmb2xkZXIgZm9yIGxvY2FsIGxpbmtzJyxcbiAgICAgIHRhc2s6IGN0eCA9PiBjcmVhdGVGb2xkZXIoY3R4LmxpbmtzVG9Mb2FkLCBuYW1lcy5kaXJGaWxlcyksXG4gICAgfSxcbiAgICB7XG4gICAgICB0aXRsZTogJ3NhdmUgcGFnZScsXG4gICAgICB0YXNrOiBjdHggPT4gc2F2ZVBhZ2UoY3R4Lm5hdGl2ZUh0bWwsIG5hbWVzLmZpbGVwYXRoLCBjdHgubGlua3NUb0xvYWQpLFxuICAgIH1dKTtcbiAgICByZXR1cm4gdGFza3MucnVuKCk7XG4gIH07XG5cbiAgcmV0dXJuIHJ1blNwaW5uZXIoKVxuICAgIC50aGVuKCgpID0+IGRlYnVnKCdlbmQgbG9hZCBhIHBhZ2UgJXMnLCBwYWdlKSlcbiAgICAuY2F0Y2goKGVycikgPT4ge1xuICAgICAgZGVidWcoYHBhZ2UgJHtwYWdlfSBub3QgbG9hZGVkIGR1ZSB0byByZWFzb246ICR7ZXJyLm1lc3NhZ2V9YCk7XG4gICAgICB0aHJvdyBlcnI7XG4gICAgfSk7XG59O1xuIl19