datapackage
Version:
Utilities to work with Data Packages as defined on specs.frictionlessdata.io
868 lines (707 loc) • 26.7 kB
JavaScript
'use strict';
var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }();
var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();
// Internal
var extractZip = function () {
var _ref6 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee3(descriptor) {
var zip, tempdir, _iteratorNormalCompletion7, _didIteratorError7, _iteratorError7, _iterator7, _step7, _step7$value, name, item, path, contents;
return regeneratorRuntime.wrap(function _callee3$(_context3) {
while (1) {
switch (_context3.prev = _context3.next) {
case 0:
if (!config.IS_BROWSER) {
_context3.next = 2;
break;
}
throw new DataPackageError('Zip is not supported in browser');
case 2:
// Load zip
zip = JSZip();
_context3.next = 5;
return promisify(require('tmp').dir)();
case 5:
tempdir = _context3.sent;
_context3.next = 8;
return zip.loadAsync(promisify(fs.readFile)(descriptor));
case 8:
if (zip.files['datapackage.json']) {
_context3.next = 10;
break;
}
throw new DataPackageError('Invalid zip with data package');
case 10:
// Save zip to tempdir
_iteratorNormalCompletion7 = true;
_didIteratorError7 = false;
_iteratorError7 = undefined;
_context3.prev = 13;
_iterator7 = _objectEntries(zip.files)[Symbol.iterator]();
case 15:
if (_iteratorNormalCompletion7 = (_step7 = _iterator7.next()).done) {
_context3.next = 32;
break;
}
_step7$value = _slicedToArray(_step7.value, 2), name = _step7$value[0], item = _step7$value[1];
// Get path/descriptor
path = tempdir + '/' + name;
if (path.endsWith('datapackage.json')) {
descriptor = path;
}
// Directory
if (!item.dir) {
_context3.next = 24;
break;
}
_context3.next = 22;
return promisify(fs.mkdir)(path);
case 22:
_context3.next = 29;
break;
case 24:
_context3.next = 26;
return item.async('nodebuffer');
case 26:
contents = _context3.sent;
_context3.next = 29;
return promisify(fs.writeFile)(path, contents);
case 29:
_iteratorNormalCompletion7 = true;
_context3.next = 15;
break;
case 32:
_context3.next = 38;
break;
case 34:
_context3.prev = 34;
_context3.t0 = _context3['catch'](13);
_didIteratorError7 = true;
_iteratorError7 = _context3.t0;
case 38:
_context3.prev = 38;
_context3.prev = 39;
if (!_iteratorNormalCompletion7 && _iterator7.return) {
_iterator7.return();
}
case 41:
_context3.prev = 41;
if (!_didIteratorError7) {
_context3.next = 44;
break;
}
throw _iteratorError7;
case 44:
return _context3.finish(41);
case 45:
return _context3.finish(38);
case 46:
return _context3.abrupt('return', descriptor);
case 47:
case 'end':
return _context3.stop();
}
}
}, _callee3, this, [[13, 34, 38, 46], [39,, 41, 45]]);
}));
return function extractZip(_x6) {
return _ref6.apply(this, arguments);
};
}();
function _objectEntries(obj) {
var entries = [];
var keys = Object.keys(obj);
for (var k = 0; k < keys.length; ++k) entries.push([keys[k], obj[keys[k]]]);
return entries;
}
function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
var fs = require('fs');
var JSZip = require('jszip');
var isEqual = require('lodash/isEqual');
var isString = require('lodash/isString');
var isBoolean = require('lodash/isBoolean');
var cloneDeep = require('lodash/cloneDeep');
var isUndefined = require('lodash/isUndefined');
var _require = require('util'),
promisify = _require.promisify;
var _require2 = require('./profile'),
Profile = _require2.Profile;
var _require3 = require('./resource'),
Resource = _require3.Resource;
var _require4 = require('./errors'),
DataPackageError = _require4.DataPackageError;
var helpers = require('./helpers');
var config = require('./config');
// Module API
/**
* Package representation
*/
var Package = function () {
_createClass(Package, [{
key: 'getResource',
/**
* Return a resource
*
* @param {string} name
* @returns {Resource|null} resource instance if exists
*/
value: function getResource(name) {
return this._resources.find(function (resource) {
return resource.name === name;
}) || null;
}
/**
* Add a resource
*
* @param {Object} descriptor
* @returns {Resource} added resource instance
*/
}, {
key: 'addResource',
value: function addResource(descriptor) {
if (!this._currentDescriptor.resources) this._currentDescriptor.resources = [];
this._currentDescriptor.resources.push(descriptor);
this._build();
return this._resources[this._resources.length - 1];
}
/**
* Remove a resource
*
* @param {string} name
* @returns {(Resource|null)} removed resource instance if exists
*/
}, {
key: 'removeResource',
value: function removeResource(name) {
var resource = this.getResource(name);
if (resource) {
var predicat = function predicat(resource) {
return resource.name !== name;
};
this._currentDescriptor.resources = this._currentDescriptor.resources.filter(predicat);
this._build();
}
return resource;
}
/**
* Infer metadata
*
* @param {string} pattern
* @returns {Object}
*/
}, {
key: 'infer',
value: function () {
var _ref = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee() {
var pattern = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : false;
var files, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, file, _iteratorNormalCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, _step2$value, index, resource, descriptor;
return regeneratorRuntime.wrap(function _callee$(_context) {
while (1) {
switch (_context.prev = _context.next) {
case 0:
if (!pattern) {
_context.next = 27;
break;
}
if (!config.IS_BROWSER) {
_context.next = 3;
break;
}
throw new DataPackageError('Browser is not supported for pattern infer');
case 3:
if (this._basePath) {
_context.next = 5;
break;
}
throw new DataPackageError('Base path is required for pattern infer');
case 5:
_context.next = 7;
return findFiles(pattern, this._basePath);
case 7:
files = _context.sent;
_iteratorNormalCompletion = true;
_didIteratorError = false;
_iteratorError = undefined;
_context.prev = 11;
for (_iterator = files[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true) {
file = _step.value;
this.addResource({ path: file });
}
_context.next = 19;
break;
case 15:
_context.prev = 15;
_context.t0 = _context['catch'](11);
_didIteratorError = true;
_iteratorError = _context.t0;
case 19:
_context.prev = 19;
_context.prev = 20;
if (!_iteratorNormalCompletion && _iterator.return) {
_iterator.return();
}
case 22:
_context.prev = 22;
if (!_didIteratorError) {
_context.next = 25;
break;
}
throw _iteratorError;
case 25:
return _context.finish(22);
case 26:
return _context.finish(19);
case 27:
// Resources
_iteratorNormalCompletion2 = true;
_didIteratorError2 = false;
_iteratorError2 = undefined;
_context.prev = 30;
_iterator2 = this.resources.entries()[Symbol.iterator]();
case 32:
if (_iteratorNormalCompletion2 = (_step2 = _iterator2.next()).done) {
_context.next = 42;
break;
}
_step2$value = _slicedToArray(_step2.value, 2), index = _step2$value[0], resource = _step2$value[1];
_context.next = 36;
return resource.infer();
case 36:
descriptor = _context.sent;
this._currentDescriptor.resources[index] = descriptor;
this._build();
case 39:
_iteratorNormalCompletion2 = true;
_context.next = 32;
break;
case 42:
_context.next = 48;
break;
case 44:
_context.prev = 44;
_context.t1 = _context['catch'](30);
_didIteratorError2 = true;
_iteratorError2 = _context.t1;
case 48:
_context.prev = 48;
_context.prev = 49;
if (!_iteratorNormalCompletion2 && _iterator2.return) {
_iterator2.return();
}
case 51:
_context.prev = 51;
if (!_didIteratorError2) {
_context.next = 54;
break;
}
throw _iteratorError2;
case 54:
return _context.finish(51);
case 55:
return _context.finish(48);
case 56:
// Profile
if (this._nextDescriptor.profile === config.DEFAULT_DATA_PACKAGE_PROFILE) {
if (this.resources.length && this.resources.every(function (resouce) {
return resouce.tabular;
})) {
this._currentDescriptor.profile = 'tabular-data-package';
this._build();
}
}
return _context.abrupt('return', this._currentDescriptor);
case 58:
case 'end':
return _context.stop();
}
}
}, _callee, this, [[11, 15, 19, 27], [20,, 22, 26], [30, 44, 48, 56], [49,, 51, 55]]);
}));
function infer() {
return _ref.apply(this, arguments);
}
return infer;
}()
/**
* Update package instance if there are in-place changes in the descriptor.
*
* @example
*
* ```javascript
* const dataPackage = await Package.load({
* name: 'package',
* resources: [{name: 'resource', data: ['data']}]
* })
*
* dataPackage.name // package
* dataPackage.descriptor.name = 'renamed-package'
* dataPackage.name // package
* dataPackage.commit()
* dataPackage.name // renamed-package
* ```
*
* @param {boolean} strict - alter `strict` mode for further work
* @throws {DataPackageError} raises any error occurred in the process
* @returns {Boolean} returns true on success and false if not modified
*/
}, {
key: 'commit',
value: function commit() {
var _ref2 = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {},
strict = _ref2.strict;
if (isBoolean(strict)) this._strict = strict;else if (isEqual(this._currentDescriptor, this._nextDescriptor)) return false;
this._currentDescriptor = cloneDeep(this._nextDescriptor);
this._build();
return true;
}
/**
* Save data package to target destination.
*
* If target path has a zip file extension the package will be zipped and
* saved entirely. If it has a json file extension only the descriptor will be saved.
*
* @param {string} target - path where to save a data package
* @param {DataPackageError} raises error if something goes wrong
* @param {boolean} returns true on success
*/
}, {
key: 'save',
value: function save(target) {
var _this = this;
return new Promise(function (resolve, reject) {
// Save descriptor to json
if (target.endsWith('.json')) {
var contents = JSON.stringify(_this._currentDescriptor, null, 4);
fs.writeFile(target, contents, function (error) {
return !error ? resolve() : reject(error);
});
// Save package to zip
} else {
// Not supported in browser
if (config.IS_BROWSER) {
throw new DataPackageError('Zip is not supported in browser');
}
// Prepare zip
var zip = new JSZip();
var descriptor = cloneDeep(_this._currentDescriptor);
var _iteratorNormalCompletion3 = true;
var _didIteratorError3 = false;
var _iteratorError3 = undefined;
try {
for (var _iterator3 = _this.resources.entries()[Symbol.iterator](), _step3; !(_iteratorNormalCompletion3 = (_step3 = _iterator3.next()).done); _iteratorNormalCompletion3 = true) {
var _step3$value = _slicedToArray(_step3.value, 2),
index = _step3$value[0],
resource = _step3$value[1];
if (!resource.name) continue;
if (!resource.local) continue;
var path = 'data/' + resource.name;
var format = resource.descriptor.format;
if (format) path = path + '.' + format.toLowerCase();
descriptor.resources[index].path = path;
zip.file(path, resource.rawRead());
}
} catch (err) {
_didIteratorError3 = true;
_iteratorError3 = err;
} finally {
try {
if (!_iteratorNormalCompletion3 && _iterator3.return) {
_iterator3.return();
}
} finally {
if (_didIteratorError3) {
throw _iteratorError3;
}
}
}
zip.file('datapackage.json', JSON.stringify(descriptor, null, 4));
// Write zip
zip.generateNodeStream({ type: 'nodebuffer', streamFiles: true }).pipe(fs.createWriteStream(target).on('error', function (error) {
return reject(error);
})).on('error', function (error) {
return reject(error);
}).on('finish', function () {
return resolve(true);
});
}
});
}
// Private
}, {
key: 'valid',
/**
* Validation status
*
* It always `true` in strict mode.
*
* @returns {Boolean} returns validation status
*/
get: function get() {
return this._errors.length === 0 && this.resources.every(function (resource) {
return resource.valid;
});
}
/**
* Validation errors
*
* It always empty in strict mode.
*
* @returns {Error[]} returns validation errors
*/
}, {
key: 'errors',
get: function get() {
var errors = cloneDeep(this._errors);
var _iteratorNormalCompletion4 = true;
var _didIteratorError4 = false;
var _iteratorError4 = undefined;
try {
for (var _iterator4 = this.resources.entries()[Symbol.iterator](), _step4; !(_iteratorNormalCompletion4 = (_step4 = _iterator4.next()).done); _iteratorNormalCompletion4 = true) {
var _step4$value = _slicedToArray(_step4.value, 2),
index = _step4$value[0],
resource = _step4$value[1];
if (!resource.valid) {
errors.push(new Error('Resource "' + (resource.name || index) + '" validation error(s)'));
}
}
} catch (err) {
_didIteratorError4 = true;
_iteratorError4 = err;
} finally {
try {
if (!_iteratorNormalCompletion4 && _iterator4.return) {
_iterator4.return();
}
} finally {
if (_didIteratorError4) {
throw _iteratorError4;
}
}
}
return errors;
}
/**
* Profile
*
* @returns {Profile}
*/
}, {
key: 'profile',
get: function get() {
return this._profile;
}
/**
* Descriptor
*
* @returns {Object} schema descriptor
*/
}, {
key: 'descriptor',
get: function get() {
// Never use this.descriptor inside this class (!!!)
return this._nextDescriptor;
}
/**
* Resources
*
* @returns {Resoruce[]}
*/
}, {
key: 'resources',
get: function get() {
return this._resources;
}
/**
* Resource names
*
* @returns {string[]}
*/
}, {
key: 'resourceNames',
get: function get() {
return this._resources.map(function (resource) {
return resource.name;
});
}
}], [{
key: 'load',
// Public
/**
* Factory method to instantiate `Package` class.
*
* This method is async and it should be used with await keyword or as a `Promise`.
*
* @param {string|Object} descriptor - package descriptor as local path, url or object.
* If ththe path has a `zip` file extension it will be unzipped
* to the temp directory first.
* @param {string} basePath - base path for all relative paths
* @param {boolean} strict - strict flag to alter validation behavior.
* Setting it to `true` leads to throwing errors on any operation
* with invalid descriptor
* @throws {DataPackageError} raises error if something goes wrong
* @returns {Package} returns data package class instance
*/
value: function () {
var _ref3 = _asyncToGenerator( /*#__PURE__*/regeneratorRuntime.mark(function _callee2() {
var descriptor = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
var _ref4 = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {},
basePath = _ref4.basePath,
_ref4$strict = _ref4.strict,
strict = _ref4$strict === undefined ? false : _ref4$strict;
var profile;
return regeneratorRuntime.wrap(function _callee2$(_context2) {
while (1) {
switch (_context2.prev = _context2.next) {
case 0:
if (!(isString(descriptor) && descriptor.endsWith('.zip'))) {
_context2.next = 4;
break;
}
_context2.next = 3;
return extractZip(descriptor);
case 3:
descriptor = _context2.sent;
case 4:
// Get base path
if (isUndefined(basePath)) {
basePath = helpers.locateDescriptor(descriptor);
}
// Process descriptor
_context2.next = 7;
return helpers.retrieveDescriptor(descriptor);
case 7:
descriptor = _context2.sent;
_context2.next = 10;
return helpers.dereferencePackageDescriptor(descriptor, basePath);
case 10:
descriptor = _context2.sent;
_context2.next = 13;
return Profile.load(descriptor.profile || config.DEFAULT_DATA_PACKAGE_PROFILE);
case 13:
profile = _context2.sent;
return _context2.abrupt('return', new Package(descriptor, { basePath: basePath, strict: strict, profile: profile }));
case 15:
case 'end':
return _context2.stop();
}
}
}, _callee2, this);
}));
function load() {
return _ref3.apply(this, arguments);
}
return load;
}()
}]);
function Package(descriptor) {
var _ref5 = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {},
basePath = _ref5.basePath,
strict = _ref5.strict,
profile = _ref5.profile;
_classCallCheck(this, Package);
// Handle deprecated resource.path.url
var _iteratorNormalCompletion5 = true;
var _didIteratorError5 = false;
var _iteratorError5 = undefined;
try {
for (var _iterator5 = (descriptor.resources || [])[Symbol.iterator](), _step5; !(_iteratorNormalCompletion5 = (_step5 = _iterator5.next()).done); _iteratorNormalCompletion5 = true) {
var resource = _step5.value;
if (resource.url) {
console.warn('Resource property "url: <url>" is deprecated.\n Please use "path: <url>" instead.');
resource.path = resource.url;
delete resource.url;
}
}
// Set attributes
} catch (err) {
_didIteratorError5 = true;
_iteratorError5 = err;
} finally {
try {
if (!_iteratorNormalCompletion5 && _iterator5.return) {
_iterator5.return();
}
} finally {
if (_didIteratorError5) {
throw _iteratorError5;
}
}
}
this._currentDescriptor = cloneDeep(descriptor);
this._nextDescriptor = cloneDeep(descriptor);
this._basePath = basePath;
this._strict = strict;
this._profile = profile;
this._resources = [];
this._errors = [];
// Build package
this._build();
}
_createClass(Package, [{
key: '_build',
value: function _build() {
// Process descriptor
this._currentDescriptor = helpers.expandPackageDescriptor(this._currentDescriptor);
this._nextDescriptor = cloneDeep(this._currentDescriptor);
// Validate descriptor
this._errors = [];
var _profile$validate = this._profile.validate(this._currentDescriptor),
valid = _profile$validate.valid,
errors = _profile$validate.errors;
if (!valid) {
this._errors = errors;
if (this._strict) {
var message = 'There are ' + errors.length + ' validation errors (see \'error.errors\')';
throw new DataPackageError(message, errors);
}
}
// Update resources
this._resources.length = (this._currentDescriptor.resources || []).length;
var _iteratorNormalCompletion6 = true;
var _didIteratorError6 = false;
var _iteratorError6 = undefined;
try {
for (var _iterator6 = (this._currentDescriptor.resources || []).entries()[Symbol.iterator](), _step6; !(_iteratorNormalCompletion6 = (_step6 = _iterator6.next()).done); _iteratorNormalCompletion6 = true) {
var _step6$value = _slicedToArray(_step6.value, 2),
index = _step6$value[0],
descriptor = _step6$value[1];
var resource = this._resources[index];
if (!resource || !isEqual(resource.descriptor, descriptor) || resource.schema && resource.schema.foreignKeys.length) {
this._resources[index] = new Resource(descriptor, {
strict: this._strict,
basePath: this._basePath,
dataPackage: this
});
}
}
} catch (err) {
_didIteratorError6 = true;
_iteratorError6 = err;
} finally {
try {
if (!_iteratorNormalCompletion6 && _iterator6.return) {
_iterator6.return();
}
} finally {
if (_didIteratorError6) {
throw _iteratorError6;
}
}
}
}
}]);
return Package;
}();
function findFiles(pattern, basePath) {
var glob = require('glob');
return new Promise(function (resolve, reject) {
var options = { cwd: basePath, ignore: 'node_modules/**' };
glob(pattern, options, function (error, files) {
if (error) reject(error);
resolve(files);
});
});
}
// System
module.exports = {
Package: Package
};