@rsksmart/rif-storage
Version:
Library integrating distributed storage projects
439 lines (438 loc) • 18.3 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __exportStar = (this && this.__exportStar) || function(m, exports) {
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __asyncValues = (this && this.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Bzz = exports.getModeProtocol = exports.BZZ_MODE_PROTOCOLS = void 0;
const stream_1 = require("stream");
const tar_stream_1 = __importDefault(require("tar-stream"));
const ky_universal_1 = __importDefault(require("ky-universal"));
const debug_1 = __importDefault(require("debug"));
const stream_normalization_1 = __importDefault(require("./utils/stream-normalization"));
const errors_1 = require("../errors");
const utils_1 = require("../utils");
const data_1 = __importDefault(require("./utils/data"));
__exportStar(require("./types"), exports);
const log = debug_1.default('swarm-mini');
exports.BZZ_MODE_PROTOCOLS = {
default: 'bzz:/',
immutable: 'bzz-immutable:/',
raw: 'bzz-raw:/'
};
function getModeProtocol(mode, defaultMode) {
return (mode && exports.BZZ_MODE_PROTOCOLS[mode]) || (defaultMode && exports.BZZ_MODE_PROTOCOLS[defaultMode]) || exports.BZZ_MODE_PROTOCOLS.default;
}
exports.getModeProtocol = getModeProtocol;
function getDownloadURL(hash, options = {}, defaultMode) {
const protocol = getModeProtocol(options.mode, defaultMode);
let url = `${protocol}${hash}/`;
if (options.path != null) {
url += options.path;
}
if (options.mode === 'raw' && options.contentType != null) {
url += `?content_type=${options.contentType}`;
}
return url;
}
function getUploadURL(options = {}) {
// Default URL to creation
let url = getModeProtocol(options.mode, 'default');
// Manifest update if hash is provided
if (options.manifestHash != null) {
url += `${options.manifestHash}/`;
if (options.path != null) {
url += options.path;
}
}
if (options.defaultPath != null) {
url += `?defaultpath=${options.defaultPath}`;
}
return url;
}
function mapDirectoryArrayToDirectory(data) {
return data.reduce((previousValue, currentValue) => {
if (utils_1.isReadable(currentValue.data) && !currentValue.size && currentValue.size !== 0) {
throw new errors_1.ValueError(`Missing "size" that is required for Readable streams (path: ${currentValue.path})`);
}
const path = currentValue.path;
previousValue[path] = {
contentType: currentValue.contentType,
data: currentValue.data,
size: currentValue.size
};
return previousValue;
}, {});
}
function kyOnlyOptions(options) {
return {
headers: options.headers,
timeout: options.timeout,
onDownloadProgress: options.onDownloadProgress
};
}
function getReason(response) {
return __awaiter(this, void 0, void 0, function* () {
const errMessage = yield response.text();
const messageMatches = /Message: (.*)$/m.exec(errMessage);
if (messageMatches && messageMatches.length === 2) {
return messageMatches[1];
}
return errMessage;
});
}
/**
* Small simple client library for Bzz part of Swarm project.
* It communicate using HTTP API.
*/
class Bzz {
constructor(config) {
const { url, timeout } = config;
this.ky = ky_universal_1.default.create({ timeout, prefixUrl: url });
}
/**
* Fetch list of entries of given manifest hash.
*
*
* @param hash
* @param options
* @throws HTTPError when hash is not a manifest
*/
list(hash, options = {}) {
return __awaiter(this, void 0, void 0, function* () {
let url = `bzz-list:/${hash}/`;
if (options.path != null) {
url += options.path;
}
try {
return (yield this.ky.get(url, kyOnlyOptions(options))).json();
}
catch (e) {
if (e.response) {
e.message = `${e.message}: ${yield getReason(e.response)}`;
}
throw e;
}
});
}
/**
* Helper method for fetching single raw file.
*
* @param hash
* @param options
*/
getFile(hash, options = {}) {
return __awaiter(this, void 0, void 0, function* () {
const url = getDownloadURL(hash, options, 'raw');
try {
const arrayBuf = yield (yield this.ky.get(url, kyOnlyOptions(options))).arrayBuffer();
return utils_1.markFile(Buffer.from(arrayBuf));
}
catch (e) {
if (e.response) {
e.message = `${e.message}: ${yield e.response.text()}`;
}
throw e;
}
});
}
/**
* Helper method for fetching directory defined by manifest.
* It employees fetching of Tar file from Swarm with all files which is then extracted on the client side.
*
* @param hash
* @param options
*/
getDirectory(hash, options) {
var e_1, _a;
return __awaiter(this, void 0, void 0, function* () {
const dir = {};
try {
for (var _b = __asyncValues(yield this.getReadable(hash, options)), _c; _c = yield _b.next(), !_c.done;) {
const file = _c.value;
const stream = file.data;
const chunks = [];
stream.on('data', (chunk) => {
chunks.push(chunk);
});
stream.on('end', () => {
dir[file.path] = {
data: Buffer.concat(chunks),
size: file.size
};
});
stream.resume();
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (_c && !_c.done && (_a = _b.return)) yield _a.call(_b);
}
finally { if (e_1) throw e_1.error; }
}
return dir;
});
}
/**
* Method for fetching file/directory from Swarm.
*
* Buffer is returned when it is single raw hash. You can use isFile() utility function to verify if file was returned.
* Directory object is returned when it is manifest hash. You can use isDirectory() utility function to verify that it is directory.
*
* @param hash
* @param options
*/
get(hash, options = {}) {
return __awaiter(this, void 0, void 0, function* () {
if (typeof hash !== 'string') {
throw new errors_1.ValueError(`hash ${hash} is not a string!`);
}
try {
const result = yield this.list(hash);
if (!result.entries && !result.common_prefixes) {
throw new errors_1.ValueError(`Hash ${hash} does not contain any files/folders!`);
}
}
catch (e) {
// Internal Server error is returned by Swarm when the hash is not Manifest
if (!e.response || e.response.status !== 500) {
throw e;
}
return this.getFile(hash, options);
}
return utils_1.markDirectory(yield this.getDirectory(hash, options));
});
}
getStream(hash, options = {}) {
return __awaiter(this, void 0, void 0, function* () {
if (options.headers == null) {
options.headers = {};
}
options.headers.accept = 'application/x-tar';
try {
const respond = yield this.ky.get(getDownloadURL(hash, options), kyOnlyOptions(options));
if (!respond.body) {
throw new Error('Respond does not have any stream body!');
}
return stream_normalization_1.default(respond.body);
}
catch (e) {
if (e.response) {
e.message = `${e.message}: ${yield getReason(e.response)}`;
}
throw e;
}
});
}
/**
* Helper function that fetch single raw file from Swarm returning Readable
*
* @param hash
* @param options
* @private
*/
getRawReadable(hash, options) {
return __awaiter(this, void 0, void 0, function* () {
options.mode = 'raw';
const stream = yield this.getStream(hash, options);
const wrapperStream = new stream_1.Readable({ objectMode: true });
// eslint-disable-next-line @typescript-eslint/no-empty-function
wrapperStream._read = () => { };
wrapperStream.push({
data: stream,
path: ''
});
wrapperStream.push(null);
return wrapperStream;
});
}
/**
* Helper function that fetch file(s)/directory (eq. hash is manifest) from Swarm
* returning Readable
*
* @param hash
* @param options
* @private
*/
getManifestReadable(hash, options) {
return __awaiter(this, void 0, void 0, function* () {
const manifestStream = yield this.getStream(hash, options);
const readable = new stream_1.Readable({ objectMode: true });
// eslint-disable-next-line @typescript-eslint/no-empty-function
readable._read = () => { };
const extract = tar_stream_1.default.extract();
extract.on('entry', (header, stream, next) => {
if (header.type === 'file') {
readable.push({
data: stream,
path: header.name,
size: header.size
});
stream.on('end', next);
}
else {
next();
}
});
extract.on('finish', () => {
readable.push(null);
});
extract.on('error', (err) => {
readable.destroy(err);
});
manifestStream.pipe(extract);
return readable;
});
}
/**
* Fetch data from Swarm and return Readable in object mode that yield
* objects in format {data: <Readable>, path: 'string', size: number | undefined}
* @param hash
* @param options
*/
getReadable(hash, options = {}) {
return __awaiter(this, void 0, void 0, function* () {
if (typeof hash !== 'string') {
throw new errors_1.ValueError(`Address ${hash} is not a string!`);
}
try {
return yield this.getManifestReadable(hash, options);
}
catch (e) {
// Internal Server error is returned by Swarm when the address is not Manifest
if (!e.response || e.response.status !== 500) {
throw e;
}
return this.getRawReadable(hash, options);
}
});
}
putItToSwarm(data, options) {
return __awaiter(this, void 0, void 0, function* () {
const url = getUploadURL(options);
try {
// @ts-ignore: In NodeJS ky = node_fetch which supports Buffer and Readable, but ky uses browser's definitions, so ignoring it for compatibilities. Suggestions how to improve this will be welcomed.
return (yield this.ky.post(url, { body: yield data_1.default(data), headers: options.headers })).text();
}
catch (e) {
if (e.response) {
e.message = `${e.message}: ${yield getReason(e.response)}`;
}
throw e;
}
});
}
// eslint-disable-next-line require-await
put(data, options = {}) {
return __awaiter(this, void 0, void 0, function* () {
if (typeof data === 'string') {
data = Buffer.from(data);
}
// Convert single element DirectoryArray
if (Array.isArray(data) && data.length === 1) {
const el = data[0];
options.contentType = options.contentType || el.contentType;
options.size = options.size || el.size;
options.fileName = options.fileName || el.path;
data = el.data;
}
if (Buffer.isBuffer(data) || utils_1.isReadable(data)) {
if (utils_1.isReadable(data) && !options.size) {
throw new errors_1.ValueError('Missing "size" that is required for Readable streams');
}
if (options.fileName) {
data = [
{
data: data,
path: options.fileName,
size: options.size,
contentType: options.contentType
}
];
options.defaultPath = options.fileName;
delete options.fileName;
delete options.size;
delete options.contentType;
}
else {
log('uploading single buffer file');
if (!options.headers) {
options.headers = {};
}
if (options.size != null) {
options.headers['content-length'] = options.size;
}
else if (Buffer.isBuffer(data)) {
options.headers['content-length'] = data.length;
}
if (!options.contentType) {
options.mode = 'raw';
}
if (options.headers['content-type'] == null && options.contentType) {
options.headers['content-type'] = options.contentType;
}
return this.putItToSwarm(data, options);
}
}
log('uploading directory');
if (options.fileName) {
throw new errors_1.ValueError('You are uploading directory, yet you specified filename that is not applicable here!');
}
if (options.size) {
throw new errors_1.ValueError('You are uploading directory, yet you specified size that is not applicable here!');
}
if ((typeof data !== 'object' && !Array.isArray(data)) || data === null || data === undefined) {
throw new TypeError('data have to be string, Readable, Buffer, DirectoryArray or Directory object!');
}
if ((Array.isArray(data) && data.length === 0) || Object.keys(data).length === 0) {
// TODO: [Q] Empty object should throw error? If not then what to return? https://github.com/rsksmart/rif-storage-js/issues/4
throw new errors_1.ValueError('You passed empty Directory');
}
if (utils_1.isTSDirectory(data, utils_1.isReadableOrBuffer)) {
Object.entries(data).forEach(([path, entry]) => {
if (path === '') {
throw new errors_1.ValueError('Empty path (name of property) is not allowed!');
}
if (utils_1.isReadable(entry.data) && !entry.size && entry.size !== 0) {
throw new errors_1.ValueError(`Missing "size" that is required for Readable streams (path: ${path})`);
}
});
return this.putItToSwarm(data, options);
}
else if (utils_1.isTSDirectoryArray(data, utils_1.isReadableOrBuffer)) {
const mappedData = mapDirectoryArrayToDirectory(data);
return this.putItToSwarm(mappedData, options);
}
else {
throw new errors_1.ValueError('Data has to be string, Buffer, Readable, Directory<string | Buffer | Readable> or DirectoryArray<string | Buffer | Readable>');
}
});
}
}
exports.Bzz = Bzz;