@rsksmart/rif-storage-pinning
Version:
Application for providing your storage space to other to use in exchange of RIF Tokens
234 lines (233 loc) • 11.1 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.IpfsProvider = exports.getDagStat = exports.PinJob = void 0;
const ipfs_http_client_1 = __importDefault(require("ipfs-http-client"));
const multiaddr_1 = __importDefault(require("multiaddr"));
const cids_1 = __importDefault(require("cids"));
const semver = __importStar(require("semver"));
const config_1 = __importDefault(require("config"));
const node_fetch_1 = __importDefault(require("node-fetch"));
const parse_duration_1 = __importDefault(require("parse-duration"));
const logger_1 = require("../logger");
const jobs_manager_1 = require("../jobs-manager");
const errors_1 = require("../errors");
const utils_1 = require("../utils");
const direct_address_model_1 = __importDefault(require("../models/direct-address.model"));
const logger = logger_1.loggingFactory('ipfs');
const REQUIRED_IPFS_VERSION = '>=0.7.0';
const NOT_PINNED_ERROR_MSG = 'not pinned or pinned indirectly';
const MIN_PIN_TIMEOUT = 60000 * 20; // 20 minutes
const RATE_MB_PER_SECOND = 0.5;
class PinJob extends jobs_manager_1.Job {
constructor(ipfs, hash, expectedSize, agreementReference) {
super(hash, agreementReference, 'ipfs - pin');
this.expectedSize = expectedSize;
this.ipfs = ipfs;
this.hash = hash;
}
getPeerIdByAgreement(agreementReference) {
return __awaiter(this, void 0, void 0, function* () {
const directAddress = yield direct_address_model_1.default.findOne({ where: { agreementReference } });
yield direct_address_model_1.default.destroy({ where: { agreementReference } });
return directAddress === null || directAddress === void 0 ? void 0 : directAddress.peerId;
});
}
getPeer() {
return __awaiter(this, void 0, void 0, function* () {
const peerId = yield this.getPeerIdByAgreement(this.agreementReference);
if (!peerId)
return undefined;
const peer = yield this.ipfs.dht.findPeer(new cids_1.default(peerId));
if (!peer)
return undefined;
return Object.assign(Object.assign({}, peer), { addresses: peer.addrs.map(addr => multiaddr_1.default(`${addr.toString()}/p2p/${peer.id}`)) });
});
}
swarmConnect() {
var _a;
return __awaiter(this, void 0, void 0, function* () {
logger.debug('In Pinning Job Swarm connect');
this.swarmAddresses = (_a = (yield this.getPeer())) === null || _a === void 0 ? void 0 : _a.addresses;
if (this.swarmAddresses) {
yield this.ipfs.swarm.connect(this.swarmAddresses);
}
});
}
swarmDisconnect() {
return __awaiter(this, void 0, void 0, function* () {
logger.debug('In Pinning Job Swarm disconnect');
// Disconnect from peer
if (this.swarmAddresses) {
yield this.ipfs.swarm.disconnect(this.swarmAddresses);
}
});
}
pinProcess(cid, { timeout }) {
return __awaiter(this, void 0, void 0, function* () {
const hash = this.hash.replace('/ipfs/', '');
yield this.swarmConnect().catch(logger.warn);
logger.info(`Pinning hash: ${hash} start`);
yield this.ipfs.pin.add(cid, { timeout });
yield this.swarmDisconnect().catch(logger.warn);
});
}
getMetaFileSize(cid) {
return this.ipfs.object.stat(cid, { timeout: config_1.default.get('ipfs.sizeFetchTimeout') })
.then(({ CumulativeSize }) => utils_1.bytesToMegabytes(CumulativeSize))
.catch(e => {
if (e.name === 'TimeoutError') {
logger.error(`Fetching size of ${cid.toString()} timed out!`);
throw new Error(`Fetching size of ${cid.toString()} timed out!`);
}
throw e;
});
}
getActualFileSize(cid) {
const timeout = config_1.default.get('ipfs.sizeFetchTimeout');
// @ts-ignore: TODO: Remove that when ipfs-js fully support this API
return this.ipfs.dag.stat(cid, { timeout: typeof timeout === 'number' ? timeout : parse_duration_1.default(timeout) })
.then((res) => {
return utils_1.bytesToMegabytes(res.Size);
})
.catch((e) => {
if (e.name === 'TimeoutError') {
logger.error(`Fetching size of ${cid.toString()} timed out!`);
throw new Error(`Fetching size of ${cid.toString()} timed out!`);
}
throw e;
});
}
_run() {
return __awaiter(this, void 0, void 0, function* () {
const hash = this.hash.replace('/ipfs/', '');
const cid = new cids_1.default(hash);
// METADATA SIZE CHECK
logger.verbose(`(${hash}) Retrieving meta size of CID`);
const metadataSizeMb = yield this.getMetaFileSize(cid); // In MB
if (metadataSizeMb.gt(this.expectedSize)) {
logger.error(`The hash ${hash} has cumulative size of ${metadataSizeMb.toString()} megabytes while it was expected to have ${this.expectedSize} megabytes.`);
throw new errors_1.HashExceedsSizeError('The hash exceeds payed size!', metadataSizeMb, this.expectedSize);
}
// PIN PROCESS
// We can be generous on the actual pinning timeout as if the CID would not be present
// in IPFS network, then the previous ipfs.object.stat() call would timeout already then.
// We are using 0.5 MB per second transfer rate, with keeping at least 20 minutes as default.
// SizeInMB * 0.5 * 1000 ==> ms
const estimatedTimeout = Math.max(MIN_PIN_TIMEOUT, metadataSizeMb.div(RATE_MB_PER_SECOND).multipliedBy(1000).toNumber());
yield this.pinProcess(cid, { timeout: estimatedTimeout });
// ACTUAL SIZE CHECK
logger.verbose(`(${hash}) Retrieving actual size of CID`);
const sizeInMb = yield this.getActualFileSize(cid); // In MB
if (sizeInMb.gt(this.expectedSize)) {
logger.error(`The hash ${hash} has cumulative size of ${sizeInMb.toString()} megabytes while it was expected to have ${this.expectedSize} megabytes.`);
// Unpin file
logger.info(`Unpin file ${hash} due to exceed size limit`);
yield this.ipfs.pin.rm(cid);
throw new errors_1.HashExceedsSizeError('The hash exceeds payed size!', sizeInMb, this.expectedSize);
}
});
}
}
exports.PinJob = PinJob;
function getDagStat(nodeUrl) {
return (cid, options) => node_fetch_1.default(`${nodeUrl}/api/v0/dag/stat?arg=${cid.toString()}&progress=false`, Object.assign({ method: 'POST' }, options))
.then(res => {
if (!res.ok) {
throw new Error(`Get dag stat for hash ${cid.toString()} error, ${res.statusText}`);
}
return res.json();
});
}
exports.getDagStat = getDagStat;
class IpfsProvider {
constructor(jobsManager, ipfs) {
this.ipfs = ipfs;
this.jobsManager = jobsManager;
}
static bootstrap(jobsManager, options) {
return __awaiter(this, void 0, void 0, function* () {
if (!options) {
// Default location of local node, lets try that one
options = '/ip4/127.0.0.1/tcp/5001';
}
// @ts-ignore: TODO: Remove this when https://github.com/ipfs/js-ipfs/pull/3456 is shipped
const ipfs = ipfs_http_client_1.default(options);
// @ts-ignore: TODO: Remove that when ipfs-js fully support this API
ipfs.dag.stat = getDagStat(typeof options === 'string' ? options : options.url);
let versionObject;
try {
versionObject = yield ipfs.version();
}
catch (e) {
if (e.code === 'ECONNREFUSED') {
throw new Error(`No running IPFS daemon on ${typeof options === 'object' ? JSON.stringify(options) : options}`);
}
throw e;
}
if (!semver.satisfies(versionObject.version, REQUIRED_IPFS_VERSION)) {
throw new Error(`Supplied IPFS node is version ${versionObject.version} while this utility requires version ${REQUIRED_IPFS_VERSION}`);
}
return new this(jobsManager, ipfs);
});
}
/**
*
* @param hash
* @param expectedSize
* @param agreementReference
*/
pin(hash, expectedSize, agreementReference) {
const job = new PinJob(this.ipfs, hash, expectedSize, agreementReference);
return this.jobsManager.run(job);
}
unpin(hash) {
return __awaiter(this, void 0, void 0, function* () {
logger.info(`Unpinning hash: ${hash}`);
hash = hash.replace('/ipfs/', '');
const cid = new cids_1.default(hash);
try {
yield this.ipfs.pin.rm(cid);
}
catch (e) {
if (e.message === NOT_PINNED_ERROR_MSG) {
throw new errors_1.NotPinnedError(`${hash} is not pinned or pinned indirectly`);
}
else {
throw e;
}
}
});
}
}
exports.IpfsProvider = IpfsProvider;