ayakashi
Version:
The next generation web scraping framework
140 lines (139 loc) • 7.19 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var g = generator.apply(thisArg, _arguments || []), i, q = [];
return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
function fulfill(value) { resume("next", value); }
function reject(value) { resume("throw", value); }
function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
};
var __asyncValues = (this && this.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
};
Object.defineProperty(exports, "__esModule", { value: true });
const path_1 = require("path");
const pipeproc_1 = require("pipeproc");
// import debug from "debug";
// const d = debug("ayakashi:scriptWrapper");
const opLog_1 = require("../opLog/opLog");
//tslint:disable
//@ts-ignore
const GeneratorFunction = Object.getPrototypeOf(function* () { }).constructor;
//@ts-ignore
const asyncGeneratorFunction = Object.getPrototypeOf(function () { return __asyncGenerator(this, arguments, function* () { }); }).constructor;
//tslint:enable
function scriptWrapper(log) {
var e_1, _a;
return __awaiter(this, void 0, void 0, function* () {
const opLog = opLog_1.getOpLog();
let scriptModule;
try {
if (["saveToSQL", "saveToJSON", "saveToCSV", "printToConsole"].indexOf(log.body.module) > -1) {
//@ts-ignore
if (log.body.input && log.body.input.value && log.body.input.value.continue === true)
return { value: { continue: true } };
scriptModule = require(path_1.resolve(log.body.appRoot, "lib", "coreScripts", log.body.module));
}
else {
scriptModule = require(path_1.resolve(log.body.projectFolder, "scripts", log.body.module));
}
if (typeof scriptModule !== "function") {
scriptModule = scriptModule.default;
}
if (typeof scriptModule !== "function") {
throw new Error(`Script <${log.body.module}> is not a function`);
}
}
catch (e) {
opLog.error(e.message);
throw e;
}
//connect to pipeproc
const pipeprocClient = pipeproc_1.PipeProc();
yield pipeprocClient.connect({ socket: `ipc://${path_1.resolve(log.body.storeProjectFolder, "ipc")}` });
opLog.info("running script", log.body.module);
try {
//@ts-ignore
if (log.body.input && log.body.input.value && log.body.input.value.continue === true)
delete log.body.input.value.continue;
const result = yield scriptModule(log.body.input.value || {}, log.body.params || {}, {
projectFolder: log.body.projectFolder,
operationId: log.body.operationId,
startDate: log.body.startDate
});
if (result instanceof GeneratorFunction || result instanceof asyncGeneratorFunction) {
//get generator results and commit them
let committedAtLeastOnce = false;
try {
for (var _b = __asyncValues(result()), _c; _c = yield _b.next(), !_c.done;) {
const val = _c.value;
if (Array.isArray(val)) {
yield pipeprocClient.commit(val.filter(v => v).map(v => {
return {
topic: log.body.saveTopic,
body: { value: v }
};
}));
committedAtLeastOnce = true;
}
else if (val) {
yield pipeprocClient.commit({
topic: log.body.saveTopic,
body: { value: val }
});
committedAtLeastOnce = true;
}
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (_c && !_c.done && (_a = _b.return)) yield _a.call(_b);
}
finally { if (e_1) throw e_1.error; }
}
if (!committedAtLeastOnce) {
return { value: { continue: true } };
}
return;
}
else if (Array.isArray(result)) {
yield pipeprocClient.commit(result.filter(re => re).map(re => {
return {
topic: log.body.saveTopic,
body: { value: re }
};
}));
return;
}
else if (result) {
return { value: result };
}
else {
return { value: { continue: true } };
}
}
catch (e) {
opLog.error(`There was an error while running script <${log.body.module}> -`, e.message, e.stack);
throw e;
}
});
}
exports.default = scriptWrapper;