ayakashi
Version:
The next generation web scraping framework
271 lines (270 loc) • 11.4 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.loadExternalPreloaders = exports.loadExternalExtractors = exports.loadExternalActions = exports.loadLocalPreloaders = exports.loadLocalExtractors = exports.loadLocalActions = exports.loadLocalProps = void 0;
const path_1 = require("path");
//@ts-ignore
const require_all_1 = __importDefault(require("require-all"));
const resolve_from_1 = __importDefault(require("resolve-from"));
const node_dir_1 = __importDefault(require("node-dir"));
const fs_1 = require("fs");
const opLog_1 = require("../opLog/opLog");
const compiler_1 = require("../preloaderCompiler/compiler");
const debug_1 = __importDefault(require("debug"));
const d = debug_1.default("ayakashi:loaders");
function loadLocalProps(ayakashiInstance, projectFolder) {
const opLog = opLog_1.getOpLog();
const localPropsDir = path_1.resolve(projectFolder, "props");
if (fs_1.existsSync(localPropsDir)) {
const props = require_all_1.default({
dirname: localPropsDir,
filter: function (fileName) {
if (fileName.includes(".js") && !fileName.includes(".map")) {
return fileName;
}
else {
return false;
}
}
});
Object.keys(props).forEach(function (propName) {
try {
if (typeof props[propName] !== "function" && props[propName].default) {
props[propName] = props[propName].default;
}
if (typeof props[propName] === "function") {
d(`autoloading prop: ${propName}`);
props[propName](ayakashiInstance);
}
else {
throw new Error("invalid_prop");
}
}
catch (e) {
opLog.error(`Local prop <${propName}> is invalid`);
throw e;
}
});
}
}
exports.loadLocalProps = loadLocalProps;
function loadLocalActions(ayakashiInstance, projectFolder) {
const opLog = opLog_1.getOpLog();
const localActionsDir = path_1.resolve(projectFolder, "actions");
if (fs_1.existsSync(localActionsDir)) {
const actions = require_all_1.default({
dirname: localActionsDir,
filter: function (fileName) {
if (fileName.includes(".js") && !fileName.includes(".map")) {
return fileName;
}
else {
return false;
}
}
});
Object.keys(actions).forEach(function (actionName) {
try {
if (typeof actions[actionName] !== "function" && actions[actionName].default) {
actions[actionName] = actions[actionName].default;
}
if (typeof actions[actionName] === "function") {
d(`autoloading action: ${actionName}`);
actions[actionName](ayakashiInstance);
}
else {
throw new Error("invalid_action");
}
}
catch (e) {
opLog.error(`Local action <${actionName}> is invalid`);
throw e;
}
});
}
}
exports.loadLocalActions = loadLocalActions;
function loadLocalExtractors(ayakashiInstance, projectFolder) {
const opLog = opLog_1.getOpLog();
const localExtractorsDir = path_1.resolve(projectFolder, "extractors");
if (fs_1.existsSync(localExtractorsDir)) {
const extractors = require_all_1.default({
dirname: localExtractorsDir,
filter: function (fileName) {
if (fileName.includes(".js") && !fileName.includes(".map")) {
return fileName;
}
else {
return false;
}
}
});
Object.keys(extractors).forEach(function (extractor) {
try {
if (typeof extractors[extractor] !== "function" && extractors[extractor].default) {
extractors[extractor] = extractors[extractor].default;
}
if (typeof extractors[extractor] === "function") {
d(`autoloading extractor: ${extractor}`);
extractors[extractor](ayakashiInstance);
}
else {
throw new Error("invalid_extractor");
}
}
catch (e) {
opLog.error(`Local extractor <${extractor}> is invalid`);
throw e;
}
});
}
}
exports.loadLocalExtractors = loadLocalExtractors;
function loadLocalPreloaders(connection, projectFolder, storeProjectFolder) {
return __awaiter(this, void 0, void 0, function* () {
const localPreloadersDir = path_1.resolve(projectFolder, "preloaders");
if (fs_1.existsSync(localPreloadersDir)) {
const localPreloaders = yield node_dir_1.default.promiseFiles(localPreloadersDir);
const localPreloaderDefinitions = localPreloaders
.filter(function (preloader) {
return preloader.includes(".js") && !preloader.includes(".map");
})
.map(function (preloader) {
return {
module: preloader,
as: null,
waitForDOM: false
};
});
yield loadPreloaders(connection, localPreloaderDefinitions, projectFolder, storeProjectFolder,
//for local preloader we use the fileName as its name
true);
}
});
}
exports.loadLocalPreloaders = loadLocalPreloaders;
function loadExternalActions(ayakashiInstance, projectFolder, actions) {
const opLog = opLog_1.getOpLog();
if (actions && Array.isArray(actions)) {
actions.forEach(function (actionModule) {
try {
let action = require(resolve_from_1.default(projectFolder, actionModule));
if (typeof action !== "function" && action.default) {
action = action.default;
}
if (typeof action === "function") {
d(`loading external action: ${actionModule}`);
action(ayakashiInstance);
}
else {
throw new Error("invalid_action");
}
}
catch (e) {
opLog.error(`Action <${actionModule}> is invalid`);
throw e;
}
});
}
}
exports.loadExternalActions = loadExternalActions;
function loadExternalExtractors(ayakashiInstance, projectFolder, extractors) {
const opLog = opLog_1.getOpLog();
if (extractors && Array.isArray(extractors)) {
extractors.forEach(function (extractorModule) {
try {
let extractor = require(resolve_from_1.default(projectFolder, extractorModule));
if (typeof extractor !== "function" && extractor.default) {
extractor = extractor.default;
}
if (typeof extractor === "function") {
d(`loading external extractor: ${extractorModule}`);
extractor(ayakashiInstance);
}
else {
throw new Error("invalid_extractor");
}
}
catch (e) {
opLog.error(`Extractor <${extractorModule}> is invalid`);
throw e;
}
});
}
}
exports.loadExternalExtractors = loadExternalExtractors;
function loadExternalPreloaders(connection, projectFolder, storeProjectFolder, preloaders) {
return __awaiter(this, void 0, void 0, function* () {
if (preloaders && Array.isArray(preloaders)) {
//@ts-ignore
const preloaderDefinitions = preloaders.map(function (preloader) {
if (typeof preloader === "string") {
return {
module: preloader,
as: null,
waitForDOM: false
};
}
else if (typeof preloader === "object" && typeof preloader.module === "string") {
return {
module: preloader.module,
as: preloader.as || null,
waitForDOM: preloader.waitForDOM || null
};
}
else {
return null;
}
//@ts-ignore
}).filter(preloader => !!preloader);
yield loadPreloaders(connection, preloaderDefinitions, projectFolder, storeProjectFolder,
//for external preloaders we don't use the fileName as its name
//we use the name provided
false);
}
});
}
exports.loadExternalPreloaders = loadExternalPreloaders;
function loadPreloaders(connection, preloaderDefinitions, projectFolder, storeProjectFolder, useFileName) {
return __awaiter(this, void 0, void 0, function* () {
const opLog = opLog_1.getOpLog();
const preloaders = yield Promise.all(preloaderDefinitions.map(function (preloaderDefinition) {
return new Promise(function (resolve, reject) {
compiler_1.compile(projectFolder, preloaderDefinition.module, "ayakashi", `${storeProjectFolder}/.cache/preloaders/`, useFileName).then(function (compiled) {
resolve({
compiled: compiled,
as: preloaderDefinition.as,
waitForDOM: preloaderDefinition.waitForDOM
});
}).catch(function (err) {
opLog.error(`Failed to compile preloader ${preloaderDefinition.module}`);
reject(err);
});
});
}));
return Promise.all(preloaders.map(function (preloader) {
return __awaiter(this, void 0, void 0, function* () {
try {
//@ts-ignore
yield connection.injectPreloader(preloader);
}
catch (e) {
//@ts-ignore
opLog.error(`Failed to load preloader ${preloader.module}`);
throw e;
}
});
}));
});
}