ayakashi/lib/prelude/actions/meta.js

The next generation web scraping framework

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.attachMetaActions = void 0;
const query_1 = require("../query/query");
const opLog_1 = require("../../opLog/opLog");
const debug_1 = __importDefault(require("debug"));
const d = debug_1.default("ayakashi:prelude:meta");
function attachMetaActions(ayakashiInstance, connection) {
    const opLog = opLog_1.getOpLog();
    ayakashiInstance.prop = function (propId) {
        if (typeof propId === "string" && ayakashiInstance.propRefs[propId]) {
            return ayakashiInstance.propRefs[propId];
        }
        else if (propId && typeof propId === "object" && propId.$$prop) {
            return propId;
        }
        else {
            return null;
        }
    };
    ayakashiInstance.evaluate = function (fn, ...args) {
        return __awaiter(this, void 0, void 0, function* () {
            return connection.evaluate(fn, ...args);
        });
    };
    ayakashiInstance.evaluateAsync = function (fn, ...args) {
        return __awaiter(this, void 0, void 0, function* () {
            return connection.evaluateAsync(fn, ...args);
        });
    };
    ayakashiInstance.defineProp = function (fn, propId) {
        const query = query_1.createQuery(ayakashiInstance, {
            propId: propId,
            triggerFn: function () {
                return ayakashiInstance.evaluate(function (propConstructor, id) {
                    const elements = propConstructor.call(this);
                    let matches;
                    //@ts-ignore
                    if (Array.isArray(elements) || elements instanceof this.window.NodeList) {
                        //@ts-ignore
                        matches = Array.from(elements);
                    }
                    else {
                        matches = [elements];
                    }
                    this.propTable[id] = {
                        //@ts-ignore
                        matches: matches
                    };
                    return matches.length;
                }, fn, query.id);
            }
        });
        ayakashiInstance.propRefs[query.id] = query;
        return query;
    };
    ayakashiInstance.pause = function () {
        return __awaiter(this, void 0, void 0, function* () {
            try {
                yield ayakashiInstance.evaluate(function () {
                    console.warn("[Ayakashi]: scraper execution is paused, run ayakashi.resume() in devtools to resume");
                    this.paused = true;
                });
                opLog.warn("scraper execution is paused, run ayakashi.resume() in devtools to resume");
                yield ayakashiInstance.waitUntil(function () {
                    return ayakashiInstance.evaluate(function () {
                        return this.paused === false;
                    });
                }, 100, 0);
                opLog.warn("scraper execution is resumed");
            }
            catch (e) {
                throw e;
            }
        });
    };
    ayakashiInstance.registerAction = function (name, actionFn) {
        d("registering action:", name);
        if (!name || typeof name !== "string" ||
            !actionFn || typeof actionFn !== "function") {
            throw new Error("Invalid action");
        }
        if (name in ayakashiInstance)
            throw new Error(`There is an action "${name}" already registered`);
        //@ts-ignore
        ayakashiInstance[name] = function (...args) {
            return actionFn.call(ayakashiInstance, ...args);
        };
    };
    ayakashiInstance.registerExtractor = function (extractorName, extractorFn, dependsOn) {
        d("registering extractor:", extractorName);
        if (!extractorName || typeof extractorName !== "string" ||
            !extractorFn || typeof extractorFn !== "function") {
            throw new Error("Invalid extractor");
        }
        if (extractorName in ayakashiInstance.extractors) {
            throw new Error(`Extractor ${extractorName} already exists`);
        }
        ayakashiInstance.extractors[extractorName] = function () {
            return __awaiter(this, void 0, void 0, function* () {
                if (dependsOn && Array.isArray(dependsOn)) {
                    yield Promise.all(dependsOn.map(function (dependency) {
                        if (dependency in ayakashiInstance.extractors) {
                            return ayakashiInstance.extractors[dependency]();
                        }
                        else {
                            return Promise.resolve();
                        }
                    }));
                }
                yield ayakashiInstance.evaluate(function (name, fn) {
                    this.extractors[name] = fn.bind(this);
                }, extractorName, extractorFn);
            });
        };
    };
}
exports.attachMetaActions = attachMetaActions;