@a11ywatch/core/proto/calls/core-crawl.js

a11ywatch central API

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.crawlStreaming = exports.coreCrawl = void 0;
const crawl_config_1 = require("../../core/streams/crawl-config");
const watcher_crawl_1 = require("../../core/actions/accessibility/watcher_crawl");
const event_1 = require("../../event");
const domain_name_1 = require("../../core/utils/domain-name");
const get_host_1 = require("../../core/utils/get-host");
const utils_1 = require("../../core/utils");
const extracter_1 = require("../../web/params/extracter");
const config_1 = require("../../config");
const scan_attempt_1 = require("../../core/controllers/users/update/scan-attempt");
const controllers_1 = require("../../core/controllers");
const crawl_tracking_1 = require("../../event/crawl-tracking");
const coreCrawl = (call) => __awaiter(void 0, void 0, void 0, function* () {
    var _a, _b;
    const { authorization, url, subdomains, tld, sitemap } = call.request;
    const userNext = (0, utils_1.getUserFromToken)(authorization);
    const userId = (_a = userNext === null || userNext === void 0 ? void 0 : userNext.payload) === null || _a === void 0 ? void 0 : _a.keyid;
    if ((0, extracter_1.validateUID)(userId) || config_1.SUPER_MODE) {
        if (!config_1.SUPER_MODE) {
            const [user] = yield (0, controllers_1.UsersController)().getUser({
                id: userId,
            });
            if ((0, scan_attempt_1.validateScanEnabled)({ user }) === false) {
                return call.end();
            }
        }
        if (crawl_tracking_1.crawlingSet.has((0, crawl_tracking_1.getKey)(url, [], userId))) {
            config_1.SUPER_MODE && console.warn(`crawl in progress for ${url}`);
            return call.end();
        }
        yield (0, exports.crawlStreaming)(yield (0, crawl_config_1.getCrawlConfig)({
            id: userId,
            url,
            role: (_b = userNext === null || userNext === void 0 ? void 0 : userNext.payload) === null || _b === void 0 ? void 0 : _b.audience,
            subdomains,
            tld,
            sitemap,
        }), call);
    }
    else {
        call.end();
    }
});
exports.coreCrawl = coreCrawl;
const crawlStreaming = (props, call) => __awaiter(void 0, void 0, void 0, function* () {
    const { url, userId, subdomains, tld, norobo, robots, proxy, sitemap } = props;
    const crawlKey = `${(0, domain_name_1.domainName)((0, get_host_1.getHostName)(url))}-${userId || 0}`;
    const crawlEvent = `crawl-${crawlKey}`;
    const crawlListener = ({ data }) => call.write({ data });
    const crawlCompleteListener = () => {
        event_1.crawlEmitter.off(crawlEvent, crawlListener);
        call.end();
    };
    event_1.crawlEmitter.on(crawlEvent, crawlListener);
    event_1.crawlTrackingEmitter.once(`crawl-complete-${crawlKey}`, crawlCompleteListener);
    yield (0, watcher_crawl_1.watcherCrawl)({
        url,
        userId,
        subdomains: !!subdomains,
        tld: !!tld,
        scan: true,
        robots: typeof norobo !== "undefined" ? norobo : robots,
        proxy,
        sitemap,
    });
});
exports.crawlStreaming = crawlStreaming;
//# sourceMappingURL=core-crawl.js.map