@a11ywatch/core/core/utils/crawl-stream-slim.js

a11ywatch central API

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.crawlHttpStreamSlim = void 0;
const watcher_crawl_1 = require("../actions/accessibility/watcher_crawl");
const crawl_tracking_1 = require("../../event/crawl-tracking");
const event_1 = require("../../event");
const get_host_1 = require("./get-host");
const names_1 = require("../../event/names");
const crawlHttpStreamSlim = (props, res, removeTrailing = true) => {
    const { url, userId, subdomains, tld, robots, agent, proxy, sitemap } = props;
    const domain = (0, get_host_1.getHostName)(url);
    const crawlEvent = (0, names_1.getActiveCrawlKey)(domain, userId);
    const key = (0, crawl_tracking_1.getKey)(domain, undefined, userId);
    return new Promise((resolve) => {
        const crawlListener = removeTrailing
            ? (source) => {
                setImmediate(() => {
                    const { data } = source;
                    if (data && !res.raw.writableEnded) {
                        const crawlSource = crawl_tracking_1.crawlingSet.has(key);
                        res.raw.write(`${JSON.stringify(data)}${!crawlSource ? "" : ","}`);
                    }
                });
            }
            : (source) => {
                setImmediate(() => {
                    const data = source === null || source === void 0 ? void 0 : source.data;
                    if (data && !res.raw.writableEnded) {
                        res.raw.write(`${JSON.stringify(data)},`);
                    }
                });
            };
        event_1.crawlEmitter.on(crawlEvent, crawlListener);
        const crawlComplete = () => {
            setImmediate(() => {
                event_1.crawlTrackingEmitter.off(crawlEvent, crawlListener);
                resolve(true);
            });
        };
        event_1.crawlTrackingEmitter.once(`crawl-complete-${key}`, crawlComplete);
        setImmediate(() => __awaiter(void 0, void 0, void 0, function* () {
            return yield (0, watcher_crawl_1.watcherCrawl)({
                url,
                userId,
                subdomains: !!subdomains,
                tld: !!tld,
                scan: true,
                robots,
                agent,
                proxy,
                sitemap,
            });
        }));
    });
};
exports.crawlHttpStreamSlim = crawlHttpStreamSlim;
//# sourceMappingURL=crawl-stream-slim.js.map