ayakashi/lib/engine/launcher.js

The next generation web scraping framework

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.launch = void 0;
const net_1 = __importDefault(require("net"));
const child_process_1 = require("child_process");
const debug_1 = __importDefault(require("debug"));
const mkdirp_1 = require("mkdirp");
const fs_1 = require("fs");
const os_1 = require("os");
const path_1 = require("path");
const d = debug_1.default("ayakashi:engine:launcher");
const DEFAULT_FLAGS = [
    // Disable built-in Google Translate service
    "--disable-translate",
    // Disable all chrome extensions entirely
    "--disable-extensions",
    // Disable various background network services, including extension updating,
    // safe browsing service, upgrade detector, translate, UMA
    "--disable-background-networking",
    // Disable fetching safebrowsing lists, likely redundant due to disable-background-networking
    "--safebrowsing-disable-auto-update",
    // Disable syncing to a Google account
    "--disable-sync",
    // Disable reporting to UMA, but allows for collection
    "--metrics-recording-only",
    // Disable installation of default apps on first run
    "--disable-default-apps",
    // Mute any audio
    "--mute-audio",
    // Skip first run wizards
    "--no-first-run",
    "--no-default-browser-check",
    "--no-service-autorun",
    // disable some features
    "--disable-smooth-scrolling",
    "--disable-suggestions-ui",
    "--disable-signin-promo",
    "--disable-password-generation",
    "--disable-cloud-import",
    "--disable-default-apps",
    // disable repost dialog
    "--disable-prompt-on-repost",
    // isolate
    // "--site-per-process",
    // other
    "--disable-gpu",
    //disable password wallet popups on linux
    "--password-store=basic"
    // "--disable-setuid-sandbox",
    // "--no-sandbox"
];
function launch(passedFlags, options) {
    return __awaiter(this, void 0, void 0, function* () {
        try {
            let userDir;
            if (options.sessionDir) {
                d("using session directory:", options.sessionDir);
                mkdirp_1.sync(options.sessionDir);
                userDir = options.sessionDir;
            }
            else {
                d("using tmp session directory");
                userDir = createTmpDir();
            }
            const flags = DEFAULT_FLAGS.concat([
                `--remote-debugging-port=${options.protocolPort}`,
                `--user-data-dir=${path_1.resolve(__dirname, userDir)}`
            ].concat(passedFlags));
            d(`Launching: \n${options.chromePath} ${flags.join(" ")}`);
            const chrome = yield spawnProcess(options.chromePath, options.protocolPort, flags);
            d(`Chrome running with pid ${chrome.pid} on port ${options.protocolPort}.`);
            return {
                pid: chrome.pid,
                port: options.protocolPort,
                process: chrome,
                forceKill: () => forceKill(chrome)
            };
        }
        catch (err) {
            throw err;
        }
    });
}
exports.launch = launch;
function isDebuggerReady(port) {
    return new Promise(function (resolve, reject) {
        const client = net_1.default.createConnection(port);
        client.once("error", err => {
            cleanup(client);
            reject(err);
        });
        client.once("connect", () => {
            cleanup(client);
            resolve();
        });
    });
}
function waitUntilReady(port) {
    return new Promise(function (resolve, reject) {
        let retries = 0;
        function poll() {
            isDebuggerReady(port).then(() => {
                d("Chrome instance is ready");
                resolve(retries);
            })
                .catch(err => {
                if (retries > 10) {
                    d("Max retries reached");
                    return reject(err);
                }
                setTimeout(poll, 500);
            });
            retries += 1;
            d(`Waiting for Chrome instance, retries: ${retries}/10`);
        }
        poll();
    });
}
function spawnProcess(chromePath, port, flags) {
    return __awaiter(this, void 0, void 0, function* () {
        try {
            d("spawning...");
            const chrome = yield child_process_1.spawn(chromePath, flags, { detached: process.platform !== "win32" });
            const chromeDebug = debug_1.default("ayakashi:chrome");
            chrome.stderr.on("data", (data) => {
                chromeDebug(`stderr: ${data}`);
            });
            chrome.stdout.on("data", (data) => {
                chromeDebug(`stdout: ${data}`);
            });
            yield waitUntilReady(port);
            return chrome;
        }
        catch (err) {
            throw err;
        }
    });
}
function cleanup(client) {
    if (client) {
        client.removeAllListeners();
        client.end();
        client.destroy();
        client.unref();
    }
}
function forceKill(chrome) {
    return new Promise((resolve, reject) => {
        chrome.on("exit", () => {
            resolve();
        });
        d("Killing chrome instance");
        try {
            process.kill(chrome.pid);
        }
        catch (err) {
            d(err);
            d(`Chrome could not be killed ${err.message}`);
            reject(err);
        }
    });
}
function createTmpDir() {
    return fs_1.mkdtempSync(path_1.join(os_1.tmpdir(), "ayakashi."));
}