ayakashi
Version:
The next generation web scraping framework
345 lines (344 loc) • 17.1 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createConnection = void 0;
const CDP = require("chrome-remote-interface");
const debug_1 = __importDefault(require("debug"));
//@ts-ignore
const input_1 = require("@ayakashi/input");
const retryOnErrorOrTimeout_1 = require("../utils/retryOnErrorOrTimeout");
const client_1 = require("../bridge/client");
const marshalling_1 = require("../utils/marshalling");
const d = debug_1.default("ayakashi:engine:connection");
function createConnection(target, bridgePort, emulatorOptions) {
return __awaiter(this, void 0, void 0, function* () {
try {
d("creating new connection", target.targetId);
const bridgeClient = client_1.getBridgeClient(bridgePort);
const client = yield retryOnErrorOrTimeout_1.retryOnErrorOrTimeOut(function () {
return __awaiter(this, void 0, void 0, function* () {
const _client = yield CDP({ target: target.webSocketDebuggerUrl });
yield Promise.all([
_client.Network.enable(),
_client.Page.enable(),
_client.DOM.enable(),
_client.CSS.enable(),
_client.DOMStorage.enable(),
_client.Security.enable(),
_client.Console.enable(),
_client.Runtime.enable()
]);
return _client;
});
});
d("connection created");
const defaultEmulatorOptions = {
width: 1920,
height: 1080,
mobile: false,
deviceScaleFactor: 0
};
yield client.Emulation.setDeviceMetricsOverride({
width: (emulatorOptions && emulatorOptions.width) || defaultEmulatorOptions.width,
height: (emulatorOptions && emulatorOptions.height) || defaultEmulatorOptions.height,
mobile: (emulatorOptions && emulatorOptions.mobile) || defaultEmulatorOptions.mobile,
deviceScaleFactor: (emulatorOptions && emulatorOptions.deviceScaleFactor) ||
defaultEmulatorOptions.deviceScaleFactor
});
yield client.Emulation.setVisibleSize({
width: (emulatorOptions && emulatorOptions.width) || defaultEmulatorOptions.width,
height: (emulatorOptions && emulatorOptions.height) || defaultEmulatorOptions.height
});
yield client.Page.setBypassCSP({ enabled: true });
const _keyBoard = new input_1.Keyboard(client);
const _mouse = new input_1.Mouse(client, _keyBoard);
const _touchScreen = new input_1.Touchscreen(client, _keyBoard);
const connection = {
target,
client,
namespace: "",
active: false,
preloaderIds: [],
preloaders: [],
unsubscribers: [],
timeouts: [],
intervals: [],
keyBoard: _keyBoard,
mouse: _mouse,
touchScreen: _touchScreen,
activate: function () {
return __awaiter(this, void 0, void 0, function* () {
d(`activating connection`);
if (connection.active)
throw new Error("connection_already_active");
connection.active = true;
d("connection activated");
});
},
release: function () {
return __awaiter(this, void 0, void 0, function* () {
d(`releasing connection`);
if (!connection.active)
throw new Error("connection_not_active");
try {
yield retryOnErrorOrTimeout_1.retryOnErrorOrTimeOut(function () {
return __awaiter(this, void 0, void 0, function* () {
yield client.Page.stopLoading();
yield removePreloaders(connection);
yield cleanup(connection);
yield bridgeClient.connectionReleased(target);
connection.active = false;
});
}, 2);
d(`connection released`);
}
catch (err) {
d(err);
d("force-releasing connection");
yield cleanup(connection);
yield bridgeClient.connectionReleased(target);
connection.active = false;
}
});
},
useNameSpace: function (ns) {
return __awaiter(this, void 0, void 0, function* () {
this.namespace = ns;
const scriptId = yield client.Page.addScriptToEvaluateOnNewDocument({
source: `
window['${this.namespace}'] = {};
window['${this.namespace}'].paused = false;
window['${this.namespace}'].resume = function() {
window['${this.namespace}'].paused = false;
};
window['${this.namespace}'].propTable = {};
window['${this.namespace}'].extractors = {};
window['${this.namespace}'].preloaders = {};
window['${this.namespace}'].document = document;
window['${this.namespace}'].window = window;
`
});
connection.preloaderIds.push(scriptId);
});
},
evaluate: function (fn, ...args) {
return __awaiter(this, void 0, void 0, function* () {
d(`evaluating expression on connection`);
return evaluate(this, false, fn, ...args);
});
},
evaluateAsync: function (fn, ...args) {
return __awaiter(this, void 0, void 0, function* () {
d(`evaluating async expression on connection`);
return evaluate(this, true, fn, ...args);
});
},
injectPreloader: function ({ compiled: { wrapper, source }, as, waitForDOM }) {
return __awaiter(this, void 0, void 0, function* () {
if (connection.active)
throw new Error("cannot_inject_preloader_into_active_connection");
if (connection.preloaders.indexOf(wrapper) > -1)
return;
connection.preloaders.push(wrapper);
const hasAlias = !!as;
//tslint:disable max-line-length
if (waitForDOM) {
const scriptId = yield client.Page.addScriptToEvaluateOnNewDocument({
source: `
(function() {
document.addEventListener("DOMContentLoaded", function() {
${source}
if (typeof ${connection.namespace}__${wrapper} !== "function" && ${connection.namespace}__${wrapper}.default) {
${connection.namespace}__${wrapper} = ${connection.namespace}__${wrapper}.default;
}
if (${hasAlias}) {
window['${connection.namespace}'].preloaders["${as}"] = ${connection.namespace}__${wrapper};
} else {
window['${connection.namespace}'].preloaders["${wrapper}"] = ${connection.namespace}__${wrapper};
}
});
}).call(window);
`
});
connection.preloaderIds.push(scriptId);
}
else {
const scriptId = yield client.Page.addScriptToEvaluateOnNewDocument({
source: `
(function() {
${source}
if (typeof ${connection.namespace}__${wrapper} !== "function" && ${connection.namespace}__${wrapper}.default) {
${connection.namespace}__${wrapper} = ${connection.namespace}__${wrapper}.default;
}
if (${hasAlias}) {
window['${connection.namespace}'].preloaders["${as}"] = ${connection.namespace}__${wrapper};
} else {
window['${connection.namespace}'].preloaders["${wrapper}"] = ${connection.namespace}__${wrapper};
}
}).call(window);
`
});
connection.preloaderIds.push(scriptId);
}
//tslint:enable
d("injected preloader:", wrapper);
});
},
pipe: {
console: function (listener) {
pipeEvent(connection, "Console", "messageAdded", function (consoleMsg) {
listener(consoleMsg.message.text);
});
},
uncaughtException: function (listener) {
pipeEvent(connection, "Runtime", "exceptionThrown", function (exception) {
listener({
text: exception.exceptionDetails.exception.description,
lineNumber: exception.exceptionDetails.lineNumber,
columnNumber: exception.exceptionDetails.columnNumber,
stackTrace: exception.exceptionDetails.stackTrace
});
});
}
}
};
return connection;
}
catch (err) {
d(err);
throw new Error("could_not_create_connection");
}
});
}
exports.createConnection = createConnection;
//awaiter polyfill extracted from typescript generated code
const awaiterPolyfill = `var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};`;
//tslint:disable no-any
function evaluate(connection, awaitPromise, fn, ...args
//tslint:enable no-any
) {
return __awaiter(this, void 0, void 0, function* () {
if (!connection.active)
throw new Error("connection_not_active");
try {
let exp;
const namespace = connection.namespace ? `window['${connection.namespace}']` : null;
if (args && args.length > 0) {
exp = `(function() {
"use strict";\n
${awaitPromise ? awaiterPolyfill : ""}\n
const serializedArgs = '${JSON.stringify(args, marshalling_1.replacer)}';\n
const args = JSON.parse(serializedArgs, ${namespace}.preloaders.marshalling.getReviver("${namespace}"));\n
let result = (${String(fn)}).apply(${namespace}, args);
if (result instanceof Promise) {
return result.then(function(resolved) {
return JSON.stringify(resolved, ${namespace}.preloaders.marshalling.replacer);
});
} else {
return JSON.stringify(result, ${namespace}.preloaders.marshalling.replacer);
}
})()`;
}
else {
exp = `
(function() {
"use strict";\n
${awaitPromise ? awaiterPolyfill : ""}\n
let result = (${String(fn)}).apply(${namespace});
if (result instanceof Promise) {
return result.then(function(resolved) {
return JSON.stringify(resolved, ${namespace}.preloaders.marshalling.replacer);
});
} else {
return JSON.stringify(result, ${namespace}.preloaders.marshalling.replacer);
}
})()`;
}
const evaled = yield connection.client.Runtime.evaluate({
expression: exp,
returnByValue: true,
awaitPromise: awaitPromise
});
if (evaled.exceptionDetails) {
//@ts-ignore
throw new EvalError(evaled.exceptionDetails.exception.description || evaled.exceptionDetails.text);
}
else {
if (typeof evaled.result.value === "string") {
const reviver = marshalling_1.getReviver("{ayakashi: {}}");
return JSON.parse(evaled.result.value, reviver);
}
else {
return evaled.result.value;
}
}
}
catch (err) {
d(err);
throw err;
}
});
}
function EvalError(message) {
this.name = "EvalError";
this.message = message;
}
EvalError.prototype = new Error();
function pipeEvent(connection, domain, eventName,
//tslint:disable no-any
listener
//tslint:enable no-any
) {
if (connection.active)
throw new Error("cannot_pipe_events_into_active_connection");
d(`piping ${domain}.${eventName}`);
connection.client.on(`${domain}.${eventName}`, listener);
connection.unsubscribers.push(function () {
connection.client.removeListener(`${domain}.${eventName}`, listener);
});
}
function removePreloaders(connection) {
return __awaiter(this, void 0, void 0, function* () {
if (connection.client && connection.client._ws && connection.client._ws.readyState !== 3) {
yield Promise.all(connection.preloaderIds
.map(preloaderId => connection.client.Page.removeScriptToEvaluateOnNewDocument(preloaderId)));
}
connection.preloaderIds = [];
});
}
function cleanup(connection) {
return __awaiter(this, void 0, void 0, function* () {
connection.unsubscribers.forEach(unsubscriber => unsubscriber());
connection.unsubscribers = [];
connection.timeouts.forEach(function (id) {
clearTimeout(id);
});
connection.intervals.forEach(function (id) {
clearInterval(id);
});
connection.timeouts = [];
connection.intervals = [];
if (connection.client && connection.client._ws && connection.client._ws.readyState !== 3) {
yield connection.client.close();
}
});
}