@extra/proxy-router
Version:
A plugin for playwright & puppeteer to route proxies dynamically.
375 lines (368 loc) • 15.9 kB
JavaScript
/*!
* @extra/proxy-router v3.1.5 by berstend
* https://github.com/berstend/puppeteer-extra/tree/master/packages/plugin-proxy-router
* @license MIT
*/
import { PuppeteerExtraPlugin } from 'puppeteer-extra-plugin';
import { Server, RequestError, redactUrl } from 'proxy-chain';
import net from 'net';
import Debug from 'debug';
const isAvailable = (options) => new Promise((resolve, reject) => {
const server = net.createServer();
server.unref();
server.on('error', reject);
server.listen(options, () => {
const { port } = server.address();
server.close(() => {
resolve(port);
});
});
});
const getPort = (options) => {
options = Object.assign({}, options);
if (typeof options.port === 'number') {
options.port = [options.port];
}
return (options.port || []).reduce((seq, port) => seq.catch(() => isAvailable(Object.assign({}, options, { port }))), Promise.reject());
};
var getPort$1 = (options) => options
? getPort(options).catch(() => getPort(Object.assign(options, { port: 0 })))
: getPort({ port: 0 });
class ProxyRouterStats {
constructor(proxyServer) {
this.proxyServer = proxyServer;
/** Log of all connections (id, proxyName, host) */
this.connectionLog = [];
this.connectionStats = new Map();
}
/** @internal */
addConnection(id, proxy, host) {
this.connectionLog.push({ id, proxy, host });
}
/** @internal */
addStats(connectionId, stats) {
this.connectionStats.set(connectionId, stats);
}
/** Get bytes transferred by proxy */
get byProxy() {
this.getStatsFromActiveConnections();
// Get unique proxy names from our actual connection logs
const proxyNames = Array.from(new Set(this.connectionLog.map(({ proxy }) => proxy)));
const getConnectionIdsForProxy = (proxyName) => this.connectionLog
.filter(({ proxy }) => proxy === proxyName)
.map(({ id }) => id);
const trafficByProxy = Object.fromEntries(proxyNames
.map((proxyName) => {
const ids = getConnectionIdsForProxy(proxyName);
const stats = ids.map((id) => this.connectionStats.get(id));
const totalBytes = stats
.map((stat) => this.calculateProxyBytes(stat))
.reduce((a, b) => a + b);
return [proxyName, totalBytes];
})
// Sort by most bytes on top
.sort((a, b) => b[1] - a[1]));
return trafficByProxy;
}
/** Get bytes transferred by host */
get byHost() {
this.getStatsFromActiveConnections();
// Get unique proxy names from our actual connection logs
const hostNames = Array.from(new Set(this.connectionLog.map(({ host }) => host)));
const getConnectionIdsForHost = (hostName) => this.connectionLog
.filter(({ host }) => host === hostName)
.map(({ id }) => id);
const trafficByHost = Object.fromEntries(hostNames
.map((hostName) => {
const ids = getConnectionIdsForHost(hostName);
const stats = ids.map((id) => this.connectionStats.get(id));
const totalBytes = stats
.map((stat) => this.calculateProxyBytes(stat))
.reduce((a, b) => a + b);
return [hostName, totalBytes];
})
// Sort by most bytes on top
.sort((a, b) => b[1] - a[1]));
return trafficByHost;
}
getStatsFromActiveConnections() {
// collect stats for active connections
this.proxyServer.getConnectionIds().forEach((connectionId) => {
const stats = this.proxyServer.getConnectionStats(connectionId);
if (stats) {
this.connectionStats.set(connectionId, stats);
}
});
}
calculateProxyBytes(stats) {
if (!stats) {
return 0;
}
return (stats.trgRxBytes || 0) + (stats.trgTxBytes || 0);
}
}
const debug = Debug('puppeteer-extra:proxy-router');
const debugVerbose = debug.extend('verbose');
const warn = console.warn.bind(console, `\n[proxy-router] %s`); // Preserves line numbers
class ProxyRouter {
constructor(opts = {}) {
var _a, _b;
this.isListening = false;
/** Internal list of failed connections to only print the same connection issue once */
this.failedConnections = [];
const proxyServerOpts = Object.assign(Object.assign({}, opts.proxyServerOpts), { prepareRequestFunction: this.handleProxyServerRequest.bind(this) });
proxyServerOpts.port = proxyServerOpts.port || 2800;
this.proxies = opts.proxies || {};
this.routeByHost = opts.routeByHost || null;
this.proxyServer = new Server(proxyServerOpts);
this.collectStats = (_a = opts.collectStats) !== null && _a !== void 0 ? _a : true;
this.stats = new ProxyRouterStats(this.proxyServer);
this.muteProxyErrors = (_b = opts.muteProxyErrors) !== null && _b !== void 0 ? _b : false;
this.muteProxyErrorsForHost = opts.muteProxyErrorsForHost || [];
debug('initialized', opts);
// Emitted when HTTP connection is closed
this.proxyServer.on('connectionClosed', ({ connectionId, stats }) => {
if (stats && this.collectStats) {
this.stats.addStats(connectionId, stats);
}
debugVerbose(`Connection ${connectionId} closed`);
});
// Emitted when a HTTP request fails
this.proxyServer.on('requestFailed', ({ request, error }) => {
if (!this.muteProxyErrors) {
warn('Request failed:', request.url, error);
}
});
// Emitted in case of a upstream proxy error (which can mean various things)
this.proxyServer.on('proxyAuthenticationFailed', ({ connectionId, str: errorStr, }) => {
// resolve the affected host and proxy
const { host, proxy } = this.stats.connectionLog.find(({ id }) => id === connectionId) || {};
const proxyUrl = !!proxy ? this.getProxyForName(proxy) : null;
const info = [errorStr];
info.push("This error can be thrown if a resource on a site simply can't be accessed (often temporarily), in this case this can be ignored.", ` - To not have errors like this printed to the console you can set 'muteProxyErrors: true' ${!!host ? `or 'muteProxyErrorsForHost: ["${host}"]'` : ''}`, 'It can also indicate incorrect proxy credentials or that the target host is blocked by the proxy.', ' - Make sure the provided proxy string and credentials are correct and the site is not blocked by the proxy (or vice versa).', " - In case the site is blocked by the proxy: Use 'routeByHost' to route the host through a different proxy or as 'DIRECT' or 'ABORT'.");
if (host && proxy) {
info.push('', `Affected target host: "${host}"`, `Affected proxy name: "${proxy}"`);
}
if (proxyUrl) {
info.push(`Affected proxy URL: "${proxyUrl}"`);
info.push('', `To test the proxy with curl: curl -v --proxy '${proxyUrl}' 'https://${host}'`, '');
if (!`${proxyUrl}`.includes('http://')) {
info.push('PS: Did you forget to prefix the proxy with "http://"?');
}
}
const probablyNoise = errorStr.includes('authenticate') && errorStr.includes('522');
const isMuted = this.muteProxyErrors || this.muteProxyErrorsForHost.includes(host);
const alreadySeen = !!this.failedConnections.find((entry) => entry.host === host && entry.proxy === proxy);
const logger = probablyNoise || isMuted || alreadySeen ? debug : warn;
logger(info.join('\n'));
if (host && proxy) {
this.failedConnections.push({ host, proxy });
}
});
// Resurface some errors that proxy-chain seems to swallow
this.proxyServer.log = (function (originalMethod, context) {
return function (connectionId, str) {
if (`${str}`.includes('Failed to authenticate upstream proxy')) {
context.emit('proxyAuthenticationFailed', {
connectionId,
str,
});
}
if (`${str}`.includes('Error: Invalid "upstreamProxyUrl" provided')) {
context.emit('proxyAuthenticationFailed', {
connectionId,
str,
});
}
if (`${str}`.includes('Failed to connect to upstream proxy')) {
context.emit('proxyAuthenticationFailed', {
connectionId,
str,
});
}
originalMethod.apply(context, [connectionId, str]);
};
})(this.proxyServer.log, this.proxyServer);
}
/** Proxy server URL of the local proxy server used for routing */
get proxyServerUrl() {
var _a;
const port = (_a = this.proxyServer) === null || _a === void 0 ? void 0 : _a.port;
if (!port || !this.isListening) {
return;
}
return `http://localhost:${port}`;
}
get effectiveProxies() {
return Object.assign({ DIRECT: null }, (this.proxies || {}));
}
/** Start the local proxy server and accept connections */
async listen() {
debug('starting server..');
if (this.serverStartPromise) {
debug('server start promise exists already');
return this.serverStartPromise;
}
this.serverStartPromise = new Promise(async (resolve) => {
if (this.isListening) {
debug('server listening already');
return resolve(this.proxyServer.port);
}
const desiredPort = this.proxyServer.port;
debug('finding available port', { desiredPort });
const availablePort = await getPort$1({ port: desiredPort });
debug('availablePort:', availablePort);
this.proxyServer.port = availablePort;
this.proxyServer.listen((err) => {
if (err === null) {
debug(`server listening on port ${this.proxyServer.port}`);
this.isListening = true;
return resolve(this.proxyServer.port);
}
warn('Unable to start local server:', err);
});
});
return this.serverStartPromise;
}
/** Stop the local proxy server */
async close() {
debug('closing..');
return new Promise((resolve) => {
this.proxyServer.close(true, (err) => {
if (err === null) {
debug('closed without error');
return resolve(null);
}
debug('closed with error', err);
return resolve(err);
});
});
}
getProxyForName(name) {
return this.effectiveProxies[name];
}
/** Handle requests to the proxy server */
async handleProxyServerRequest({ request, hostname: host, port, connectionId, isHttp, }) {
let proxyName = 'DEFAULT';
if (!!this.routeByHost) {
const fnResult = await this.routeByHost({ host, isHttp, port });
if (typeof fnResult === 'string' && !!fnResult) {
proxyName = fnResult;
}
}
if (this.collectStats) {
this.stats.addConnection(connectionId, proxyName, host);
}
let proxyUrl = this.getProxyForName(proxyName);
debugVerbose('handleProxyServerRequest', host, proxyName, redactProxyUrl(proxyUrl));
if (proxyName === 'ABORT') {
throw new RequestError('Request aborted', 400);
}
if (!proxyUrl && proxyUrl !== null) {
warn(`No proxy configured for proxy name "${proxyName}" - configuration error?`);
proxyUrl = null;
}
return {
upstreamProxyUrl: proxyUrl,
};
}
}
function redactProxyUrl(input) {
if (!input || typeof input !== 'string') {
return `${input}`;
}
try {
return redactUrl(input);
}
catch (err) {
return `${input}`;
}
}
/** Standalone proxy router not requiring plugin events */
const ProxyRouterStandalone = ProxyRouter;
class ExtraPluginProxyRouter extends PuppeteerExtraPlugin {
constructor(opts) {
super(opts);
/** The name of the automation framework used */
this.framework = null;
// Disable the puppeteer compat shim when used with playwright-extra
this.noPuppeteerShim = true;
this.debug('Initialized', this.opts);
this.router = new ProxyRouter(this.opts);
}
get name() {
return 'proxy-router';
}
get defaults() {
return {
collectStats: true,
proxyServerOpts: {
port: 2800,
},
};
}
// Make accessing router methods shorter
/** Get or set proxies at runtime */
get proxies() {
return this.router.proxies;
}
set proxies(proxies) {
this.router.proxies = proxies;
}
/** Retrieve traffic statistics */
get stats() {
return this.router.stats;
}
/** Get or set the `routeByHost` function at runtime */
get routeByHost() {
return this.router.routeByHost;
}
set routeByHost(fn) {
this.router.routeByHost = fn;
}
get proxyBypassListString() {
return (this.opts.proxyBypassList || []).join(',') || undefined;
}
async onPluginRegistered(args) {
this.framework =
(args === null || args === void 0 ? void 0 : args.framework) === 'playwright' ? 'playwright' : 'puppeteer';
this.debug('plugin registered', this.framework);
}
async beforeLaunch(options = {}) {
this.debug('beforeLaunch - before', options);
await this.router.listen();
const proxyUrl = this.router.proxyServerUrl;
if (!proxyUrl) {
throw new Error('No local proxy server available');
}
if (this.framework === 'playwright') {
const pwOptions = options;
pwOptions.proxy = {
server: proxyUrl,
bypass: this.proxyBypassListString,
};
}
else if (this.framework === 'puppeteer') {
const pptrOptions = options;
pptrOptions.args = pptrOptions.args || [];
pptrOptions.args.push(`--proxy-server=${proxyUrl}`);
if (this.proxyBypassListString) {
pptrOptions.args.push(`--proxy-bypass-list=${this.proxyBypassListString}`);
}
}
else {
this.debug('Unsupported framework, not setting proxy');
}
this.debug('beforeLaunch - after', options);
}
async onDisconnected() {
await this.router.close().catch(this.debug);
}
}
/** Default export, ExtraPluginProxyRouter */
const defaultExport = (options) => {
return new ExtraPluginProxyRouter(options || {});
};
export default defaultExport;
export { ExtraPluginProxyRouter, ProxyRouter, ProxyRouterStandalone, ProxyRouterStats };
//# sourceMappingURL=index.esm.js.map