@luminati-io/luminati-proxy
Version:
A configurable local proxy for luminati.io
1,421 lines (1,350 loc) • 110 kB
JavaScript
#!/usr/bin/env node
// LICENSE_CODE ZON ISC
'use strict'; /*jslint node:true, esnext:true*/
const events = require('events');
const fs = require('fs');
const path = require('path');
const os = require('os');
const dns = require('dns');
configure_dns();
const url = require('url');
const express = require('express');
const compression = require('compression');
const body_parser = require('body-parser');
const request = require('request').defaults({gzip: true});
const net = require('net');
const http = require('http');
const https = require('https');
const http_shutdown = require('http-shutdown');
const util = require('util');
const forge = require('node-forge');
const {Netmask} = require('netmask');
const cookie = require('cookie');
const cookie_filestore = require('tough-cookie-file-store');
const check_node_version = require('check-node-version');
const pkg = require('../package.json');
const zconfig = require('../util/config.js');
const zerr = require('../util/zerr.js');
const etask = require('../util/etask.js');
const conv = require('../util/conv.js');
const zcountry = require('../util/country.js');
const string = require('../util/string.js');
const file = require('../util/file.js');
const date = require('../util/date.js');
const user_agent = require('../util/user_agent.js');
const lpm_config = require('../util/lpm_config.js');
const zurl = require('../util/url.js');
const zutil = require('../util/util.js');
const {get_perm, is_static_proxy, is_mobile, is_unblocker,
get_password, get_gb_cost} = require('../util/zones.js');
const logger = require('./logger.js').child({category: 'MNGR'});
const consts = require('./consts.js');
const Proxy_port = require('./proxy_port.js');
const ssl = require('./ssl.js');
const cities = require('./cities.js');
const perr = require('./perr.js');
const util_lib = require('./util.js');
const web_socket = require('ws');
const Loki = require('./loki.js');
const Zagent_api = require('./zagent_api.js');
const Timeouts = require('./timeouts.js');
const Lpm_f = require('./lpm_f.js');
const Lpm_conn = require('./lpm_conn.js');
const Cache_report = require('./cache_report.js');
const Cluster_mgr = require('./cluster_mgr.js');
const puppeteer = require('./puppeteer.js');
const Config = require('./config.js');
const Stat = require('./stat.js');
const is_darwin = process.platform=='darwin';
let zos;
if (!lpm_config.is_win && !is_darwin)
zos = require('../util/os.js');
if (process.env.LPM_DEBUG)
require('longjohn');
try { require('heapdump'); } catch(e){}
let cookie_jar;
zerr.set_level('CRIT');
const qw = string.qw;
const E = module.exports = Manager;
// XXX krzysztof: why is it generated here, not in ssl.js?
let keys = forge.pki.rsa.generateKeyPair(2048);
keys.privateKeyPem = forge.pki.privateKeyToPem(keys.privateKey);
keys.publicKeyPem = forge.pki.publicKeyToPem(keys.publicKey);
function configure_dns(){
const google_dns = ['8.8.8.8', '8.8.4.4'];
const original_dns = dns.getServers();
const servers = google_dns.concat(original_dns.filter(
d=>!google_dns.includes(d)));
// dns.setServers cashes node if there is an in-flight dns resolution
// should be done before any requests are made
// https://github.com/nodejs/node/issues/14734
dns.setServers(servers);
}
E.default = Object.assign({}, lpm_config.manager_default);
const check_running = argv=>etask(function*(){
const tasks = yield util_lib.get_lpm_tasks();
if (!tasks.length)
return;
if (!argv.dir)
{
logger.notice(`LPM is already running (${tasks[0].pid})`);
logger.notice('You need to pass a separate path to the directory for '
+'this LPM instance. Use --dir flag');
process.exit();
}
});
const apply_explicit_mgr_opt = (_defaults, args)=>{
args = zutil.clone_deep(args);
const ips_fields = ['whitelist_ips', 'www_whitelist_ips',
'extra_ssl_ips'];
ips_fields.forEach(f=>{
if (args[f])
args[f] = [...new Set([..._defaults[f]||[], ...args[f]||[]])];
});
return Object.assign(_defaults, args);
};
const empty_wss = {
close: ()=>null,
broadcast: (data, type)=>{
logger.debug('wss is not ready, %s will not be emitted', type);
},
};
function Manager(argv, run_config={}){
events.EventEmitter.call(this);
logger.notice([
`Running Luminati Proxy Manager`,
`PID: ${process.pid}`,
`Node: ${process.versions.node}`,
`Version: ${pkg.version}`,
`Build date: ${zconfig.CONFIG_BUILD_DATE}`,
`Os version: ${os.platform()} ${os.arch()} ${os.release()}`,
`Host name: ${os.hostname()}`,
].join('\n'));
try {
this.cluster_mgr = new Cluster_mgr(this);
this.proxy_ports = {};
this.zones = [];
this.argv = argv;
this.mgr_opts = zutil.pick(argv, ...lpm_config.mgr_fields);
this.opts = zutil.pick(argv, ...Object.keys(lpm_config.proxy_fields));
this.config = new Config(this, E.default, {filename: argv.config});
const conf = this.config.get_proxy_configs();
const explicit_mgr_opt = this.argv.explicit_mgr_opt||{};
this._defaults = apply_explicit_mgr_opt(conf._defaults,
explicit_mgr_opt);
this.proxies = conf.proxies;
this.config_ts = conf.ts||date();
this.pending_www_ips = new Set();
this.pending_ips = new Set();
this.config.save({skip_cloud_update: 1});
this.loki = new Loki(argv.loki);
this.timeouts = new Timeouts();
this.ensure_socket_close = util_lib.ensure_socket_close.bind(null,
this.timeouts);
this.long_running_ets = [];
this.async_reqs_queue = [];
this.async_active = 0;
this.tls_warning = false;
this.lpm_users = [];
this.conn = {};
this.config_changes = [];
this.wss = empty_wss;
this.is_upgraded = run_config.is_upgraded;
this.backup_exist = run_config.backup_exist;
this.conflict_shown = false;
this.lpm_conn = new Lpm_conn();
this.lpm_f = new Lpm_f(this);
this.lpm_f.on('server_conf', server_conf=>{
logger.notice('Updated server configuration');
this.server_conf = server_conf;
});
this.stat = new Stat(this);
this.cache_report = new Cache_report();
this.on('error', (err, fatal)=>{
let match;
if (match = err.message.match(/EADDRINUSE.+:(\d+)/))
return this.show_port_conflict(match[1], argv.force);
const err_msg = err.raw ? err.message : 'Unhandled error: '+err;
logger.error(err_msg);
const handle_fatal = ()=>{
if (fatal)
this.stop(err_msg);
};
if (!perr.enabled || err.raw)
handle_fatal();
else
{
let error = zerr.e2s(err);
if (typeof error=='object')
{
try { error = JSON.stringify(err); }
catch(e){ error = err && err.message || err; }
}
this.perr('crash', {error});
handle_fatal();
}
});
} catch(e){
logger.error('constructor: %s', zerr.e2s(e));
throw e;
}
}
function get_content_type(data){
if (data.response_body=='unknown')
return 'unknown';
let content_type;
let res = 'other';
try {
const headers = JSON.parse(data.response_headers);
content_type = headers['content-type']||'';
} catch(e){ content_type = ''; }
if (content_type.match(/json/))
res = 'xhr';
else if (content_type.match(/html/))
res = 'html';
else if (content_type.match(/javascript/))
res = 'js';
else if (content_type.match(/css/))
res = 'css';
else if (content_type.match(/image/))
res = 'img';
else if (content_type.match(/audio|video/))
res = 'media';
else if (content_type.match(/font/))
res = 'font';
return res;
}
util.inherits(Manager, events.EventEmitter);
E.prototype.show_port_conflict = etask._fn(function*(_this, port, force){
if (_this.conflict_shown)
return;
_this.conflict_shown = true;
yield _this._show_port_conflict(port, force);
});
E.prototype._show_port_conflict = etask._fn(function*(_this, port, force){
const tasks = yield util_lib.get_lpm_tasks();
if (!tasks.length)
return logger.error(`There is a conflict on port ${port}`);
const pid = tasks[0].pid;
logger.notice(`LPM is already running (${pid}) and uses port ${port}`);
if (!force)
{
logger.notice('If you want to kill other instances use --force flag');
return process.exit();
}
logger.notice('Trying to kill it and restart.');
for (const t of tasks)
process.kill(t.ppid, 'SIGTERM');
_this.restart();
});
E.prototype.handle_usage = function(data){
let url_parts;
if (url_parts = data.url.match(/^([^/]+?):(\d+)$/))
{
data.protocol = url_parts[2]==443 ? 'https' : 'http';
data.hostname = url_parts[1];
}
else
{
const {protocol, hostname} = url.parse(data.url);
data.protocol = (protocol||'https:').slice(0, -1);
data.hostname = hostname;
}
if (!data.hostname)
this.perr('empty_hostname', data);
if (!util_lib.is_ip(data.hostname||''))
data.hostname = (data.hostname||'').split('.').slice(-2).join('.');
data.content_type = get_content_type(data);
data.success = +(data.status_code && (data.status_code=='unknown' ||
consts.SUCCESS_STATUS_CODE_RE.test(data.status_code)));
const proxy = this.proxy_ports[data.port];
if (proxy && proxy.status!='ok' && data.success)
this.proxy_ports[data.port].status = 'ok';
this.stat.process(data);
if (!this.argv.www || this.argv.high_perf)
return;
if (this._defaults.request_stats)
this.loki.stats_process(data, zutil.get(proxy, 'opt.gb_cost', 0));
this.logs_process(data);
this.lpm_f.sync_recent_stats();
};
E.prototype.handle_usage_abort = function(data){
if (!this.argv.www || this.argv.high_perf)
return;
this.logs_process(data);
};
E.prototype.handle_usage_start = function(data){
if (!Number(this._defaults.logs))
return;
const req = {
uuid: data.uuid,
details: {
port: data.port,
context: data.context,
timestamp: data.timestamp,
timeline: [],
},
request: {
url: data.url,
method: data.method,
headers: headers_to_a(data.headers),
},
response: {content: {}},
};
this.wss.broadcast(req, 'har_viewer_start');
};
E.prototype.logs_process = function(data){
const har_req = this.har([data]).log.entries[0];
const max_logs = Number(this._defaults.logs);
if (!max_logs)
return this.emit('request_log', har_req);
this.wss.broadcast(har_req, 'har_viewer');
this.emit('request_log', har_req);
this.loki.request_process(data, max_logs);
};
E.prototype.stop_servers = etask._fn(
function*mgr_stop_servers(_this){
let servers = [];
const stop_server = server=>servers.push(etask(function*mgr_stop_server(){
try {
yield server.stop();
} catch(e){
logger.error('Failed to stop server: %s', e.message);
}
}));
if (_this.www_server)
stop_server(_this.www_server);
if (_this.zagent_server)
stop_server(_this.zagent_server);
Object.values(_this.proxy_ports).forEach(proxy_port=>{
if (proxy_port.opt.proxy_type!='duplicate')
stop_server(proxy_port);
});
_this.wss.close();
yield etask.all(servers);
});
E.prototype.stop = etask._fn(
function*mgr_stop(_this, reason, force, restart){
_this.timeouts.clear();
_this.long_running_ets.forEach(et=>et.return());
yield _this.perr(restart ? 'restart' : 'exit', {reason});
yield _this.loki.save();
_this.loki.stop();
if (reason!='config change')
_this.config.save({skip_cloud_update: 1});
if (reason instanceof Error)
reason = zerr.e2s(reason);
logger.notice('Manager stopped: %s', reason);
_this.lpm_f.close();
_this.lpm_conn.close();
yield _this.stop_servers();
_this.cluster_mgr.kill_workers();
if (!restart)
_this.emit('stop', reason);
});
const headers_to_a = h=>Object.entries(h).map(p=>({name: p[0], value: p[1]}));
E.prototype.har = function(entries){
return {log: {
version: '1.2',
creator: {name: 'Luminati Proxy', version: pkg.version},
pages: [],
entries: entries.map(entry=>{
const req = JSON.parse(entry.request_headers||'{}');
const res = JSON.parse(entry.response_headers||'{}');
const timeline = JSON.parse(entry.timeline||null)||[{}];
entry.request_body = entry.request_body||'';
const start = timeline[0].create;
return {
uuid: entry.uuid,
details: {
context: entry.context,
out_bw: entry.out_bw,
in_bw: entry.in_bw,
bw: entry.bw||entry.out_bw+entry.in_bw,
proxy_peer: entry.proxy_peer,
protocol: entry.protocol,
port: entry.port,
timestamp: entry.timestamp,
content_type: entry.content_type,
success: entry.success,
timeline: timeline.map(t=>({
// first_byte: may be undefined when there was no body
// connect: may be undefined for http requests
blocked: t.create-start,
wait: t.connect-t.create||0,
ttfb: t.first_byte-(t.connect||start)||0,
receive: t.end-(t.first_byte||t.connect||start),
port: t.port,
})),
super_proxy: entry.super_proxy,
username: entry.username,
password: entry.password,
remote_address: entry.remote_address,
rules: entry.rules,
},
startedDateTime: new Date(start||0).toISOString(),
time: timeline.slice(-1)[0].end-start,
request: {
method: entry.method,
url: entry.url,
host: entry.hostname,
httpVersion: 'unknown',
cookies: [],
headers: headers_to_a(req),
headersSize: -1,
postData: {
mimeType: req['content-type']||req['Content-Type']||'',
text: entry.request_body,
},
bodySize: entry.request_body.length||0,
queryString: [],
},
response: {
status: entry.status_code,
statusText: entry.status_message||'',
httpVersion: 'unknown',
cookies: [],
headers: headers_to_a(res),
content: {
size: entry.content_size||0,
mimeType: res['content-type']||'unknown',
text: entry.response_body||'',
},
headersSize: -1,
bodySize: entry.content_size,
redirectURL: '',
},
cache: {},
// XXX krzysztof: check if can be removed and still be a
// correct HAR file
timings: {
blocked: 0,
dns: 0,
ssl: 0,
connect: 0,
send: 0,
wait: 0,
receive: 0,
},
serverIPAddress: entry.super_proxy,
comment: entry.username,
};
}),
}};
};
E.prototype.get_zones = function(req, res){
const zones = this.zones.map(z=>({
name: z.zone,
perm: z.perm,
plan: z.plan || {},
password: z.password,
refresh_cost: z.refresh_cost,
})).filter(p=>p.plan && !p.plan.disable);
return {zones, def: this._defaults.zone};
};
E.prototype.get_zones_api = function(req, res){
res.json(this.get_zones());
};
E.prototype.get_consts_api = function(req, res){
const proxy = Object.entries(lpm_config.proxy_fields).reduce(
(acc, [k, v])=>Object.assign(acc, {[k]: {desc: v}}), {});
Object.getOwnPropertyNames(E.default)
.filter(E.default.propertyIsEnumerable.bind(E.default))
.forEach(k=>proxy[k] && Object.assign(proxy[k], {def: E.default[k]}));
if (proxy.zone)
proxy.zone.def = this._defaults.zone;
proxy.dns.values = ['', 'local', 'remote'];
const ifaces = Object.keys(os.networkInterfaces())
.map(iface=>({key: iface, value: iface}));
ifaces.unshift({key: 'All', value: '0.0.0.0'});
ifaces.unshift({key: 'dynamic (default)', value: ''});
proxy.iface.values = ifaces;
res.json({proxy, consts});
};
E.prototype.enable_ssl_api = etask._fn(
function*mgr_enable_ssl(_this, req, res){
const port = req.body.port;
let proxies = _this.proxies.slice();
if (port)
proxies = proxies.filter(p=>p.port==port);
for (let i in proxies)
{
const p = proxies[i];
if (p.port!=_this._defaults.dropin_port && !p.ssl)
yield _this.proxy_update(p, {ssl: true});
}
res.send('ok');
});
E.prototype.update_ips_api = etask._fn(
function*mgr_update_ips(_this, req, res){
const ips = req.body.ips||[];
const vips = req.body.vips||[];
const proxy = _this.proxies.find(p=>p.port==req.body.port);
yield _this.proxy_update(proxy, {ips, vips});
res.send('ok');
});
E.prototype.report_bug_api = etask._fn(
function*mgr_report_bug(_this, req, res){
let log_file = '';
const config_file = Buffer.from(_this.config.get_string())
.toString('base64');
if (file.exists(logger.lpm_filename))
{
let buffer = fs.readFileSync(logger.lpm_filename);
buffer = buffer.slice(buffer.length-50000);
log_file = buffer.toString('base64');
}
const reqs = _this.filtered_get({query: {limit: 100}}).items.map(x=>({
url: x.url,
status_code: x.status_code,
}));
const har = JSON.stringify(reqs);
const browser = user_agent.guess_browser(req.get('user-agent')).browser;
const response = yield _this.api_request({
method: 'POST',
endpoint: '/lpm/report_bug',
form: {report: {config: config_file, log: log_file, har,
desc: req.body.desc, lpm_v: pkg.version, email: req.body.email,
browser, os: util_lib.format_platform(os.platform())}},
});
res.status(response.statusCode).json(response.body);
});
E.prototype.get_fixed_whitelist = function(){
return (this.opts.whitelist_ips||[]).concat(
this._defaults.www_whitelist_ips||[]);
};
E.prototype.get_default_whitelist = function(){
return this.get_fixed_whitelist().concat(this._defaults.whitelist_ips||[]);
};
E.prototype.set_www_whitelist_ips = function(ips){
const prev = this.get_default_whitelist();
ips = [...new Set(ips)];
ips.forEach(ip=>this.pending_www_ips.delete(ip));
if (!ips.length)
delete this._defaults.www_whitelist_ips;
else
this._defaults.www_whitelist_ips = ips;
this.set_whitelist_ips(this._defaults.whitelist_ips||[], prev);
};
E.prototype.set_whitelist_ips = function(ips, prev){
const fixed_whitelist = this.get_fixed_whitelist();
ips = [...new Set(ips)];
ips.forEach(ip=>this.pending_ips.delete(ip));
ips = ips.filter(ip=>!fixed_whitelist.includes(ip));
prev = prev||this.get_default_whitelist();
if (!ips.length)
delete this._defaults.whitelist_ips;
else
{
this._defaults.whitelist_ips = ips.map(ip=>{
try {
const _ip = new Netmask(ip);
const mask = _ip.bitmask==32 ? '' : '/'+_ip.bitmask;
return _ip.base+mask;
} catch(e){ return null; }
}).filter(ip=>ip!==null && ip!='127.0.0.1');
}
this.update_ports({
whitelist_ips: {
default: 1,
prev,
curr: this.get_default_whitelist(),
},
});
};
E.prototype.error_handler = function error_handler(source, err){
if (!err.code && err.stack)
logger.error(err.stack.split('\n').slice(0, 2).join('\n'));
else if (err.code=='EMFILE')
return logger.error('EMFILE: out of file descriptors');
else
logger.error(err.message);
err.source = source;
this.emit('error', err);
};
E.prototype.complete_proxy_config = function(conf){
const c = Object.assign({}, E.default, this._defaults, conf);
const zone = this.zones.find(z=>z.zone==c.zone);
const {plan, perm} = zone||{};
c.ssl_perm = !!(plan && plan.ssl);
c.state_perm = !!perm && perm.split(' ').includes('state');
const lpm_user = this.lpm_users.find(u=>c.user && u.email==c.user);
if (lpm_user)
c.user_password = lpm_user.password;
c.hosts = this.hosts;
c.cn_hosts = this.cn_hosts;
c.keys = keys;
c.extra_ssl_ips = [
...new Set([...c.extra_ssl_ips||[],
...this.argv.extra_ssl_ips||[]]),
];
return c;
};
E.prototype.create_single_proxy = etask._fn(
function*mgr_create_single_proxy(_this, conf){
conf = _this.complete_proxy_config(conf);
logger.notice('Starting port %s', conf.port);
const proxy = new Proxy_port(conf);
proxy.on('tls_error', ()=>{
if (_this.tls_warning)
return;
_this.tls_warning = true;
_this.wss.broadcast({payload: true, path: 'tls_warning'}, 'global');
});
proxy.on('ready', ()=>{
logger.notice('Port %s ready', conf.port);
});
proxy.on('stopped', ()=>{
logger.notice('Port %s stopped', conf.port);
});
proxy.on('updated', ()=>{
logger.notice('Port %s updated', conf.port);
});
proxy.on('usage_start', data=>{
_this.handle_usage_start(data);
});
proxy.on('usage', data=>{
_this.handle_usage(data);
});
proxy.on('usage_abort', data=>{
_this.handle_usage_abort(data);
});
proxy.on('refresh_ip', data=>{
_this.refresh_ip(data.ip, data.vip, data.port);
});
proxy.on('banip_global', opt=>{
_this.banip(opt.ip, opt.domain, opt.ms);
});
proxy.on('save_config', ()=>{
_this.config.save();
});
proxy.on('add_static_ip', data=>etask(function*(){
const proxy_conf = _this.proxies.find(p=>p.port==data.port);
const proxy_port = _this.proxy_ports[data.port];
if ((proxy_conf.ips||[]).includes(data.ip))
return;
if (!proxy_conf.ips)
proxy_conf.ips = [];
if (!proxy_conf.pool_size)
return;
if (proxy_conf.ips.length>=proxy_conf.pool_size)
return;
proxy_conf.ips.push(data.ip);
proxy_port.update_config({ips: proxy_conf.ips});
_this.add_config_change('add_static_ip', data.port, data.ip);
yield _this.config.save();
}));
proxy.on('remove_static_ip', data=>etask(function*(){
const proxy_conf = _this.proxies.find(p=>p.port==data.port);
const proxy_port = _this.proxy_ports[data.port];
if (!(proxy_conf.ips||[]).includes(data.ip))
return;
proxy_conf.ips = proxy_conf.ips.filter(ip=>ip!=data.ip);
proxy_port.update_config({ips: proxy_conf.ips});
_this.add_config_change('remove_static_ip', data.port, data.ip);
yield _this.config.save();
}));
proxy.on('add_pending_ip', ip=>{
_this.pending_ips.add(ip);
});
proxy.on('error', err=>{
_this.error_handler('Port '+conf.port, err);
});
_this.proxy_ports[conf.port] = proxy;
proxy.start();
const task = this;
proxy.on('ready', task.continue_fn());
proxy.on('error', task.continue_fn());
yield this.wait();
return proxy;
});
E.prototype.add_config_change = function(key, area, payload){
this.config_changes.push({key, area, payload});
};
E.prototype.validate_proxy = function(proxy){
const port_in_range = (port, multiply, taken)=>{
multiply = multiply||1;
return port<=taken && port+multiply-1>=taken;
};
if (this.argv.www &&
port_in_range(proxy.port, proxy.multiply, this.argv.www))
{
return 'Proxy port conflict UI port';
}
if (Object.values(this.proxy_ports).length+(proxy.multiply||1)>
this._defaults.ports_limit)
{
return 'number of many proxy ports exceeding the limit: '
+this._defaults.ports_limit;
}
if (this.proxy_ports[proxy.port])
return 'Proxy port already exists';
};
E.prototype.init_proxy = etask._fn(function*mgr_init_proxy(_this, proxy){
const error = _this.validate_proxy(proxy);
if (error)
return {proxy_port: proxy, proxy_err: error};
const zone_name = proxy.zone || _this._defaults.zone;
proxy.password = get_password(proxy, zone_name, _this.zones) ||
_this.argv.password || _this._defaults.password;
proxy.gb_cost = get_gb_cost(zone_name, _this.zones);
proxy.whitelist_ips = [...new Set(
_this.get_default_whitelist().concat(proxy.whitelist_ips||[]))];
const conf = Object.assign({}, proxy);
lpm_config.numeric_fields.forEach(field=>{
if (conf[field])
conf[field] = +conf[field];
});
conf.static = is_static_proxy(zone_name, _this.zones);
conf.mobile = is_mobile(zone_name, _this.zones);
conf.unblock = is_unblocker(zone_name, _this.zones);
const proxies = _this.multiply_port(conf);
const proxy_ports = yield etask.all(proxies.map(
_this.create_single_proxy.bind(_this)));
const proxy_port = proxy_ports[0];
proxy_port.dups = proxy_ports.slice(1);
return {proxy_port};
});
E.prototype.multiply_port = function(master){
const multiply = master.multiply||1;
const proxies = [master];
const ips = master.ips||[];
const vips = master.vips||[];
const users = master.users||[];
for (let i=1; i<multiply; i++)
{
const dup = Object.assign({}, master, {
proxy_type: 'duplicate',
master_port: master.port,
port: master.port+i,
});
if (dup.multiply_ips)
{
dup.ip = ips[i%ips.length];
// XXX krzysztof: get rid of this redundancy
dup.ips = [dup.ip];
}
if (dup.multiply_vips)
{
dup.vip = vips[i%vips.length];
// XXX krzysztof: get rid of this redundancy
dup.vips = [dup.vip];
}
if (dup.multiply_users)
dup.user = users[i%users.length];
proxies.push(dup);
}
if (master.multiply_ips)
{
master.ip = ips[0];
// XXX krzysztof: check why we need vips property
master.ips = [master.ip];
}
if (master.multiply_vips)
{
master.vip = vips[0];
// XXX krzysztof: check why we need vips property
master.vips = [master.vip];
}
if (master.multiply_users)
master.user = users[0];
return proxies;
};
E.prototype.check_any_whitelisted_ips = function(){
const get_not_whitelisted_payload = ()=>{
if ((this._defaults.whitelist_ips||[]).some(util_lib.is_any_ip) ||
(this._defaults.www_whitelist_ips||[]).some(util_lib.is_any_ip))
{
return {type: 'defaults'};
}
const proxy_port = this.proxies.find(p=>
(p.whitelist_ips||[]).some(util_lib.is_any_ip));
return proxy_port ? {type: 'proxy', port: proxy_port.port} : null;
};
this.wss.broadcast({payload: get_not_whitelisted_payload(),
path: 'not_whitelisted'}, 'global');
};
E.prototype.create_new_proxy = etask._fn(function*(_this, conf){
this.on('uncaught', e=>{
logger.error('proxy create: '+zerr.e2s(e));
this.throw(e);
});
if (!conf.proxy_type && conf.port!=_this._defaults.dropin_port)
conf.proxy_type = 'persist';
conf = util_lib.omit_by(conf, v=>!v && v!==0 && v!==false);
const {proxy_port, proxy_err} = yield _this.init_proxy(conf);
if (conf.proxy_type=='persist' && !proxy_err)
{
_this.proxies.push(conf);
yield _this.config.save();
if (conf.ext_proxies)
yield _this.ext_proxy_created(conf.ext_proxies);
_this.check_any_whitelisted_ips();
}
else if (proxy_err)
{
logger.warn('Could not create proxy port %s: %s', proxy_port.port,
proxy_err);
}
return {proxy_port, proxy_err};
});
E.prototype.proxy_delete = etask._fn(function*_proxy_delete(_this, port, opt){
opt = opt||{};
const proxy = _this.proxy_ports[port];
if (!proxy)
throw new Error('this proxy does not exist');
if (proxy.opt.proxy_type=='duplicate')
throw new Error('cannot delete this port');
if (proxy.deleting)
throw new Error('this proxy is already being stopped and deleted');
proxy.deleting = true;
yield proxy.stop();
[proxy, ...proxy.dups].forEach(p=>{
// needed in order to prevent other APIs from getting orphan dups
delete _this.proxy_ports[p.opt.port];
p.destroy();
});
if (proxy.opt.proxy_type!='persist')
return;
const idx = _this.proxies.findIndex(p=>p.port==port);
if (idx==-1)
return;
_this.proxies.splice(idx, 1);
if (!opt.skip_config_save)
yield _this.config.save(opt);
_this.check_any_whitelisted_ips();
});
const get_free_port = proxies=>{
const proxy_ports = Array.isArray(proxies) ?
proxies.map(x=>x.port) : Object.keys(proxies);
if (!proxy_ports.length)
return 24000;
return Math.max(...proxy_ports)+1;
};
E.prototype.proxy_dup_api = etask._fn(
function*mgr_proxy_dup_api(_this, req, res, next){
this.on('uncaught', next);
const port = req.body.port;
const proxy = zutil.clone_deep(_this.proxies.filter(p=>p.port==port)[0]);
proxy.port = get_free_port(_this.proxy_ports);
yield _this.create_new_proxy(proxy);
res.json({proxy});
});
E.prototype.proxy_create_api = etask._fn(
function*mgr_proxy_create_api(_this, req, res, next){
this.on('uncaught', next);
const port = +req.body.proxy.port;
const {ext_proxies} = req.body.proxy;
const errors = yield _this.proxy_check({port, ext_proxies});
if (errors.length)
return res.status(400).json({errors});
const proxy = Object.assign({}, req.body.proxy, {port});
_this.add_config_change('create_proxy_port', port, req.body.proxy);
const {proxy_port, proxy_err} = yield _this.create_new_proxy(proxy);
if (proxy_err)
return res.status(400).json({errors: [{msg: proxy_err}]});
res.json({data: proxy_port.opt});
});
E.prototype.proxy_update = etask._fn(
function*mgr_proxy_update(_this, old_proxy, new_proxy){
const multiply_changed = new_proxy.multiply!==undefined &&
new_proxy.multiply!=old_proxy.multiply;
const port_changed = new_proxy.port && new_proxy.port!=old_proxy.port;
_this.add_config_change('update_proxy_port', old_proxy.port, new_proxy);
if (port_changed || multiply_changed)
return yield _this.proxy_remove_and_create(old_proxy, new_proxy);
return yield _this.proxy_update_in_place(old_proxy, new_proxy);
});
E.prototype.proxy_update_in_place = etask._fn(
function*(_this, old_proxy, new_proxy){
const old_opt = _this.proxies.find(p=>p.port==old_proxy.port);
Object.assign(old_opt, new_proxy);
yield _this.config.save();
for (let i=1; i<(old_opt.multiply||1); i++)
_this.proxy_ports[old_proxy.port+i].update_config(new_proxy);
const proxy_port = _this.proxy_ports[old_proxy.port];
return {proxy_port: proxy_port.update_config(new_proxy)};
});
E.prototype.proxy_remove_and_create = etask._fn(
function*(_this, old_proxy, new_proxy){
const old_server = _this.proxy_ports[old_proxy.port];
const banlist = old_server.banlist;
const old_opt = _this.proxies.find(p=>p.port==old_proxy.port);
yield _this.proxy_delete(old_proxy.port, {skip_cloud_update: 1});
const proxy = Object.assign({}, old_proxy, new_proxy, {banlist});
const {proxy_port, proxy_err} = yield _this.create_new_proxy(proxy);
if (proxy_err)
{
yield _this.create_new_proxy(old_opt);
return {proxy_err};
}
proxy_port.banlist = banlist;
return {proxy_port: proxy_port.opt};
});
E.prototype.proxy_update_api = etask._fn(
function*mgr_proxy_update_api(_this, req, res, next){
this.on('uncaught', next);
logger.info('proxy_update_api');
const old_port = req.params.port;
const old_proxy = _this.proxies.find(p=>p.port==old_port);
if (!old_proxy)
{
return res.status(400).json(
{errors: [{msg: `No proxy at port ${old_port}`}]});
}
if (old_proxy.proxy_type!='persist')
return res.status(400).json({errors: [{msg: 'Proxy is read-only'}]});
// XXX krzysztof: get rid of proxy check, move this logic inside
// validate_proxy
const errors = yield _this.proxy_check(Object.assign({}, old_proxy,
req.body.proxy), old_port);
if (errors.length)
return res.status(400).json({errors});
const {proxy_port, proxy_err} = yield _this.proxy_update(old_proxy,
req.body.proxy);
if (proxy_err)
return res.status(400).json({errors: [{msg: proxy_err}]});
res.json({data: proxy_port});
});
E.prototype.api_url_update_api = etask._fn(
function*mgr_api_url_update_api(_this, req, res){
const api_domain = _this._defaults.api_domain =
req.body.url.replace(/https?:\/\/(www\.)?/, '');
_this.conn.domain = yield _this.check_domain();
if (!_this.conn.domain)
return void res.json({res: false});
yield _this.logged_update();
_this.update_lpm_users(yield _this.lpm_users_get());
_this.add_config_change('update_api_domain', 'defaults', api_domain);
yield _this.config.save();
res.json({res: true});
});
E.prototype.proxy_banips_api = function(req, res){
const port = req.params.port;
const proxy = this.proxy_ports[port];
if (!proxy)
return res.status(400).send(`No proxy at port ${port}`);
let {ips, domain, ms=0} = req.body||{};
ips = (ips||[]).filter(ip=>util_lib.is_ip(ip) || util_lib.is_eip(ip));
if (!ips.length)
return res.status(400).send('No ips provided');
ips.forEach(ip=>proxy.banip(ip, ms, domain));
return res.status(204).end();
};
E.prototype.global_banip_api = function(req, res){
const {ip, domain, ms=0} = req.body||{};
if (!ip || !(util_lib.is_ip(ip) || util_lib.is_eip(ip)))
return res.status(400).send('No IP provided');
this.banip(ip, domain, ms);
return res.status(204).end();
};
E.prototype.banip = function(ip, domain, ms){
Object.values(this.proxy_ports).forEach(p=>{
p.banip(ip, ms, domain);
});
};
E.prototype.proxy_banip_api = function(req, res){
const port = req.params.port;
const proxy = this.proxy_ports[port];
if (!proxy)
return res.status(400).send(`No proxy at port ${port}`);
const {ip, domain, ms=0} = req.body||{};
if (!ip || !(util_lib.is_ip(ip) || util_lib.is_eip(ip)))
return res.status(400).send('No IP provided');
proxy.banip(ip, ms, domain);
return res.status(204).end();
};
E.prototype.proxy_unbanip_api = function(req, res){
const port = req.params.port;
const server = this.proxy_ports[port];
if (!server)
throw `No proxy at port ${port}`;
const {ip, domain} = req.body;
if (!ip || !(util_lib.is_ip(ip) || util_lib.is_eip(ip)))
return res.status(400).send('No IP provided');
const {ips: banned_ips} = this.get_banlist(server, true);
if (!banned_ips.some(({ip: banned_ip})=>banned_ip==ip))
return res.status(400).send('IP is not banned');
server.unbanip(ip, domain);
return res.json(this.get_banlist(server, true));
};
E.prototype.proxy_unbanips_api = function(req, res){
const port = req.params.port;
const server = this.proxy_ports[port];
if (!server)
throw `No proxy at port ${port}`;
server.unbanips();
return res.status(200).send('OK');
};
E.prototype.get_banlist = function(server, full){
if (full)
{
return {ips: [...server.banlist.cache.values()].map(
b=>({ip: b.ip, domain: b.domain, to: b.to_date}))};
}
return {ips: [...server.banlist.cache.keys()]};
};
E.prototype.get_banlist_api = function(req, res){
const port = req.params.port;
if (!port)
return res.status(400).send('port number is missing');
const server = this.proxy_ports[port];
if (!server)
return res.status(400).send('server does not exist');
res.json(this.get_banlist(server, req.query.full));
};
E.prototype.get_sessions_api = function(req, res){
const {port} = req.params;
const server = this.proxy_ports[port];
if (!server)
return res.status(400).send('server does not exist');
res.json({});
};
E.prototype.proxy_delete_wrapper = etask._fn(
function*mgr_proxy_delete_wrapper(_this, ports, opt){
if (ports.length)
{
yield etask.all(ports.map(p=>_this.proxy_delete(p, opt), _this));
_this.loki.requests_clear(ports);
_this.loki.stats_clear_by_ports(ports);
}
});
E.prototype.proxy_delete_api = etask._fn(
function*mgr_proxy_delete_api(_this, req, res, next){
this.on('uncaught', next);
logger.info('proxy_delete_api');
const port = +req.params.port;
_this.add_config_change('remove_proxy_port', port);
yield _this.proxy_delete_wrapper([port]);
res.sendStatus(204);
});
E.prototype.proxies_delete_api = etask._fn(
function*mgr_proxies_delete_api(_this, req, res, next){
this.on('uncaught', next);
logger.info('proxies_delete_api');
const ports = req.body.ports||[];
ports.forEach(port=>_this.add_config_change('remove_proxy_port', port));
yield _this.proxy_delete_wrapper(ports, {skip_cloud_update: 1});
yield _this.config.save();
res.sendStatus(204);
});
E.prototype.refresh_sessions_api = function(req, res){
const port = req.params.port;
const proxy_port = this.proxy_ports[port];
if (!proxy_port)
return res.status(400, 'Invalid proxy port').end();
const session_id = this.refresh_server_sessions(port);
if (proxy_port.opt.rotate_session)
return res.status(204).end();
res.json({session_id: `${port}_${session_id}`});
};
E.prototype.refresh_server_sessions = function(port){
const proxy_port = this.proxy_ports[port];
return proxy_port.refresh_sessions();
};
E.prototype.proxy_status_get_api = etask._fn(
function*mgr_proxy_status_get_api(_this, req, res, next){
this.on('uncaught', next);
const port = req.params.port;
const proxy = _this.proxy_ports[port];
if (!proxy)
return res.json({status: 'Unknown proxy'});
if (proxy.opt && proxy.opt.zone)
{
const db_zone = _this.zones.find(z=>z.zone==proxy.opt.zone)||{};
if ((db_zone.plan||{}).disable)
return res.json({status: 'Disabled zone'});
}
if (proxy.opt && proxy.opt.smtp && proxy.opt.smtp.length)
return res.json({status: 'ok', status_details: [{msg: 'SMTP proxy'}]});
const force = req.query.force!==undefined
&& req.query.force!=='false' && req.query.force!=='0';
const fields = ['status'];
if (proxy.opt && proxy.opt.proxy_type=='persist')
{
fields.push('status_details');
if (!proxy.status_details)
{
proxy.status_details = yield _this.proxy_check(proxy.opt,
proxy.opt.port);
}
}
if (force && proxy.status)
proxy.status = undefined;
for (let cnt=0; proxy.status===null && cnt<=22; cnt++)
yield etask.sleep(date.ms.SEC);
if (proxy.status===null)
return res.json({status: 'Unexpected lock on status check.'});
if (proxy.status)
return res.json(zutil.pick(proxy, ...fields));
yield _this.test_port(proxy, req.headers);
res.json(zutil.pick(proxy, ...fields));
});
E.prototype.test_port = etask._fn(function*lum_test(_this, proxy, headers){
proxy.status = null;
let success = false;
let error = '';
try {
const r = yield util_lib.json({
url: _this._defaults.test_url,
method: 'GET',
proxy: `http://127.0.0.1:${proxy.opt.port}`,
timeout: 20*date.ms.SEC,
headers: {
'x-hola-context': 'STATUS CHECK',
'x-hola-agent': lpm_config.hola_agent,
'user-agent': util_lib.user_agent,
'x-lpm-fake': headers['x-lpm-fake'],
},
});
success = r.statusCode==200;
error = r.headers['x-luminati-error'] || r.headers['x-lpm-error'];
if (/ECONNREFUSED/.test(error))
{
error = 'connection refused (may have been caused due to firewall '
+'settings)';
}
} catch(e){
etask.ef(e);
if (e.code=='ESOCKETTIMEDOUT')
error = 'timeout (may have been caused due to firewall settings)';
}
proxy.status = error || (success ? 'ok' : 'error');
});
E.prototype.get_browser_opt = function(port){
const proxy = this.proxy_ports[port]||{};
const browser_opt = {};
const proxy_opt = proxy.opt && zutil.pick(proxy.opt,
qw`timezone country resolution webrtc`) || {};
const {timezone, country, resolution, webrtc} = proxy_opt;
if (timezone=='auto')
browser_opt.timezone = country && zcountry.code2timezone(`${country}`);
else if (timezone)
browser_opt.timezone = timezone;
if (resolution)
{
const [width, height] = resolution.split('x').map(Number);
browser_opt.resolution = {width, height};
}
if (webrtc)
browser_opt.webrtc = webrtc;
return browser_opt;
};
E.prototype.open_browser_api = etask._fn(
function*mgr_open_browser_api(_this, req, res, next){
this.on('uncaught', next);
if (!puppeteer)
return res.status(400).send('Puppeteer not installed');
let responded = false;
if (!puppeteer.ready)
{
res.status(206).send('Fetching chromium');
responded = true;
}
const {port} = req.params;
try {
const browser_opt = _this.get_browser_opt(port);
yield puppeteer.open_page(_this._defaults.test_url, port, browser_opt);
} catch(e){
logger.error('open_browser_api: %s', e.message);
}
if (!responded)
res.status(200).send('OK');
});
E.prototype.proxy_port_check = etask._fn(
function*mgr_proxy_port_check(_this, port, duplicate, old_port, old_duplicate){
duplicate = +duplicate || 1;
port = +port;
old_port = +old_port;
let start = port;
const end = port+duplicate-1;
const old_end = old_port && old_port+(+old_duplicate||1)-1;
const ports = [];
for (let p = start; p <= end; p++)
{
if (old_port && old_port<=p && p<=old_end)
continue;
if (p==_this.argv.www)
return p+' in use by the UI/API and UI/WebSocket';
if (_this.proxy_ports[p])
return p+' in use by another proxy';
ports.push(p);
}
try {
yield etask.all(ports.map(p=>etask(function*inner_check(){
const server = http.createServer();
server.on('error', e=>{
if (e.code=='EADDRINUSE')
this.throw(new Error(p + ' in use by another app'));
if (e.code=='EACCES')
{
this.throw(new Error(p + ' cannot be used due to '
+'permission restrictions'));
}
this.throw(new Error(e));
});
http_shutdown(server);
server.listen(p, '0.0.0.0', this.continue_fn());
yield this.wait();
yield etask.nfn_apply(server, '.forceShutdown', []);
})));
} catch(e){
etask.ef(e);
return e.message;
}
});
E.prototype.proxy_check = etask._fn(
function*mgr_proxy_check(_this, new_proxy_config, old_proxy_port){
const old_proxy = old_proxy_port && _this.proxy_ports[old_proxy_port]
&& _this.proxy_ports[old_proxy_port].opt || {};
const info = [];
const {port, zone, multiply, whitelist_ips, ext_proxies} =
new_proxy_config;
const effective_zone = zone||E.default.zone;
if (port!==undefined)
{
if (!port || +port<1000)
{
info.push({
msg: 'Invalid port. It must be a number >= 1000',
field: 'port',
});
}
else
{
const in_use = yield _this.proxy_port_check(port, multiply,
old_proxy_port, old_proxy.multiply);
if (in_use)
info.push({msg: 'port '+in_use, field: 'port'});
}
}
if (zone!==undefined)
{
if (_this.zones.length)
{
let db_zone = _this.zones.filter(i=>i.zone==zone)[0];
if (!db_zone)
db_zone = _this.zones.filter(i=>i.zone==effective_zone)[0];
if (!db_zone)
{
info.push({msg: 'the provided zone name is not valid.',
field: 'zone'});
}
else if (db_zone.ips==='')
{
info.push({msg: 'the zone has no IPs in whitelist',
field: 'zone'});
}
else if (!db_zone.plan || db_zone.plan.disable)
info.push({msg: 'zone disabled', field: 'zone'});
}
}
if (whitelist_ips!==undefined)
{
if (_this.argv.zagent && whitelist_ips.some(util_lib.is_any_ip))
{
info.push({
msg: 'Not allowed to set \'any\' or 0.0.0.0/0 as a '
+'whitelisted IP in cloud LPM',
field: 'whitelist_ips',
});
}
}
if (ext_proxies!==undefined && ext_proxies.length>consts.MAX_EXT_PROXIES)
{
info.push({
msg: `Maximum external proxies size ${consts.MAX_EXT_PROXIES} `
+'exceeded',
field: 'ext_proxies',
});
}
for (let field in new_proxy_config)
{
const val = new_proxy_config[field];
if ((typeof val=='string' || val instanceof String) &&
val.length>consts.MAX_STRING_LENGTH)
{
info.push({
msg: 'Maximum string length exceeded',
field,
});
}
}
return info;
});
E.prototype.proxy_tester_api = function(req, res){
const port = req.params.port;
const proxy = this.proxy_ports[port];
if (!proxy)
return res.status(500).send(`proxy port ${port} not found`);
let response_sent = false;
const handle_log = req_log=>{
if (req_log.details.context!='PROXY TESTER TOOL')
return;
this.removeListener('request_log', handle_log);
response_sent = true;
res.json(req_log);
};
this.on('request_log', handle_log);
const opt = Object.assign(zutil.pick(req.body, ...qw`url headers body`),
{followRedirect: false});
if (opt.body && typeof opt.body!='string')
opt.body = JSON.stringify(opt.body);
const password = proxy.opt.password;
const user = 'tool-proxy_tester';
const basic = Buffer.from(user+':'+password).toString('base64');
opt.headers = opt.headers||{};
opt.headers['proxy-authorization'] = 'Basic '+basic;
opt.headers['user-agent'] = req.get('user-agent');
if (+port)
{
opt.proxy = 'http://127.0.0.1:'+port;
if (proxy.opt && proxy.opt.ssl)
opt.ca = ssl.ca.cert;
if (proxy.opt && proxy.opt.unblock)
opt.rejectUnauthorized = false;
}
request(opt, err=>{
if (!err)
return;
this.removeListener('request_log', handle_log);
logger.error('proxy_tester_api: %s', err.message);
if (!response_sent)
res.status(500).send(err.message);
});
};
E.prototype.get_all_locations_api = etask._fn(
function*mgr_get_all_locations(_this, req, res, next){
this.on('uncaught', next);
const data = yield cities.all_locations(_this.api_request.bind(_this));
let shared_countries;
if (process.env.ZLXC)
shared_countries = {body: ['th', 'us']};
else
{
shared_countries = yield _this.api_request({
endpoint: '/lpm/shared_block_countries',
force: 1,
});
}
res.json(Object.assign({}, data, {
shared_countries: shared_countries && shared_countries.body,
countries_by_code: data.countries.reduce((acc, e)=>
Object.assign(acc, {[e.country_id]: e.country_name}), {}),
}));
});
E.prototype.get_all_carriers_api = etask._fn(
function*mgr_get_all_carriers(_this, req, res, next){
this.on('uncaught', next);
const c_res = yield _this.api_request({
endpoint: '/lpm/carriers',
no_throw: 1,
force: 1,
});
if (c_res.statusCode==200)
return res.json(c_res.body);
logger.warn('Unable to get carriers: %s %s %s', c_res.statusCode,
c_res.statusMessage, c_res.body);
res.json([]);
});
E.prot