@luminati-io/luminati-proxy
Version:
A configurable local proxy for luminati.io
1,466 lines (1,390 loc) • 95.8 kB
JavaScript
#!/usr/bin/env node
// LICENSE_CODE ZON ISC
'use strict'; /*jslint node:true, esnext:true*/
const _ = require('lodash');
const events = require('events');
const fs = require('fs');
const path = require('path');
const os = require('os');
const dns = require('dns');
configure_dns();
const url = require('url');
const stringify = require('json-stable-stringify');
const express = require('express');
const compression = require('compression');
const body_parser = require('body-parser');
const request = require('request').defaults({gzip: true});
const http = require('http');
const util = require('util');
const {Netmask} = require('netmask');
const logger = require('./logger.js').child({category: 'MNGR'});
const http_shutdown = require('http-shutdown');
const consts = require('./consts.js');
const Proxy_port = require('./proxy_port.js');
const ssl = require('./ssl.js');
const pkg = require('../package.json');
const swagger = require('./swagger.json');
const zconfig = require('../util/config.js');
const zerr = require('../util/zerr.js');
const etask = require('../util/etask.js');
const rand = require('../util/rand.js');
const zws = require('../util/ws.js');
const string = require('../util/string.js');
const file = require('../util/file.js');
const date = require('../util/date.js');
const user_agent = require('../util/user_agent.js');
const lpm_config = require('../util/lpm_config.js');
const zurl = require('../util/url.js');
const cookie = require('cookie');
const cookie_filestore = require('tough-cookie-file-store');
const check_node_version = require('check-node-version');
const cities = require('./cities.js');
const perr = require('./perr.js');
const util_lib = require('./util.js');
const web_socket = require('ws');
const Loki = require('./loki.js');
const puppeteer = require('./puppeteer.js');
const {get_perm, is_static_proxy, is_mobile, is_unblocker,
get_password} = require('../util/zones.js');
const cluster = require('cluster');
const Config = require('./config.js');
const is_darwin = process.platform=='darwin';
let zos;
if (!lpm_config.is_win && !is_darwin)
zos = require('../util/os.js');
if (process.env.LPM_DEBUG)
require('longjohn');
let cookie_jar;
zerr.set_level('CRIT');
const qw = string.qw;
const E = module.exports = Manager;
swagger.info.version = pkg.version;
function configure_dns(){
const google_dns = ['8.8.8.8', '8.8.4.4'];
const original_dns = dns.getServers();
const servers = google_dns.concat(original_dns.filter(
d=>!google_dns.includes(d)));
// dns.setServers cashes node if there is an in-flight dns resolution
// should be done before any requests are made
// https://github.com/nodejs/node/issues/14734
dns.setServers(servers);
}
E.default = Object.assign({}, lpm_config.manager_default);
const check_running = argv=>etask(function*(){
const tasks = yield util_lib.get_lpm_tasks();
if (!tasks.length)
return;
if (!argv.dir)
{
logger.notice(`LPM is already running (${tasks[0].pid})`);
logger.notice('You need to pass a separate path to the directory for '
+'this LPM instance. Use --dir flag');
process.exit();
}
});
function Manager(argv, run_config={}){
events.EventEmitter.call(this);
logger.notice([
`Running Luminati Proxy Manager`,
`PID: ${process.pid}`,
`Version: ${pkg.version}`,
`Build date: ${zconfig.CONFIG_BUILD_DATE}`,
`Os version: ${os.platform()} ${os.arch()} ${os.release()}`,
`Host name: ${os.hostname()}`,
].join('\n'));
try {
this.proxy_ports = {};
this.argv = argv;
this.mgr_opts = _.pick(argv, lpm_config.mgr_fields);
this.opts = _.pick(argv, _.keys(lpm_config.proxy_fields));
this.config = new Config(this, E.default, {filename: argv.config});
const conf = this.config.get_proxy_configs();
this._total_conf = conf;
this._defaults = conf._defaults;
this.proxies = conf.proxies;
this.pending_www_ips = new Set();
this.pending_ips = new Set();
this.config.save();
this.first_actions = this.get_first_actions_data();
this.loki = new Loki(argv.loki);
this.features = new Set();
this.feature_used('start');
this.long_running_ets = [];
this.async_reqs_queue = [];
this.async_active = 0;
this.tls_warning = false;
this.lpm_users = [];
this.wss = {
close: ()=>null,
broadcast: (data, type)=>{
logger.debug('wss is not ready, %s will not be emitted', type);
},
};
this.is_upgraded = run_config.is_upgraded;
this.backup_exist = run_config.backup_exist;
this.conflict_shown = false;
this.on('error', (e, fatal)=>{
let match;
if (match = e.message.match(/EADDRINUSE.+:(\d+)/))
return this.show_port_conflict(match[1], argv.force);
logger.error(e.raw ? e.message : 'Unhandled error: '+e);
const handle_fatal = ()=>{
if (fatal)
this.stop();
};
if (!perr.enabled || e.raw)
handle_fatal();
else
{
util_lib.perr('crash', {error: zerr.e2s(e)});
handle_fatal();
}
});
} catch(e){
logger.error('constructor: %s', zerr.e2s(e));
throw e;
}
}
function get_content_type(data){
if (data.response_body=='unknown')
return 'unknown';
let content_type;
let res = 'other';
try {
const headers = JSON.parse(data.response_headers);
content_type = headers['content-type']||'';
} catch(e){ content_type = ''; }
if (content_type.match(/json/))
res = 'xhr';
else if (content_type.match(/html/))
res = 'html';
else if (content_type.match(/javascript/))
res = 'js';
else if (content_type.match(/css/))
res = 'css';
else if (content_type.match(/image/))
res = 'img';
else if (content_type.match(/audio|video/))
res = 'media';
else if (content_type.match(/font/))
res = 'font';
return res;
}
util.inherits(Manager, events.EventEmitter);
E.prototype.show_port_conflict = etask._fn(function*(_this, port, force){
if (_this.conflict_shown)
return;
_this.conflict_shown = true;
yield _this._show_port_conflict(port, force);
});
E.prototype._show_port_conflict = etask._fn(function*(_this, port, force){
const tasks = yield util_lib.get_lpm_tasks();
if (!tasks.length)
return logger.error(`There is a conflict on port ${port}`);
const pid = tasks[0].pid;
logger.notice(`LPM is already running (${pid}) and uses port ${port}`);
if (!force)
{
logger.notice('If you want to kill other instances use --force flag');
return process.exit();
}
logger.notice('Trying to kill it and restart.');
for (const t of tasks)
process.kill(t.ppid, 'SIGTERM');
_this.restart();
});
E.prototype.handle_usage = function(data){
if (!this.argv.www || this.argv.high_perf)
return;
let url_parts;
if (url_parts = data.url.match(/^([^/]+?):(\d+)$/))
{
data.protocol = url_parts[2]==443 ? 'https' : 'http';
data.hostname = url_parts[1];
}
else
{
const {protocol, hostname} = url.parse(data.url);
data.protocol = (protocol||'https:').slice(0, -1);
data.hostname = hostname;
}
if (!data.hostname)
util_lib.perr('empty_hostname', data);
data.hostname = (data.hostname||'').split('.').slice(-2).join('.');
data.content_type = get_content_type(data);
data.success = +(data.status_code && (data.status_code=='unknown' ||
consts.SUCCESS_STATUS_CODE_RE.test(data.status_code)));
const proxy = this.proxy_ports[data.port];
if (proxy && proxy.status!='ok' && data.success)
this.proxy_ports[data.port].status = 'ok';
if (this._defaults.request_stats)
this.loki.stats_process(data);
this.logs_process(data);
};
E.prototype.handle_usage_abort = function(data){
if (!this.argv.www || this.argv.high_perf)
return;
this.logs_process(data);
};
E.prototype.handle_usage_start = function(data){
if (!Number(this._defaults.logs))
return;
const req = {
uuid: data.uuid,
details: {
port: data.port,
context: data.context,
timestamp: data.timestamp,
timeline: [],
},
request: {
url: data.url,
method: data.method,
headers: headers_to_a(data.headers),
},
response: {content: {}},
};
this.wss.broadcast(req, 'har_viewer_start');
};
E.prototype.logs_process = function(data){
const har_req = this.har([data]).log.entries[0];
const max_logs = Number(this._defaults.logs);
if (!max_logs)
return this.emit('request_log', har_req);
this.wss.broadcast(har_req, 'har_viewer');
this.emit('request_log', har_req);
this.loki.request_process(data, max_logs);
};
E.prototype.get_first_actions_data = function(){
const filename = lpm_config.first_actions;
if (file.exists(filename))
{
try {
const data = JSON.parse(fs.readFileSync(filename).toString());
if (!data || !data.sent || !data.pending)
return {sent: {}, sending: {}, pending: []};
return Object.assign(data, {sending: {}});
} catch(e){ logger.error('get first actions: %s', zerr.e2s(e)); }
}
return {sent: {}, sending: {}, pending: []};
};
E.prototype.save_first_actions = function(){
fs.writeFileSync(lpm_config.first_actions,
stringify(_.omit(this.first_actions, 'sending')));
};
E.prototype.stop_servers = etask._fn(
function*mgr_stop_servers(_this){
let servers = [];
const stop_server = server=>servers.push(etask(function*mgr_stop_server(){
try {
yield server.stop();
} catch(e){
logger.error('Failed to stop server: %s', e.message);
}
}));
if (_this.www_server)
stop_server(_this.www_server);
_.values(_this.proxy_ports).forEach(stop_server);
_this.wss.close();
yield etask.all(servers);
});
E.prototype.stop = etask._fn(
function*mgr_stop(_this, reason, force, restart){
util_lib.clear_timeouts();
_this.long_running_ets.forEach(et=>et.return());
yield util_lib.perr(restart ? 'restart' : 'exit', {reason});
yield _this.loki.save();
_this.loki.stop();
if (reason!='config change')
yield _this.config.save();
if (reason instanceof Error)
reason = zerr.e2s(reason);
logger.notice('Manager stopped: %s', reason);
yield _this.stop_servers();
const task = this;
cluster.disconnect(()=>{
task.continue();
});
yield this.wait();
if (!restart)
_this.emit('stop', reason);
});
const headers_to_a = h=>_.toPairs(h).map(p=>({name: p[0], value: p[1]}));
E.prototype.har = function(entries){
return {log: {
version: '1.2',
creator: {name: 'Luminati Proxy', version: pkg.version},
pages: [],
entries: entries.map(entry=>{
const req = JSON.parse(entry.request_headers||'{}');
const res = JSON.parse(entry.response_headers||'{}');
const timeline = JSON.parse(entry.timeline||null)||[{}];
entry.request_body = entry.request_body||'';
const start = timeline[0].create;
return {
uuid: entry.uuid,
details: {
context: entry.context,
out_bw: entry.out_bw,
in_bw: entry.in_bw,
bw: entry.bw||entry.out_bw+entry.in_bw,
proxy_peer: entry.proxy_peer,
protocol: entry.protocol,
port: entry.port,
timestamp: entry.timestamp,
content_type: entry.content_type,
success: entry.success,
timeline: timeline.map(t=>({
blocked: t.create-start,
wait: t.connect-t.create||0,
receive: t.end-t.connect||t.end-t.create,
port: t.port,
})),
super_proxy: entry.super_proxy,
username: entry.username,
password: entry.password,
remote_address: entry.remote_address,
rules: entry.rules,
},
startedDateTime: new Date(start||0).toISOString(),
time: timeline.slice(-1)[0].end-start,
request: {
method: entry.method,
url: entry.url,
host: entry.hostname,
httpVersion: 'unknown',
cookies: [],
headers: headers_to_a(req),
headersSize: -1,
postData: {
mimeType: req['content-type']||req['Content-Type']||'',
text: entry.request_body,
},
bodySize: entry.request_body.length||0,
queryString: [],
},
response: {
status: entry.status_code,
statusText: entry.status_message||'',
httpVersion: 'unknown',
cookies: [],
headers: headers_to_a(res),
content: {
size: entry.content_size||0,
mimeType: res['content-type']||'unknown',
text: entry.response_body||'',
},
headersSize: -1,
bodySize: entry.content_size,
redirectURL: '',
},
cache: {},
// XXX krzysztof: this is to be added. timeline is broken
timings: {
blocked: 0,
dns: 0,
ssl: 0,
connect: 0,
send: 0,
wait: 0,
receive: 0,
},
serverIPAddress: entry.super_proxy,
comment: entry.username,
};
}),
}};
};
E.prototype.get_zones_api = function(req, res){
const zones = this.zones.map(z=>({
name: z.zone,
perm: z.perm,
plan: z.plan || {},
password: z.password,
})).filter(p=>p.plan && !p.plan.disable);
res.json({zones, def: this._defaults.zone});
};
E.prototype.get_consts_api = function(req, res){
const proxy = _.mapValues(lpm_config.proxy_fields, desc=>({desc}));
_.forOwn(E.default, (def, prop)=>{
if (proxy[prop])
proxy[prop].def = def;
});
if (proxy.zone)
proxy.zone.def = this._defaults.zone;
_.merge(proxy, {dns: {values: ['', 'local', 'remote']}});
const ifaces = Object.keys(os.networkInterfaces())
.map(iface=>({key: iface, value: iface}));
ifaces.unshift({key: 'All', value: '0.0.0.0'});
ifaces.unshift({key: 'Default (dynamic)', value: ''});
proxy.iface.values = ifaces;
const notifs = this.lum_conf && this.lum_conf.lpm_notifs || [];
const logins = this.lum_conf && this.lum_conf.logins || [];
res.json({proxy, notifs, logins});
};
E.prototype.enable_ssl_api = etask._fn(
function*mgr_enable_ssl(_this, req, res){
const port = req.body.port;
let proxies = _this.proxies.slice();
if (port)
proxies = proxies.filter(p=>p.port==port);
for (let i in proxies)
{
const p = proxies[i];
if (p.port!=22225 && !p.ssl)
yield _this.proxy_update(p, Object.assign(p, {ssl: true}));
}
res.send('ok');
});
E.prototype.update_ips_api = etask._fn(
function*mgr_update_ips(_this, req, res){
const ips = req.body.ips||[];
const vips = req.body.vips||[];
const proxy = _this.proxies.find(p=>p.port==req.body.port);
yield _this.proxy_update(proxy, Object.assign(proxy, {ips, vips}));
res.send('ok');
});
E.prototype.update_notifs_api = etask._fn(
function*mgr_update_notif(_this, req, res){
_this.lum_conf.lpm_notifs = _this.lum_conf.lpm_notifs || [];
const notifs = req.body.notifs;
notifs.forEach(updated_notif=>{
const stored_notif = _this.lum_conf.lpm_notifs.find(
n=>n._id==updated_notif.id);
if (stored_notif)
stored_notif.status = updated_notif.status;
});
const response = yield _this.api_request({
method: 'POST',
url: `${_this._defaults.api}/update_lpm_notifs`,
form: {notifs},
});
res.json(response.body);
});
E.prototype.send_rule_mail = etask._fn(
function*mgr_send_rule_mail(_this, port, to, _url){
const subject = `Luminati: Rule was triggered`;
const text = `Hi,\n\nYou are getting this email because you asked to get `
+`notified when Luminati rules are triggered.\n\n`
+`Request URL: ${_url}\n`
+`Port: ${port}\n\n`
+`You can resign from receiving these notifications in the proxy port `
+`configuration page in the Rule tab. If your LPM is running on localhost `
+`you can turn it off here: `
+`http://127.0.0.1:${_this.opts.www}/proxy/${port}/rules\n\n`
+`Luminati`;
const response = yield _this.api_request({
method: 'POST',
url: `${_this._defaults.api}/send_rule_mail`,
form: {to, subject, text},
});
return response.body;
});
E.prototype.report_bug_api = etask._fn(
function*mgr_report_bug(_this, req, res){
let log_file = '';
const config_file = Buffer.from(_this.config.get_string())
.toString('base64');
if (file.exists(logger.lpm_filename))
{
let buffer = fs.readFileSync(logger.lpm_filename);
buffer = buffer.slice(buffer.length-50000);
log_file = buffer.toString('base64');
}
const reqs = _this.filtered_get({query: {limit: 100}}).items.map(x=>({
url: x.url,
status_code: x.status_code,
}));
const har = JSON.stringify(reqs);
const browser = user_agent.guess_browser(req.get('user-agent')).browser;
const response = yield _this.api_request({
method: 'POST',
url: `${_this._defaults.api}/report_bug`,
form: {report: {config: config_file, log: log_file, har,
desc: req.body.desc, lpm_v: pkg.version, email: req.body.email,
browser, os: util_lib.format_platform(os.platform())}},
});
res.status(response.statusCode).json(response.body);
});
E.prototype.get_fixed_whitelist = function(){
return (this.opts.whitelist_ips||[]).concat(
this._defaults.www_whitelist_ips||[]);
};
E.prototype.get_default_whitelist = function(){
return this.get_fixed_whitelist().concat(this._defaults.whitelist_ips||[]);
};
E.prototype.set_www_whitelist_ips = function(ips){
const prev = this.get_default_whitelist();
ips = [...new Set(ips)];
ips.forEach(ip=>this.pending_www_ips.delete(ip));
if (!ips.length)
delete this._defaults.www_whitelist_ips;
else
this._defaults.www_whitelist_ips = ips;
this.set_whitelist_ips(this._defaults.whitelist_ips||[], prev);
};
E.prototype.set_whitelist_ips = function(ips, prev){
const fixed_whitelist = this.get_fixed_whitelist();
ips = [...new Set(ips)];
ips.forEach(ip=>this.pending_ips.delete(ip));
ips = ips.filter(ip=>!fixed_whitelist.includes(ip));
prev = prev||this.get_default_whitelist();
if (!ips.length)
delete this._defaults.whitelist_ips;
else
{
this._defaults.whitelist_ips = ips.map(ip=>{
try {
const _ip = new Netmask(ip);
const mask = _ip.bitmask==32 ? '' : '/'+_ip.bitmask;
return _ip.base+mask;
} catch(e){ return null; }
}).filter(ip=>ip!==null && ip!='127.0.0.1');
}
this.update_ports({whitelist_ips: {default: 1, prev,
curr: this.get_default_whitelist()}});
};
E.prototype.error_handler = function error_handler(source, err){
if (!err.code && err.stack)
logger.error(err.stack.split('\n').slice(0, 2).join('\n'));
else if (err.code=='EMFILE')
return logger.error('EMFILE: out of file descriptors');
else
logger.error(err.message);
err.source = source;
this.emit('error', err);
};
E.prototype.complete_proxy_config = function(conf){
const c = Object.assign({}, E.default, this._defaults, conf);
c.whitelist_ips = [...new Set(
this.get_default_whitelist().concat(c.whitelist_ips||[]))];
const zone = this.zones.find(z=>z.zone==c.zone);
const plan = zone && zone.plan;
c.ssl_perm = !!(plan && plan.ssl);
const lpm_user = this.lpm_users.find(u=>c.user && u.email==c.user);
if (lpm_user)
c.user_password = lpm_user.password;
c.hosts = this.hosts;
return c;
};
E.prototype.create_single_proxy = etask._fn(
function*mgr_create_single_proxy(_this, conf){
conf = _this.complete_proxy_config(conf);
logger.notice('Starting port %s', conf.port);
const proxy = new Proxy_port(conf);
proxy.on('error', err=>{
_this.error_handler('Port '+conf.port, err);
});
proxy.on('tls_error', ()=>{
if (_this.tls_warning)
return;
_this.tls_warning = true;
_this.feature_used('tls.error_detected');
_this.wss.broadcast({payload: true, path: 'tls_warning'}, 'global');
});
proxy.on('ready', ()=>{
logger.notice('Port %s ready', conf.port);
});
proxy.on('stopped', ()=>{
logger.notice('Port %s stopped', conf.port);
});
proxy.on('usage_start', data=>{
_this.handle_usage_start(data);
});
proxy.on('usage', data=>{
_this.handle_usage(data);
});
proxy.on('usage_abort', data=>{
_this.handle_usage_abort(data);
});
proxy.on('send_rule_mail', data=>{
_this.send_rule_mail(data.port, data.email, data.url);
});
proxy.on('first_lpm_action', data=>{
_this.first_lpm_action(data.action, data.ua);
});
proxy.on('refresh_ip', data=>{
_this.refresh_ip(data.ip, data.port);
});
proxy.on('banip_global', opt=>{
_this.banip(opt.ip, opt.domain, opt.ms);
});
proxy.on('save_config', ()=>{
_this.config.save();
});
proxy.on('feature_used', feature=>{
_this.feature_used(feature);
});
proxy.on('add_static_ip', data=>{
const proxy_conf = _this.proxies.find(p=>p.port==data.port);
const serv = _this.proxy_ports[data.port];
if ((proxy_conf.ips||[]).includes(data.ip))
return;
if (!proxy_conf.ips)
proxy_conf.ips = [];
if (!proxy_conf.pool_size)
return;
if (proxy_conf.ips.length>=proxy_conf.pool_size)
return;
proxy_conf.ips.push(data.ip);
serv.update_config({ips: proxy_conf.ips});
_this.config.save();
});
proxy.on('remove_static_ip', data=>{
const proxy_conf = _this.proxies.find(p=>p.port==data.port);
const serv = _this.proxy_ports[data.port];
if (!(proxy_conf.ips||[]).includes(data.ip))
return;
proxy_conf.ips = proxy_conf.ips.filter(ip=>ip!=data.ip);
serv.update_config({ips: proxy_conf.ips});
_this.config.save();
});
proxy.on('add_pending_ip', ip=>{
_this.pending_ips.add(ip);
});
_this.proxy_ports[conf.port] = proxy;
proxy.start();
const task = this;
proxy.on('ready', ()=>{
task.continue();
});
yield this.wait();
return proxy;
});
E.prototype.create_proxy = etask._fn(
function*mgr_create_proxy(_this, proxy, ua){
if (proxy.proxy_type=='persist')
_this.first_lpm_action('create_proxy_port', ua);
proxy = _.omitBy(proxy, v=>!v && v!==0 && v!==false);
const zone_name = proxy.zone || _this._defaults.zone;
proxy.password = get_password(proxy, zone_name, _this.zones) ||
_this.argv.password || _this._defaults.password;
const conf = Object.assign({}, proxy);
lpm_config.numeric_fields.forEach(field=>{
if (conf[field])
conf[field] = +conf[field];
});
conf.static = is_static_proxy(zone_name, _this.zones);
conf.mobile = is_mobile(zone_name, _this.zones);
conf.unblock = is_unblocker(zone_name, _this.zones);
const proxies = _this.multiply_port(conf);
const servers = yield etask.all(proxies.map(
_this.create_single_proxy.bind(_this)));
const server = servers[0];
server.dups = servers.slice(1);
return server;
});
E.prototype.multiply_port = function(master){
const multiply = master.multiply||1;
const proxies = [master];
const ips = master.ips||[];
const vips = master.vips||[];
const users = master.users||[];
for (let i=1; i<multiply; i++)
{
const dup = Object.assign({}, master, {
proxy_type: 'duplicate',
master_port: master.port,
port: master.port+i,
});
if (dup.multiply_ips)
{
dup.ip = ips[i%ips.length];
// XXX krzysztof: get rid of this redundancy
dup.ips = [dup.ip];
}
if (dup.multiply_vips)
{
dup.vip = vips[i%vips.length];
// XXX krzysztof: get rid of this redundancy
dup.vips = [dup.vip];
}
if (dup.multiply_users)
dup.user = users[i%users.length];
proxies.push(dup);
}
if (master.multiply_ips)
{
master.ip = ips[0];
// XXX krzysztof: check why we need vips property
master.ips = [master.ip];
}
if (master.multiply_vips)
{
master.vip = vips[0];
// XXX krzysztof: check why we need vips property
master.vips = [master.vip];
}
if (master.multiply_users)
master.user = users[0];
return proxies;
};
E.prototype.proxy_create = etask._fn(
function*mgr_proxy_create(_this, proxy, ua){
this.on('uncaught', e=>{
logger.error('proxy create: '+zerr.e2s(e));
});
if (!proxy.proxy_type && proxy.port!=22225)
proxy.proxy_type = 'persist';
const server = yield _this.create_proxy(proxy, ua);
if (proxy.proxy_type=='persist')
{
_this.proxies.push(proxy);
_this.config.save();
if (proxy.ext_proxies)
yield _this.ext_proxy_created(proxy.ext_proxies);
}
return server;
});
E.prototype.proxy_delete = etask._fn(function*mgr_proxy_delete(_this, port){
this.on('uncaught', e=>{
logger.error('proxy delete: '+zerr.e2s(e));
});
const proxy = _this.proxy_ports[port];
if (!proxy)
return;
proxy.stop();
proxy.once('stopped', this.continue_fn());
yield this.wait();
[proxy, ...proxy.dups].forEach(p=>{
// needed in order to prevent other APIs from getting orphan dups
delete _this.proxy_ports[p.opt.port];
p.destroy();
});
if (proxy.opt.proxy_type=='persist')
{
const idx = _this.proxies.findIndex(p=>p.port==port);
if (idx==-1)
return;
_this.proxies.splice(idx, 1);
_this.config.save();
}
});
const get_free_port = proxies=>{
if (Array.isArray(proxies))
proxies = proxies.map(x=>x.port);
else
proxies = Object.keys(proxies);
if (!proxies.length)
return 24000;
return +_.max(proxies)+1;
};
E.prototype.proxy_dup_api = etask._fn(
function*mgr_proxy_dup_api(_this, req, res){
const port = req.body.port;
const proxy = _.cloneDeep(_this.proxies.filter(p=>p.port==port)[0]);
proxy.port = get_free_port(_this.proxy_ports);
yield _this.proxy_create(proxy);
res.json({proxy});
});
E.prototype.proxy_create_api = etask._fn(
function*mgr_proxy_create_api(_this, req, res){
const port = +req.body.proxy.port;
do {
const errors = yield _this.proxy_check({port});
if (errors.length)
return res.status(400).json({errors});
break;
} while (true);
const proxy = Object.assign({}, req.body.proxy, {port});
const server = yield _this.proxy_create(proxy, req.get('user-agent'));
res.json({data: server.opt});
});
E.prototype.proxy_update = etask._fn(
function*mgr_proxy_update(_this, old_proxy, new_proxy){
const old_port = old_proxy.port;
const old_server = _this.proxy_ports[old_port];
const banlist = old_server.banlist;
yield _this.proxy_delete(old_port);
let proxy = Object.assign({}, old_proxy, new_proxy);
proxy = _.omitBy(proxy, v=>v==='');
const server = yield _this.proxy_create(proxy);
server.banlist = banlist;
return server.opt;
});
E.prototype.proxy_update_api = etask._fn(
function*mgr_proxy_update_api(_this, req, res){
logger.info('proxy_update_api');
const old_port = req.params.port;
const old_proxy = _this.proxies.find(p=>p.port==old_port);
if (!old_proxy)
{
return res.status(400).json(
{errors: [{msg: `No proxy at port ${old_port}`}]});
}
if (old_proxy.proxy_type!='persist')
return res.status(400).json({errors: [{msg: 'Proxy is read-only'}]});
const errors = yield _this.proxy_check(req.body.proxy, old_port);
if (errors.length)
return res.status(400).json({errors});
const data = yield _this.proxy_update(old_proxy, req.body.proxy);
res.json({data});
});
E.prototype.api_url_update_api = etask._fn(
function*mgr_api_url_update_api(_this, req, res){
let new_url = 'https://'+req.body.url.replace(/https?:\/\/(www\.)?/, '');
_this._defaults.api = new_url;
_this.conn.domain = yield _this.check_domain();
if (!_this.conn.domain)
return void res.json({res: false});
yield _this.logged_update();
yield _this.retry_failed_first_actions();
yield _this.sync_recent_stats();
_this.update_lpm_users(yield _this.lpm_users_get());
_this.config.save();
res.json({res: true});
});
E.prototype.proxy_banips_api = function(req, res){
const port = req.params.port;
const proxy = this.proxy_ports[port];
if (!proxy)
return res.status(400).send(`No proxy at port ${port}`);
const {ips, domain, ms=0} = req.body||{};
if (!ips || !ips.length || !ips.every(util_lib.is_ip))
return res.status(400).send('No ips provided');
ips.every(ip=>proxy.banip(ip, ms, domain));
return res.status(204).end();
};
// XXX krzysztof: make a separate module for all the banips API
E.prototype.banip_api = function(req, res){
const {ip, domain, ms=0} = req.body||{};
if (!ip || !util_lib.is_ip(ip))
return res.status(400).send('No IP provided');
this.banip(ip, domain, ms);
return res.status(204).end();
};
E.prototype.banip = function(ip, domain, ms){
Object.values(this.proxy_ports).forEach(p=>{
p.banip(ip, ms, domain);
});
};
E.prototype.proxy_banip_api = function(req, res){
const port = req.params.port;
const proxy = this.proxy_ports[port];
if (!proxy)
return res.status(400).send(`No proxy at port ${port}`);
const {ip, domain, ms=0} = req.body||{};
if (!ip || !(util_lib.is_ip(ip) || util_lib.is_eip(ip)))
return res.status(400).send('No IP provided');
proxy.banip(ip, ms, domain);
return res.status(204).end();
};
E.prototype.proxy_unbanip_api = function(req, res){
const port = req.params.port;
const server = this.proxy_ports[port];
if (!server)
throw `No proxy at port ${port}`;
const {ip, domain} = req.body;
if (!util_lib.is_ip(ip))
throw `No ip provided`;
server.unbanip(ip, domain);
return res.json(this.get_banlist(server, true));
};
E.prototype.get_banlist = function(server, full){
if (full)
{
return {ips: [...server.banlist.cache.values()].map(
b=>({ip: b.ip, domain: b.domain, to: b.to_date}))};
}
return {ips: [...server.banlist.cache.keys()]};
};
E.prototype.get_banlist_api = function(req, res){
const port = req.params.port;
if (!port)
return res.status(400).send('port number is missing');
const server = this.proxy_ports[port];
if (!server)
return res.status(400).send('server does not exist');
res.json(this.get_banlist(server, req.query.full));
};
E.prototype.get_sessions_api = function(req, res){
const {port} = req.params;
const server = this.proxy_ports[port];
if (!server)
return res.status(400).send('server does not exist');
res.json({});
};
E.prototype.proxy_delete_wrapper = etask._fn(
function*mgr_proxy_delete_wrapper(_this, ports){
if (ports.length)
{
yield etask.all(ports.map(_this.proxy_delete, _this));
_this.loki.requests_clear(ports);
_this.loki.stats_clear_by_ports(ports);
}
});
E.prototype.proxy_delete_api = etask._fn(
function*mgr_proxy_delete_api(_this, req, res){
logger.info('proxy_delete_api');
const port = +req.params.port;
yield _this.proxy_delete_wrapper([port]);
res.sendStatus(204);
});
E.prototype.proxies_delete_api = etask._fn(
function*mgr_proxies_delete_api(_this, req, res){
logger.info('proxies_delete_api');
const ports = req.body.ports||[];
yield _this.proxy_delete_wrapper(ports);
res.sendStatus(204);
});
E.prototype.refresh_sessions_api = function(req, res){
const port = req.params.port;
if (!this.proxy_ports[port])
return res.status(400, 'Invalid proxy port').end();
this.refresh_server_sessions(port);
res.status(204).end();
};
E.prototype.refresh_server_sessions = function(port){
const proxy_port = this.proxy_ports[port];
return proxy_port.refresh_sessions();
};
E.prototype.proxy_status_get_api = etask._fn(
function*mgr_proxy_status_get_api(_this, req, res){
const port = req.params.port;
const proxy = _this.proxy_ports[port];
if (!proxy)
return res.json({status: 'Unknown proxy'});
if (proxy.opt && proxy.opt.smtp && proxy.opt.smtp.length)
return res.json({status: 'ok', status_details: [{msg: 'SMTP proxy'}]});
const force = req.query.force!==undefined
&& req.query.force!=='false' && req.query.force!=='0';
const fields = ['status'];
if (proxy.opt && proxy.opt.proxy_type=='persist')
{
fields.push('status_details');
if (!proxy.status_details)
{
proxy.status_details = yield _this.proxy_check(proxy.opt,
proxy.opt.port);
}
}
if (force && proxy.status)
proxy.status = undefined;
for (let cnt=0; proxy.status===null && cnt<=22; cnt++)
yield etask.sleep(date.ms.SEC);
if (proxy.status===null)
return res.json({status: 'Unexpected lock on status check.'});
if (proxy.status)
return res.json(_.pick(proxy, fields));
yield _this.test_port(proxy, req.headers);
res.json(_.pick(proxy, fields));
});
E.prototype.test_port = etask._fn(function*lum_test(_this, proxy, headers){
proxy.status = null;
let success = false;
let error = '';
try {
const r = yield util_lib.json({
url: _this._defaults.test_url,
method: 'GET',
proxy: `http://127.0.0.1:${proxy.opt.port}`,
timeout: 20*date.ms.SEC,
headers: {
'x-hola-context': 'STATUS CHECK',
'x-hola-agent': lpm_config.hola_agent,
'user-agent': util_lib.user_agent,
'x-lpm-fake': headers['x-lpm-fake'],
},
});
success = r.statusCode==200;
error = r.headers['x-luminati-error'] || r.headers['x-lpm-error'];
if (/ECONNREFUSED/.test(error))
{
error = 'connection refused (may have been caused due to firewall '
+'settings)';
}
} catch(e){
etask.ef(e);
if (e.code=='ESOCKETTIMEDOUT')
error = 'timeout (may have been caused due to firewall settings)';
}
proxy.status = error || (success ? 'ok' : 'error');
});
E.prototype.open_browser_api = etask._fn(
function*mgr_open_browser_api(_this, req, res){
if (!puppeteer)
return res.status(400).send('Puppeteer not installed');
let responded = false;
if (!puppeteer.ready)
{
res.status(206).send('Fetching chromium');
responded = true;
}
const {port} = req.params;
try {
yield puppeteer.open_page(_this._defaults.test_url, port);
} catch(e){
logger.error('open_browser_api: %s', e.message);
}
if (!responded)
res.status(200).send('OK');
});
E.prototype.proxy_port_check = etask._fn(
function*mgr_proxy_port_check(_this, port, duplicate, old_port, old_duplicate){
duplicate = +duplicate || 1;
port = +port;
let start = port;
const end = port+duplicate-1;
const old_end = old_port && old_port + (old_duplicate||1) -1;
const ports = [];
for (let p = start; p <= end; p++)
{
if (old_port && old_port<=p && p<=old_end)
continue;
if (p==_this.argv.www)
return p+' in use by the UI/API and UI/WebSocket';
if (_this.proxy_ports[p])
return p+' in use by another proxy';
ports.push(p);
}
try {
yield etask.all(ports.map(p=>etask(function*proxy_port_check(){
const server = http.createServer();
server.on('error', e=>{
if (e.code=='EADDRINUSE')
this.throw(new Error(p + ' in use by another app'));
if (e.code=='EACCES')
{
this.throw(new Error(p + ' cannot be used due to '
+'permission restrictions'));
}
this.throw(new Error(e));
});
http_shutdown(server);
server.listen(p, '0.0.0.0', this.continue_fn());
yield this.wait();
yield etask.nfn_apply(server, '.forceShutdown', []);
})));
} catch(e){
etask.ef(e);
return e.message;
}
});
E.prototype.proxy_check = etask._fn(
function*mgr_proxy_check(_this, new_proxy_config, old_proxy_port){
const old_proxy = old_proxy_port && _this.proxy_ports[old_proxy_port]
&& _this.proxy_ports[old_proxy_port].opt || {};
const info = [];
const port = new_proxy_config.port;
const zone = new_proxy_config.zone;
const effective_zone = zone||E.default.zone;
const multiply = new_proxy_config.multiply;
if (port!==undefined)
{
if (!port)
info.push({msg: 'invalid port', field: 'port'});
else
{
const in_use = yield _this.proxy_port_check(port, multiply,
old_proxy_port, old_proxy.multiply);
if (in_use)
info.push({msg: 'port '+in_use, field: 'port'});
}
}
if (zone!==undefined)
{
if (_this.zones.length)
{
let db_zone = _this.zones.filter(i=>i.zone==zone)[0];
if (!db_zone)
db_zone = _this.zones.filter(i=>i.zone==effective_zone)[0];
if (!db_zone)
{
info.push({msg: 'the provided zone name is not valid.',
field: 'zone'});
}
else if (db_zone.ips==='')
{
info.push({msg: 'the zone has no IPs in whitelist',
field: 'zone'});
}
else if (!db_zone.plan || db_zone.plan.disable)
info.push({msg: 'zone disabled', field: 'zone'});
}
}
return info;
});
E.prototype.refresh_zones_api = etask._fn(
function*refresh_zones(_this, req, res){
_this.feature_used('refresh_zones');
const logged = yield _this.logged_update(req.get('user-agent'));
if (logged)
return res.status(200).send('OK');
res.status(400).send('You need to log in again');
});
E.prototype.feature_used = function(key){
this.features.add(key);
};
E.prototype.async_req_api = function(req, res){
if (!req.body.url)
return res.status(400).send(`url is required parameter`);
if (!req.body.callback_url)
return res.status(400).send(`callback_url is required parameter`);
const port = req.params.port;
const proxy = this.proxy_ports[port];
if (!proxy)
return res.status(500).send(`proxy port ${port} not found`);
const make_callback = (_url, method, body, status_code, metadata,
is_err)=>
{
const params = {metadata};
if (is_err)
params.error = body;
else
params.response = {body, status_code};
method = method||'POST';
const _opt = {url: _url, method};
if (method=='GET')
_opt.qs = params;
else
_opt.form = params;
request(_opt, (err, response, _body)=>{
if (err)
logger.error('async_req_api (callback): %s', err.message);
this.async_active--;
if (!this.async_reqs_queue.length)
return;
const {req_opt, cb_opt} = this.async_reqs_queue.shift();
make_request(req_opt, cb_opt);
});
};
const make_request = (req_opt, cb_opt)=>{
this.async_active++;
request(req_opt, (err, response, body)=>{
if (err)
{
logger.error('async_req_api (proxy req): %s', err.message);
return make_callback(cb_opt.url, cb_opt.method, err.message, 0,
cb_opt.metadata, true);
}
make_callback(cb_opt.url, cb_opt.method, body, response.statusCode,
cb_opt.metadata, false);
});
};
const req_opt = {
proxy: 'http://127.0.0.1:'+port,
followRedirect: false,
timeout: 20*date.ms.SEC,
url: req.body.url,
method: req.body.method||'GET',
headers: req.body.headers,
};
if (proxy.opt.ssl)
req_opt.ca = ssl.ca.cert;
if (proxy.opt.unblock)
req_opt.rejectUnauthorized = false;
const cb_opt = {
url: req.body.callback_url,
method: req.body.callback_method,
metadata: req.body.metadata,
};
if (this.async_active>=100)
{
this.async_reqs_queue.push({req_opt, cb_opt});
res.status(200).send(`Request scheduled`);
}
else
{
make_request(req_opt, cb_opt);
res.status(200).send(`Request sent`);
}
};
E.prototype.proxy_tester_api = function(req, res){
this.feature_used('proxy_tester');
const port = req.params.port;
const proxy = this.proxy_ports[port];
if (!proxy)
return res.status(500).send(`proxy port ${port} not found`);
let response_sent = false;
const handle_log = req_log=>{
if (req_log.details.context!='PROXY TESTER TOOL')
return;
this.removeListener('request_log', handle_log);
response_sent = true;
res.json(req_log);
};
this.on('request_log', handle_log);
const opt = Object.assign(_.pick(req.body, qw`url headers body`), {
followRedirect: false,
});
if (opt.body && typeof opt.body!='string')
opt.body = JSON.stringify(opt.body);
const password = proxy.opt.password;
const user = 'tool-proxy_tester';
const basic = Buffer.from(user+':'+password).toString('base64');
opt.headers = opt.headers||{};
opt.headers['proxy-authorization'] = 'Basic '+basic;
opt.headers['user-agent'] = req.get('user-agent');
if (+port)
{
opt.proxy = 'http://127.0.0.1:'+port;
if (proxy.opt && proxy.opt.ssl)
opt.ca = ssl.ca.cert;
if (proxy.opt && proxy.opt.unblock)
opt.rejectUnauthorized = false;
}
request(opt, err=>{
if (!err)
return;
this.removeListener('request_log', handle_log);
logger.error('proxy_tester_api: %s', err.message);
if (!response_sent)
res.status(500).send(err.message);
});
};
E.prototype.get_all_locations_api = etask._fn(
function*mgr_get_all_locations(_this, req, res){
const data = yield cities.all_locations();
const shared_countries = yield _this.api_request({
url: `${_this._defaults.api}/users/zone/shared_block_countries`});
res.json(Object.assign(data,
{shared_countries: shared_countries && shared_countries.body}));
});
E.prototype.get_all_carriers_api = etask._fn(
function*mgr_get_all_carriers(_this, req, res){
const c_res = yield etask.nfn_apply(request, [{
url: `https://client.${_this._defaults.api_domain}/api/carriers`,
headers: {'user-agent': util_lib.user_agent},
json: true,
}]);
if (c_res.statusCode==200)
return res.json(c_res.body);
logger.warn('Unable to get carriers: %s %s %s', c_res.statusCode,
c_res.statusMessage, c_res.body);
res.json([]);
});
E.prototype.logs_suggestions_api = function(req, res){
if (this.argv.high_perf)
return res.json({ports: [], status_codes: [], protocols: []});
const ports = this.loki.colls.port.chain().data().map(r=>r.key);
const protocols = this.loki.colls.protocol.chain().data().map(r=>r.key);
const status_codes = this.loki.colls.status_code.chain().data()
.map(r=>r.key);
const suggestions = {ports, status_codes, protocols};
res.json(suggestions);
};
E.prototype.logs_reset_api = function(req, res){
const ports = req.query.port && [+req.query.port] || undefined;
this.loki.stats_clear();
this.loki.requests_clear(ports);
res.send('ok');
};
E.prototype.logs_get_api = function(req, res){
if (this.argv.high_perf)
return {};
const result = this.filtered_get(req);
res.json(Object.assign({}, this.har(result.items), {total: result.total,
skip: result.skip, sum_out: result.sum_out, sum_in: result.sum_in}));
};
E.prototype.logs_har_get_api = function(req, res){
res.setHeader('content-disposition', 'attachment; filename=data.har');
const result = this.filtered_get(req);
res.send(JSON.stringify(this.har(result.items), null, 4));
};
E.prototype.logs_resend_api = function(req, res){
const ids = req.body.uuids;
for (let i in ids)
{
const r = this.loki.request_get_by_id(ids[i]);
let proxy;
if (!(proxy = this.proxy_ports[r.port]))
continue;
const opt = {
proxy: 'http://127.0.0.1:'+r.port,
url: r.url,
method: 'GET',
headers: JSON.parse(r.request_headers),
followRedirect: false,
};
if (proxy.opt.ssl)
opt.ca = ssl.ca.cert;
request(opt);
}
res.send('ok');
};
E.prototype.filtered_get = function(req){
if (this.argv.high_perf)
return {};
const skip = +req.query.skip||0;
const limit = +req.query.limit||0;
const query = {};
if (req.query.port_from && req.query.port_to)
query.port = {'$between': [req.query.port_from, req.query.port_to]};
if (req.query.search)
{
query.$or = [{url: {'$regex': RegExp(req.query.search)}},
{username: {'$regex': RegExp('session-[^-]*'+req.query.search)}}];
}
let status_code;
if (status_code = req.query.status_code)
{
if (/^\d\*\*$/.test(status_code))
query.status_code = {'$regex': RegExp(`^${status_code[0]}`)};
else
query.status_code = +status_code;
}
['port', 'content_type', 'protocol'].forEach(param=>{
let val;
if (val = req.query[param])
{
if (param=='port')
val = +val;
query[param] = val;
}
});
const sort = {field: req.query.sort||'uuid', desc: !!req.query.sort_desc};
const items = this.loki.requests_get(query, sort, limit, skip);
const total = this.loki.requests_count(query);
const sum_in = this.loki.requests_sum_in(query);
const sum_out = this.loki.requests_sum_out(query);
return {total, skip, limit, items, sum_in, sum_out};
};
E.prototype.node_version_api = etask._fn(
function*mgr_node_version(_this, req, res){
if (process.versions && !!process.versions.electron)
return res.json({is_electron: true});
const chk = yield etask.nfn_apply(check_node_version,
[{node: pkg.recomendedNode}]);
res.json({
current: chk.versions.node.version,
satisfied: chk.versions.node.isSatisfied,
recommended: pkg.recomendedNode,
});
});
E.prototype.last_version_api = etask._fn(
function*mgr_last_version(_this, req, res){
const r = yield util_lib.get_last_version(_this._defaults.api);
res.json({version: r.ver, newer: r.newer, versions: r.versions});
});
E.prototype.get_params = function(){
const args = [];
for (let k in this.argv)
{
const val = this.argv[k];
if (qw`$0 h help version p ? v _ explicit_opt rules native_args
daemon_opt`.includes(k))
{
continue;
}
if (lpm_config.credential_fields.includes(k))
continue;
if (typeof val=='object'&&_.isEqual(val, E.default[k])||
val===E.default[k])
{
continue;
}
if (lpm_config.boolean_fields.includes(k)||val===false)
{
args.push(`--${val?'':'no-'}${k}`);
continue;
}
[].concat(val).forEach(v=>{
if (k!='_')
args.push('--'+k);
args.push(v);
});
}
if (!this.argv.config)
{
// must provide these as args to enable login w/o config
for (let k of lpm_config.