@luminati-io/luminati-proxy
Version:
A configurable local proxy for brightdata.com
1,377 lines (1,364 loc) • 92.4 kB
JavaScript
// LICENSE_CODE ZON
'use strict'; /*jslint node:true, browser:true, es6:true*/
(function(){
let define;
let next_tick;
let is_node = typeof module=='object' && module.exports && module.children;
var is_rn = typeof global=='object' && !!global.nativeRequire ||
typeof navigator=='object' && navigator.product=='ReactNative';
if (is_rn)
{
define = require('./require_node.js').define(module, '../',
require('/util/conv.js'), require('/util/etask.js'),
require('/util/events.js'), require('/util/string.js'),
require('/util/zerr.js'), require('/util/util.js'),
require('/util/date.js'), require('/util/url.js'));
}
else if (!is_node)
{
define = self.define;
next_tick = (function(){
var can_set_immediate = typeof window!=='undefined'
&& window.setImmediate;
var can_post = typeof window!=='undefined'
&& window.postMessage && window.addEventListener;
if (can_set_immediate)
return function(f){ return window.setImmediate(f); };
if (can_post)
{
var queue = [];
window.addEventListener('message', function(ev){
var source = ev.source;
if ((source===window || source===null)
&& ev.data==='process-tick')
{
ev.stopPropagation();
if (queue.length>0)
{
var fn = queue.shift();
fn();
}
}
}, true);
return function(fn){
queue.push(fn);
window.postMessage('process-tick', '*');
};
}
return function(fn){ setTimeout(fn, 0); };
})();
}
else
define = require('./require_node.js').define(module, '../');
next_tick = next_tick || process.nextTick;
define(['/util/conv.js', '/util/etask.js', '/util/events.js',
'/util/string.js', '/util/zerr.js', '/util/util.js', '/util/date.js',
'/util/url.js'],
function(conv, etask, events, string, zerr, zutil, date, zurl){
const ef = etask.ef, assign = Object.assign;
const E = {}, E_t = {};
// for security reasons 'func' is disabled by default
const zjson_opt_default = {func: false, date: true, re: true};
const is_win = /^win/.test((is_node||is_rn) && process.platform);
const is_darwin = is_node && process.platform=='darwin';
const is_k8s = is_node && !!process.env.CLUSTER_NAME;
const default_user_agent = is_node ? (()=>{
const zconf = require('./config.js');
const conf = require('./conf.js');
return `Hola ${conf.app}/${zconf.ZON_VERSION}`;
})() : undefined;
const internalize = string.internalize_pool();
const DEBUG_STR_LEN = 4096, DEFAULT_WIN_SIZE = 1048576;
const SEC = 1000, MIN = 60*SEC, VFD_SZ = 8, VFD_BIN_SZ = 12;
let zcounter; // has to be lazy because zcounter.js itself uses this module
const net = is_node ? require('net') : null;
const SERVER_NO_MASK_SUPPORT = 'server_no_mask_support';
const BUFFER_CONTENT = 10;
const BUFFER_IPC_CALL = 11;
let SnappyStream, UnsnappyStream;
const EventEmitter = is_node ? require('events').EventEmitter
: events.EventEmitter;
const json_event = flag=>flag ? 'zjson' : 'json';
function noop(){}
const BUFFER_MESSAGES_CONTENT = 20;
const BUFFER_MESSAGES_TYPE_BIN = 0;
const BUFFER_MESSAGES_TYPE_STRING = 1;
const BUFFER_MESSAGES_TYPE_JSON = 2;
const BUFFER_MESSAGES_BIN_SZ = 4;
const BUFFER_MESSAGES_MAX_LENGTH = Math.pow(2, 32)-1;
class Buffers_array {
static is_buffer(buf){
if (!(buf instanceof Buffer) || buf.length<3*BUFFER_MESSAGES_BIN_SZ)
return false;
let prefix = buf.readUInt32BE(0);
let msg_type = buf.readUInt32BE(BUFFER_MESSAGES_BIN_SZ);
let length = buf.readUInt32BE(BUFFER_MESSAGES_BIN_SZ*2);
return prefix===0 && length==buf.length
&& msg_type==BUFFER_MESSAGES_CONTENT;
}
static parse(buf){
let offset = 3*BUFFER_MESSAGES_BIN_SZ, messages = [];
while (offset<buf.length)
{
let msg_type = buf.readUInt8(offset);
let msg_len = buf.readUInt32BE(offset+1);
let msg_value = buf.slice(offset+1+BUFFER_MESSAGES_BIN_SZ,
offset+1+BUFFER_MESSAGES_BIN_SZ+msg_len);
if (msg_type==BUFFER_MESSAGES_TYPE_STRING
|| msg_type==BUFFER_MESSAGES_TYPE_JSON)
{
msg_value = msg_value.toString();
}
messages.push(msg_value);
offset = offset+1+BUFFER_MESSAGES_BIN_SZ+msg_len;
}
return messages;
}
constructor(){
this.clean();
}
push(value){
let type;
if (value instanceof Buffer)
type = BUFFER_MESSAGES_TYPE_BIN;
else if (typeof value=='string')
{
type = BUFFER_MESSAGES_TYPE_STRING;
value = Buffer.from(value);
}
else
{
type = BUFFER_MESSAGES_TYPE_JSON;
value = Buffer.from(JSON.stringify(value));
}
let new_bytes_size = this._bytes_size+1+BUFFER_MESSAGES_BIN_SZ
+value.length;
if (new_bytes_size>BUFFER_MESSAGES_MAX_LENGTH)
throw new Error(`Buffers_array overflow ${new_bytes_size}`);
this._bytes_size += 1+BUFFER_MESSAGES_BIN_SZ+value.length;
this._buffer.push({type, value});
}
clean(){
this._buffer = [];
this._bytes_size = 3*BUFFER_MESSAGES_BIN_SZ;
}
get_buffer(){
let buf = Buffer.allocUnsafe(this._bytes_size);
buf.writeUInt32BE(0, 0);
buf.writeUInt32BE(BUFFER_MESSAGES_CONTENT, BUFFER_MESSAGES_BIN_SZ);
buf.writeUInt32BE(this._bytes_size, BUFFER_MESSAGES_BIN_SZ*2);
let offset = 3*BUFFER_MESSAGES_BIN_SZ;
for (let {type, value} of this._buffer)
{
buf.writeUInt8(type, offset);
buf.writeUInt32BE(value.length, offset+1);
value.copy(buf, offset = offset+1+BUFFER_MESSAGES_BIN_SZ);
offset += value.length;
}
return buf;
}
}
const make_ipc_server_class = ws_opt=>{
if (ws_opt.ipc_server && !ws_opt.ipc_server_class)
ws_opt.ipc_server_class = IPC_server_base.build(ws_opt);
};
const make_ipc_client_class = ws_opt=>{
if (ws_opt.ipc_client && !ws_opt.ipc_client_class)
ws_opt.ipc_client_class = IPC_client_base.build(ws_opt);
};
const counter_cache = {};
const make_counter = opt=>{
const zcounter_glob = opt && opt.zcounter_glob;
const cache_key = zcounter_glob ? 'glob_counter' : 'local_counter';
if (counter_cache[cache_key])
return counter_cache[cache_key];
const counter = {};
['inc', 'inc_level', 'avg', 'max', 'min', 'set_level'].forEach(m=>{
counter[m] = zcounter_glob ? zcounter[`glob_${m}`] : zcounter[m];
counter[m] = counter[m].bind(zcounter);
});
counter_cache[cache_key] = counter;
return counter;
};
class WS extends EventEmitter {
constructor(opt){
super();
make_ipc_server_class(opt);
make_ipc_client_class(opt);
this.ws = undefined;
this.data = opt.data;
this.connected = false;
this.reason = undefined;
this.listen_bin_throttle = opt.listen_bin_throttle;
this.zc_rx = opt.zcounter=='rx' || opt.zcounter=='all';
this.zc_tx = opt.zcounter=='tx' || opt.zcounter=='all';
this.zc_tx_per_cmd = this.zc_tx && opt.zcounter_tx_per_cmd;
this.zc_mux = opt.zcounter=='mux' || opt.zcounter=='all';
this.msg_log = assign({}, {treshold_size: null, print_size: 100},
opt.msg_log);
this.zc = opt.zc_label || (opt.zcounter ?
opt.label ? internalize(`${opt.label}_ws`) : 'ws' : undefined);
const zjson_opt = assign({}, zjson_opt_default, opt.zjson_opt);
this.zjson_opt_send = assign({}, zjson_opt, opt.zjson_opt_send);
this.zjson_opt_receive = assign({}, zjson_opt, opt.zjson_opt_receive);
this.label = opt.label;
this.remote_label = undefined;
this.local_addr = undefined;
this.local_port = undefined;
this.remote_addr = undefined;
this.remote_port = undefined;
this.remote_forwarded = false;
this.status = 'disconnected';
if (this.ping = is_node && opt.ping!=false)
{
this.ping_interval = typeof opt.ping_interval=='function'
? opt.ping_interval()
: opt.ping_interval || (is_k8s ? 30000 : 60000);
this.ping_timeout = typeof opt.ping_timeout=='function'
? opt.ping_timeout() : opt.ping_timeout || 10000;
this.ping_timer = undefined;
this.ping_expire_timer = undefined;
this.ping_last = 0;
}
this.pong_received = true;
this.refresh_ping_on_msg = opt.refresh_ping_on_msg!==false;
this.idle_timeout = opt.idle_timeout;
this.idle_timer = undefined;
this.ipc = opt.ipc_client_class ? new opt.ipc_client_class(this)
: undefined;
if (opt.ipc_server_class)
new opt.ipc_server_class(this);
this.bin_methods = opt.bin_methods || this.ipc && this.ipc.bin_methods;
this.time_parse = opt.time_parse;
this.mux = opt.mux ? new Mux(this) : undefined;
if ((this.zc || this.zc_mux) && !zcounter)
zcounter = require('./zcounter.js');
if (zcounter)
this._counter = make_counter(opt);
this.max_backpressure = opt.max_backpressure;
this._send_throttle_t = undefined;
this._buffers = undefined;
}
get_bin_prefix(msg){
return this.bin_methods && msg instanceof Buffer &&
msg.readUInt32BE(0)===0 ? msg.readUInt32BE(4) : undefined;
}
_clean_throttle(){
if (this._send_throttle_t)
{
clearTimeout(this._send_throttle_t);
this._send_throttle_t = undefined;
}
this._buffers = undefined;
}
_send_throttle(msg, throttle_ts){
this._buffers = this._buffers || new Buffers_array();
this._buffers.push(msg);
this._send_throttle_t = this._send_throttle_t || setTimeout(()=>{
this._send_throttle_t = undefined;
this.ws.send(this._buffers.get_buffer(), this.uws2 ? true
: undefined);
this._buffers.clean();
}, throttle_ts);
}
send(msg, opt){
if (zerr.is.debug())
{
zerr.debug(typeof msg=='string'
? `${this}> str: ${string.trunc(msg, DEBUG_STR_LEN)}`
: `${this}> buf: ${msg.length} bytes`);
}
if (!this.connected)
{
if (zerr.is.info())
zerr.info(`${this}: sending failed: disconnected`);
return false;
}
// workaround for ws library: the socket is already closing,
// but a notification has not yet been emitted
if (this.ws.readyState==2) // ws.CLOSING
{
if (zerr.is.info())
zerr.info(`${this}: sending failed: socket closing`);
return false;
}
this._update_idle();
if (opt && opt.bin_throttle)
this._send_throttle(msg, opt.bin_throttle);
else
this.ws.send(msg, this.uws2 ? typeof msg!='string' : opt);
if (this.zc_tx)
{
this._counter.inc(`${this.zc}_tx_msg`);
this._counter.inc(`${this.zc}_tx_bytes`, msg.length);
this._counter.avg(`${this.zc}_tx_bytes_per_msg`, msg.length);
}
if (this.zc_tx_per_cmd && opt && opt.cmd)
{
this._counter.inc(`${this.zc}.${opt.cmd}_tx_msg`);
this._counter.inc(`${this.zc}.${opt.cmd}_tx_bytes`, msg.length);
this._counter.avg(`${this.zc}.${opt.cmd}_tx_bytes_per_msg`,
msg.length);
}
return true;
}
bin(data){
data.msg = data.msg||Buffer.alloc(0);
let cmd = data.cmd;
let buf = Buffer.alloc(16 + cmd.length + data.msg.length);
// [0|BUFFER_IPC_CALL|cookie|cmd length|...cmd bin|...cmd result bin]
buf.writeUInt32BE(0, 0);
buf.writeUInt32BE(BUFFER_IPC_CALL, 4);
buf.writeUInt32BE(data.cookie, 8);
buf.writeUInt32BE(cmd.length, 12);
buf.write(cmd, 16);
data.msg.copy(buf, 16+cmd.length);
return this.send(buf, {cmd: data.cmd});
}
json(data){ return this.send(JSON.stringify(data), {cmd: data.cmd}); }
zjson(data){
return this.send(conv.JSON_stringify(data, this.zjson_opt_send),
{cmd: data.cmd});
}
_check_status(){
let prev = this.status;
this.status = this.ws
? this.connected ? 'connected' : 'connecting'
: 'disconnected';
if (this.status!=prev)
{
this.emit(this.status);
this.emit('status', this.status);
if (this.status=='disconnected')
this._on_disconnected();
}
}
_on_disconnected(){ this.emit('destroyed'); }
_assign(ws){
this.ws = ws;
this.uws2 = this.ws.uws2;
this.ws.onopen = this._on_open.bind(this);
this.ws.onclose = this._on_close.bind(this);
this.ws.onmessage = this._on_message.bind(this);
(this.ws.handlers||this.ws).onerror = this._on_error.bind(this);
if (is_node)
{
this.ws.on('upgrade', this._on_upgrade.bind(this));
this.ws.on('unexpected-response',
this._on_unexpected_response.bind(this));
}
if (this.ping)
this.ws.on('pong', this._on_pong.bind(this));
}
abort(code, reason){
this.reason = reason||code;
let msg = `${this}: closed locally`;
if (this.reason)
msg += ` (${this.reason})`;
// chrome and ff doesn't allow code outside 1000 and 3000-4999
if (!is_node && !is_rn && !(code==1000 || code>=3000 && code<5000))
code += 3000;
this._close(true, code, reason);
zerr.warn(msg);
if (this.zc && code)
this._counter.inc(`${this.zc}_err_${code}`);
this._check_status();
}
_close(close, code, reason){
if (!this.ws)
return;
if (this.ping)
{
clearTimeout(this.ping_timer);
clearTimeout(this.ping_expire_timer);
this.ping_timer = undefined;
this.ping_expire_timer = undefined;
this.ping_last = 0;
this.pong_received = true;
this.ws.removeAllListeners('pong');
}
this.ws.onopen = undefined;
this.ws.onclose = undefined;
this.ws.onmessage = undefined;
(this.ws.handlers||this.ws).onerror = noop;
if (is_node)
{
this.ws.removeAllListeners('unexpected-response');
this.ws.removeAllListeners('upgrade');
}
this.local_addr = undefined;
this.local_port = undefined;
this.remote_addr = undefined;
this.remote_port = undefined;
if (close)
{
if (this.ws.terminate && (!this.connected || code==-2))
{
zerr.info(`${this}: ws.terminate`);
this.ws.terminate();
}
else
{
zerr.info(`${this}: ws.close`);
this.ws.close(code, reason);
}
}
this.ws = undefined;
this.connected = false;
this._clean_throttle();
}
toString(){
let res = this.label ? `${this.label} WS` : 'WS';
if (this.remote_label || this.remote_addr)
res += ' (';
if (this.remote_label)
res += this.remote_label;
if (this.remote_label && this.remote_addr)
res += ' ';
if (this.remote_addr)
res += this.remote_addr;
if (this.remote_label || this.remote_addr)
res += ')';
return res;
}
test_ping({ping_timeout, pong_last_expire, ping_cb}){
if (pong_last_expire && date.monotonic()-this
.pong_last<pong_last_expire)
{
return true;
}
if (!this._test_ping_et)
{
try { this.ws.ping(); }
catch(e){ return false; }
if (ping_cb)
ping_cb();
this._test_ping_et = etask.wait();
}
let _this = this;
return etask(function*(){
this.on('finally', ()=>{
if (_this._test_ping_et && !etask
.is_final(_this._test_ping_et))
{
_this._test_ping_et.return();
}
_this._test_ping_et = null;
});
this.alarm(ping_timeout, ()=>this.return(false));
yield this.wait_ext(_this._test_ping_et);
return true;
});
}
_on_open(){
if (this.connected)
return;
this.connected = true;
let sock = this.ws._socket||{};
// XXX pavlo: uws lib doesn't have these properties in _socket:
// https://github.com/hola/uWebSockets-bindings/blob/master/nodejs/src/uws.js#L276
// get them from upgrade request
this.local_addr = internalize(sock.localAddress);
this.local_port = sock.localPort;
if (this.remote_addr==sock.remoteAddress)
this.remote_forwarded = false;
if (!this.remote_forwarded)
{
this.remote_addr = sock.remoteAddress;
this.remote_port = sock.remotePort;
}
zerr.notice(`${this}: connected`);
if (this.ping)
{
this.pong_received = true; // skip first ping expiration
this.ping_timer = setTimeout(()=>this._ping(), this.ping_interval);
this.ping_expire_timer = setTimeout(()=>this._ping_expire(),
this.ping_timeout);
}
this._check_status();
}
_on_close(event){
this.reason = event.reason||event.code;
zerr.notice('%s: closed by remote (reason=%s, event=%O)', this,
this.reason, zutil.omit(event, 'target'));
this._close();
this._check_status();
}
_on_error(event){
this.reason = event.message||'Network error';
if (!is_error_event_silent(event))
{
zerr('%s: got error event (reason=%s, event=%O)', this,
this.reason, zutil.omit(event, 'target'));
}
if (this.zc)
this._counter.inc(`${this.zc}_err`);
this._close();
this._check_status();
}
_on_unexpected_response(req, resp){
this._on_error({message: 'unexpected response'});
this.emit('unexpected-response');
}
_on_upgrade(resp){ zerr.debug(`${this}: upgrade conn`); }
_on_message(event){
let msg = event.data;
if (msg instanceof ArrayBuffer)
msg = Buffer.from(Buffer.from(msg)); // make a copy
if (!this.listen_bin_throttle || !Buffers_array.is_buffer(msg))
return this._on_message_base(msg);
for (const data of Buffers_array.parse(msg))
this._on_message_base(data);
}
_on_message_base(msg){
if (zerr.is.debug())
{
zerr.debug(typeof msg=='string'
? `${this}< str: ${string.trunc(msg, DEBUG_STR_LEN)}`
: `${this}< buf: ${msg.length} bytes`);
}
if (this.zc_rx)
{
this._counter.inc(`${this.zc}_rx_msg`);
this._counter.inc(`${this.zc}_rx_bytes`, msg.length);
this._counter.avg(`${this.zc}_rx_bytes_per_msg`, msg.length);
this._counter.max(`${this.zc}_rx_bytes_per_msg_max`, msg.length);
}
try {
if (!this._parse_message(msg))
this.abort(1003, 'Unexpected message');
this._update_idle();
if (this.refresh_ping_on_msg)
this._refresh_ping_timers();
} catch(e){ ef(e);
zerr(`${this}: ${zerr.e2s(e)}`);
return this.abort(1011, e.message);
}
}
_parse_message(msg){
let handled = false;
const bin_event = this._parse_bin_message(msg);
if (bin_event)
{
this.emit('json', bin_event);
handled = true;
}
if (typeof msg=='string')
{
let parsed;
if (this._events.zjson)
{
if (this.zc && this.time_parse)
{
const t = date.monotonic();
parsed = conv.JSON_parse(msg, this.zjson_opt_receive);
const d = date.monotonic()-t;
this._counter.avg(`${this.zc}_parse_zjson_ms`, d);
this._counter.max(`${this.zc}_parse_zjson_max_ms`, d);
}
else
parsed = conv.JSON_parse(msg, this.zjson_opt_receive);
this.emit('zjson', parsed);
handled = true;
}
if (this._events.json)
{
if (this.zc && this.time_parse && !parsed)
{
const t = date.monotonic();
parsed = JSON.parse(msg);
const d = date.monotonic()-t;
this._counter.avg(`${this.zc}_parse_json_ms`, d);
this._counter.max(`${this.zc}_parse_json_max_ms`, d);
}
else
parsed = parsed||JSON.parse(msg);
this.emit('json', parsed);
handled = true;
}
if (this._events.text)
{
this.emit('text', msg);
handled = true;
}
if (this.msg_log.treshold_size &&
msg.length>=this.msg_log.treshold_size)
{
zerr.warn(`${this}: Message length treshold`
+` ${this.msg_log.treshold_size} exceeded:`
+` ${msg.substr(0, this.msg_log.print_size)}`);
}
}
else if (this._events.bin)
{
this.emit('bin', msg);
handled = true;
}
if (this._events.raw)
{
this.emit('raw', msg);
handled = true;
}
return handled;
}
_parse_bin_message(msg){
const type = this.get_bin_prefix(msg);
if (type==BUFFER_CONTENT)
{
const bin_event_len = msg.readUInt32BE(VFD_SZ);
const bin_event = JSON.parse(
msg.slice(VFD_BIN_SZ, VFD_BIN_SZ+bin_event_len).toString());
bin_event.msg = msg.slice(VFD_BIN_SZ+bin_event_len);
return bin_event;
}
if (type==BUFFER_IPC_CALL)
{
const cookie = msg.readUInt32BE(8);
const cmd_length = msg.readUInt32BE(12);
const cmd = msg.slice(16, 16+cmd_length).toString();
const res = msg.slice(16+cmd_length);
return {type: 'ipc_call', cookie, cmd, msg: res, bin: true};
}
}
_on_pong(){
this.pong_received = true;
this.pong_last = date.monotonic();
if (zerr.is.debug())
{
zerr.debug(
`${this}< pong (rtt ${date.monotonic()-this.ping_last}ms)`);
}
if (this.zc)
{
this._counter.avg(
`${this.zc}_ping_ms`, this.pong_last-this.ping_last);
}
if (this._test_ping_et)
{
if (!etask.is_final(this._test_ping_et))
this._test_ping_et.return();
this._test_ping_et = null;
}
}
_ping(){
// don't send new ping if ping_timeout > ping_interval (weird case)
if (!this.pong_received)
return;
this.pong_received = false;
// workaround for ws library: the socket is already closing,
// but a notification has not yet been emitted
if (!this.connected || this.ws.readyState==2) // ws.CLOSING
return;
try { this.ws.ping(); }
catch(e){ // rarer case: don't crash - post more logs
return zerr('Ping attempt fail, for'
+` ${JSON.stringify(this.inspect())} ${zerr.e2s(e)}`);
}
this._refresh_ping_timers();
this.ping_last = date.monotonic();
if (zerr.is.debug())
zerr.debug(`${this}> ping (max ${this.ping_timeout}ms)`);
}
_ping_expire(){
if (this.pong_received)
return;
this.abort(1002, 'Ping timeout');
}
_refresh_ping_timers(){
if (!this.ping_timer || !this.ping_expire_timer)
return;
if (zutil.is_timer_refresh)
{
this.ping_timer.refresh();
this.ping_expire_timer.refresh();
return;
}
clearTimeout(this.ping_timer);
this.ping_timer = setTimeout(()=>this._ping(), this.ping_interval);
clearTimeout(this.ping_expire_timer);
this.ping_expire_timer = setTimeout(()=>this._ping_expire(),
this.ping_timeout);
}
_idle(){
if (this.zc)
this._counter.inc(`${this.zc}_idle_timeout`);
this.emit('idle_timeout');
this.abort(1002, 'Idle timeout');
}
_update_idle(){
if (!this.idle_timeout)
return;
clearTimeout(this.idle_timer);
this.idle_timer = setTimeout(this._idle.bind(this), this.idle_timeout);
}
close(code, reason){
if (!this.ws)
return;
let msg = `${this}: closed locally`;
if (reason||code)
msg += ` (${reason||code})`;
this._close(true, code, reason);
zerr.notice(msg);
if (this.zc && code)
this._counter.inc(`${this.zc}_err_${code}`);
this._check_status();
}
inspect(){
return {
class: this.constructor.name,
label: this.toString(),
status: this.status,
reason: this.reason,
local_addr: this.local_addr,
local_port: this.local_port,
remote_addr: this.remote_addr,
remote_port: this.remote_port,
};
}
}
class Client extends WS {
constructor(url, opt={}){
if (opt.mux)
opt.mux = assign({}, opt.mux);
super(opt);
this.status = 'connecting';
// XXX bruno: enable also for opt.zcounter=='all' after prd validations
if (opt.zcounter=='disconnected')
this._monitor_disconnected();
this.impl = client_impl(opt);
this.url = url;
this.servername = opt.servername;
this.retry_interval = opt.retry_interval||10000;
this.retry_max = opt.retry_max||this.retry_interval;
this.retry_random = opt.retry_random;
this.next_retry = this.retry_interval;
this.no_retry = opt.no_retry;
this.retry_chances = opt.retry_chances;
this._retry_count = 0;
this.lookup = opt.lookup;
this.lookup_ip = opt.lookup_ip;
this.fallback = opt.fallback &&
assign({retry_threshold: 1, retry_mod: 5}, opt.fallback);
this.headers = undefined;
this.deflate = !!opt.deflate;
this.agent = opt.agent;
if (opt.e1008_exit_reason)
{
this.e1008_exit_reason = opt.e1008_exit_reason;
this.e1008_ts_start = date.monotonic();
}
this.reason = undefined;
this.reconnect_timer = undefined;
this.server_no_mask_support = false;
this.handshake_timeout = opt.handshake_timeout===undefined
? 50000 : opt.handshake_timeout;
this.handshake_timer = undefined;
if (this.zc)
{
this._counter.inc_level(`level_${this.zc}_online`, 0, 'sum',
'sum');
}
if (opt.proxy)
{
let _lib = require('https-proxy-agent');
this.agent = new _lib(opt.proxy);
}
if (is_node)
{
this.headers = assign(
{'User-Agent': opt.user_agent||default_user_agent},
opt.headers,
{'client-no-mask-support': 1});
}
this._connect();
}
_monitor_disconnected(){
let disconnected_metric = `${this.zc}_disconnected_ms`;
let on_status = status=>{
if (status=='connected')
{
if (this.monitor_disconnected_et)
this.monitor_disconnected_et.return();
return;
}
if (this.monitor_disconnected_et)
return;
let disconnected_ts = Date.now();
this.monitor_disconnected_et = etask.interval(10*SEC, ()=>{
this._counter.set_level(disconnected_metric,
Date.now()-disconnected_ts);
});
this.monitor_disconnected_et.finally(()=>{
this._counter.set_level(disconnected_metric, 0);
this.monitor_disconnected_et = null;
});
};
on_status(this.status);
this.on('status', on_status);
let on_destroyed = ()=>{
if (this.monitor_disconnected_et)
this.monitor_disconnected_et.return();
this.off('status', on_status);
this.off('destroyed', on_destroyed);
};
this.on('destroyed', on_destroyed);
}
// we don't want WS to emit 'destroyed', Client controls it by itself,
// because it supports reconnects
_on_disconnected(){}
send(msg){
return super.send(msg,
this.server_no_mask_support ? {mask: false} : undefined);
}
_on_message(ev){
if (!this.server_no_mask_support && ev.data===SERVER_NO_MASK_SUPPORT)
{
this.server_no_mask_support = true;
return void this.emit(SERVER_NO_MASK_SUPPORT);
}
return super._on_message(ev);
}
_assign(ws){
super._assign(ws);
if (this.zc)
{
this._counter.inc_level(`level_${this.zc}_online`, 1, 'sum',
'sum');
}
}
_close(close, code, reason){
if (this.zc && this.ws)
{
this._counter.inc_level(`level_${this.zc}_online`, -1, 'sum',
'sum');
}
if (this.handshake_timer)
this.handshake_timer = clearTimeout(this.handshake_timer);
super._close(close, code, reason);
}
_connect(){
this.reason = undefined;
this.reconnect_timer = undefined;
this.server_no_mask_support = false;
let opt = {headers: this.headers};
let url = this.url, lookup_ip = this.lookup_ip, fb = this.fallback, v;
if (fb && fb.url && this._retry_count%fb.retry_mod>fb.retry_threshold)
{
url = fb.url;
lookup_ip = fb.lookup_ip;
}
if (!is_rn)
{
// XXX vladislavl: it won't work for uws out of box
opt.agent = this.agent;
opt.perMessageDeflate = this.deflate;
opt.servername = this.servername;
opt.lookup = this.lookup;
if (lookup_ip && net && (v = net.isIP(lookup_ip)))
{
opt.lookup = (h, o, cb)=>{
cb = cb||o;
next_tick(()=>cb(undefined, o && o.all ?
[{address: lookup_ip, family: v}] : lookup_ip, v));
};
}
}
if (this.zc)
this._counter.inc(`${this.zc}_fallback`, url==this.url ? 0 : 1);
zerr.notice(`${this}: connecting to ${url}`);
this._assign(new this.impl(url, undefined, opt));
if (this.handshake_timeout)
{
this.handshake_timer = setTimeout(
()=>this.abort(1002, 'Handshake timeout'),
this.handshake_timeout);
}
this._check_status();
}
_reconnect(){
if (this.no_retry)
return false;
if (this.retry_chances && this._retry_count >= this.retry_chances-1)
{
this.close();
this.emit('out_of_retries');
return false;
}
this.emit('reconnecting');
let delay = this.next_retry;
if (typeof delay=='function')
delay = delay();
else
{
let coeff = this.retry_random ? 1+Math.random() : 2;
this.next_retry = Math.min(Math.round(delay*coeff),
typeof this.retry_max=='function'
? this.retry_max() : this.retry_max);
}
if (zerr.is.info())
zerr.info(`${this}: will retry in ${delay}ms`);
this._retry_count++;
this.reconnect_timer = setTimeout(()=>this._connect(), delay);
}
_on_open(){
if (this.handshake_timer)
this.handshake_timer = clearTimeout(this.handshake_timer);
this.next_retry = this.retry_interval;
this._retry_count = 0;
super._on_open();
}
_handle_e1008(event){
if (!this.e1008_exit_reason)
return;
if (event.code != 1008 || this.e1008_exit_reason != event.reason)
return;
if (!this.no_retry && date.monotonic()-this.e1008_ts_start < 60000)
return;
zerr.zexit(`Got '${event.reason}' from ${event.target.url}`);
}
_on_close(event){
this._handle_e1008(event);
this._reconnect();
super._on_close(event);
}
_on_error(event){
this._reconnect();
super._on_error(event);
}
abort(code, reason){
this._reconnect();
super.abort(code, reason);
}
close(code, reason){
if (this.ws)
this.emit('destroyed');
super.close(code, reason);
if (this.reconnect_timer)
{
clearTimeout(this.reconnect_timer);
this.reconnect_timer = undefined;
this.emit('destroyed');
}
}
inspect(){
return assign(super.inspect(), {
url: this.url,
lookup_ip: this.lookup_ip,
retry_interval: this.retry_interval,
retry_max: this.retry_max,
next_retry: this.next_retry,
handshake_timeout: this.handshake_timeout,
deflate: this.deflate,
reconnecting: !!this.reconnect_timer,
fallback: this.fallback,
});
}
}
class Server {
constructor(opt={}, handler=undefined){
if (opt.mux)
opt.mux = assign({}, {dec_vfd: true}, opt.mux);
make_ipc_server_class(opt);
make_ipc_client_class(opt);
this.handler = handler;
let ws_opt = {
server: opt.http_server,
host: opt.host||'0.0.0.0',
port: opt.port,
noServer: !opt.http_server && !opt.port,
path: opt.path,
clientTracking: false,
perMessageDeflate: !!opt.deflate,
};
if (opt.max_payload)
ws_opt.maxPayload = opt.max_payload;
if (opt.verify)
ws_opt.verifyClient = opt.verify;
const impl = server_impl(opt);
this.ws_server = new impl.Server(ws_opt);
this.server_no_mask_support = impl.server_no_mask_support;
this.opt = opt;
this.label = opt.label;
this.connections = new Set();
this.zc = opt.zcounter!=false ? opt.label ? `${opt.label}_ws` : 'ws'
: undefined;
this.ws_server.addListener('connection', this.accept.bind(this));
if (opt.port)
zerr.notice(`${this}: listening on port ${opt.port}`);
if (!zcounter)
zcounter = require('./zcounter.js');
this._counter = make_counter(opt);
// ensure the metric exists, even if 0
if (this.zc)
this._counter.inc_level(`level_${this.zc}_conn`, 0, 'sum', 'sum');
}
toString(){ return this.label ? `${this.label} WS server` : 'WS server'; }
upgrade(req, socket, head){
this.ws_server.handleUpgrade(req, socket, head,
ws=>this.accept(ws, req));
}
accept(ws, req=ws.upgradeReq){
if (!ws._socket.remoteAddress)
{
ws.onerror = noop;
return zerr.warn(`${this}: dead incoming connection`);
}
let headers = req && req.headers || {};
if (this.opt.origin_whitelist)
{
if (!this.opt.origin_whitelist.includes(headers.origin))
{
if (ws._socket.destroy)
ws._socket.destroy();
else if (ws.terminate)
ws.terminate();
return zerr.notice('incoming conn from %s rejected',
headers.origin||'unknown origin');
}
zerr.notice('incoming conn from %s', headers.origin);
}
let zws = new WS(this.opt);
if (this.opt.conn_max_event_listeners!==undefined)
zws.setMaxListeners(this.opt.conn_max_event_listeners);
if (this.opt.trust_forwarded)
{
let forwarded = headers['x-real-ip'];
if (!forwarded)
{
forwarded = headers['x-forwarded-for'];
if (forwarded)
{
let ips = forwarded.split(',');
forwarded = ips[ips.length-1];
}
}
if (forwarded)
{
zws.remote_addr = forwarded;
zws.remote_forwarded = true;
}
}
let ua = headers['user-agent'];
let m = /^Hola (.+)$/.exec(ua);
zws.remote_label = m ? m[1] : ua ? 'web' : undefined;
zws._assign(ws);
zws._on_open();
if (this.server_no_mask_support && headers['client-no-mask-support'])
zws.send(SERVER_NO_MASK_SUPPORT);
this.connections.add(zws);
if (this.zc)
{
this._counter.inc(`${this.zc}_conn`);
this._counter.inc_level(`level_${this.zc}_conn`, 1, 'sum', 'sum');
}
zws.addListener('disconnected', ()=>{
this.connections.delete(zws);
if (this.zc)
{
this._counter.inc_level(`level_${this.zc}_conn`, -1, 'sum',
'sum');
}
});
if (this.handler)
{
try {
zws.data = this.handler(zws, req);
} catch(e){ ef(e);
zerr(zerr.e2s(e));
return zws.close(1011, String(e));
}
}
return zws;
}
broadcast(msg){
if (zerr.is.debug())
{
zerr.debug(typeof msg=='string'
? `${this}> broadcast str: ${string.trunc(msg, DEBUG_STR_LEN)}`
: `${this}> broadcast buf: ${msg.length} bytes`);
}
for (let zws of this.connections)
zws.send(msg);
}
broadcast_json(data){
if (this.connections.size)
this.broadcast(JSON.stringify(data));
}
broadcast_zjson(data){
if (this.connections.size)
this.broadcast(conv.JSON_stringify(data, this.zjson_opt_send));
}
close(code, reason){
zerr.notice(`${this}: closed`);
this.ws_server.close();
for (let zws of this.connections)
zws.close(code, reason);
}
inspect(){
let connections = [];
for (let c of this.connections)
connections.push(c.inspect());
return {
class: this.constructor.name,
label: this.toString(),
opt: this.opt,
connections,
};
}
}
class Uws2_impl extends EventEmitter {
constructor(ws_opt, handlers = {onconnection: noop}){
super();
this.lib = require('uws2');
this.verifyClient = ws_opt.verifyClient;
this.handlers = handlers;
this.listen_socket = null;
const app_opt = {};
if (ws_opt.ssl_conf)
{
app_opt.key_file_name = ws_opt.ssl_conf.key;
app_opt.cert_file_name = ws_opt.ssl_conf.cert;
}
const ws_handler = {
idleTimeout: Math.min(ws_opt.srv_timeout, 960),
maxPayloadLength: ws_opt.maxPayloadLength||1024*1024*100,
maxBackpressure: 0,
upgrade: (res, req, context)=>{
let req_aborted = {aborted: false};
res.onAborted(()=>{ req_aborted.aborted = true; });
const _handlers = Object.assign({}, this.handlers);
const emitter = new EventEmitter();
emitter.on('error', (...args)=>
_handlers.onerror && _handlers.onerror(...args));
const headers = {};
// XXX igors: do we need to filter by allowed headers ?
req.forEach((key, value)=>headers[key] = value);
if (!this.verify(res, headers))
return;
const user_data = {
headers,
handlers: _handlers,
emitter,
uws2: true,
_socket: {
remoteAddress: Buffer.from(Buffer.from(
res.getRemoteAddressAsText())).toString(),
localPort: ws_opt.port,
},
on: (event, handler)=>emitter.on(event, handler),
emit: (event, msg)=>emitter.emit(event, msg),
removeAllListeners: ev=>emitter.removeAllListeners(ev),
};
res.upgrade(user_data, req.getHeader('sec-websocket-key'),
req.getHeader('sec-websocket-protocol'),
req.getHeader('sec-websocket-extensions'), context);
},
open: ws=>this.handlers.onconnection &&
this.handlers.onconnection(ws),
drain: ws=>{
if (!ws.pending_mux_et)
return;
for (let task of ws.pending_mux_et)
{
if (ws.getBufferedAmount()>this.max_backpressure)
break;
task.continue(null, true);
}
},
close: (ws, code, message)=>{
if (ws.pending_mux_et)
ws.pending_mux_et = null;
message = Buffer.from(message).toString();
if (message=='WebSocket timed out from inactivity')
{
if (this.handlers.ontimeout)
this.handlers.ontimeout(ws);
}
return ws.onclose && ws.onclose({code, message});
},
message: (ws, message, is_bin)=>{
if (!is_bin)
message = Buffer.from(Buffer.from(message));
return ws.onmessage && ws.onmessage({
data: is_bin ? message : message.toString()});
},
pong: (ws, message)=>ws.emitter.emit('pong', {ws, message}),
};
const req_handler = (res, req)=>{
if (!this.handlers.onreq)
return res.end();
let req_aborted = {aborted: false};
res.onAborted(()=>{ req_aborted.aborted = true; });
this.handlers.onreq(req, res, req_aborted);
};
this.server = this.lib[ws_opt.ssl_conf ? 'SSLApp' : 'App'](app_opt)
.ws('/', ws_handler).any('/*', req_handler);
if (ws_opt.ssl_conf && ws_opt.ssl_conf.sni)
{
for (let servername in ws_opt.ssl_conf.sni)
{
this.server.addServerName('*.'+servername, {
key_file_name: ws_opt.ssl_conf.sni[servername]+'.key',
cert_file_name: ws_opt.ssl_conf.sni[servername]+'.crt'
}).domain('*.'+servername)
.ws('/', ws_handler).any('/*', req_handler);
}
}
this.server.listen(ws_opt.host||'0.0.0.0', ws_opt.port, token=>{
if (token)
{
this.emit('listening');
this.listen_socket = token;
}
});
}
on(event, handler){
if (event=='listening' && this.listen_socket)
handler();
else
super.on(event, handler);
}
close(){
if (this._server_closed)
return;
this._server_closed = true;
this.server.close();
this.listen_socket = null;
this.emit('close');
}
close_listen_socket(){
if (!this.listen_socket)
return;
this.lib.us_listen_socket_close(this.listen_socket);
this.listen_socket = null;
this.emit('close');
}
verify(res, headers){
if (!this.verifyClient)
return true;
let verify_res, arg = {
origin: headers.origin,
secure: false,
req: {headers}
};
const cb = (v_res, status='403', reason='Unauthorized', _headers={})=>{
verify_res = v_res;
if (!v_res)
{
res.cork(()=>{
res.writeStatus(status);
for (let h in _headers)
res.writeHeader(h, _headers[h]);
res.end(reason);
});
}
};
if (this.verifyClient.length==1)
{
verify_res = this.verifyClient(arg);
if (!verify_res)
res.end();
}
else
this.verifyClient(arg, cb);
return verify_res;
}
}
class Server_uws2 {
constructor(opt={}, handler=undefined){
if (opt.mux)
opt.mux = assign({}, {dec_vfd: true}, opt.mux);
make_ipc_server_class(opt);
make_ipc_client_class(opt);
this.handler = handler;
let ws_opt = {
host: opt.host,
port: opt.port,
ssl_conf: opt.ssl_conf,
srv_timeout: opt.srv_timeout,
on_timeout: opt.on_timeout,
req_handler: opt.req_handler,
};
if (opt.max_payload)
ws_opt.maxPayloadLength = opt.max_payload;
if (opt.verify)
ws_opt.verifyClient = opt.verify;
this.ws_server = new Uws2_impl(ws_opt, {
onconnection: this.accept.bind(this),
ontimeout: opt.on_timeout,
onreq: opt.req_handler,
});
this.server_no_mask_support = false;
opt.max_backpressure = this.ws_server.max_backpressure = (is_node
? process.env.MAX_BACKPRESSURE : null) || 50*1024;
this.opt = opt;
this.label = opt.label;
this.connections = new Set();
this.zc = opt.zcounter!=false ? opt.label ? `${opt.label}_ws` : 'ws'
: undefined;
if (opt.port)
zerr.notice(`${this}: listening on port ${opt.port}`);
if (!zcounter)
zcounter = require('./zcounter.js');
this._counter = make_counter(opt);
// ensure the metric exists, even if 0
if (this.zc)
this._counter.inc_level(`level_${this.zc}_conn`, 0, 'sum', 'sum');
}
wait_listen(){
let listen = etask.wait();
this.ws_server.once('listening', ()=>listen.return());
return listen;
}
add_handler(opt, pattern, handler){
let def_res = {
json(obj){
this.writeStatus('200').writeHeader('Content-Type',
'application/json');
this.end(JSON.stringify(obj));
},
uncaught(url, e){
let err = zerr.e2s(e);
zerr.notice(`${url} uncaught ${err}`);
this.writeStatus('502').end(err);
},
};
let param_count = 0;
// XXX igors: find the better way to find parameters amount
if (typeof pattern=='string')
param_count = pattern.split(':').length-1;
this.ws_server.server[opt](pattern, (res, req)=>{
let active_et;
res.onAborted(()=>{
zerr.notice(`${opt.toUpperCase()} ${pattern} aborted`);
if (active_et)
active_et.return();
});
let url = `${opt.toUpperCase()} ${req.getUrl()}${req.getQuery()}`;
return active_et = etask(function*(){
try {
assign(res, def_res);
let len = req.getHeader('content-length');
let ctype = req.getHeader('content-type');
// need to clone parameters, otherwise they will be
// unaccessible
req.query = zurl.qs_parse(req.getQuery()||'');
if (param_count)
{
req.params = new Array(param_count);
for (let p = 0; p<param_count; p++)
req.params[p] = req.getParameter(p);
}
if (len)
{
let body = [], wait_body = etask.wait(20*SEC);
res.onData((ch, is_last)=>{
body.push(Buffer.from(Buffer.from(ch)));
if (is_last)
wait_body.return(Buffer.concat(body));
});
let obj_body = yield wait_body;
if (ctype=='application/json')
req.body = JSON.parse(String(obj_body));
else if (ctype=='text/html')
req.body = String(obj_body);