luminati-proxy-stripped
Version:
A configurable local proxy for luminati.io
881 lines (859 loc) • 30.2 kB
JavaScript
// LICENSE_CODE ZON ISC
; /*jslint react:true, es6:true*/
import React from 'react';
import Pure_component from '../../www/util/pub/pure_component.js';
import ajax from '../../util/ajax.js';
import setdb from '../../util/setdb.js';
import zescape from '../../util/escape.js';
import csv from '../../util/csv.js';
import etask from '../../util/etask.js';
import classnames from 'classnames';
import filesaver from 'file-saver';
import {get_static_country} from './util.js';
import _ from 'lodash';
import $ from 'jquery';
import Proxy_add from './proxy_add.js';
import Proxy_blank from './proxy_blank.js';
import {Modal, Checkbox, Pagination_panel, Link_icon,
Tooltip, Modal_dialog, Tooltip_bytes} from './common.js';
import {withRouter} from 'react-router-dom';
let country_names = {};
const Targeting_cell = ({proxy})=>{
const flag_with_title = (country, title)=>{
let country_name = country_names[country];
return <Tooltip title={country_name}>
<span>
<span className={'flag-icon flag-icon-'+country}/>
{title}
</span>
</Tooltip>;
};
const static_country = get_static_country(proxy);
if (static_country&&static_country!='any'&&static_country!='*')
return flag_with_title(static_country, static_country.toUpperCase());
let val = proxy.country;
if (!val||val=='any'||val=='*')
{
return <Tooltip title="Any">
<img src="/img/flag_any_country.svg"/>
</Tooltip>;
}
val = val.toUpperCase();
const state = proxy.state&&proxy.state.toUpperCase();
if (!state)
return flag_with_title(proxy.country, val);
if (!proxy.city)
return flag_with_title(proxy.country, `${val} (${state})`);
return flag_with_title(proxy.country,
`${val} (${state}), ${proxy.city}`);
};
const Status_cell = ({proxy, status, status_details})=>{
const details = status_details && status_details.map(d=>d.msg).join(',');
if (!status)
{
return <Tooltip title="Status of this proxy is being tested">
Testing</Tooltip>;
}
else if (status=='error')
return <Tooltip title={details}>Error</Tooltip>;
else if (status=='ok'&&details)
{
return <Tooltip title={details}>
<span>
OK
<div className="ic_warning"/>
</span>
</Tooltip>;
}
else if (status=='ok'&&!details)
return <Tooltip title="This proxy works correctly">OK</Tooltip>;
};
const Boolean_cell = ({proxy, col})=>{
if (proxy[col]===true)
return <img src="/img/ic_checkmark.svg"/>;
return <img src="/img/ic_off.svg"/>;
};
const Session_cell = ({proxy})=>{
if (proxy.session===true)
return <i className="fa fa-random"/>;
else if (proxy.session)
return proxy.session;
};
const Type_cell = ({proxy})=>{
let val, tip;
if (proxy.ext_proxies)
{
val = 'External';
tip = 'Proxy port configured with external IP and credentials';
}
else
{
val = 'Luminati';
tip = 'Proxy port using your Luminati account';
}
return <Tooltip title={tip}>{val}</Tooltip>;
};
const Last_req_cell = ({proxy})=>{
const val = (proxy.last_req||{}).url||'—';
return <Tooltip title={val}>{val}</Tooltip>;
};
const Port_cell = ({proxy, master_port})=>{
let val;
if (!master_port&&proxy.multiply&&proxy.multiply>1)
val = proxy.port+':1..'+proxy.multiply;
else
val = proxy.port;
const title = `${proxy.port} is a proxy port that refers to a specific
virtual location on a computer. You can use it as a virtual proxy to
sends requests`;
return <Tooltip title={title}>{val}</Tooltip>;
};
const Success_rate_cell = ({proxy})=>{
const val = !proxy.reqs ? '—' :
(proxy.success/proxy.reqs*100).toFixed(2)+'%';
const title = `total: ${proxy.reqs}, success: ${proxy.success}`;
return <Tooltip title={title}>{val}</Tooltip>;
};
const Reqs_cell = ({proxy})=>{
const reqs = proxy.reqs||0;
return <Tooltip title={`${reqs} requests sent through this proxy port`}>
{reqs}</Tooltip>;
};
const columns = [
{
key: 'port',
title: 'Proxy port',
sticky: true,
render: Port_cell,
tooltip: 'A proxy port is a number that refers to a specific virtual '
+'location on a computer. Create and configure proxy ports, then '
+'connect the crawler to send requests through the port',
ext: true,
dynamic: true,
},
{
key: 'proxy_type',
title: 'Type',
render: Type_cell,
tooltip: 'Type of connected proxy - Luminati proxy or external proxy '
+'(non-Luminati)',
sticky: true,
ext: true,
},
{
key: '_status',
title: 'Status',
type: 'status',
render: Status_cell,
sticky: true,
tooltip: 'Real time proxy status',
ext: true,
dynamic: true,
},
{
key: 'iface',
title: 'Interface',
type: 'options',
tooltip: 'Specific network interface on which the local machine is '
+'running. Switch interfaces in the proxy configuration page',
ext: true,
},
{
key: 'multiply',
title: 'Multiple',
type: 'number',
tooltip: 'Number of multiplied proxy ports. A proxy port can be '
+'multiplied in the proxy configuration page',
ext: true,
},
{
key: 'history',
title: 'Logs',
render: Boolean_cell,
tooltip: 'Last 1K requests are automatically logged for easy '
+'debugging. To save all requests, Enable Logs in proxy '
+'configuration page',
ext: true,
},
{
key: 'ssl',
title: 'SSL Log',
render: Boolean_cell,
tooltip: 'In order to log HTTPS requests, enable SSL Logs in proxy '
+'configuration',
ext: true,
},
{
key: 'zone',
title: 'Zone',
type: 'options',
default: true,
tooltip: 'Specific Luminati zone configured for this proxy. Switch '
+'zones in proxy configuration page.',
},
{
key: 'secure_proxy',
title: 'SSL for super proxy',
render: Boolean_cell,
ext: true,
},
{
key: 'country',
title: 'Targeting',
type: 'options',
default: true,
render: Targeting_cell,
tooltip: 'Exit node (IP) GEO location',
},
{
key: 'asn',
title: 'ASN',
type: 'number',
tooltip: 'ASN uniquely identifies each network on the internet. You '
+'can target exit nodes (IPs) on specific ASNs',
},
{
key: 'ip',
title: 'Datacenter IP',
type: 'text',
},
{
key: 'vip',
title: 'gIP',
type: 'number',
tooltip: 'A gIP is a group of exclusive residential IPs. Using gIPs '
+'ensures that nobody else uses the same IPs with the same target '
+'sites as you do.',
},
{
key: 'max_requests',
title: 'Max requests',
type: 'text',
ext: true,
},
{
key: 'session_duration',
title: 'Session duration (sec)',
type: 'text',
ext: true,
},
{
key: 'pool_size',
title: 'Pool size',
type: 'number',
ext: true,
},
{
key: 'pool_type',
title: 'Pool type',
type: 'options',
ext: true,
},
{
key: 'sticky_ip',
title: 'Sticky IP',
render: Boolean_cell,
ext: true,
},
{
key: 'keep_alive',
title: 'Keep-alive',
type: 'number',
ext: true,
},
{
key: 'seed',
title: 'Seed',
type: 'text',
},
{
key: 'session',
title: 'Session',
render: Session_cell,
},
{
key: 'allow_proxy_auth',
title: 'Request authentication',
render: Boolean_cell,
},
{
key: 'proxy_count',
title: 'Mininum super proxies',
type: 'number',
},
{
key: 'race_reqs',
title: 'Race request',
type: 'number',
},
{
key: 'dns',
title: 'DNS',
type: 'options',
},
{
key: 'log',
title: 'Log Level',
type: 'options',
ext: true,
},
{
key: 'proxy_switch',
title: 'Autoswitch super proxy on failure',
type: 'number',
},
{
key: 'throttle',
title: 'Throttle concurrent connections',
type: 'number',
ext: true,
},
{
key: 'request_timeout',
title: 'Request timeout (sec)',
type: 'number',
ext: true,
},
{
key: 'debug',
title: 'Debug info',
type: 'options',
},
{
key: 'null_response',
title: 'NULL response',
type: 'text',
ext: true,
},
{
key: 'bypass_proxy',
title: 'Bypass proxy',
type: 'text',
ext: true,
},
{
key: 'direct_include',
title: 'Direct include',
type: 'text',
},
{
key: 'success_rate',
title: 'Success',
tooltip: 'The ratio of successful requests out of total requests. A '
+'request considered as successful if the server of the '
+'destination website responded',
render: Success_rate_cell,
default: true,
ext: true,
dynamic: true,
},
{
key: 'in_bw',
title: 'BW up',
render: ({proxy})=>Tooltip_bytes({bytes: proxy.out_bw}),
sticky: true,
ext: true,
tooltip: 'Data transmitted to destination website. This includes'
+'request headers, request data, response headers, response data',
dynamic: true,
},
{
key: 'out_bw',
title: 'BW down',
render: ({proxy})=>Tooltip_bytes({bytes: proxy.in_bw}),
sticky: true,
ext: true,
tooltip: 'Data transmitted to destination website. This includes'
+'request headers, request data, response headers, response data',
dynamic: true,
},
{
key: 'reqs',
title: 'Requests',
sticky: true,
render: Reqs_cell,
ext: true,
tooltip: 'Number of all requests sent from this proxy port',
dynamic: true,
},
{
key: 'last_req.url',
title: 'Last request',
sticky: true,
render: Last_req_cell,
ext: true,
tooltip: 'Last request that was sent on this proxy port',
dynamic: true,
},
];
class Columns_modal extends Pure_component {
constructor(props){
super(props);
this.state = {selected_cols: []};
}
componentDidMount(){
const selected_cols = this.props.selected_cols.reduce((acc, e)=>
Object.assign(acc, {[e]: true}), {});
this.setState({selected_cols, saved_cols: selected_cols});
}
on_change({target: {value}}){
this.setState(prev=>({selected_cols: {
...prev.selected_cols,
[value]: !prev.selected_cols[value],
}}));
}
click_ok(){
const new_columns = Object.keys(this.state.selected_cols).filter(c=>
this.state.selected_cols[c]);
this.props.update_selected_cols(new_columns);
window.localStorage.setItem('columns', JSON.stringify(
this.state.selected_cols));
}
click_cancel(){ this.setState({selected_cols: this.state.saved_cols}); }
select_all(){
this.setState({selected_cols: columns.filter(c=>!c.sticky)
.reduce((acc, e)=>Object.assign(acc, {[e.key]: true}), {})});
}
select_none(){ this.setState({selected_cols: {}}); }
select_default(){
this.setState({selected_cols: columns.filter(c=>!c.sticky&&c.default)
.reduce((acc, e)=>Object.assign(acc, {[e.key]: true}), {})});
}
render(){
const header = <div className="header_buttons">
<button onClick={this.select_all.bind(this)}
className="btn btn_lpm">Check all</button>
<button onClick={this.select_none.bind(this)}
className="btn btn_lpm">Uncheck all</button>
<button onClick={this.select_default.bind(this)}
className="btn btn_lpm">Default</button>
</div>;
return <Modal id="edit_columns" custom_header={header}
click_ok={this.click_ok.bind(this)}
cancel_clicked={this.click_cancel.bind(this)}>
<div className="row columns">
{columns.filter(col=>!col.sticky).map(col=>
<div key={col.key} className="col-md-6">
<Checkbox text={col.title} value={col.key}
on_change={this.on_change.bind(this)}
checked={!!this.state.selected_cols[col.key]}/>
</div>
)}
</div>
</Modal>;
}
}
const save_pagination = (table, opt={})=>{
const curr = JSON.parse(window.localStorage.getItem('pagination'))||{};
curr[table] = curr[table]||{};
if (opt.page)
curr[table].page = opt.page;
if (opt.items)
curr[table].items = opt.items;
window.localStorage.setItem('pagination', JSON.stringify(curr));
};
const get_pagination = table=>{
const curr = JSON.parse(window.localStorage.getItem('pagination'))||{};
return {items: 10, page: 0, ...curr[table]||{}};
};
class Proxies extends Pure_component {
constructor(props){
super(props);
let from_storage = JSON.parse(window.localStorage.getItem(
'columns'))||{};
from_storage = Object.keys(from_storage).filter(c=>from_storage[c]);
const default_cols = columns.filter(c=>c.default).map(col=>col.key);
const pagination = get_pagination('proxies');
this.state = {
selected_cols: from_storage.length&&from_storage||default_cols,
items_per_page: pagination.items,
cur_page: 0,
proxies: [],
filtered_proxies: [],
displayed_proxies: [],
loaded: false,
};
setdb.set('head.proxies.update', this.update.bind(this));
}
componentDidMount(){
this.setdb_on('head.proxies_running', proxies=>{
if (!proxies)
return;
proxies = this.prepare_proxies(proxies||[]);
const filtered_proxies = this.filter_proxies(proxies,
this.props.master_port);
this.setState({proxies, filtered_proxies, loaded: true},
this.paginate);
});
this.setdb_on('head.locations', locations=>{
if (!locations)
return;
const countries = locations.countries_by_code;
country_names = countries;
this.setState({countries});
});
window.setTimeout(this.req_status.bind(this));
}
componentWillReceiveProps(props){
if (props.master_port!=this.props.master_port)
{
const filtered_proxies = this.filter_proxies(this.state.proxies,
props.master_port);
this.setState({filtered_proxies}, this.paginate);
}
}
filter_proxies(proxies, mp){
return proxies.filter(p=>{
if (mp)
return ''+p.port==mp||''+p.master_port==mp;
return p.proxy_type!='duplicate';
});
}
prepare_proxies(proxies){
proxies.sort(function(a, b){ return a.port>b.port ? 1 : -1; });
for (let i=0; i<proxies.length; i++)
{
const cur = proxies[i];
if (Array.isArray(cur.proxy)&&cur.proxy.length==1)
cur.proxy = cur.proxy[0];
cur._status_details = cur._status_details||[];
}
return proxies;
}
update_items_per_page(items_per_page){
this.setState({items_per_page}, ()=>this.paginate(0));
save_pagination('proxies', {items: items_per_page});
}
req_status(){
const _this = this;
this.etask(function*(){
this.on('uncaught', e=>console.log(e));
const params = {};
if (_this.props.master_port)
params.master_port = _this.props.master_port;
const url = zescape.uri('/api/recent_stats', params);
const stats = yield ajax.json({url});
setdb.set('head.recent_stats', stats);
if (!_this.state.proxies.length ||
_.isEqual(stats, _this.state.stats))
{
yield etask.sleep(1000);
_this.req_status();
return;
}
_this.setState(prev=>{
const new_proxies = prev.proxies.map(p=>{
let stat = {reqs: 0, success: 0, in_bw: 0, out_bw: 0};
if (''+p.port in stats.ports)
stat = stats.ports[p.port];
p.in_bw = stat.in_bw;
p.out_bw = stat.out_bw;
p.reqs = stat.reqs;
p.success = stat.success;
p.last_req = {url: stat.url, ts: stat.timestamp};
return p;
}).reduce((acc, p)=>({...acc, [p.port]: p}), {});
if (!_this.props.master_port)
{
for (let p of Object.values(new_proxies))
{
if (p.proxy_type=='duplicate')
{
const master_port = new_proxies[p.master_port];
master_port.in_bw += p.in_bw;
master_port.out_bw += p.out_bw;
master_port.reqs += p.reqs;
master_port.success += p.success;
if (!master_port.last_req.ts ||
p.last_req.ts &&
p.last_req.ts>master_port.last_req.ts)
{
master_port.last_req = p.last_req;
}
}
}
}
const filtered_proxies = _this.filter_proxies(
Object.values(new_proxies), _this.props.master_port);
return {proxies: Object.values(new_proxies), filtered_proxies,
stats};
}, ()=>_this.paginate(_this.state.cur_page));
yield etask.sleep(1000);
_this.req_status();
});
}
update(){
this.etask(function*(){
const proxies = yield ajax.json({url: '/api/proxies_running'});
setdb.set('head.proxies_running', proxies);
});
}
download_csv(){
const data = this.state.proxies.map(p=>['127.0.0.1:'+p.port]);
filesaver.saveAs(csv.to_blob(data), 'proxies.csv');
}
edit_columns(){ $('#edit_columns').modal('show'); }
update_selected_columns(new_columns){
this.setState({selected_cols: new_columns});
}
proxy_add = ()=>{ $('#add_new_proxy_modal').modal('show'); };
paginate(page=-1){
page = page>-1 ? page : this.state.cur_page;
const pages = Math.ceil(
this.state.filtered_proxies.length/this.state.items_per_page);
const cur_page = Math.min(pages, page);
const displayed_proxies = this.state.filtered_proxies.slice(
cur_page*this.state.items_per_page,
(cur_page+1)*this.state.items_per_page);
this.setState({
displayed_proxies,
cur_page,
});
}
page_change(page){
this.paginate(page-1);
save_pagination('proxies', {page});
}
render(){
const cols = columns.filter(col=>
this.state.selected_cols.includes(col.key)||col.sticky);
const displayed_proxies = this.state.displayed_proxies;
if (!this.state.countries)
return null;
if (this.state.loaded&&!this.state.filtered_proxies.length)
return <Proxy_blank/>;
return <div className="panel proxies_panel">
<div className="panel_heading">
<h2>
Proxies
<Tooltip title="Add new proxy">
<button className="btn btn_lpm btn_lpm_small add_proxy_btn"
onClick={this.proxy_add}>
New proxy
<i className="glyphicon glyphicon-plus"/>
</button>
</Tooltip>
</h2>
</div>
{this.state.loaded && displayed_proxies.length &&
<div className="panel_body with_table">
<Proxies_pagination entries={this.state.filtered_proxies}
cur_page={this.state.cur_page}
items_per_page={this.state.items_per_page}
page_change={this.page_change.bind(this)}
edit_columns={this.edit_columns.bind(this)}
update_items_per_page={this.update_items_per_page.bind(this)}
download_csv={this.download_csv.bind(this)}
top/>
<div className="proxies_table_wrapper">
<table className="table table-hover">
<thead>
<tr>
<th className="head_actions">
<Tooltip title="Delete/duplicate/refresh sessions">
Actions
</Tooltip>
</th>
{cols.map(col=>
<th key={col.key} className={'col_'+col.key}>
<Tooltip title={col.tooltip}>
{col.title}</Tooltip>
</th>
)}
</tr>
</thead>
<tbody>
{displayed_proxies.map(proxy=>
<Proxy_row
key={proxy.port}
go={this.state.go}
update_proxies={this.update.bind(this)}
proxy={proxy}
cols={cols}
master_port={this.props.master_port}/>
)}
</tbody>
</table>
</div>
<Proxies_pagination entries={this.state.filtered_proxies}
cur_page={this.state.cur_page}
items_per_page={this.state.items_per_page}
page_change={this.page_change.bind(this)}
edit_columns={this.edit_columns.bind(this)}
update_items_per_page={this.update_items_per_page.bind(this)}
download_csv={this.download_csv.bind(this)}
bottom/>
</div>
}
<Proxy_add/>
<Columns_modal selected_cols={this.state.selected_cols}
update_selected_cols={this.update_selected_columns.bind(this)}/>
</div>;
}
}
const Proxies_pagination = ({entries, items_per_page, cur_page, bottom,
page_change, edit_columns, download_csv, top, update_items_per_page})=>
<Pagination_panel entries={entries} items_per_page={items_per_page}
cur_page={cur_page} page_change={page_change} top={top} bottom={bottom}
update_items_per_page={update_items_per_page}>
<Link_icon tooltip="Edit columns" on_click={edit_columns} id="filter"/>
<Link_icon tooltip="Download all proxy ports as CSV"
on_click={download_csv} id="download"/>
</Pagination_panel>;
const Proxy_row = withRouter(class Proxy_row extends Pure_component {
constructor(props){
super(props);
this.state = {status: this.props.proxy._status};
}
componentDidMount(){ this.get_status(); }
get_status(opt={}){
const {proxy} = this.props;
const _this = this;
return this.etask(function*(){
this.on('uncaught', e=>{
_this.setState({status: 'error', status_details:
[{msg: 'Failed to get proxy status'}]});
});
const params = {};
if (proxy.proxy_type!='duplicate')
params.with_details = true;
if (opt.force)
{
_this.setState({status: undefined});
params.force = true;
}
const uri = '/api/proxy_status/'+proxy.port;
const url = zescape.uri(uri, params);
const res = yield ajax.json({url});
if (res.status=='ok')
{
_this.setState({status: res.status,
status_details: res.status_details});
}
else
{
let errors = res.status_details.filter(s=>s.lvl=='err');
errors = errors.length ? errors : [{msg: res.status}];
_this.setState({status: 'error', status_details: errors});
}
});
}
edit(){
if (this.props.proxy.proxy_type!='persist')
return;
if (!this.props.master_port && this.props.proxy.multiply &&
this.props.proxy.multiply>1)
{
this.props.history.push(`/overview/${this.props.proxy.port}`);
}
else
this.props.history.push(`/proxy/${this.props.proxy.port}`);
}
render(){
const proxy = this.props.proxy;
const cell_class = col=>classnames(col.key.replace(/\./g, '_'), {
default_cursor: this.props.proxy.proxy_type!='persist',
});
const row_class = classnames('proxy_row',
{default: proxy.port==22225});
return <tr className={row_class}>
<Actions proxy={proxy}
get_status={this.get_status.bind(this)}
update_proxies={this.props.update_proxies}/>
{this.props.cols.map(col=>
<Cell
key={col.key}
proxy={proxy}
col={col}
status={this.state.status}
status_details={this.state.status_details}
master_port={this.props.master_port}
on_click={this.edit.bind(this)}
className={cell_class(col)}/>
)}
</tr>;
}
});
class Cell extends React.Component {
shouldComponentUpdate(){ return !!this.props.col.dynamic; }
render(){
const {proxy, col, master_port, status, status_details, on_click,
className} = this.props;
let val;
if (!col.ext && proxy.ext_proxies)
val = '—';
else if (col.render)
{
val = col.render({proxy, col: col.key, master_port, status,
status_details})||null;
}
else
val = _.get(proxy, col.key)||null;
return <td onClick={on_click} className={className}>{val}</td>;
}
}
class Actions extends Pure_component {
state = {open_delete_dialog: false};
open_delete_dialog = ()=>{ this.setState({open_delete_dialog: true}); };
close_delete_dialog = ()=>{ this.setState({open_delete_dialog: false}); };
delete_proxy(){
const _this = this;
this.etask(function*(){
yield ajax.json({url: '/api/proxies/'+_this.props.proxy.port,
method: 'DELETE'});
yield _this.props.update_proxies();
_this.close_delete_dialog();
});
}
refresh_sessions(){
const _this = this;
this.etask(function*(){
const url = '/api/refresh_sessions/'+_this.props.proxy.port;
yield ajax.json({url, method: 'POST'});
yield _this.props.get_status({force: true});
});
}
duplicate(){
const _this = this;
this.etask(function*(){
this.on('uncaught', e=>console.log(e));
// XXX krzysztof: switch fetch->ajax
yield window.fetch('/api/proxy_dup', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({port: _this.props.proxy.port}),
});
yield _this.props.update_proxies();
});
}
render(){
const persist = this.props.proxy.proxy_type=='persist';
const delete_title = `Are you sure you want to delete proxy port
${this.props.proxy.port}?`;
return <td className="proxies_actions">
<Action_icon id="trash"
on_click={this.open_delete_dialog.bind(this)}
tooltip="Delete" invisible={!persist}/>
<Action_icon id="duplicate" on_click={this.duplicate.bind(this)}
tooltip="Duplicate proxy port"
invisible={!persist}/>
<Action_icon id="refresh"
on_click={this.refresh_sessions.bind(this)}
tooltip="Refresh Sessions"/>
<Modal_dialog title={delete_title}
open={this.state.open_delete_dialog}
ok_clicked={this.delete_proxy.bind(this)}
cancel_clicked={this.close_delete_dialog.bind(this)}/>
</td>;
}
}
const Action_icon = ({on_click, disabled, invisible, id, tooltip,
tooltip_disabled})=>
{
const classes = classnames('action_icon', {disabled, invisible});
if (disabled)
tooltip = tooltip_disabled;
return <Link_icon tooltip={tooltip} on_click={on_click} id={id}
classes={classes} invisible={invisible} disabled={disabled}/>;
};
export default Proxies;