whistle
Version:
HTTP, HTTP2, HTTPS, Websocket debugging proxy
842 lines (804 loc) • 23.8 kB
JavaScript
var Pac = require('node-pac');
var net = require('net');
var LRU = require('lru-cache');
var path = require('path');
var iconv = require('iconv-lite');
var parseUrl = require('../util/parse-url-safe');
var extend = require('extend');
var parseQuery = require('querystring').parse;
var lookup = require('./dns');
var Rules = require('./rules');
var rulesUtil = require('./util');
var util = require('../util');
var logger = require('../util/logger');
var fileMgr = require('../util/file-mgr');
var config = require('../config');
var values = rulesUtil.values;
var globalRules = rulesUtil.rules;
var tplCache = new LRU({ max: 36 });
var clientCerts = new LRU({ max: 60, maxAge: 1000 * 60 * 60 });
var rules = new Rules();
var tempRules = new Rules();
var cachedPacs = {};
var RULES_HEADER = 'x-whistle-rule-value';
var KV_HEADER = 'x-whistle-key-value';
var NAME_HEADER = 'x-whistle-rule-name';
var KEY_HEADER = 'x-whistle-rule-key';
var HOST_HEADER = 'x-whistle-rule-host';
var LOCALHOST = '127.0.0.1';
var resolveReqRules = rules.resolveReqRules.bind(rules);
var AUTH_RE = /^([^\\/]+)@/;
var PLUGIN_VAR_RE = /^[a-z\d_\-]+\./;
var COMMENT_RE = /^\s*#/;
var VALUE_RE = /^\s*``/m;
var BRACKET_RE = /[(\[]/;
var SCRIPT_RE = /\b(?:rules|values)\b/;
var SEP_CIPHER_RE = /[^a-z\d:!-]/i;
var CERT_RE = /^\s*-----/;
function isRulesContent(ctn) {
return !BRACKET_RE.test(ctn) || COMMENT_RE.test(ctn) || VALUE_RE.test(ctn) || !SCRIPT_RE.test(ctn);
}
rules._isGlobal = true;
exports.Rules = Rules;
exports.getEnabledRules = function() {
return rules._enabledList || [];
};
exports.getMFlag = function() {
return rules._mflag;
};
exports.parse = rules.parse.bind(rules);
exports.append = rules.append.bind(rules);
exports.resolveSNICallback = rules.resolveSNICallback.bind(rules);
exports.resolveHost = rules.resolveHost.bind(rules);
exports.getGRuleList = rules.getGRuleList.bind(rules);
exports.resolveInternalHost = rules.resolveInternalHost.bind(rules);
exports.resolveProxy = rules.resolveProxy.bind(rules);
exports.resolveEnable = rules.resolveEnable.bind(rules);
exports.hasReqScript = rules.hasReqScript.bind(rules);
exports.resolveDisable = rules.resolveDisable.bind(rules);
exports.resolvePipe = rules.resolvePipe.bind(rules);
exports.resolveRule = rules.resolveRule.bind(rules);
exports.resolveResRules = rules.resolveResRules.bind(rules);
exports.resolveBodyFilter = rules.resolveBodyFilter.bind(rules);
exports.lookupHost = rules.lookupHost.bind(rules);
exports.resolveLocalRule = rules.resolveLocalRule.bind(rules);
exports.clearAppend = rules.clearAppend.bind(rules);
exports.resolveTplVar = Rules.resolveTplVar;
exports.disableDnsCache = function () {
Rules.disableDnsCache();
};
var dnsResolve = function (host, callback) {
return lookup(host, callback || util.noop, true);
};
var PROXY_HOSTS_RE = /[?&]proxyHosts?(?:&|$)/i;
var P_HOST_RE = /[?&]host=([\w.:-]+)(?:&|$)/i;
function isProxyHost(req, proxy, host) {
if (proxy) {
req._proxyTunnel =
isLineProp(proxy, 'proxyTunnel') ||
isLineProp(host, 'proxyTunnel') ||
util.isEnable(req, 'proxyTunnel') ||
isProxyEnable(req, 'proxyTunnel');
if (isLineProp(proxy, 'proxyHost')) {
return true;
}
}
return isLineProp(host, 'proxyHost');
}
function isProxyEnable(req, name) {
var props = req._proxyProps;
if (props === undefined) {
props = rules.resolveProxyProps(req);
req._proxyProps = props || null;
}
return props && util.isEnable(props, name);
}
function isLineProp(rule, name) {
return rule && rule.lineProps[name];
}
function resolveRulesFromList(list, fnName, req, options) {
var rule;
for (var i = 0, len = list.length; i < len; i++) {
var mgr = list[i];
var _rule = mgr && mgr[fnName](req, options);
if (_rule) {
if (util.isImportant(_rule)) {
return _rule;
}
rule = rule || _rule;
}
}
return rule;
}
exports.getProxy = function (url, req, callback) {
if (!req) {
return callback();
}
var reqRules = req.rules;
req.curUrl = url;
if (!reqRules) {
return rules.lookupHost(req, callback);
}
delete reqRules.proxy;
delete reqRules.pac;
var pRules = req.pluginRules;
var fRules = req.rulesFileMgr;
var hRules = req.headerRulesMgr;
var filter = extend(
rules.resolveFilter(req),
pRules && pRules.resolveFilter(req),
fRules && fRules.resolveFilter(req),
hRules && hRules.resolveFilter(req)
);
var ignoreProxy;
var proxy;
var pacRule;
var host = rules.getHost(req, pRules, fRules, hRules);
var hostValue = util.getMatcherValue(host) || '';
if (config.multiEnv || req._headerRulesFirst) {
proxy = resolveRulesFromList([pRules, hRules, rules, fRules], 'resolveProxy', req, hostValue);
} else {
proxy = resolveRulesFromList([pRules, rules, fRules, hRules], 'resolveProxy', req, hostValue);
}
var proxyHostOnly;
var proxyHost = util.isEnable(req, 'proxyHost') || isProxyEnable(req, 'proxyHost');
if (proxy) {
proxyHostOnly = isLineProp(proxy, 'proxyHostOnly');
proxyHost = proxyHost || proxyHostOnly;
var protocol = proxy.matcher.substring(0, proxy.matcher.indexOf(':'));
ignoreProxy =
!filter['ignore:' + protocol] &&
(util.isIgnored(filter, 'proxy') || util.isIgnored(filter, protocol));
if (ignoreProxy) {
proxy = null;
} else {
proxyHost = proxyHost || PROXY_HOSTS_RE.test(proxy.matcher);
}
}
if (!ignoreProxy) {
proxyHost = isProxyHost(req, proxy, host) || proxyHost; // 不能调换顺序
}
var setHost = function () {
if (!host || req._isProxyReq) {
return false;
}
reqRules.host = host;
var hostname = util.removeProtocol(host.matcher, true);
if (!net.isIP(hostname)) {
req.curUrl = hostname || url;
rules.lookupHost(req, function (err, ip) {
callback(err, ip, host.port, host);
});
} else {
callback(null, hostname, host.port, host);
}
return true;
};
var resolvePacRule = function () {
if (pacRule != null) {
return;
}
if (util.isIgnored(filter, 'pac')) {
pacRule = false;
return;
}
if (config.multiEnv || req._headerRulesFirst) {
pacRule =
(pRules && pRules.resolvePac(req)) ||
(hRules && hRules.resolvePac(req)) ||
rules.resolvePac(req) ||
(fRules && fRules.resolvePac(req)) ||
false;
} else {
pacRule =
(pRules && pRules.resolvePac(req)) ||
rules.resolvePac(req) ||
(fRules && fRules.resolvePac(req)) ||
(hRules && hRules.resolvePac(req)) ||
false;
}
if (pacRule) {
proxyHostOnly = isLineProp(pacRule, 'proxyHostOnly');
proxyHost =
proxyHost || proxyHostOnly || isLineProp(pacRule, 'proxyHost');
}
};
if (host) {
if (!proxyHost && !ignoreProxy) {
resolvePacRule();
}
if (proxyHost) {
req._phost = parseUrl(util.setProtocol(util.joinIpPort(host.matcher, host.port)));
} else if (
!isLineProp(proxy, 'proxyFirst') &&
!isLineProp(host, 'proxyFirst') &&
!util.isEnable(req, 'proxyFirst') &&
!isProxyEnable(req, 'proxyFirst')
) {
return setHost();
}
proxyHost = true;
reqRules.host = host;
req._enableProxyHost = true;
}
if (ignoreProxy || (proxyHostOnly && !req._phost)) {
req.curUrl = url;
return setHost() || rules.lookupHost(req, callback);
}
if (proxy) {
if (!req._phost && P_HOST_RE.test(proxy.matcher)) {
req._phost = parseUrl(util.setProtocol(RegExp.$1));
}
reqRules.proxy = proxy;
return callback();
}
resolvePacRule();
var pacUrl = util.getMatcherValue(pacRule);
if (!pacUrl) {
return setHost() || callback();
}
var auth;
if (AUTH_RE.test(pacUrl)) {
auth = RegExp.$1;
pacUrl = pacUrl.substring(auth.length + 1);
}
if (!util.isUrl(pacUrl) && !(pacUrl = util.joinPath(pacRule.root, pacUrl))) {
return setHost() || callback();
}
req._pacAuth = auth;
var pac = cachedPacs[pacUrl];
if (pac) {
delete cachedPacs[pacUrl];
cachedPacs[pacUrl] = pac;
} else {
var list = Object.keys(cachedPacs);
if (list.length >= 10) {
delete cachedPacs[list[0]];
}
cachedPacs[pacUrl] = pac = new Pac(pacUrl, dnsResolve);
}
reqRules.pac = pacRule;
return pac.findWhistleProxyForURL(
url.replace('tunnel:', 'https:'),
function (err, rule) {
if (rule) {
tempRules._file = pacRule.file;
tempRules.parse(pacRule.rawPattern + ' ' + rule);
req.curUrl = url;
if ((proxy = tempRules.resolveProxy(req, hostValue))) {
proxyHost = isProxyHost(req, proxy) || proxyHost; // 不能调换顺序
req._proxyTunnel =
req._proxyTunnel || isLineProp(pacRule, 'proxyTunnel');
var protocol = proxy.matcher.substring(0, proxy.matcher.indexOf(':'));
if (!util.isIgnored(filter, protocol)) {
reqRules.proxy = proxy;
reqRules.proxy.raw = pacRule.raw;
}
}
}
if (reqRules.proxy) {
(!proxyHost && setHost()) || callback();
} else {
req.curUrl = url;
setHost() || rules.lookupHost(req, callback);
}
logger.error(err);
}
);
};
function tpl(str, data) {
if (
typeof str !== 'string' ||
str.indexOf('<%') === -1 ||
str.indexOf('%>') === -1
) {
return str + '';
}
var key = str;
var fn = tplCache.get(key);
if (!fn) {
str = str
.replace(/[\u2028\u2029]/g, '')
.replace(/\t/g, ' ')
.replace(/\r?\n|\r/g, '\t')
.split('<%')
.join('\u2028')
.replace(/((^|%>)[^\u2028]*)'/g, '$1\r')
.replace(/\u2028=(.*?)%>/g, '\',$1,\'')
.split('\u2028')
.join('\');')
.split('%>')
.join('p.push(\'')
.split('\r')
.join('\\\'');
try {
fn = new Function(
'obj',
'var p=[],print=function(){p.push.apply(p,arguments);};' +
'with(obj){p.push(\'' +
str +
'\');}return p.join(\'\');'
);
} catch (e) {
fn = e;
throw e;
} finally {
tplCache.set(key, fn);
}
} else if (typeof fn !== 'function') {
throw fn;
}
return fn(data || {}).replace(/\t/g, '\n');
}
function getScriptContext(req, res, body, pattern, frameOpts) {
var ip = req.clientIp || LOCALHOST;
var ctx = req.scriptContenxt;
if (!ctx) {
var headers = extend(true, {}, req.headers);
ctx = req.scriptContenxt = {
Buffer: Buffer,
pattern: pattern,
version: config.version,
port: config.port,
uiHost: 'local.wproxy.org',
uiPort: config.uiport,
url: req.fullUrl,
fullUrl: req.fullUrl,
method: util.toUpperCase(req.method) || 'GET',
httpVersion: req.httpVersion || '1.1',
decodeBuffer: function (buf, encoding) {
return iconv.decode(buf, encoding || 'utf8');
},
encodeString: function (str, encoding) {
return iconv.encode(str, encoding || 'utf8');
},
encodingExists: function (encoding) {
return iconv.encodingExists(encoding);
},
isLocalAddress: function (_ip) {
return util.isLocalAddress(_ip || ip);
},
ip: ip,
clientIp: ip,
clientPort: req.clientPort,
headers: headers,
reqHeaders: headers,
body: body || '',
reqScriptData: {},
res: null
};
}
if (frameOpts) {
ctx.ctx = {
sendToServer: frameOpts.sendToServer,
sendToClient: frameOpts.sendToClient,
handleSendToClientFrame: null,
handleSendToServerFrame: null
};
} else {
ctx.rules = [];
ctx.values = {};
}
ctx.value = req.globalValue;
ctx.getValue = function (key, onlyValues) {
var value = !onlyValues && req._inlineValues && req._inlineValues[key];
return typeof value === 'string' ? value : values.get(key);
};
ctx.parseUrl = parseUrl;
ctx.parseQuery = parseQuery;
ctx.tpl = ctx.render = tpl;
if (res) {
ctx.statusCode = res.statusCode;
ctx.serverIp = req.hostIp || LOCALHOST;
ctx.resHeaders = extend(true, {}, res.headers);
} else {
ctx.statusCode = '';
ctx.serverIp = '';
ctx.resHeaders = '';
}
return ctx;
}
function getReqPayload(req, res, cb) {
if (res) {
return cb();
}
if (req.getPayload && util.hasRequestBody(req)) {
if (typeof req._reqBody === 'string') {
cb(req._reqBody);
} else {
req.getPayload(function (_, payload) {
cb(fileMgr.decode(payload));
});
}
} else {
cb();
}
}
function execRulesScript(script, req, res, body, pattern, frameOpts) {
var context = getScriptContext(req, res, body, pattern, frameOpts);
if (!util.execScriptSync(script, context)) {
return '';
}
if (frameOpts) {
return context.ctx;
}
return Array.isArray(context.rules) ? {
rules: context.rules.join('\n').trim(),
values: context.values
} : '';
}
var CTX_RE = /\bctx\b/;
exports.getFrameScriptCtx = function(script, req, res, options, cb) {
util.getRuleValue(script, function (text) {
if (!text || !CTX_RE.test(text)) {
return cb();
}
cb(execRulesScript(text, req, res, '', script.rawPattern, options));
}, null, null, null, req);
};
function handleDynamicRules(script, req, res, cb) {
util.getRuleValue(script, function (list) {
var scriptItem, index, text;
if (list) {
index = script.scriptIndex;
scriptItem = script.list[index];
text = scriptItem && list[index];
}
if (!scriptItem || isRulesContent(text)) {
return cb(list && list.join('\n'));
}
getReqPayload(req, res, function (body) {
var result = execRulesScript(text, req, res, body, script.rawPattern);
list[index] = result.rules;
cb(list.join('\n'), result.values);
});
}, null, null, null, req);
}
function resolvePluginVars(req, list) {
var varMap = {};
var globalValue;
var pList;
list.forEach(function (item) {
if (item.matcher[0] !== 'P') {
if (!globalValue) {
globalValue = item;
}
return;
}
var value = util.getMatcherValue(item);
var index = value.indexOf(PLUGIN_VAR_RE.test(value) ? '.' : '=');
if (index !== -1) {
var name = value.substring(0, index);
var plugin = getPluginName(name);
if (!plugin) {
return;
}
value = value.substring(index + 1);
if (value) {
pList = pList || [];
pList.push(item);
var list = varMap[name] || [];
varMap[name] = list;
if (list.indexOf(value) === -1) {
list.push(value);
}
}
}
});
req.rules = req.rules || {};
req._pluginVars = varMap;
req.rules.P = pList;
req.rules.G = globalValue;
req.globalValue = util.getMatcherValue(globalValue);
}
exports.resolvePluginVars = resolvePluginVars;
exports.resolveRulesFile = function (req, callback) {
!req._resolvedG && req.rules.G && resolvePluginVars(req, req.rules.G.list);
req._resolvedG = true;
var rule = req.rules.rulesFile;
handleDynamicRules(rule, req, null, function (text, vals) {
if (text) {
vals = util.toPrivateValues(vals, rule.file);
var rulesFileMgr = new Rules(vals);
rulesFileMgr._file = rule.file;
rulesFileMgr.parse(text);
req.rulesFileMgr = rulesFileMgr;
req.curUrl = req.fullUrl;
text = req.rulesFileMgr.resolveReqRules(req);
}
// 不能放到if里面
util.mergeRules(req, text);
callback();
});
};
exports.resolveResRulesFile = function (req, res, callback) {
var rule = req.rules && req.rules.resScript;
handleDynamicRules(
rule,
req,
res,
function (text, vals) {
text = text && text.trim();
callback(
text && {
file: rule.file,
text: text,
values: util.toPrivateValues(vals, rule.file)
}
);
}
);
};
function getValue(req, key, keep) {
var value = req.headers[key];
if (value) {
if (!req.fromComposer && !keep && (config.strict || (!config.enableRequestHeaderRules && !config.multiEnv))) {
value = null;
} else {
req.rulesHeaders[key] = value;
}
delete req.headers[key];
}
try {
return value && decodeURIComponent(value);
} catch (e) {}
return value;
}
function getPluginName(key) {
return exports.getPlugin(key);
}
function initHeaderRules(req, needBodyFilters) {
if (req._bodyFilters !== undefined) {
return;
}
req._bodyFilters = null;
req.rulesHeaders = {};
var isPluginReq = req.isPluginReq && !req._isProxyReq;
req._headerRulesFirst = isPluginReq;
var rulesHeader = getValue(req, RULES_HEADER, isPluginReq);
var hostHeader = util.trimStr(getValue(req, HOST_HEADER));
var nameHeader = config.multiEnv && util.trimStr(getValue(req, NAME_HEADER));
var keyHeader = util.trimStr(getValue(req, KEY_HEADER));
var kvHeader = getValue(req, KV_HEADER, isPluginReq);
var ruleValue = util.trimStr(rulesHeader);
if (hostHeader) {
ruleValue = ruleValue + '\n' + hostHeader;
}
if (keyHeader) {
keyHeader = util.trimStr(values.get(keyHeader));
if (keyHeader) {
ruleValue = keyHeader + '\n' + ruleValue;
}
}
if (nameHeader) {
nameHeader = util.trimStr(globalRules.get(nameHeader));
if (nameHeader) {
ruleValue = ruleValue + '\n' + nameHeader;
}
}
var curVars = rules._globalPluginVars;
var globalVars = {};
var value;
req._globalPluginVars = globalVars;
Object.keys(curVars).forEach(function (key) {
if (getPluginName(key)) {
value = curVars[key];
globalVars[key] = value;
}
});
if (ruleValue) {
var file = 'Header Rules';
var vals = util.toPrivateValues(util.parseJSON(kvHeader), file);
var rulesMgr = new Rules(vals);
rulesMgr._file = file;
rulesMgr.parse(ruleValue);
req.headerRulesMgr = rulesMgr;
var bodyFilters = needBodyFilters && rulesMgr._rules._bodyFilters;
if (bodyFilters && bodyFilters.length) {
req._bodyFilters = rules._rules._bodyFilters.concat(bodyFilters);
}
var headerVars = rulesMgr._globalPluginVars;
Object.keys(headerVars).forEach(function (key) {
var headerVal = getPluginName(key) && headerVars[key];
if (headerVal) {
var curVal = curVars[key];
globalVars[key] = curVal ? curVal.concat(headerVal) : headerVal;
}
});
}
}
exports.initHeaderRules = initHeaderRules;
function initRules(req) {
var fullUrl = req.fullUrl || util.getFullUrl(req);
req.curUrl = fullUrl;
initHeaderRules(req);
if (req.headerRulesMgr) {
if (config.multiEnv || req._headerRulesFirst) {
req.rules = resolveReqRules(req);
util.mergeRules(req, req.headerRulesMgr.resolveReqRules(req));
} else {
req.rules = req.headerRulesMgr.resolveReqRules(req);
util.mergeRules(req, resolveReqRules(req));
}
} else {
req.rules = resolveReqRules(req);
}
return req.rules;
}
exports.initRules = initRules;
var HTTPS_RE = /^(?:ws|http)s:\/\//;
function checkCache(cacheKey, callback) {
var result = clientCerts.peek(cacheKey);
if (result) {
if (result.pending) {
result.push(callback);
} else {
callback(result);
}
return true;
} else {
result = [callback];
result.pending = true;
clientCerts.set(cacheKey, result);
}
}
function getTlsOptions(req, cb) {
var cipher = req.rules.cipher;
if (!cipher) {
return cb();
}
cipher.list.forEach(function (rule) {
var value = rule && util.getMatcherValue(rule);
if (value && !SEP_CIPHER_RE.test(value)) {
rule.jsonObject = { ciphers: value };
}
});
util.parseRuleJson(cipher, function (data) {
if (!data) {
return cb();
}
var opts;
var addOption = function(name, value) {
if (!opts) {
opts = {};
cipher.__tlsOptions = function () {};
cipher.__tlsOptions._opts = opts;
}
opts[name] = value;
};
var addStrOption = function(name, opName) {
var value = (opName && data[opName]) || data[name];
if (util.isString(value)) {
addOption(name, value);
}
};
var addNumOption = function(name) {
var value = +data[name];
if (value >= 0) {
addOption(name, value);
}
};
addStrOption('ciphers', 'cipher');
addStrOption('secureProtocol');
addStrOption('maxVersion');
addStrOption('minVersion');
addStrOption('ca');
addStrOption('crl');
addStrOption('allowPartialTrustChain');
addStrOption('sessionIdContext');
addStrOption('sigalgs');
addStrOption('dhparam');
addStrOption('ecdhCurve');
addNumOption('secureOptions');
addNumOption('sessionTimeout');
if (data.honorCipherOrder) {
addOption('honorCipherOrder', true);
}
cb(data);
}, req);
}
function isCert(str) {
return CERT_RE.test(str);
}
exports.getClientCert = function (req, cb) {
if (!req) {
return cb();
}
req.curUrl = req.realUrl || req.fullUrl;
if (!HTTPS_RE.test(req.curUrl)) {
return cb();
}
getTlsOptions(req, function(options) {
var pRules = req.pluginRules;
var fRules = req.rulesFileMgr;
var hRules = req.headerRulesMgr;
var rule;
if (config.multiEnv || req._headerRulesFirst) {
rule = resolveRulesFromList([pRules, hRules, rules, fRules], 'resolveClientCert', req);
} else {
rule = resolveRulesFromList([pRules, rules, fRules, hRules], 'resolveClientCert', req);
}
if (rule) {
req.rules.clientCert = rule;
rule = util.parseJSON(rule.matcher.substring(17));
options = rule ? extend({}, rule, options) : options;
}
if (!options) {
return cb();
}
var base = util.getString(options.base);
var key = util.getString(options.key);
var cert = util.getString(options.cert);
var cacheKey;
try {
if (key && cert) {
if (base) {
key = path.join(base, key);
cert = path.join(base, cert);
}
cacheKey = 'cert\n' + key + '\n' + cert;
if (isCert(key) && isCert(cert)) {
return cb(key, cert, false, cacheKey);
}
if (
checkCache(cacheKey, function (data) {
if (data) {
cb(data[0], data[1], false, cacheKey);
} else {
cb();
}
})
) {
return;
}
return fileMgr.readFileList([key, cert], function (data) {
var list = clientCerts.peek(cacheKey);
if (data[0] && data[1] && data[0].length && data[1].length) {
clientCerts.set(cacheKey, data);
} else {
clientCerts.del(cacheKey);
data = '';
}
list.forEach(function (fn) {
fn(data);
});
});
}
var pwd = util.getString(options.pwd || options.passphrase);
var pfx = util.getString(options.pfx);
if (pfx) {
if (base) {
pfx = path.join(base, key);
}
cacheKey = 'pfx\n' + pwd + '\n' + pfx;
if (isCert(pfx)) {
return cb(pwd, pfx, true, cacheKey);
}
if (
checkCache(cacheKey, function (buf) {
if (buf) {
cb(pwd, buf, true, cacheKey);
} else {
cb();
}
})
) {
return;
}
return fileMgr.readFile(pfx, function (buf) {
var list = clientCerts.peek(cacheKey);
if (buf && buf.length) {
clientCerts.set(cacheKey, buf);
} else {
clientCerts.del(cacheKey);
buf = '';
}
list.forEach(function (fn) {
fn(buf);
});
});
}
} catch (e) {}
cb();
});
};