UNPKG

whistle

Version:

HTTP, HTTP2, HTTPS, Websocket debugging proxy

842 lines (804 loc) 23.8 kB
var Pac = require('node-pac'); var net = require('net'); var LRU = require('lru-cache'); var path = require('path'); var iconv = require('iconv-lite'); var parseUrl = require('../util/parse-url-safe'); var extend = require('extend'); var parseQuery = require('querystring').parse; var lookup = require('./dns'); var Rules = require('./rules'); var rulesUtil = require('./util'); var util = require('../util'); var logger = require('../util/logger'); var fileMgr = require('../util/file-mgr'); var config = require('../config'); var values = rulesUtil.values; var globalRules = rulesUtil.rules; var tplCache = new LRU({ max: 36 }); var clientCerts = new LRU({ max: 60, maxAge: 1000 * 60 * 60 }); var rules = new Rules(); var tempRules = new Rules(); var cachedPacs = {}; var RULES_HEADER = 'x-whistle-rule-value'; var KV_HEADER = 'x-whistle-key-value'; var NAME_HEADER = 'x-whistle-rule-name'; var KEY_HEADER = 'x-whistle-rule-key'; var HOST_HEADER = 'x-whistle-rule-host'; var LOCALHOST = '127.0.0.1'; var resolveReqRules = rules.resolveReqRules.bind(rules); var AUTH_RE = /^([^\\/]+)@/; var PLUGIN_VAR_RE = /^[a-z\d_\-]+\./; var COMMENT_RE = /^\s*#/; var VALUE_RE = /^\s*``/m; var BRACKET_RE = /[(\[]/; var SCRIPT_RE = /\b(?:rules|values)\b/; var SEP_CIPHER_RE = /[^a-z\d:!-]/i; var CERT_RE = /^\s*-----/; function isRulesContent(ctn) { return !BRACKET_RE.test(ctn) || COMMENT_RE.test(ctn) || VALUE_RE.test(ctn) || !SCRIPT_RE.test(ctn); } rules._isGlobal = true; exports.Rules = Rules; exports.getEnabledRules = function() { return rules._enabledList || []; }; exports.getMFlag = function() { return rules._mflag; }; exports.parse = rules.parse.bind(rules); exports.append = rules.append.bind(rules); exports.resolveSNICallback = rules.resolveSNICallback.bind(rules); exports.resolveHost = rules.resolveHost.bind(rules); exports.getGRuleList = rules.getGRuleList.bind(rules); exports.resolveInternalHost = rules.resolveInternalHost.bind(rules); exports.resolveProxy = rules.resolveProxy.bind(rules); exports.resolveEnable = rules.resolveEnable.bind(rules); exports.hasReqScript = rules.hasReqScript.bind(rules); exports.resolveDisable = rules.resolveDisable.bind(rules); exports.resolvePipe = rules.resolvePipe.bind(rules); exports.resolveRule = rules.resolveRule.bind(rules); exports.resolveResRules = rules.resolveResRules.bind(rules); exports.resolveBodyFilter = rules.resolveBodyFilter.bind(rules); exports.lookupHost = rules.lookupHost.bind(rules); exports.resolveLocalRule = rules.resolveLocalRule.bind(rules); exports.clearAppend = rules.clearAppend.bind(rules); exports.resolveTplVar = Rules.resolveTplVar; exports.disableDnsCache = function () { Rules.disableDnsCache(); }; var dnsResolve = function (host, callback) { return lookup(host, callback || util.noop, true); }; var PROXY_HOSTS_RE = /[?&]proxyHosts?(?:&|$)/i; var P_HOST_RE = /[?&]host=([\w.:-]+)(?:&|$)/i; function isProxyHost(req, proxy, host) { if (proxy) { req._proxyTunnel = isLineProp(proxy, 'proxyTunnel') || isLineProp(host, 'proxyTunnel') || util.isEnable(req, 'proxyTunnel') || isProxyEnable(req, 'proxyTunnel'); if (isLineProp(proxy, 'proxyHost')) { return true; } } return isLineProp(host, 'proxyHost'); } function isProxyEnable(req, name) { var props = req._proxyProps; if (props === undefined) { props = rules.resolveProxyProps(req); req._proxyProps = props || null; } return props && util.isEnable(props, name); } function isLineProp(rule, name) { return rule && rule.lineProps[name]; } function resolveRulesFromList(list, fnName, req, options) { var rule; for (var i = 0, len = list.length; i < len; i++) { var mgr = list[i]; var _rule = mgr && mgr[fnName](req, options); if (_rule) { if (util.isImportant(_rule)) { return _rule; } rule = rule || _rule; } } return rule; } exports.getProxy = function (url, req, callback) { if (!req) { return callback(); } var reqRules = req.rules; req.curUrl = url; if (!reqRules) { return rules.lookupHost(req, callback); } delete reqRules.proxy; delete reqRules.pac; var pRules = req.pluginRules; var fRules = req.rulesFileMgr; var hRules = req.headerRulesMgr; var filter = extend( rules.resolveFilter(req), pRules && pRules.resolveFilter(req), fRules && fRules.resolveFilter(req), hRules && hRules.resolveFilter(req) ); var ignoreProxy; var proxy; var pacRule; var host = rules.getHost(req, pRules, fRules, hRules); var hostValue = util.getMatcherValue(host) || ''; if (config.multiEnv || req._headerRulesFirst) { proxy = resolveRulesFromList([pRules, hRules, rules, fRules], 'resolveProxy', req, hostValue); } else { proxy = resolveRulesFromList([pRules, rules, fRules, hRules], 'resolveProxy', req, hostValue); } var proxyHostOnly; var proxyHost = util.isEnable(req, 'proxyHost') || isProxyEnable(req, 'proxyHost'); if (proxy) { proxyHostOnly = isLineProp(proxy, 'proxyHostOnly'); proxyHost = proxyHost || proxyHostOnly; var protocol = proxy.matcher.substring(0, proxy.matcher.indexOf(':')); ignoreProxy = !filter['ignore:' + protocol] && (util.isIgnored(filter, 'proxy') || util.isIgnored(filter, protocol)); if (ignoreProxy) { proxy = null; } else { proxyHost = proxyHost || PROXY_HOSTS_RE.test(proxy.matcher); } } if (!ignoreProxy) { proxyHost = isProxyHost(req, proxy, host) || proxyHost; // 不能调换顺序 } var setHost = function () { if (!host || req._isProxyReq) { return false; } reqRules.host = host; var hostname = util.removeProtocol(host.matcher, true); if (!net.isIP(hostname)) { req.curUrl = hostname || url; rules.lookupHost(req, function (err, ip) { callback(err, ip, host.port, host); }); } else { callback(null, hostname, host.port, host); } return true; }; var resolvePacRule = function () { if (pacRule != null) { return; } if (util.isIgnored(filter, 'pac')) { pacRule = false; return; } if (config.multiEnv || req._headerRulesFirst) { pacRule = (pRules && pRules.resolvePac(req)) || (hRules && hRules.resolvePac(req)) || rules.resolvePac(req) || (fRules && fRules.resolvePac(req)) || false; } else { pacRule = (pRules && pRules.resolvePac(req)) || rules.resolvePac(req) || (fRules && fRules.resolvePac(req)) || (hRules && hRules.resolvePac(req)) || false; } if (pacRule) { proxyHostOnly = isLineProp(pacRule, 'proxyHostOnly'); proxyHost = proxyHost || proxyHostOnly || isLineProp(pacRule, 'proxyHost'); } }; if (host) { if (!proxyHost && !ignoreProxy) { resolvePacRule(); } if (proxyHost) { req._phost = parseUrl(util.setProtocol(util.joinIpPort(host.matcher, host.port))); } else if ( !isLineProp(proxy, 'proxyFirst') && !isLineProp(host, 'proxyFirst') && !util.isEnable(req, 'proxyFirst') && !isProxyEnable(req, 'proxyFirst') ) { return setHost(); } proxyHost = true; reqRules.host = host; req._enableProxyHost = true; } if (ignoreProxy || (proxyHostOnly && !req._phost)) { req.curUrl = url; return setHost() || rules.lookupHost(req, callback); } if (proxy) { if (!req._phost && P_HOST_RE.test(proxy.matcher)) { req._phost = parseUrl(util.setProtocol(RegExp.$1)); } reqRules.proxy = proxy; return callback(); } resolvePacRule(); var pacUrl = util.getMatcherValue(pacRule); if (!pacUrl) { return setHost() || callback(); } var auth; if (AUTH_RE.test(pacUrl)) { auth = RegExp.$1; pacUrl = pacUrl.substring(auth.length + 1); } if (!util.isUrl(pacUrl) && !(pacUrl = util.joinPath(pacRule.root, pacUrl))) { return setHost() || callback(); } req._pacAuth = auth; var pac = cachedPacs[pacUrl]; if (pac) { delete cachedPacs[pacUrl]; cachedPacs[pacUrl] = pac; } else { var list = Object.keys(cachedPacs); if (list.length >= 10) { delete cachedPacs[list[0]]; } cachedPacs[pacUrl] = pac = new Pac(pacUrl, dnsResolve); } reqRules.pac = pacRule; return pac.findWhistleProxyForURL( url.replace('tunnel:', 'https:'), function (err, rule) { if (rule) { tempRules._file = pacRule.file; tempRules.parse(pacRule.rawPattern + ' ' + rule); req.curUrl = url; if ((proxy = tempRules.resolveProxy(req, hostValue))) { proxyHost = isProxyHost(req, proxy) || proxyHost; // 不能调换顺序 req._proxyTunnel = req._proxyTunnel || isLineProp(pacRule, 'proxyTunnel'); var protocol = proxy.matcher.substring(0, proxy.matcher.indexOf(':')); if (!util.isIgnored(filter, protocol)) { reqRules.proxy = proxy; reqRules.proxy.raw = pacRule.raw; } } } if (reqRules.proxy) { (!proxyHost && setHost()) || callback(); } else { req.curUrl = url; setHost() || rules.lookupHost(req, callback); } logger.error(err); } ); }; function tpl(str, data) { if ( typeof str !== 'string' || str.indexOf('<%') === -1 || str.indexOf('%>') === -1 ) { return str + ''; } var key = str; var fn = tplCache.get(key); if (!fn) { str = str .replace(/[\u2028\u2029]/g, '') .replace(/\t/g, ' ') .replace(/\r?\n|\r/g, '\t') .split('<%') .join('\u2028') .replace(/((^|%>)[^\u2028]*)'/g, '$1\r') .replace(/\u2028=(.*?)%>/g, '\',$1,\'') .split('\u2028') .join('\');') .split('%>') .join('p.push(\'') .split('\r') .join('\\\''); try { fn = new Function( 'obj', 'var p=[],print=function(){p.push.apply(p,arguments);};' + 'with(obj){p.push(\'' + str + '\');}return p.join(\'\');' ); } catch (e) { fn = e; throw e; } finally { tplCache.set(key, fn); } } else if (typeof fn !== 'function') { throw fn; } return fn(data || {}).replace(/\t/g, '\n'); } function getScriptContext(req, res, body, pattern, frameOpts) { var ip = req.clientIp || LOCALHOST; var ctx = req.scriptContenxt; if (!ctx) { var headers = extend(true, {}, req.headers); ctx = req.scriptContenxt = { Buffer: Buffer, pattern: pattern, version: config.version, port: config.port, uiHost: 'local.wproxy.org', uiPort: config.uiport, url: req.fullUrl, fullUrl: req.fullUrl, method: util.toUpperCase(req.method) || 'GET', httpVersion: req.httpVersion || '1.1', decodeBuffer: function (buf, encoding) { return iconv.decode(buf, encoding || 'utf8'); }, encodeString: function (str, encoding) { return iconv.encode(str, encoding || 'utf8'); }, encodingExists: function (encoding) { return iconv.encodingExists(encoding); }, isLocalAddress: function (_ip) { return util.isLocalAddress(_ip || ip); }, ip: ip, clientIp: ip, clientPort: req.clientPort, headers: headers, reqHeaders: headers, body: body || '', reqScriptData: {}, res: null }; } if (frameOpts) { ctx.ctx = { sendToServer: frameOpts.sendToServer, sendToClient: frameOpts.sendToClient, handleSendToClientFrame: null, handleSendToServerFrame: null }; } else { ctx.rules = []; ctx.values = {}; } ctx.value = req.globalValue; ctx.getValue = function (key, onlyValues) { var value = !onlyValues && req._inlineValues && req._inlineValues[key]; return typeof value === 'string' ? value : values.get(key); }; ctx.parseUrl = parseUrl; ctx.parseQuery = parseQuery; ctx.tpl = ctx.render = tpl; if (res) { ctx.statusCode = res.statusCode; ctx.serverIp = req.hostIp || LOCALHOST; ctx.resHeaders = extend(true, {}, res.headers); } else { ctx.statusCode = ''; ctx.serverIp = ''; ctx.resHeaders = ''; } return ctx; } function getReqPayload(req, res, cb) { if (res) { return cb(); } if (req.getPayload && util.hasRequestBody(req)) { if (typeof req._reqBody === 'string') { cb(req._reqBody); } else { req.getPayload(function (_, payload) { cb(fileMgr.decode(payload)); }); } } else { cb(); } } function execRulesScript(script, req, res, body, pattern, frameOpts) { var context = getScriptContext(req, res, body, pattern, frameOpts); if (!util.execScriptSync(script, context)) { return ''; } if (frameOpts) { return context.ctx; } return Array.isArray(context.rules) ? { rules: context.rules.join('\n').trim(), values: context.values } : ''; } var CTX_RE = /\bctx\b/; exports.getFrameScriptCtx = function(script, req, res, options, cb) { util.getRuleValue(script, function (text) { if (!text || !CTX_RE.test(text)) { return cb(); } cb(execRulesScript(text, req, res, '', script.rawPattern, options)); }, null, null, null, req); }; function handleDynamicRules(script, req, res, cb) { util.getRuleValue(script, function (list) { var scriptItem, index, text; if (list) { index = script.scriptIndex; scriptItem = script.list[index]; text = scriptItem && list[index]; } if (!scriptItem || isRulesContent(text)) { return cb(list && list.join('\n')); } getReqPayload(req, res, function (body) { var result = execRulesScript(text, req, res, body, script.rawPattern); list[index] = result.rules; cb(list.join('\n'), result.values); }); }, null, null, null, req); } function resolvePluginVars(req, list) { var varMap = {}; var globalValue; var pList; list.forEach(function (item) { if (item.matcher[0] !== 'P') { if (!globalValue) { globalValue = item; } return; } var value = util.getMatcherValue(item); var index = value.indexOf(PLUGIN_VAR_RE.test(value) ? '.' : '='); if (index !== -1) { var name = value.substring(0, index); var plugin = getPluginName(name); if (!plugin) { return; } value = value.substring(index + 1); if (value) { pList = pList || []; pList.push(item); var list = varMap[name] || []; varMap[name] = list; if (list.indexOf(value) === -1) { list.push(value); } } } }); req.rules = req.rules || {}; req._pluginVars = varMap; req.rules.P = pList; req.rules.G = globalValue; req.globalValue = util.getMatcherValue(globalValue); } exports.resolvePluginVars = resolvePluginVars; exports.resolveRulesFile = function (req, callback) { !req._resolvedG && req.rules.G && resolvePluginVars(req, req.rules.G.list); req._resolvedG = true; var rule = req.rules.rulesFile; handleDynamicRules(rule, req, null, function (text, vals) { if (text) { vals = util.toPrivateValues(vals, rule.file); var rulesFileMgr = new Rules(vals); rulesFileMgr._file = rule.file; rulesFileMgr.parse(text); req.rulesFileMgr = rulesFileMgr; req.curUrl = req.fullUrl; text = req.rulesFileMgr.resolveReqRules(req); } // 不能放到if里面 util.mergeRules(req, text); callback(); }); }; exports.resolveResRulesFile = function (req, res, callback) { var rule = req.rules && req.rules.resScript; handleDynamicRules( rule, req, res, function (text, vals) { text = text && text.trim(); callback( text && { file: rule.file, text: text, values: util.toPrivateValues(vals, rule.file) } ); } ); }; function getValue(req, key, keep) { var value = req.headers[key]; if (value) { if (!req.fromComposer && !keep && (config.strict || (!config.enableRequestHeaderRules && !config.multiEnv))) { value = null; } else { req.rulesHeaders[key] = value; } delete req.headers[key]; } try { return value && decodeURIComponent(value); } catch (e) {} return value; } function getPluginName(key) { return exports.getPlugin(key); } function initHeaderRules(req, needBodyFilters) { if (req._bodyFilters !== undefined) { return; } req._bodyFilters = null; req.rulesHeaders = {}; var isPluginReq = req.isPluginReq && !req._isProxyReq; req._headerRulesFirst = isPluginReq; var rulesHeader = getValue(req, RULES_HEADER, isPluginReq); var hostHeader = util.trimStr(getValue(req, HOST_HEADER)); var nameHeader = config.multiEnv && util.trimStr(getValue(req, NAME_HEADER)); var keyHeader = util.trimStr(getValue(req, KEY_HEADER)); var kvHeader = getValue(req, KV_HEADER, isPluginReq); var ruleValue = util.trimStr(rulesHeader); if (hostHeader) { ruleValue = ruleValue + '\n' + hostHeader; } if (keyHeader) { keyHeader = util.trimStr(values.get(keyHeader)); if (keyHeader) { ruleValue = keyHeader + '\n' + ruleValue; } } if (nameHeader) { nameHeader = util.trimStr(globalRules.get(nameHeader)); if (nameHeader) { ruleValue = ruleValue + '\n' + nameHeader; } } var curVars = rules._globalPluginVars; var globalVars = {}; var value; req._globalPluginVars = globalVars; Object.keys(curVars).forEach(function (key) { if (getPluginName(key)) { value = curVars[key]; globalVars[key] = value; } }); if (ruleValue) { var file = 'Header Rules'; var vals = util.toPrivateValues(util.parseJSON(kvHeader), file); var rulesMgr = new Rules(vals); rulesMgr._file = file; rulesMgr.parse(ruleValue); req.headerRulesMgr = rulesMgr; var bodyFilters = needBodyFilters && rulesMgr._rules._bodyFilters; if (bodyFilters && bodyFilters.length) { req._bodyFilters = rules._rules._bodyFilters.concat(bodyFilters); } var headerVars = rulesMgr._globalPluginVars; Object.keys(headerVars).forEach(function (key) { var headerVal = getPluginName(key) && headerVars[key]; if (headerVal) { var curVal = curVars[key]; globalVars[key] = curVal ? curVal.concat(headerVal) : headerVal; } }); } } exports.initHeaderRules = initHeaderRules; function initRules(req) { var fullUrl = req.fullUrl || util.getFullUrl(req); req.curUrl = fullUrl; initHeaderRules(req); if (req.headerRulesMgr) { if (config.multiEnv || req._headerRulesFirst) { req.rules = resolveReqRules(req); util.mergeRules(req, req.headerRulesMgr.resolveReqRules(req)); } else { req.rules = req.headerRulesMgr.resolveReqRules(req); util.mergeRules(req, resolveReqRules(req)); } } else { req.rules = resolveReqRules(req); } return req.rules; } exports.initRules = initRules; var HTTPS_RE = /^(?:ws|http)s:\/\//; function checkCache(cacheKey, callback) { var result = clientCerts.peek(cacheKey); if (result) { if (result.pending) { result.push(callback); } else { callback(result); } return true; } else { result = [callback]; result.pending = true; clientCerts.set(cacheKey, result); } } function getTlsOptions(req, cb) { var cipher = req.rules.cipher; if (!cipher) { return cb(); } cipher.list.forEach(function (rule) { var value = rule && util.getMatcherValue(rule); if (value && !SEP_CIPHER_RE.test(value)) { rule.jsonObject = { ciphers: value }; } }); util.parseRuleJson(cipher, function (data) { if (!data) { return cb(); } var opts; var addOption = function(name, value) { if (!opts) { opts = {}; cipher.__tlsOptions = function () {}; cipher.__tlsOptions._opts = opts; } opts[name] = value; }; var addStrOption = function(name, opName) { var value = (opName && data[opName]) || data[name]; if (util.isString(value)) { addOption(name, value); } }; var addNumOption = function(name) { var value = +data[name]; if (value >= 0) { addOption(name, value); } }; addStrOption('ciphers', 'cipher'); addStrOption('secureProtocol'); addStrOption('maxVersion'); addStrOption('minVersion'); addStrOption('ca'); addStrOption('crl'); addStrOption('allowPartialTrustChain'); addStrOption('sessionIdContext'); addStrOption('sigalgs'); addStrOption('dhparam'); addStrOption('ecdhCurve'); addNumOption('secureOptions'); addNumOption('sessionTimeout'); if (data.honorCipherOrder) { addOption('honorCipherOrder', true); } cb(data); }, req); } function isCert(str) { return CERT_RE.test(str); } exports.getClientCert = function (req, cb) { if (!req) { return cb(); } req.curUrl = req.realUrl || req.fullUrl; if (!HTTPS_RE.test(req.curUrl)) { return cb(); } getTlsOptions(req, function(options) { var pRules = req.pluginRules; var fRules = req.rulesFileMgr; var hRules = req.headerRulesMgr; var rule; if (config.multiEnv || req._headerRulesFirst) { rule = resolveRulesFromList([pRules, hRules, rules, fRules], 'resolveClientCert', req); } else { rule = resolveRulesFromList([pRules, rules, fRules, hRules], 'resolveClientCert', req); } if (rule) { req.rules.clientCert = rule; rule = util.parseJSON(rule.matcher.substring(17)); options = rule ? extend({}, rule, options) : options; } if (!options) { return cb(); } var base = util.getString(options.base); var key = util.getString(options.key); var cert = util.getString(options.cert); var cacheKey; try { if (key && cert) { if (base) { key = path.join(base, key); cert = path.join(base, cert); } cacheKey = 'cert\n' + key + '\n' + cert; if (isCert(key) && isCert(cert)) { return cb(key, cert, false, cacheKey); } if ( checkCache(cacheKey, function (data) { if (data) { cb(data[0], data[1], false, cacheKey); } else { cb(); } }) ) { return; } return fileMgr.readFileList([key, cert], function (data) { var list = clientCerts.peek(cacheKey); if (data[0] && data[1] && data[0].length && data[1].length) { clientCerts.set(cacheKey, data); } else { clientCerts.del(cacheKey); data = ''; } list.forEach(function (fn) { fn(data); }); }); } var pwd = util.getString(options.pwd || options.passphrase); var pfx = util.getString(options.pfx); if (pfx) { if (base) { pfx = path.join(base, key); } cacheKey = 'pfx\n' + pwd + '\n' + pfx; if (isCert(pfx)) { return cb(pwd, pfx, true, cacheKey); } if ( checkCache(cacheKey, function (buf) { if (buf) { cb(pwd, buf, true, cacheKey); } else { cb(); } }) ) { return; } return fileMgr.readFile(pfx, function (buf) { var list = clientCerts.peek(cacheKey); if (buf && buf.length) { clientCerts.set(cacheKey, buf); } else { clientCerts.del(cacheKey); buf = ''; } list.forEach(function (fn) { fn(buf); }); }); } } catch (e) {} cb(); }); };