ks3/lib/request.js

本代码库为`金山云存储KS3`服务.主要提供`KS3 nodejs SDK`和`命令行工具`.

var requestPromise = require('request');
var config = require('./../config');
var util = require('./util');
var xml2json = util.xml2json;
var debug = require('debug')('ks3:request');
var UAParser = require('ua-parser-js');
var dns = require('dns');
var auth = require('./auth');
var { RetryState } = require('./retry');
var { RETRY_STRATEGY } = require('./const');
// 内存缓存，存储DNS查询结果
var dnsCache = new Map();
var dnsCacheTimeout = null;
var enableDnsCache = false;

function makeRequest(req, body, headers = {}, cb) {
    let self = this;
    var maxRetryTimes = this.maxRetryTimes; // 最大重试次数
    req.retryTimes = req.retryTimes || 0;
    let uri = req.uri;
    enableDnsCache = this.dnsCache; //是否启用DNS缓存
    dnsCacheTimeout = this.dnsCacheTimeout; // DNS缓存时间
    let options = {
        uri,
        headers: headers,
        method: req.method,
        timeout: req.timeout || 1000 * 60 * 2,
    }
    //配置代理
    if (this.proxyUrl) {
        options.proxy = this.proxyUrl;
    }
    // 配置代理认证的Proxy-Authorization头，因为ks3的认证header也是Authorization，所以这里用Proxy-Authorization
    if (this.proxyAuth && this.proxyAuth.pass && this.proxyAuth.user) {
        options.headers['Proxy-Authorization'] =
            'Basic ' +
            Buffer.from(
                this.proxyAuth.user + ':' + this.proxyAuth.pass
            ).toString('base64');
    }
    options.followRedirect = this.followRedirect;
    let readStreamBody = null;
    if (body && typeof body.pipe === 'function') {
        // 如果body是一个流对象 则将其赋值给readStreamBody 便于pipe
        readStreamBody = body;
        options.body = null;
    }else if (Buffer.isBuffer(body) || typeof body === 'string') {
        options.body = body;
        options.headers['Content-Length'] = Buffer.byteLength(body, 'utf8')
    } else if (util.isObject(body)) {
        options.data = body;
    }
    let hasReturned = false;
    // 如果启用DNS缓存，解析域名获取family
    if(enableDnsCache){
        //node高版本默认ipv6，为兼容版本问题，默认只使用ipv4
        const family = 4;
        options.family = family;
        options.lookup = (hostname, options, callback) => customDnsLookup(hostname, options.family, callback, enableDnsCache);
    }
    let sender = requestPromise(options);
    if (readStreamBody) {
        readStreamBody.pipe(sender);
    }
    const streamHandler = function (err, response, body) {
        if (hasReturned) {
            cb(err, null, response, body)
            return
        }

        hasReturned = true

        debug('err', err)
        if (response) debug('statusCode', response.statusCode)

        if (!cb) {
            return
        }

        let data = ''
        if (!!body && JSON.stringify(body) !== '{}') {
            data = body
            if (!req.rawBody) {
                let dataStr = body.toString();
                data = dataStr;
                const dataType = req.dataType || config.dataType;
                if (dataType === 'json') {
                    dataStr = dataStr.replace(/\s\w+\:\w+=\"\S+\"/g, '');
                    const json = xml2json.parser(dataStr)
                        ? xml2json.parser(dataStr)
                        : dataStr;
                    data = json;
                }
            }
        }

        if (sender) {
            sender.removeAllListeners && sender.removeAllListeners();
            sender.on('error', function () {});
            sender = null;
        }

        debug('data', data)
        // 增加原始数据流,因为在写文件的时候
        // `buf.toString` 会有问题
        // err: 异常信息
        // data: 转格式后的数据
        // response: response
        // body: 原始数据
        cb(err, data, response, body)
    }

    sender.on('error', function (err) {
        const canRetry = shouldRetry(err, self.retryStrategy);
        if (canRetry && !readStreamBody && maxRetryTimes > req.retryTimes) {
            headers.date = util.getDate();
            req.date = headers.date;
            const token = auth.generateAuth(self.ak, self.sk, req, body);
            headers.Authorization = token;
            setTimeout(() => {
                req.retryTimes++;
                makeRequest.call(self, req, body, headers, cb);
            }, getRetryDelay.call(self, req.retryTimes));
            return
        }
        streamHandler(util.error(err));
    })
    sender.on('response', function (response) {
        let statusCode = response.statusCode;
        let requestId = response.caseless.get('x-kss-request-id');
        let responseHeaders = response.headers;
        let curReqHost = response.client._host; // 当前请求的host
        let isSuccess = statusCode >= 200 && statusCode < 300;
        let dataList = [];
        //如果有 Output，直接将流pipe到 Output
        if (req.Output) {
            req.Output.on('error', function (err) {
                sender && sender.abort && sender.abort();
                streamHandler(err, null, null);
            })
            sender.pipe(req.Output) //此时直接将数据流送往 Output
            //提前返回，不再占用 dataList 内存
            return sender;
        }
        // 手动处理重定向
        if (
            statusCode > 300 &&
            statusCode < 400 &&
            options.followRedirect &&
            response.headers.location &&
            !readStreamBody
        ) {
            sender.removeAllListeners && sender.removeAllListeners();
            sender.abort();
            req.uri = response.headers.location;
            console.log('redirect to:', response.headers, req.uri);
            return makeRequest.call(self, req, body, headers, cb);
        }

        //增加重试机制
        const canRetry = shouldRetry(response, self.retryStrategy)
        if (canRetry && !readStreamBody && maxRetryTimes > req.retryTimes) {
            headers.date = util.getDate();
            req.date = headers.date;
            const token = auth.generateAuth(self.ak, self.sk, req, body);
            headers.Authorization = token;
            setTimeout(() => {
                req.retryTimes++;
                console.log('retry:', req.retryTimes, req);
                makeRequest.call(self, req, body, headers, cb);
            }, getRetryDelay.call(self, req.retryTimes));
            return
        }

        sender.on('data', function (data) {
            dataList.push(data);
        })

        sender.on('end', function () {
            let bodyBuf = Buffer.from('');
            try {
                bodyBuf = Buffer.concat(dataList);
            } catch (e) {
                console.error('end error:', e);
                streamHandler(util.error(e));
                return
            }
            let body = bodyBuf;

            // 如果返回有结果则需要解析xml
            let json = {};
            try {
                if (!isSuccess) {
                    body = bodyBuf.toString();
                    json =
                        (body &&
                            body.indexOf('<') > -1 &&
                            body.indexOf('>') > -1 &&
                            xml2json.parser(body)) ||
                        {};
                }
            } catch (e) {}

            // 处理返回值
            let xmlError = json && json.Error;
            if (isSuccess) {
                streamHandler(null, response, body);
            } else if (xmlError) {
                // 正常返回了 xml body，且有 Error 节点
                xmlError = util.lowerFirst(xmlError);
                streamHandler(
                    util.error(xmlError, responseHeaders, {
                        requestId: requestId,
                        host: curReqHost,
                        statusCode: statusCode,
                        code: statusCode,
                        error: body,
                    }),
                    response,
                    body
                )
            } else if (statusCode) {
                // 有错误的状态码 如307
                streamHandler(
                    util.error(
                        {
                            message: response.statusMessage,
                            code: '' + statusCode,
                        },
                        responseHeaders,
                        {
                            host: curReqHost,
                            statusCode: statusCode,
                            requestId: requestId,
                            code: statusCode,
                        }
                    ),
                    response,
                    body
                )
            } else {
                // 无状态码，或者获取不到状态码
                streamHandler(
                    util.error(new Error('statusCode error')),
                    response,
                    body
                )
            }
            dataList = null;

            // streamHandler(null, response, body)
        })
    })

    // upload progress
    if (req.onProgress && typeof req.onProgress === 'function') {
        let contentLength = data.headers['Content-Length'];
        let time0 = Date.now();
        let size0 = 0;
        sender.on('drain', function () {
            let time1 = Date.now();
            let loaded = 0;
            try {
                // 已经上传的字节数 = socket当前累计发送的字节数 - 头部长度 - socket以前发送的字节数
                loaded =
                    sender.req.connection.bytesWritten -
                    sender.req._header.length -
                    (sender.req.connection._lastBytesWritten || 0)
            } catch (e) {}
            let total = contentLength;
            let speed =
                parseInt(((loaded - size0) / ((time1 - time0) / 1000)) * 100) /
                    100 || 0;
            let percent = parseInt((loaded / total) * 100) / 100 || 0;
            time0 = time1;
            size0 = loaded;
            req.onProgress({
                loaded: loaded,
                total: total,
                speed: speed,
                percent: percent,
            })
        })
    }

    // download progress
    if (
        req.onDownloadProgress &&
        typeof req.onDownloadProgress === 'function'
    ) {
        let time0 = Date.now();
        let size0 = 0;
        let loaded = 0;
        let total = 0;
        sender.on('response', function (res) {
            total = res.headers['content-length'];
            sender.on('data', function (chunk) {
                loaded += chunk.length;
                let time1 = Date.now();
                let speed =
                    parseInt(
                        ((loaded - size0) / ((time1 - time0) / 1000)) * 100
                    ) / 100 || 0;
                let percent = parseInt((loaded / total) * 100) / 100 || 0;
                time0 = time1;
                size0 = loaded;
                req.onDownloadProgress({
                    loaded: loaded,
                    total: total,
                    speed: speed,
                    percent: percent,
                })
            })
        })
    }

    return sender
}



function sendRequest(reqParams, body, token, cb) {
    let ua = '';
    let uaParser = new UAParser();
    if (uaParser.getResult().ua) {
        ua = config.ua + ' ' + uaParser.getResult().ua;
    }
    // add debug mode
    requestPromise.debug = config.isDebug;
    const headers = {
        'Content-Type': reqParams.type || config.contentType,
        'User-Agent': ua || reqParams.ua || config.ua,
        // warning: 这个地方需要注意,因为后端的token产生需要`date`,所以需要在请求的时候,把用于计算授权的`date`再传给后端,
        // 不能自己生成,否则token就报错了
        date: reqParams.date,
        ...(reqParams.headers || {}),
    }
    // console.log('this.sk:', this.sk, 'this.ak:', this.ak)
    // console.log('headers:', headers)
    if (this.ak && this.sk) {
        headers.Authorization = token;
    }

    // set default timeout from init config
    reqParams.timeout = this.timeout
    return makeRequest.call(this, reqParams, body, headers, cb);
}

function customDnsLookup(hostname, family, callback, enableDnsCache = true) {
    if (!hostname) {
        return callback(new Error('Hostname is required'));
    }

    hostname = util.convertToASCII(hostname); // 将域名转换为 ASCII 格式
    const now = Date.now(); // 当前时间戳
    // 如果启用了DNS缓存，并且缓存中存在有效的记录
    if (enableDnsCache && dnsCache.has(hostname)) {
        const cacheEntry = dnsCache.get(hostname);
        if (cacheEntry.expires > now) {
            const { address, family } = cacheEntry;
            return callback(null, address, family);
        } else {
            // 移除过期的缓存记录
            dnsCache.delete(hostname);
        }
    }

    // 执行实际的DNS查询
    dns.lookup(hostname, { family }, (err, address) => {
        if (err) {
            return callback(err);
        }
        // 如果启用了 DNS 缓存
        if (enableDnsCache) {
            dnsCache.set(hostname, {
                address,
                family,
                expires: Date.now() + dnsCacheTimeout,
            })
        }
        return callback(null, address, family);
    })
}

function shouldRetry(response, retryStrategy) {
    if (!retryStrategy) {
        return false;
    }
    if (!response) {
        return true;
    }
    if (response && !response.statusCode) {
        return true;
    }
    //通过状态码判断是否需要重试
    if ([408, 429, 500, 502, 503, 504].indexOf(response.statusCode) > -1) {
        return true;
    }
    return false;
}

function getRetryDelay(retryTimes) {
    // 可以根据实际情况自定义哪些错误可以重试
    const retryStrategyClass = RETRY_STRATEGY[this.retryStrategy];
    var retryWaitStrategy = new retryStrategyClass();
    var retryState = new RetryState(retryTimes + 1);
    // 调用策略的 invoke 方法
    var retryDelay = retryWaitStrategy.invoke(retryState);
    return retryDelay;
}

module.exports = sendRequest;