ks3-js-sdk/src/advance.js

js sdk for ks3

var utils = require('./utils.js');
var submitRequest = require("./requestSender");
var EventProxy = require('./event').EventProxy;
var session = require('./session');
var { getUploadIdAndPartList, uploadSliceList, uploadSliceComplete } = require("./chunk");

/**
 * 获取请求地址
 * @params {*} params - { region, bucket, domain, resource, protocol, key }
 * @returns
 * @description
 * -  `${this.config.protocol}://${bucket}.${domain}/${resource}`
 * - resource: key + suffix
 */
function getUrl(params) {
    var region = params.region || this.config.region;
    var bucket = params.bucket || this.config.bucket;
    var domain = params.domain || '';
    var resource = params.resource || '';
    var key = params.key
    var protocol = params.protocol || this.config.protocol
    if (protocol) {
        protocol += ':'
    } else {
        protocol = location.protocol === 'http' ? 'http:' : 'https:'
    }

    if (!domain) {
        const ENDPOINT_KEY = Object.keys(utils.ENDPOINT)
        if (ENDPOINT_KEY.indexOf(region) > -1) {
            domain = utils.ENDPOINT[region];
        } else {
            // 输入的region不在既定的map中 则设置为默认的domain
            domain = region;
        }
    }else{
        domain = region;
    }

    // 去掉域名最后的斜杆
    if (domain.slice(-1) === '/') {
        domain = domain.slice(0, -1);
    }

    if (!this.config.forcePathStyle) {
        domain = bucket + '.' + domain;
    } else {
        domain = domain + '/' + bucket
    }

    var url = `${protocol}//${domain}`
    if (key) {
        url += `/${utils.encodeKey(key)}`
    }

    if (resource) {
        url += `${resource}`
    }
    return url;
}


/**
 * 高级上传文件
 * @param params
 *     @params  {String}    params.bucket              bucket名称，必须
 *     @params  {String}    params.key                 object key，必须
 *     @params  {String}    params.region              region名称，必须
 *     @params  {String}    params.acl                 文件预设ACL(private | public-read)，默认值private，非必须
 *     @params  {String}    params.body                上传的文件，非必须
 *     @params  {Boolean}   params.encryption          是否加密文件，非必须
 *     @params  {String}    params.storageClass        对象的存储类型,非必须
 *     @params  {String}    params.headers             自定义header信息，非必须
 *     @params  {String}    params.sliceSize           多大的文件使用分片上传，默认 10MB
 *     @params  {Function}  params.onTaskReady         上传任务准备完成回调，非必须
 *     @params  {Function}  params.onProgress          上传进度监听函数，非必须
 *     @params  {Function}  params.onFileFinish        上传任务准备完成回调，非必须
 * @param callback (err, data)
 */
async function uploadFile(params, callback) {
    var self = this;

    // 判断多大的文件使用分片上传
    var sliceSize = params.sliceSize || 10*1024*1024;
    var taskList = [];

    
    var Body = params.body;
    var fileSize = Body.size || Body.length || 0;
    var fileInfo = { taskId: '' };

    // 整理 option，用于返回给回调
    utils.each(params, function (v, k) {
        if (typeof v !== 'object' && typeof v !== 'function') {
            fileInfo[k] = v;
        }
    });

    // 处理文件 onTaskReady
    var _onTaskReady = params.onTaskReady;
    var onTaskReady = function (tid) {
        fileInfo.taskId = tid;
        _onTaskReady && _onTaskReady(tid);
    };
    params.onTaskReady = onTaskReady;
    
    // 添加上传任务,超过阈值使用分块上传，小于等于则简单上传
    var api = fileSize > sliceSize ? 'sliceUploadFile' : 'putUploadFile';

    // 处理文件完成
    var _onFileFinish = params.onFileFinish;
    var onFileFinish = function (err, data) {
        _onFileFinish && _onFileFinish(err, data, fileInfo);
        callback && callback(err, data);
    };
    
    taskList.push({
        api,
        params,
        callback: onFileFinish
    })

    // 添加任务至队列
    self._addTasks(taskList);
}

/**
 * 新 put 上传方法，基于 requestSender，支持暂停
 * @param params
 *     @params  {String}    params.bucket              bucket名称，必须
 *     @params  {String}    params.region              region名称，必须
 *     @params  {String}    params.key                 object key，必须
 *     @params  {String}    params.body                上传的文件，必须
 *     @params  {String}    params.acl                 文件预设ACL(private | public-read)，默认值private，非必须
 *     @params  {Boolean}   params.encryption          是否加密文件，非必须
 *     @params  {String}    params.storageClass        对象的存储类型,非必须
 *     @params  {String}    params.headers             自定义header信息，非必须
 *     @params  {Function}  params.onProgress          上传进度监听函数，非必须
 * @param callback callback(err, res)
 * */
async function putUploadFile(params, callback) {
    var self = this;
    var fileSize = params.contentLength; // contentLength 不必填，addTask 的时候自动计算并添加到 params 中
    if (params.key === null || params.key === undefined) {
        throw Error('require the params of key!');
    }
    if (params.body === null || params.body === undefined) {
        throw Error('require the params of body!');
    }
    if (!params.bucket && !this.config.bucket) {
        throw Error('require the params of bucket or defaultConfig of bucket!');
    }
    if (!params.region && !this.config.region) {
        throw Error('require the params of region or defaultConfig of region!');
    }
    var headers = params.headers || {};
    if (!headers['Content-Type'] && !headers['content-type']) {
        headers['Content-Type'] = (params.body && params.body.type) ? params.body.type : '';
    }
    if (params.acl) {
        if (['private', 'public-read'].indexOf(params.acl) == -1) {
            throw Error('The value of acl can only be private or public-read!');
        }
        headers['x-kss-acl'] = params.acl;
    }
    if (params.encryption) {
        headers['x-kss-server-side-encryption'] = 'AES256';
    }
    if (params.storageClass) {
        headers['x-kss-storage-class'] = params.storageClass; // 存储类型
    }
    var onProgress = utils.throttleOnProgress.call(this, 0, params.onProgress);
    var bucket = params.bucket || this.config.bucket;
    var region = params.region || this.config.region;
    var url = getUrl.bind(this)({
        bucket,
        region,
        key: params.key
    })
    var reqParams = {
        taskId: params.taskId,
        method: 'PUT',
        bucket,
        url,
        qs: {},
        headers: headers,
        body: params.body,
        onProgress,
        onCancel: params.onCancel
    }
    if (!reqParams.headers['x-kss-date']) {
        reqParams.headers['x-kss-date'] = params.date ? params.date : new Date().toUTCString();
    }
    reqParams.date = reqParams.headers['x-kss-date'];
    var signature = await utils.getAuth(this.config.SK, reqParams.bucket, params.key, '', reqParams.method, reqParams.headers, reqParams.date, params);
    reqParams.headers['Authorization'] = `KSS ${this.config.AK}:${signature}`;
    if (params.authorization) {
        reqParams.headers['Authorization'] = params.authorization;
    }

    submitRequest.call(self, reqParams, function (err, data) {
        if (err) {
            onProgress(null, true);
            callback && callback(err, data);
            return
        }
        onProgress({loaded: fileSize, total: fileSize}, true);
        callback && callback(err, data);
    })
}

/**
 * 分片上传对象（断点续传）
 * @param params
 *     @params  {String}    params.bucket              bucket名称，必须
 *     @params  {String}    params.region              region名称，必须
 *     @params  {String}    params.key                 object key，必须
 *     @params  {String}    params.body                上传的文件，必须
 *     @params  {String}    params.acl                 文件预设ACL(private | public-read)，默认值private，非必须
 *     @params  {Boolean}   params.encryption          是否加密文件，非必须
 *     @params  {Number}   params.maxPartNumber        分块最大数量
 *     @params  {String}    params.storageClass        对象的存储类型,非必须
 *     @params  {Number}    params.chunkSize           分块上传的最小单位，默认 5MB，会根据文件大小动态调整
 *     @params  {String}    params.headers             自定义header信息，非必须
 *     @params  {Function}  params.onProgress          上传进度监听函数，非必须
 * @param callback callback(err, data)
 */
async function sliceUploadFile(params, callback) {
    var self = this;
    var ep = new EventProxy();
    var taskId = params.taskId;
    var bucket = params.bucket || this.config.bucket;
    var region = params.region || this.config.region;
    var key = params.key;
    var body = params.body;
    var chunkSize = params.chunkSize || self.config.chunkSize; // 单片大小
    var chunkRetryTimes = self.config.retryTimes || 3; // 单片大小
    var MaxPartNumber = params.maxPartNumber || 10000;
    var asyncLimit = Math.min(self.config.parallelLimit || 1, 256); // 分片并发数量
    var storageClass = params.storageClass; // 存储类型
    var encryption = params.encryption;
    var UploadIdCacheLimit = 50; // uploadId 缓存最大数量，一般不修改故不开放给用户
    var fileSize;

    var onProgress;

    if (!key) {
        throw Error('require the params of key!');
    }
    if (!body) {
        throw Error('require the params of body!');
    }
    if (!bucket) {
        throw Error('require the params of bucket or defaultConfig of bucket!');
    }
    if (!region) {
        throw Error('require the params of region or defaultConfig of region!');
    }

    ep.on('error', function (errData) {
        if (!self._isRunningTask(taskId)) return;
        return callback(errData);
    });
    
    // 上传分块完成，开始 uploadSliceComplete 操作
    ep.on('upload_complete', function (UploadCompleteData) {
        var _UploadCompleteData = utils.extend(
            {
                UploadId: params.UploadData.UploadId || '',
            },
            UploadCompleteData
        );
        callback(null, _UploadCompleteData);
    });
    
    // 上传分块完成，开始 uploadSliceComplete 操作
    ep.on('upload_slice_complete', function (UploadData) {
        var metaHeaders = {};
        uploadSliceComplete.call(
            self,
            {
                bucket: bucket,
                region: region,
                key: key,
                UploadId: UploadData.UploadId,
                SliceList: UploadData.SliceList,
                Headers: params.headers || {},
                signature: params.signature
            },
            function (err, data) {
                if (!self._isRunningTask(taskId)) return;
                session.removeUsing(UploadData.UploadId);
                if (err) {
                    onProgress(null, true);
                    return ep.emit('error', err);
                }
                session.removeUploadId.call(self, UploadData.UploadId);
                onProgress({ loaded: fileSize, total: fileSize }, true);
                ep.emit('upload_complete', data);
            }
        );
    });
    
    // 获取 UploadId 完成，开始上传每个分片
    ep.on('get_upload_data_finish', function (UploadData) {
        // 处理 UploadId 缓存
        var uuid = session.getFileId(body, params.chunkSize, bucket, key);
        uuid && session.saveUploadId.call(self, uuid, UploadData.UploadId, UploadIdCacheLimit); // 缓存 UploadId
        session.setUsing(UploadData.UploadId); // 标记 UploadId 为正在使用

        // 获取 UploadId
        onProgress(null, true); // 任务状态开始 uploading
        uploadSliceList.call(
            self,
            {
                taskId: taskId,
                bucket: bucket,
                region: region,
                key: key,
                Body: body,
                FileSize: fileSize,
                SliceSize: chunkSize,
                asyncLimit: asyncLimit,
                chunkRetryTimes: chunkRetryTimes,
                encryption: encryption,
                UploadData: UploadData,
                Headers: params.Headers,
                onProgress: onProgress,
                signature: params.signature
            },
            function (err, data) {
                if (!self._isRunningTask(taskId)) return;
                if (err) {
                    onProgress(null, true);
                    return ep.emit('error', err);
                }
                ep.emit('upload_slice_complete', data);
            }
        );
    });

    // 开始获取文件 UploadId，里面会视情况计算 ETag，并比对，保证文件一致性，也优化上传
    ep.on('get_file_size_finish', function () {
        onProgress = utils.throttleOnProgress.call(self, fileSize, params.onProgress);

        if (params.UploadData && params.UploadData.UploadId) {
            ep.emit('get_upload_data_finish', params.UploadData);
        } else {
            var _params =
                {
                    taskId: taskId,
                    bucket: bucket,
                    region: region,
                    key: key,
                    headers: params.headers,
                    storageClass: storageClass,
                    body: body,
                    fileSize: fileSize, // 文件大小
                    sliceSize: chunkSize, // 分片大小
                    ...params
                }

            // 没有 uploadId，第一次上传    
            getUploadIdAndPartList.call(self, _params, function (err, UploadData) {
                if (!self._isRunningTask(taskId)) return;
                if (err) return ep.emit('error', err);
                params.UploadData.UploadId = UploadData.UploadId;
                params.UploadData.PartList = UploadData.PartList;
                ep.emit('get_upload_data_finish', params.UploadData);
            });
        }
    });


    // 获取上传文件大小
    fileSize = params.contentLength;
    delete params.contentLength;
    !params.headers && (params.headers = {});
    utils.each(params.Headers, function (item, key) {
        if (key.toLowerCase() === 'content-length') {
            delete params.headers[key];
        }
    });
    
    // 控制分片大小
    (function () {
        var SIZE = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 1024 * 2, 1024 * 4, 1024 * 5];
        var autoChunkSize = 1024 * 1024;// 初始 1MB
        for (var i = 0; i < SIZE.length; i++) {
            autoChunkSize = SIZE[i] * 1024 * 1024;
            if (fileSize / autoChunkSize <= MaxPartNumber) break;
        }
        params.chunkSize = params.sliceSize = chunkSize = Math.max(chunkSize, autoChunkSize);
    })();

    // 开始上传
    if (fileSize === 0) {
        params.body = '';
        params.contentLength = 0;
        params.skipTask = true;
        self.putUploadFile(params, callback);
    } else {
        ep.emit('get_file_size_finish');
    }
}

// 上传分片方法: 基于 requestSender，支持暂停
async function uploadSlicePart(params, callback){
    if (params.key === null || params.key === undefined) {
        throw Error('require the params of key!');
    }
    if(!params.bucket && !this.config.bucket){
        throw Error('require the params of bucket or defaultConfig of bucket!');
    }
    if(!params.region && !this.config.region){
        throw Error('require the params of region or defaultConfig of region!');
    }
    if(!params.partNumber){
        throw Error('require the params of partNumber!');
    }
    if(!params.uploadId){
        throw Error('require the params of uploadId!');
    }
    var headers = {};
    if(params.uploadType === 'copyPart'){
        headers['x-kss-copy-source'] = '/' + params.sourceBucket + '/' + utils.encodeKey(params.sourceKey);
        headers['x-kss-copy-source-range'] = params.range;
    }
    headers = Object.assign(headers, params.headers);
    var resource = `?partNumber=${params.partNumber}&uploadId=${params.uploadId}`;
    var bucket = params.bucket  || this.config.bucket;
    var region = params.region || this.config.region;
    var url = getUrl.bind(this)({
        bucket,
        region,
        key: params.key,
        resource: resource
    })
    // var onProgress = utils.throttleOnProgress.call(this, 0, params.onProgress);
    var reqParams = {
        taskId: params.taskId,
        method: 'PUT',
        bucket,
        url,
        qs: {},
        headers: headers,
        onProgress: params.onProgress
    }
    if(params.uploadType === 'upload'){
        reqParams.body = params.body
    }
    if(!reqParams.headers['x-kss-date']) {
        reqParams.headers['x-kss-date'] = params.date ? params.date : new Date().toUTCString();
    }
    reqParams.date = reqParams.headers['x-kss-date'];
    var signature = await utils.getAuth(this.config.SK, reqParams.bucket, params.key , resource, reqParams.method, reqParams.headers, reqParams.date, params);
    reqParams.headers['Authorization'] = `KSS ${this.config.AK}:${signature}`;

    var self = this;
    submitRequest.call(self, reqParams, function (err, data) {
        callback && callback(err, data);
    })
}


var API_MAP = {
    putUploadFile,
    sliceUploadFile,
    uploadSlicePart,
    uploadFile
};

module.exports.init = function (ks3, task) {
    task.transferToTaskMethod(API_MAP, 'putUploadFile');
    task.transferToTaskMethod(API_MAP, 'sliceUploadFile');
    utils.each(API_MAP, function (fn, apiName) {
        ks3.prototype[apiName] = API_MAP[apiName];
    });
};