ks3-js-sdk/src/chunk.js

js sdk for ks3

var util = require('./utils')

var EventProxy = require('./event').EventProxy;
var session = require('./session');
const Async = require("async");

// 获取上传任务的 UploadId 和 分片列表
export function getUploadIdAndPartList(params, callback) {
    var taskId = params.taskId;
    var bucket = params.bucket;
    var region = params.region;
    var key = params.key;
    var storageClass = params.storageClass;
    var self = this;

    // 计算 ETag
    var ETagMap = {};
    var FileSize = params.fileSize;
    var SliceSize = params.sliceSize; // 这里的 sliceSize 就是 chunkSize
    var SliceCount = Math.ceil(FileSize / SliceSize);
    var FinishSliceCount = 0; // 结束分片的数量
    var FinishSize = 0; // 结束分片的总大小
    var getChunkETag = function (PartNumber, callback) {
        var start = SliceSize * (PartNumber - 1);
        var end = Math.min(start + SliceSize, FileSize);
        var ChunkSize = end - start;

        if (ETagMap[PartNumber]) {
            callback(null, {
                PartNumber: PartNumber,
                ETag: ETagMap[PartNumber],
                Size: ChunkSize,
            });
        } else {
            util.fileSliceForUpload(params.body, start, end, false, function (chunkItem) {
                util.getFileMd5(chunkItem, function (err, md5) {
                    if (err) return callback(util.error(err));
                    var ETag = String(md5);
                    ETagMap[PartNumber] = ETag;
                    FinishSliceCount += 1;
                    FinishSize += ChunkSize;
                    callback(null, {
                        PartNumber: PartNumber,
                        ETag: ETag,
                        Size: ChunkSize,
                    });
                });
            });
        }
    };

    // 通过和文件的 md5 对比，判断 UploadId 是否可用
    var isAvailableUploadList = function (PartList, callback) {
        var PartCount = PartList.length;
        // 如果没有分片，通过
        if (PartCount === 0) {
            return callback(null, true);
        }
        // 检查分片数量
        if (PartCount > SliceCount) {
            return callback(null, false);
        }
        // 检查分片大小
        if (PartCount > 1) {
            var PartSliceSize = Math.max(PartList[0].Size, PartList[1].Size);
            if (PartSliceSize !== SliceSize) {
                return callback(null, false);
            }
        }
        // 逐个分片计算并检查 ETag 是否一致
        var next = function (index) {
            if (index < PartCount) {
                var Part = PartList[index];
                getChunkETag(Part.PartNumber, function (err, chunk) {
                    if (chunk && chunk.ETag === Part.ETag && chunk.Size === Part.Size) {
                        next(index + 1);
                    } else {
                        callback(null, false);
                    }
                });
            } else {
                callback(null, true);
            }
        };
        next(0);
    };

    var ep = new EventProxy();
    ep.on('error', function (errData) {
        if (!self._isRunningTask(taskId)) return;
        return callback(errData);
    });

    // 存在 UploadId 构造待上传 partList 列表
    ep.on('upload_id_available', function (UploadData) {
        // 转换成 map
        var map = {};
        var list = [];
        util.each(UploadData.PartList, function (item) {
            map[item.PartNumber] = item;
        });
        for (var PartNumber = 1; PartNumber <= SliceCount; PartNumber++) {
            var item = map[PartNumber];
            if (item) {
                item.PartNumber = PartNumber;
                item.Uploaded = true;
            } else {
                item = {
                    PartNumber: PartNumber,
                    ETag: null,
                    Uploaded: false,
                };
            }
            list.push(item);
        }
        UploadData.PartList = list;
        callback(null, UploadData);
    });

    // 不存在 UploadId, 初始化生成 UploadId
    ep.on('no_available_upload_id', function () {
        if (!self._isRunningTask(taskId)) return;
        var _params = util.extend(
            {
                bucket: bucket,
                region: region,
                key: key,
                storageClass: storageClass,
                body: params.body,
                encryption: params.encryption,
                acl: params.acl,
                headers: params.headers
            },
            params
        );
        var headers = params.Headers;
        _params.Headers = headers;
        // 初始化分片上传
        self.initiateMultipartUpload(_params).then(res => {
            if (!self._isRunningTask(taskId)) return
            if (res.error) return ep.emit('error', res.error)
            if (!res.body) return ep.emit('error', new Error('no such upload id'))

            var data = res.body.InitiateMultipartUploadResult;
            var UploadId = data.UploadId;
            if (!UploadId) {
                return callback(util.error(new Error('no such upload id')));
            }
            ep.emit('upload_id_available', {UploadId: UploadId, PartList: []});
        })
    });

    // 如果已存在 UploadId，找一个可以用的 UploadId
    ep.on('has_and_check_upload_id', function (UploadIdList) {
        // 串行地，找一个内容一致的 UploadId
        UploadIdList = UploadIdList.reverse();
        Async.eachLimit(
            UploadIdList,
            1,
            function (UploadId, asyncCallback) {
                if (!self._isRunningTask(taskId)) return;
                // 如果正在上传，跳过
                if (session.using[UploadId]) {
                    asyncCallback(); // 检查下一个 UploadId
                    return;
                }
                // 判断 UploadId 是否可用
                wholeMultipartListPart.call(
                    self,
                    {
                        bucket: bucket,
                        region: region,
                        key: key,
                        UploadId: UploadId,
                        tracker: params.tracker,
                    },
                    function (err, PartListData) {
                        if (!self._isRunningTask(taskId)) return;
                        if (err) {
                            session.removeUsing(UploadId);
                            return ep.emit('error', err);
                        }
                        // 已上传的分片
                        var PartList = PartListData.PartList;
                        PartList.forEach(function (item) {
                            item.PartNumber *= 1;
                            item.Size *= 1;
                            item.ETag = item.ETag || '';
                        });
                        isAvailableUploadList(PartList, function (err, isAvailable) {
                            if (!self._isRunningTask(taskId)) return;
                            if (err) return ep.emit('error', err);
                            if (isAvailable) {
                                asyncCallback({
                                    UploadId: UploadId,
                                    PartList: PartList,
                                }); // 马上结束
                            } else {
                                asyncCallback(); // 检查下一个 UploadId
                            }
                        });
                    }
                );
            },
            function (AvailableUploadData) {
                if (!self._isRunningTask(taskId)) return;
                if (AvailableUploadData && AvailableUploadData.UploadId) {
                    ep.emit('upload_id_available', AvailableUploadData);
                } else {
                    ep.emit('no_available_upload_id');
                }
            }
        );
    });

    // 在本地缓存找可用的 UploadId
    ep.on('seek_local_avail_upload_id', function (RemoteUploadIdList) {
        // 在本地找可用的 UploadId
        var uuid = session.getFileId(params.body, params.chunkSize, bucket, key);
        var LocalUploadIdList = session.getUploadIdList.call(self, uuid);
        if (!uuid || !LocalUploadIdList) {
            ep.emit('has_and_check_upload_id', RemoteUploadIdList);
            return;
        }
        var next = function (index) {
            // 如果本地找不到可用 UploadId，再一个个遍历校验远端
            if (index >= LocalUploadIdList.length) {
                ep.emit('has_and_check_upload_id', RemoteUploadIdList);
                return;
            }
            var UploadId = LocalUploadIdList[index];
            // 如果不在远端 UploadId 列表里，跳过并删除
            if (!util.isInArray(RemoteUploadIdList, UploadId)) {
                session.removeUploadId.call(self, UploadId);
                next(index + 1);
                return;
            }
            // 如果正在上传，跳过
            if (session.using[UploadId]) {
                next(index + 1);
                return;
            }
            // 判断 UploadId 是否存在线上
            wholeMultipartListPart.call(
                self,
                {
                    bucket: bucket,
                    region: region,
                    key: key,
                    UploadId: UploadId,
                    // tracker: params.tracker,
                },
                function (err, PartListData) {
                    if (!self._isRunningTask(taskId)) return;
                    if (err) {
                        // 如果 UploadId 获取会出错，跳过并删除
                        session.removeUploadId.call(self, UploadId);
                        next(index + 1);
                    } else {
                        // 找到可用 UploadId
                        ep.emit('upload_id_available', {
                            UploadId: UploadId,
                            PartList: PartListData.PartList,
                        });
                    }
                }
            );
        };
        next(0);
    });

    // 获取线上 UploadId 列表
    ep.on('get_remote_upload_id_list', function () {
        // 获取符合条件的 UploadId 列表，因为同一个文件可以有多个上传任务。
        wholeMultipartList.call(self,
            {
                bucket: bucket,
                region: region,
                key: key
            },
            function (err, data) {
                if (!self._isRunningTask(taskId)) return;
                if (err) return ep.emit('error', err);
                // 整理远端 UploadId 列表
                var RemoteUploadIdList = util
                    .filter(data.UploadList, function (item) {
                        return (
                            item.Key === key && (!storageClass || item.StorageClass.toUpperCase() === storageClass.toUpperCase())
                        );
                    })
                    .reverse()
                    .map(function (item) {
                        return item.UploadId || item.UploadID;
                    });
                if (RemoteUploadIdList.length) {
                    ep.emit('seek_local_avail_upload_id', RemoteUploadIdList);
                } else {
                    // 远端没有 UploadId，清理缓存的 UploadId
                    var uuid = session.getFileId(params.body, params.ChunkSize, bucket, key)
                    var LocalUploadIdList;
                    if (uuid && (LocalUploadIdList = session.getUploadIdList.call(self, uuid))) {
                        util.each(LocalUploadIdList, function (UploadId) {
                            session.removeUploadId.call(self, UploadId);
                        });
                    }
                    ep.emit('no_available_upload_id');
                }
            }
        );
    });

    // 开始找可用 UploadId
    ep.emit('get_remote_upload_id_list');
}

// 获取符合条件的全部上传任务 (条件包括 bucket, Region, Prefix)
function wholeMultipartList(params, callback) {
    var self = this;
    var uploadList = [];
    var sendParams = {
        bucket: params.bucket,
        region: params.region,
        prefix: params.key,
    };
    var next = function () {
        // 通过 key 寻找已存在的上传列表
        self.listMultipartUploads(sendParams).then(res => {
            if (res.error) {
                return callback(res.error);
            } else if (res.body && res.body.ListMultipartUploadsResult) {
                const data = res.body.ListMultipartUploadsResult;
                // 检查 data.Upload 是否存在且为数组
                if (data.Upload && Array.isArray(data.Upload)) {
                    uploadList = data.Upload;
                } else if (data.Upload && typeof data.Upload === 'object') {
                    uploadList.push(data.Upload);
                }

                // 用户空间中正在进行的分块上传任务数大于此操作设定的上限值
                if (data.IsTruncated === 'true') {
                    // 列表不完整
                    sendParams['key-marker'] = data.NextKeyMarker;
                    sendParams['upload-id-marker'] = data.NextUploadIdMarker;
                    next();
                } else {
                    callback(null, {UploadList: uploadList});
                }
            }
        }).catch(err => {
            return callback(err)
        })
    };
    next();
}

// 获取指定上传任务的分块列表
function wholeMultipartListPart(params, callback) {
    var self = this;
    var PartList = [];
    var sendParams = {
        bucket: params.bucket,
        region: params.region,
        key: params.key,
        uploadId: params.UploadId,
    };
    var next = function () {
        self.listParts(sendParams).then(res => {
            if (res.error) {
                callback(res.error)
            } else {
                var data = res.body['ns2:ListPartsResult'];
                PartList.push.apply(PartList, data['ns2:Part'] || []);
                if (data['ns2:IsTruncated'] === 'true') {
                    // 列表不完整
                    sendParams['part-number-marker'] = data['ns2:NextPartNumberMarker'];
                    next();
                } else {
                    callback(null, {PartList: PartList});
                }
            }
        }).catch(err => {
            callback(err)
        })
    };
    next();
}


// 上传文件分块，包括
/*
 UploadId (上传任务编号)
 asyncLimit (并发量)，
 UploadData (上传的分块数组)，
 FilePath (本地文件的位置)，
 SliceSize (文件分块大小)
 FileSize (文件大小)
 onProgress (上传成功之后的回调函数)
 */
export function uploadSliceList(params, cb) {
    var self = this;
    var taskId = params.taskId;
    var bucket = params.bucket;
    var region = params.region;
    var key = params.key;
    var UploadData = params.UploadData;
    var FileSize = params.FileSize;
    var SliceSize = params.SliceSize;
    var chunkParallel = params.asyncLimit;
    var chunkRetryTimes = params.chunkRetryTimes;
    var Body = params.Body;
    var SliceCount = Math.ceil(FileSize / SliceSize); // 分片数量 = 文件大小 / 分片大小
    var FinishSize = 0;
    var encryption = params.encryption;
    var Headers = params.Headers;
    var needUploadSlices = util.filter(UploadData.PartList, function (SliceItem) {
        if (SliceItem['Uploaded']) {
            FinishSize += SliceItem['PartNumber'] >= SliceCount ? FileSize % SliceSize || SliceSize : SliceSize;
        }
        return !SliceItem['Uploaded'];
    });
    var onProgress = params.onProgress;

    Async.eachLimit(
        needUploadSlices,
        chunkParallel,
        function (SliceItem, asyncCallback) {
            if (!self._isRunningTask(taskId)) return;
            var PartNumber = SliceItem['PartNumber'];
            var currentSize =
                Math.min(FileSize, SliceItem['PartNumber'] * SliceSize) - (SliceItem['PartNumber'] - 1) * SliceSize;
            var preAddSize = 0;
            uploadSliceItem.call(
                self,
                {
                    taskId: taskId,
                    bucket: bucket,
                    region: region,
                    key: key,
                    SliceSize: SliceSize,
                    FileSize: FileSize,
                    PartNumber: PartNumber,
                    chunkRetryTimes: chunkRetryTimes,
                    encryption: encryption,
                    Body: Body,
                    UploadData: UploadData,
                    Headers: Headers,
                    onProgress: function (data) {
                        FinishSize += data.loaded - preAddSize;
                        preAddSize = data.loaded;
                        onProgress({loaded: FinishSize, total: FileSize});
                    },
                    tracker: params.tracker,
                },
                function (err, res) {
                    if (!self._isRunningTask(taskId)) return;
                    if (err) {
                        FinishSize -= preAddSize;
                    } else {
                        FinishSize += currentSize - preAddSize;
                        SliceItem.ETag = res.headers.etag;
                    }
                    onProgress({loaded: FinishSize, total: FileSize});
                    asyncCallback(err || null, res);
                }
            );
        },
        function (err) {
            if (!self._isRunningTask(taskId)) return;
            if (err) return cb(err);
            cb(null, {
                UploadId: UploadData.UploadId,
                SliceList: UploadData.PartList,
            });
        }
    );
}

// 上传指定分片
function uploadSliceItem(params, callback) {
    var self = this;
    var taskId = params.taskId;
    var bucket = params.bucket;
    var region = params.region;
    var key = params.key;
    var FileSize = params.FileSize;
    var FileBody = params.Body;
    var PartNumber = params.PartNumber * 1;
    var SliceSize = params.SliceSize;
    var encryption = params.encryption;
    var UploadData = params.UploadData;
    var Headers = params.Headers || {};
    var ChunkRetryTimes = params.chunkRetryTimes;

    var start = SliceSize * (PartNumber - 1);

    var ContentLength = SliceSize;

    var end = start + SliceSize;

    if (end > FileSize) {
        end = FileSize;
        ContentLength = end - start;
    }

    var PartItem = UploadData.PartList[PartNumber - 1];
    Async.retry(
        ChunkRetryTimes,
        function (tryCallback) {
            if (!self._isRunningTask(taskId)) return;
            util.fileSliceForUpload(FileBody, start, end, true, function (Body) {
                var sliceParams = {
                    taskId: taskId,
                    bucket: bucket,
                    region: region,
                    key: key,
                    contentLength: ContentLength,
                    partNumber: PartNumber,
                    uploadId: UploadData.UploadId,
                    encryption: encryption,
                    body: Body,
                    uploadType: "upload",
                    headers: Headers,
                    onProgress: params.onProgress,
                };
                self.uploadSlicePart(sliceParams, (err, data) => {
                    if (!self._isRunningTask(taskId)) return;
                    if (err || data.error) {
                        return tryCallback(err);
                    }
                    PartItem.Uploaded = true;
                    tryCallback(null, data);
                })
            })
        },
        function (err, data) {
            if (!self._isRunningTask(taskId)) return;
            return callback(err, data);
        }
    );
}


// 完成分块上传
export function uploadSliceComplete(params, callback) {
    var bucket = params.bucket;
    var region = params.region;
    var key = params.key;
    var UploadId = params.UploadId;
    var SliceList = params.SliceList;
    var self = this;
    var ChunkRetryTimes = this.config.ChunkRetryTimes + 1;
    var Headers = params.Headers;
    var Parts = SliceList.map(function (item) {
        return {
            PartNumber: item.PartNumber,
            ETag: item.ETag,
        };
    });

    // 转换格式
    var senderParts = transformData(Parts);

    // 完成上传的请求也做重试
    Async.retry(
        ChunkRetryTimes,
        function (tryCallback) {
            self.completeMultipart(
                {
                    bucket: bucket,
                    region: region,
                    key: key,
                    uploadId: UploadId,
                    parts: senderParts,
                    headers: Headers,
                }
            ).then(res => {
                if (res.error) {
                    return tryCallback(res.error);
                }
                return tryCallback(null, res);
            })
        },
        function (err, data) {
            callback(err, data);
        }
    );
}


// 转换函数
function transformData(sourceArray) {
    return sourceArray.map(item => ({
        partNumber: item.PartNumber,
        etag: item.ETag,
    }));
}