ks3/lib/advance.js

本代码库为`金山云存储KS3`服务.主要提供`KS3 nodejs SDK`和`命令行工具`.

const EventDispatcer = require('./common/event')
const async = require('./common/async')
const utils = require('./util')
const config = require('./../config');
const fs = require('fs');
var debug = require('debug')('ks3:advance');
/**
 * 分块拷贝大文件
 * @param {String} params.Bucket
 * @param {String} params.Key
 * @param {String} params.SourceBucket
 * @param {String} params.SourceKey
 * @param {String} params.ACL
 * @param {String} params.StorageClass
 * @param {String} params.onProgress
 * @param {String} params.Headers
 * 
 * @description 不支持客户端加密的数据大文件分块拷贝
 */
function sliceCopyFile (params, cb) {
    const self = this
    const eventBus = new EventDispatcer()

    var onProgress = null
    var fileSize = 0
    var uploadId = ''
    var finishedSize = 0 // 成功的文件大小
    self.config({
        dataType: 'json'
    })
    var copyChunkParallelLimit = params.ParallelLimit || 10

    const { SourceBucket, SourceKey, Headers } = params
    // 数据准备 sse-c时headObject会报错 故不支持客户端加密类型的拷贝
    self.object.headObject({
        Bucket: SourceBucket,
        Key: SourceKey,
        Endpoint: params.Endpoint
    }, function (err, data, res) {
        if (err) {
            cb(err)
            return
        }
        fileSize = res.caseless.get('content-length')
        if (!fileSize) {
            cb(new Error('[sliceCopyFile] get object content-length error'))
            return
        }

        fileSize = Number(fileSize)
        if (fileSize < config.uploadMaxSize) {
            params.headers = params.Headers;
            return self.object.copyObject(params, cb)
        }
        onProgress = utils.throttleOnProgress.call(self, fileSize, params.onProgress)
        let originHeaderInfo = {}

        const headerKeys = ['cache-control', 'content-disposition', 'content-encoding', 'content-type', 'expires', 'x-kss-storage-class', 'x-kss-server-side-encryption']
        Object.keys(res.headers).forEach(item => {
            const kssMetaPrefix = 'x-kss-meta'
            let k = item.toLowerCase();
            const val = res.headers[k]
            if (k.indexOf(kssMetaPrefix) == 0 && k.length > kssMetaPrefix.length) {
                originHeaderInfo[k] = val
            }
            // 提取headers里面的acl、storageClass、meta、加密
            if (headerKeys.indexOf(k) > -1) {
                originHeaderInfo[k] = val
            }
        })
        if (params.ACL) originHeaderInfo['x-kss-acl'] = params.ACL

        const chunks = utils.getSliceChunks(fileSize, config.chunkSize, config.chunkMaxNum)
        eventBus.emit('slice-copy-prepared', {
            chunks,
            originHeaders: originHeaderInfo
        })
    })

    eventBus.on('slice-copy-prepared', d => {
        const { chunks, originHeaders} = d
        
        self.object.multitpart_upload_init({
            Bucket: params.Bucket,
            Key: params.Key,
            headers: Object.assign(originHeaders, Headers),
            Endpoint: params.Endpoint
        }, (err, data, res) => {
            if (err) return cb(err)
            uploadId = data.InitiateMultipartUploadResult.UploadId;
            chunks.map(item => { item.UploadId = uploadId })
            eventBus.emit('slice-copy-start', chunks)
        })
    })

    eventBus.on('slice-copy-start', chunks => {
        async.eachLimit(chunks, copyChunkParallelLimit, function (sliceItem, callback) {
            const { PartNumber, Range, UploadId, ChunkStart, ChunkEnd } = sliceItem
            const chunkSize = ChunkEnd - ChunkStart
            self.object.uploadPartCopy({
                SourceBucket, SourceKey,
                Bucket: params.Bucket,
                Key: params.Key,
                PartNumber, Range, UploadId,
                Endpoint: params.Endpoint
            }, function (err, data, res) {
                if (err) {
                    return callback(err)
                }
                finishedSize += chunkSize
                onProgress({
                    loaded: finishedSize,
                    total: fileSize
                })
                const etag = data.CopyObjectResult.ETag
                sliceItem.ETag = etag.replace(/"/g, '')
                callback(err || null, data)
            })
        }, function (err) {
            if (err) {
                onProgress(null, true)
                console.log('request error should abort upload：', uploadId)
                self.object.upload_abort({
                    UploadId: uploadId,
                    Bucket: params.Bucket,
                    Key: params.Key,
                    Endpoint: params.Endpoint
                }, function (err, data, res) {
                    if (!err) console.log('cancel upload success')
                })
                return cb(err)
            }
            if (chunks && chunks.length) eventBus.emit('slice-copy-complete', chunks)
        })
    })
    

    eventBus.on('slice-copy-complete', chunks => {
        const etags = chunks.map(item => { return item.ETag;})
        const opt = {
            Bucket: params.Bucket,
            Key: params.Key,
            PartNumber: etags.length,
            UploadId: uploadId,
            body: utils.generateCompleteXML(etags),
            Endpoint: params.Endpoint,
            //拷贝支持覆盖，complete时也需要传入headers[x-kss-forbid-overwrite]
            header: params.Headers
        }
        self.object.upload_complete(opt, function(err, data, res) {
            if (err) {
                onProgress(null, true)
                return cb(err)
            }
            onProgress({
                loaded: finishedSize,
                total: finishedSize
            }, true)
            cb(err, data, res)
        })
    })

}

/**
 * 分块下载
 * @param {String} params.Bucket
 * @param {String} params.Key
 * @param {String} params.FilePath 下载位置
 * @param {String} params.ParallelLimit 并发数
 * @param {*} callback 
 */
function downloadFile (params, callback) {

    const self = this
    const eventBus = new EventDispatcer()
    var onProgress = null
    var fileSize = 0
    var uploadId = ''
    var parallelLimit = params.ParallelLimit || 5
    var finishedSize = 0 // 成功的文件大小
    var stopped = false;
    
    var filePath = params.FilePath

    const { Bucket, Key, Headers } = params

    self.object.headObject({
        Bucket,
        Key,
        Endpoint: params.Endpoint
    }, function(err, data, res) {
        if (err) {
            callback(err)
            return
        }
        fileSize = res.caseless.get('content-length')

        if (!fileSize) {
            cb(new Error('[downloadFile] get object content-length error'))
            return
        }

        // 先校验文件是否可写入
        try {
            fs.writeFileSync(filePath, '')
        } catch (error) {
            callback(error)
            return;
        }

        fileSize = Number(fileSize)
        if (fileSize < config.chunkSize) {
            self.object.get(params, function (err, data, resp) {
                if (err) {
                    callback(err)
                    return;
                } else {
                    fs.writeFileSync(filePath, data)
                }
            })
        }

        onProgress = utils.throttleOnProgress.call(self, fileSize, params.onProgress)
        const chunks = utils.getSliceChunks(fileSize, config.chunkSize, config.chunkMaxNum)
        eventBus.emit('slice-download-start', chunks)
    })

    eventBus.on('slice-download-start', function (chunks) {
        async.eachLimit(chunks, parallelLimit, function (sliceItem, callback) {
            if (stopped) return
            
            const { Range, ChunkStart, ChunkEnd } = sliceItem
            const dataSize = ChunkEnd - ChunkStart
            console.log('请求的区间：', Range)
            self.object.get({
                Bucket,
                Key,
                headers: {
					Range: Range
				},
                Endpoint: params.Endpoint
            }, function (err, data, res) {
                console.log('获取到的长度：', data.length)
                fs.open(filePath, 'r+', function (err, fd) {
                    if (err) {
                        callback(err)
                        return
                    }
                    console.log('开始写入的位置：', `${ChunkStart}-${ChunkEnd}`, dataSize)
                    // write(fd, buffer, offset, length, position, callback)
                    var cb = function (err, buffer) {
                        if (err) {
                            callback(err)
                            return
                        } else {
                            // 更新进度
                            finishedSize += dataSize
                            onProgress({
                                loaded: finishedSize,
                                total: fileSize
                            })

                            if (finishedSize == fileSize) fs.close(fd)

                            callback(err || null, data)
                        }
                    }
                    console.log('data.length, ChunkStart: ', data.length, ChunkStart)


                    var len = data.length
                    var position = ChunkStart
                    var buffers = Buffer.from(data)

                    fs.write(fd, buffers, 0, len, position, cb)
                })
            })

        }, function(err){
            if (err) {
                onProgress(null, true)
                console.log('request error ：', err)
                return callback(err)
            }
            eventBus.emit('slice-download-complete', chunks)
        })
    })

    eventBus.on('error', function () {
        stopped = true;
    })

    eventBus.on('slice-download-complete', function () {
        callback(null, '')
    })
}


module.exports = {
    sliceCopyFile,
    downloadFile
}