ks3/examples/object.js

本代码库为`金山云存储KS3`服务.主要提供`KS3 nodejs SDK`和`命令行工具`.

var KS3 = require('..');
const { ak, sk, bucketName, endpoint } = require('./const');

var path = require('path');
var fs = require('fs');
var mime = require('mime');
var util = require('../lib/util')
var config = require('../config')
var request = require('../lib/request');
var key = 'a.mp4';
var bigFile = path.join(__dirname, '../test/assets/' + key);

var filePath = bigFile;
var fileName = (function(){
	var s = filePath.split('/');
	return s[s.length-1];
})();

var cb = function (err, data, res, body) {
    if (err) {
        console.log('==========> err:', err)
    } else {
        console.log('==========> data:', JSON.stringify(data))
        console.log('==========> statusCode:', res.statusCode)
        console.log('==========> body:', body)
        console.log('==========> meta:', util.getObjectMetadata(res))
    }
}

// 准备工作
// var key = `test2_${fileName}`;
// var client = new KS3({ak, sk, bucketName, endpoint, inited: false});
var client = require('./client').client;
var chunkSize = config.chunkSize

var stat = fs.statSync(filePath)
var fileSize = stat.size;
// 总块数
var count = parseInt(fileSize / chunkSize) + ((fileSize % chunkSize == 0 ? 0: 1));

// 文件类型
var contentType = mime.lookup(key)


var uploadId = 'fb5f9b287a894b8da0608777b10853b5'

// 上传
function upload () {
    const headers = {
        'x-kss-server-side-encryption': 'AES256'
    } 
    const initParams = { Key: key, Bucket: bucketName, headers }
    client.object.multitpart_upload_init( initParams, function(err, data, res) {

        // data = util.xml2json.parser(data)
        if (err) {
            console.log(err)
            return;
        }

        uploadId = data.InitiateMultipartUploadResult.UploadId
        console.log(`获取到上传id： ${uploadId}`)

        let chunkNum = 0;

        let etags = []

        console.log('开始分块上传...')

        up();

        function up () {
            let start = chunkNum * chunkSize
            console.log(`开始上传 第${chunkNum} 块文件...`)
            // 是否全部上传
            if (chunkNum < count) {
                util.getChunk(filePath, chunkSize, start, function (buffer) {
                    const params = {
                        Bucket: bucketName,
                        Key: key,
                        UploadId: uploadId,
                        PartNumber: chunkNum + 1, // 传值从1开始
                        body: buffer,
                        type: contentType
                    }
                    console.log('分块上传参数：', params)
                    client.object.upload_part(params, function(err, data, res) {
                        if (err) {
                            if (err.code === 404) {
                                throw new Error(data)
                            } else {
                                throw err;
                            }
                        } else {
                            const etag = res.headers.etag;
                            console.log(`上传成功，获取etag信息：${etag}`)
                            etags.push(etag)
                            chunkNum++;
                            up();
                        }
                    })
                })
            } else {
                // 合并上传的分块
                const params = {
                    Bucket: bucketName,
                    Key: key,
                    PartNumber: count,
                    type: contentType,
                    UploadId: uploadId,
                    body: util.generateCompleteXML(etags)
                }
                console.log('合并参数：', params)
                client.object.upload_complete(params, function(err, data, res) {
                    if (err) throw err;
                    console.log('上传成功，合并文件：', err, data )
                })
            }
        }
    });
}

function queryChunks () {
    const params = {
        Key: key,
        Bucket: bucketName,
        UploadId: uploadId
    }
    client.object.listParts(params, function (err, data, res) {
        console.log(err)
        console.log('data:', data)
    })
}

function resume () {
    client.object.resume_upload({
        Key: key, Bucket: bucketName,
        filePath,
        resumeUploadId: '2b8f5775acb14f57a1340c40b1c2165b',
        resumeNo: 30
    }, function (done, total) {
        console.log('done, total: ', done, total)
    })
    
}

// 复制object
function copyObject () {
    client.object.copyObject({
        Key: '5MB_file',
        sourceBucket: 'zwy-test-shanghai',
        sourceKey: '5MB_file',
        headers: {
            'x-kss-forbid-overwrite': false
        }
    }, cb)
}


// 解冻object
function restore () {
    client.object.restore({
        Bucket: bucketName,
        Key: 'through2.js',
    }, cb)
}


function rename () {
    client.object.rename({
        // Bucket: bucketName,
        Key: '1.151G1',
        NewKey: '111/\uffff\ufffe'
    }, cb)
}

function modifyStorageClass () {
    client.object.modifyStorageClass({
        Bucket: bucketName,
        Key: 'through2.js',
        storageClass: 'STANDARD_IA'
    }, cb)
}

function getACLType () {
    client.object.getACLType({
        Bucket: bucketName,
        Key: 'through2.js',
    }, cb)
}


function generateUrl () {
    client.object.generatePresignedUrl({
        Bucket: bucketName,
        Key: '2024刑诉法名师精讲课11 回避（对对对）.mp4 .csv',
        Expiration: 5 * 60,
        Sign: true
    }, function (res, data) {
        console.log('url: ', data)
    })
}

// 生成上传外链传不计算签名参数
function generateUploadUrlByHttpParameters () {
    client.config({
        isDebug: true
    })
    client.object.generatePresignedUrl({
        Bucket: bucketName,
        Key: key,
        Expiration: 5 * 60,
        Method: 'GET',
        Endpoint: 'ks3-cn-shanghai.ksyuncs.com',
        Sign: false,
        QueryParams: {
            'max-keys': 1000,
            prefix: 'a',
        },
    }, function (err, data) {
        console.log(data)
    })
}


// 生成上传外链传计算签名参数
function generateUploadUrlByResource () {
    client.config({
        isDebug: true
    })
    client.object.generatePresignedUrl({
        Bucket: bucketName,
        Key: key,
        Expiration: 5 * 60,
        Method: 'POST',
        Endpoint: 'ks3-cn-shanghai.ksyuncs.com',
        QueryParams: {
            uploadId: '0f431272ef7743018d1894f24c4bd7c5',
            partNumber: 1
        },
        // Sign: false,
    }, function (err, data) {
        console.log('url----', data)
    })
}

function getAcl () {
    client.object.getAcl({
        Bucket: bucketName,
        Key: key
    }, cb)
}


// key = '#####.png'
function head () {
    client.object.head({
        Key: 'a.mp4',
    }, cb)
}


function getObjectTagging () {
    client.object.getObjectTagging({
        Key: key
    }, cb)
}

function deleteObjectTagging () {
    client.object.deleteObjectTagging({
        Key: key
    }, cb)
}

function putObjectTagging () {
    const tags = [{
        key: 'string1',
        value: 'string1'
    }, {
        key: 'string2',
        value: 'string2'
    }, {
        key: 'string3',
        value: 'string'
    }, {
        key: 'string4',
        value: 'string'
    }, {
        key: 'string5',
        value: 'string'
    }, {
        key: 'string6',
        value: 'string'
    }, {
        key: 'string7',
        value: 'string'
    }, {
        key: 'string8',
        value: 'string'
    }, {
        key: 'string9',
        value: 'string'
    }]
    client.object.putObjectTagging({
        Key: key,
        Taggings: tags
    }, cb)
}


function putStream () {
    const fout = fs.createReadStream(filePath)
    const stat = fs.statSync(filePath)
    client.object.put({
        Key: 'a',
        Body: fout,
        headers: {
            'Content-Length': stat.size,
            // 'x-kss-server-side-encryption': 'AES256'
        }
    }, cb)
}

function putString () {
    client.object.put({
        Key: 'aaa/a../../\b/..\b/.\b/\b\\\\',
        Body: "<!DOCTYPE html><html lang='en'><head><meta charset='UTF-8' /><title>测试</title></head><body>testing page for gulp-ks3</body></html>",
        headers: {
            'Content-Type':'text/html'
        },
    }, cb)
}

function putBuffer () {
    client.object.put({
        Key: 'put-buffer',
        Body: Buffer.from('this is a 都是独生 buffer content')
    }, cb)
}

function putDirectory () {
    client.object.put({
        Key: 'a/',
    }, cb)
}

// set callback
function putCallBack () {
    const fout = fs.createReadStream(filePath)
    const param = {
        Key: 'callback.mp4',
        Body: fout,
        headers: {
            "x-kss-callbackurl":"",
            'x-kss-callbackbody': 'etag=${etag}&objectSize=${objectSize}&key=${key}',
            'x-kss-callbackauth': '1',
            'kss-location': 'test'
        }
    }
    client.object.put(param, cb)
}

/**
 * 外链上传测试
 */
async function uploadByUrl () {
    const body = await fs.createReadStream(filePath)
    const stat = await fs.statSync(filePath)
    client.object.generatePresignedUrl({
        Bucket: bucketName,
        Key: 'stream.mp4',
        expiration: 5 * 60,
        method: 'put'
    }, function (res, data) {
        console.log('url: ', data)
        request({
            method: 'put',
            uri: data,
            headers: {
                'Content-Length': stat.size
            }
        }, body, '', cb);
    })
}


/**
 * upload a object with content type && get the object
 */
function getContentType () {
    var buf = Buffer.from('<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8" /><title>测试</title></head><body>testing page for gulp-ks3</body></html>');
    var key = 'test_buffer';
    client.object.put({
            Key: key,
            Body: buf,
            headers: {
                'Content-Type':'text/html'
            }
        },
        function(err, data, res) {
            if(err) throw err
            client.object.get({
                Bucket:bucketName,
                Key:key
            },function(err,data,res){
                const contentType = res.caseless.get('content-type')
                console.log(contentType)
            });
        }
    );
}

function putFile (){
    client.object.put({
        Key: '1.docx',
        FilePath: '/Users/zhangweiyue/Desktop/projects/new-sdk/test-go/test.html',
    }, cb)
}

function encryptionSSES3 () {
    client.config({
        isDebug: true
    })
    client.object.put({
        FilePath: filePath,
        Key: 'sse-s3-a.mp4',
        ServerSideEncryption: 'AES256'
    }, cb)
}

function encryptionSSEC () {
    client.object.put({
        FilePath: filePath,
        Key: 'ss3-c.mp4',
        SSECustomerAlgorithm: 'AES256',
        SSECustomerKey: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', 
        SSECustomerKeyMD5:'cLyPS3KoaSFGi/joRB3OUQ=='
    }, cb)
}

function fetch () {
    client.object.fetch({
        Key: '10mb-1.zip',
        SourceUrl: 'http://xxx.com/10mb.zip',
        ACL: 'public-read',
        Taggings: [{
            key: 'tag1',
            value: 'value1'
        }, {
            key: 'tag2',
            value: 'value2'
        }]
    }, cb)
}

function getObjectToLocal () {
    client.object.get({
        Key: '1.docx',
        Output: './1.docx'
    }, function (err, data, res) {
        console.error(err)
        if (!err) console.log(res.caseless.get('content-length'))
    })
}

function getObjectToCache () {
    client.object.get({
        Key: 'stream.mp4'
    }, function (err, data, res) {
        console.log(data)
        // fs.writeFileSync('./tmp1-stream.mp4', data)
    })
}

function getObjectStream (){
    client.object.get({
        Key: 'stream.mp4',
        Output: fs.createWriteStream('./tmp2-stream.mp4')
    }, function (err, data, res) {
        if (err) {
            console.log('下载失败')
        } else {
            console.log('下载成功')
        }
    })
}


// 下载到本地内存
function getDataToMem () {
    client.object.get({
        Key: '离骚.txt'
    }, function (err, data, res) {
        console.log(data)
    })
}

function getObjectByRange () {
    client.object.get({
        Key: '1.docx',
        headers: {
            Range: 'bytes=0-9'
        }
    }, function (err, data, res) {
        if (err) {
            console.log('下载失败')
        } else {
            fs.writeFileSync('./离骚.docx', data)
            console.log('下载成功')
        }
    })
}


function recoverObject(){
    client.object.recoverObject({
        Key: "11/mmm/bundled/about.png",
        RetentionId: "29819397128893_MjcxMjAwMDAwMDAwMDAxNzQyNUEwMDAwMDAwMDAwM0YyMTZB",
        Overwrite: true
    }, cb)
}

function clearObject(){
    client.object.clearObject({
        Key: "11/mmm/bundled/about.png",
        RetentionId: "29819397128893_MjcxMjAwMDAwMDAwMDAxNzQyNUEwMDAwMDAwMDAwM0YyMTZB"
    }, cb)
}

function getObjectReturnStream() {
    let request = client.object.get({
        Key: "a",
        ReturnStream: true
    })
    request.on('data', function (chunk) {
    })
    request.on('response', function (response) {
        console.log('response:', response.statusCode)
    })
}

// queryChunks();
// upload()
copyObject();
// restore();
// rename();
// modifyStorageClass();
// getACLType();
// generateUrl();
// getAcl()
// head()
// putObjectTagging() 
// getObjectTagging()
// deleteObjectTagging()
//文件流上传
// putStream()
//字符串上传
// putString()
//Buffer上传
// putBuffer()
//创建目录
// putDirectory()
// putCallBack()
// 外链上传测试
// uploadByUrl()
// 上传文件
// putFile()
// getContentType()
// encryptionSSES3()
// encryptionSSEC()
// fetch()
// 下载到本地
// getObjectToLocal()
// getObjectToCache()
// 流式下载
// getObjectStream()
//指定Range获取文件内容
// getObjectByRange()
//生成外链
// generateUploadUrlByHttpParameters()
// generateUploadUrlByResource()

// getObjectReturnStream()

// recoverObject()
// clearObject()