UNPKG

ks3

Version:

本代码库为`金山云存储KS3`服务.主要提供`KS3 nodejs SDK`和`命令行工具`.

1,049 lines (960 loc) 27.2 kB
var fs = require('fs') var mime = require('mime') var crypto = require('crypto') var xml2js = require('xml2js') const { URL } = require('whatwg-url') var xmlParser = new xml2js.Parser({ explicitArray: false, ignoreAttrs: true }) const constants = require('./const') const dns = require('dns') function isArray(obj) { return Object.prototype.toString.call(obj) === '[object Array]' } function isObject(obj) { return Object.prototype.toString.call(obj) === '[object Object]' } function getType(obj) { return Object.prototype.toString .call(obj) .replace(/(.*? |])/g, '') .toLowerCase() } function firstUpperCase(key, exclude = []) { if (exclude.indexOf(key) > -1) return obj return key.replace(/^./, (_) => _.toUpperCase()) } /** * 将xml解析成json对象 */ var xml2json = { parser: function (bodyStr) { var json = {} xmlParser.parseString(bodyStr, function (err, res) { json = res }) return json }, } /** * 将obj的key首字母转成大写 * @param {*} obj * @param {*} options {exclude: []} 不做处理的key */ function capitalize(obj, options = {}) { if (obj === null || typeof obj !== 'object') return obj let tmp = null if (isArray(obj)) { tmp = [] for (let i = 0, len = obj.length; i < len; i++) { tmp.push(capitalize(obj[i])) } } else { tmp = {} Object.keys(obj).forEach((key) => { const k = firstUpperCase(key, options.exclude) tmp[k] = capitalize(obj[key], options.exclude) }) } return tmp } /** * 将obj的key首字母转成小写 */ function lowerFirst(obj) { if (obj === null || typeof obj !== 'object') return obj let tmp = [] if (isArray(obj)) { tmp = [] for (let i = 0, len = obj.length; i < len; i++) { tmp.push(lowerFirst(obj[i])) } } else { tmp = {} Object.keys(obj).forEach((key) => { const nk = key.replace(/^./, (_) => _.toLowerCase()) tmp[nk] = lowerFirst(obj[key]) }) } return tmp } //提取header时忽略header大小写 function ignoreLowerCase(options, key) { let value = '' for (let item in options) { if (item.toLowerCase() === key) { value = options[item] } } return value } /** * 获取字符串的字符长度 * @param {*} str */ function getStrBytesCount(str) { let counts = 0 for (let i = 0, len = str.length; i < len; i++) { const char = str.charAt(i) // /[^\u0000-\u00FF]/ (匹配非单字节字符 ) if (/[^\u0000-\u00FF]/.test(char)) { counts += 2 } else { counts += 1 } } return counts } /** * 校验bucketName的可用性 */ function verifyBucket(name) { var flag = true var len = name.length var re = /^[a-z0-9]+([\.-]*([a-z0-9])*)*[a-z0-9]+$/ if (len >= 3 && len <= 63) { flag = re.test(name) } else { flag = false } return flag } /** * 验证ACL权限 */ function verifyAcl(acl) { var flag = null var acls = [ 'private', 'public-read', 'public-read-write', 'authenticated-read', 'bucket-owner-read', 'bucket-owner-full-control', ] var reg = new RegExp('^' + acl + '$', 'i') acls.forEach(function (name) { if (reg.test(name)) { flag = name.toLowerCase() return flag } }) return flag } /** * 验证存储类型 */ function verifyBucketType(type) { var flag = null var types = ['NORMAL', 'ARCHIVE', 'IA'] var reg = new RegExp('^' + type + '$', 'i') types.forEach(function (name) { if (reg.test(name)) { flag = name.toLowerCase() return flag } }) return flag } /** * 验证对象的存储类型 * @param {*} type */ function verifyStorageClass(type) { var flag = null var types = ['STANDARD', 'STANDARD_IA', 'ARCHIVE'] var reg = new RegExp('^' + type + '$', 'i') types.forEach(function (name) { if (reg.test(name)) { flag = name.toLowerCase() return flag } }) return flag } /** * key 进行encodeURIComponent编码,'/'不能被编码 * */ function encodeKey(key) { var newKey = encodeURIComponent(key) newKey = newKey.replace(/%2F/g, '/') if (newKey.startsWith('/')) newKey = newKey.replace('/', '%2F') // '//'needs convert to '/%2F' if (newKey.includes('//')) newKey = newKey.replace(/\/\//g, '/%2F') if (newKey.includes("'")) newKey = newKey.replace(/\'/g, '%27') return newKey } /** * 处理外链的编码 * @param {*} result * @returns * @description * - 有些字符是不会被转码的 所以需要手动转码, 例如:"%2C!%3D%3F%2B()'~%C2%B0" * */ function encodePresignedUrlKey(urlKey) { let result = encodeKey(urlKey) const specialKeyMap = { '(': '%28', ')': '%29', '!': '%21', // '~': '%7E', 该字符不可转 "'": '%27', } Object.keys(specialKeyMap).forEach((key) => { if (result.indexOf(key) > -1) { let tempKey = key // () 是正则的组 需要特别处理 if (key == '(' || key == ')') key = `\\${key}` const regExp = new RegExp(key, 'gim') result = result.replace(regExp, specialKeyMap[tempKey]) } }) return result } //拷贝对象 function deepClone(obj) { return JSON.parse(JSON.stringify(obj)) } /** * 遍历文件 */ function walkFile(p, opt = {}, fileList) { fileList = fileList || [] let ignore = opt.ignore || '' const escapedTypes = ignore.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') // 转义特殊字符 const regexStr = new RegExp(`\\.(${escapedTypes})$`, 'i') // 生成正则表达式字符串 var _config = { isDeep: opt.isDeep || false, ignore: regexStr, } var dirList = fs.readdirSync(p) dirList.forEach(function (item) { if (!_config.ignore.test(item)) { if (_config.isDeep) { if (fs.statSync(p + '/' + item).isDirectory()) { fileList.push(p + '/' + item) walkFile(p + '/' + item, opt, fileList) } else { fileList.push(p + '/' + item) } } else { fileList.push(p + '/' + item) } } }) return fileList } /** * 遍历文件夹 */ function walkDir(p, opt) { var fileList = walkFile(p, opt) var dirList = [] fileList.forEach(function (item) { if (fs.statSync(item).isDirectory()) { dirList.push(item) } }) return dirList } /** * 获取分块数据 * @param {*} filePath 文件路径 * @param {*} chunkSize 分块大小 * @param {*} start 开始位置 * @param {*} cb 回调方法 */ function getChunk(filePath, chunkSize, start, cb) { fs.open(filePath, 'r', function (err, fd) { fs.fstat(fd, function (err, stat) { var bufferSize = stat.size var index = start / chunkSize if (start + chunkSize > bufferSize) { chunkSize = bufferSize - start } var buffer = Buffer.allocUnsafe(chunkSize) fs.read( fd, buffer, 0, chunkSize, start, function (err, byteRead, buf) { cb(buf) } ) }) }) } /** * 获取文件 按照等比例分块后的块数 * * @param {*} filePath 文件路径 * @param {*} chunkSize 分块大小 */ function getTotalChunk(filePath, chunkSize) { var stat = fs.statSync(filePath) var fileSize = stat.size if (!fileSize) return fileSize // 总块数 var count = parseInt(fileSize / chunkSize) + (fileSize % chunkSize == 0 ? 0 : 1) return count } /** * 分块完成时 需要通过提供的xml文件进行分块组装,在xml文件中,块号必须使用升序排列,必须提供每个块的ETag值 * @param {*} etags etags数组 * @returns */ function generateCompleteXML(etags) { var str = '' var index = etags.length if (index > 0) { str = '<CompleteMultipartUpload>' for (var i = 1; i <= index; i++) { str += '<Part><PartNumber>' + i + '</PartNumber><ETag>' + etags[i - 1] + '</ETag></Part>' } str += '</CompleteMultipartUpload>' } return str } /** * 通过key获取对应的content-type值 * @param {*} key * @returns */ function getContentTypeByKey(key) { return mime.lookup(key) || '' } /** * 获取分块上传的etags * @param {*} xml xml json数据 * @returns */ function getETagsFromXml(xml) { let json = xml2json.parser(xml) let tags = [] let nodes = json['ns2:ListPartsResult']['ns2:Part'] if (nodes) { if (!Array.isArray(nodes)) { let node = nodes nodes = [node] } nodes.forEach((node) => { let tag = {} if (node.PartNumber) { tag.PartNumber = node.PartNumber } if (node.ETag) { tag.ETag = node.ETag } if (JSON.stringify(tag) !== '{}') tags.push(tag) }) } return tags } /** * 获取过期时间 * @param {*} expries 过期时长 单位s */ function getExpiresTime(expries) { const currentTimestamp = new Date().getTime() const date = new Date(currentTimestamp + expries * 1000) return date.getTime().toString().substr(0, 10) } function sortObjectByKeys(obj) { // 将对象的键(属性名)转换为数组 const keys = Object.keys(obj) // 对键数组进行排序 keys.sort((a, b) => a.localeCompare(b)) // 创建一个新的对象,按照排序后的键顺序添加属性 const sortedObj = {} keys.forEach((key) => { sortedObj[key] = obj[key] }) return sortedObj } /** * 从queryParams提取出SubResource和query * @param {*} obj */ function extractSubResourceAndQuery(queryParams) { let obj = sortObjectByKeys(queryParams) let subResource = {} let query = {} for (let key in obj) { if (constants['RESOURCE_LIST'].indexOf(key) > -1) { subResource[key] = obj[key] } else { query[key] = obj[key] } } return { subResource: paramsEncode(subResource), query: paramsEncode(query), } } /** * 将obj转成k=v&k=v格式 * @param {*} obj */ function paramsEncode(obj) { var str = '' for (var key in obj) { if (str !== '') { str += '&' } str += obj[key] ? key + '=' + encodeURIComponent(obj[key]) : key } return str } /** * 获取文件的md5 * @param {*} file * @param {*} cb * @returns * @description * 先使用md5获取文件指纹,再通过base64将结果转成指定格式,上传object时如遇大文件比较耗时, * 故上传操作不建议使用 */ function getFileMd5(file, cb) { var md5 = crypto.createHash('md5') const stream = fs.createReadStream(file) stream.on('data', (chunk) => { md5.update(chunk) }) stream.on('error', function (err) { cb(err) }) stream.on('end', () => { const res = md5.digest('base64') cb(res) }) } /** * 计算指定字符串的Md5值 * @param {*} str * @returns */ function getStringMd5(str) { if (!str) { throw new error('计算字符串md5值失败, str不能为空') } return crypto.createHash('md5').update(str).digest('base64') } function toArray(obj) { if (!obj) return [] if (isArray(obj)) return obj return [obj] } /** * 转化成CORS所要求的xml参数 * @param {*} rules */ function convertToCORSXML(rules) { let xml = '<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' const parseMethod = (val) => { xml += `<AllowedMethod>${val}</AllowedMethod>` } const parseOrigin = (val) => { xml += `<AllowedOrigin>${val}</AllowedOrigin>` } const parseHeader = (val) => { xml += `<AllowedHeader>${val}</AllowedHeader>` } const parseExposeHeader = (val) => { xml += `<ExposeHeader>${val}</ExposeHeader>` } for (let i = 0, len = rules.length; i < len; i++) { const rule = rules[i] xml += '<CORSRule>' toArray(rule.AllowedMethod).forEach(parseMethod) toArray(rule.AllowedOrigin).forEach(parseOrigin) toArray(rule.AllowedHeader).forEach(parseHeader) toArray(rule.ExposeHeader).forEach(parseExposeHeader) if (rule.MaxAgeSeconds) { xml += `<MaxAgeSeconds>${rule.MaxAgeSeconds}</MaxAgeSeconds>` } xml += `</CORSRule>` } xml += '</CORSConfiguration>' return xml } /** * 转成Replication所要求的xml参数 * @param {*} rule * {prefix: '', deleteMarkerStatus: '', targetBucket: '', historicalObjectReplication: ''} */ function convertToReplicationXML(rule) { rule = capitalize(rule) let xml = '<Replication xmlns="http://s3.amazonaws.com/doc/2006-03-01/">' const parsePrefix = (val) => { xml += `<prefix>${val}</prefix>` } if (rule.Prefix) { toArray(rule.Prefix).forEach(parsePrefix) } xml += `<DeleteMarkerStatus>${rule.DeleteMarkerStatus}</DeleteMarkerStatus>` xml += `<targetBucket>${rule.TargetBucket}</targetBucket>` if (rule.HistoricalObjectReplication) { xml += `<HistoricalObjectReplication>${rule.HistoricalObjectReplication}</HistoricalObjectReplication>` } xml += '</Replication>' return xml } /** * 将json 转成xml * @param {*} obj * @returns */ function objToXml(obj) { let xml = '' obj = capitalize(obj) if (getType(obj) === 'object') { Object.keys(obj).forEach((key) => { const value = obj[key] const type = getType(value) if (type !== 'undefined' && type !== 'null') { if (type === 'string' || type === 'number') { xml += `<${key}>${value}</${key}>` } else if (type === 'array') { xml += value .map((item) => { return `<${key}>${objToXml(item)}</${key}>` }) .join('') } else if (type === 'object') { xml += `<${key}>${objToXml(value)}</${key}>` } else { xml += `<${key}>${value.toString()}</${key}>` } } }) } else { xml += obj.toString() } return xml } function checkDateAndDays(obj) { obj = capitalize(obj) const { Days, Date } = obj if (Days && !/^[1-9][0-9]*$/.test(Days)) { throw new Error('Days must be a positive integer') } else if ( Date && !/\d{4}-\d{2}-\d{2}T00:00:00\+08:00/.test(Date) && !/\d{4}-\d{2}-\d{2}T00:00:00.000\+08:00/.test(Date) ) { throw new Error( `[${Date}] ` + 'Date must be Date and tranform to iso8601 format, and should be beijing Date' ) } } /*** * 检查生命周期的规则是否合法 */ function checkLifecycleRule(rule) { rule = capitalize(rule) const { ID, Filter, Expiration, Status } = rule let { Transition } = rule const { Prefix, Tag } = Filter.And || Filter if (ID && getStrBytesCount(ID) > 255) { throw new Error('the ID max length is 255 bytes') } if (Prefix && isArray(Prefix) && Prefix.length > 1) { throw new Error('Prefix can only has one value in single rule') } if (!Status) { throw new Error('require the Status') } else { if (!['Enabled', 'Disabled'].includes(Status)) throw new Error('Status value only in Enable/Disabled') } if (Expiration) { // Date和Days只能二选一,并且不能不选,必须填写其中一个。 if (Expiration.Date && Expiration.Days) throw new Error( 'Expiration only has one element which in Date/Days' ) if (!Expiration.Date && !Expiration.Days) throw new Error('Expiration must has one element') checkDateAndDays(Expiration) } if (Transition) { // transition maybe has more than one if (!isArray(Transition)) Transition = [Transition] Transition.forEach((item) => { if (item.Date && item.Days) throw new Error( 'Transition only has one element which in Date/Days' ) if (!item.Date && !item.Days) throw new Error('Transition must has one element') checkDateAndDays(item) }) } } const COMMON_RULES = [ { pattern: /^\S.*/, msg: 'can not start with space char', }, { pattern: /.*\S$/, msg: 'can not end with space char', }, { pattern: /^[a-zA-Z0-9 +-=._:/@]+$/, msg: 'tag can contain letters, numbers, spaces, and the following symbols: (+), (-), (=), (.), (_), (:), (/), and (@)', }, ] const TAG_RULES = { key: [ ...COMMON_RULES, { pattern: /^.{1,128}$/, msg: 'tag key can be a maximum of 128 bytes in length', }, ], value: [ ...COMMON_RULES, { pattern: /^.{1,256}$/, msg: 'tag key value can be a maximum of 256 bytes in length', }, ], } function checkValid(tag) { const { Key, Value } = tag TAG_RULES.key.forEach((item) => { if (!Key) throw new Error('key is can not null') if (!item.pattern.test(Key)) throw new Error(item.msg) }) TAG_RULES.value.forEach((item) => { if (!Value) throw new Error('value is can not null') if (!item.pattern.test(Value)) throw new Error(item.msg) }) } /** * 检查tag是否合法 * @param {*} tags */ function checkTag(tags) { tags = capitalize(tags) if (tags.length > 10) throw new Error('Taggings length max value is 10') let keys = tags.map((x) => x.Key) keys = [...new Set(keys)] // Key不可重复 if (keys.length != tags.length) throw new Error('key cannot repeat') tags.forEach((tag) => { checkValid(tag) }) } //切割header里的tag转码 function splitTag(tag) { let headerTags = tag.split('&') // 分割成键值对数组 let encodeTags = '' headerTags.forEach((tag, index) => { let firstEqualIndex = tag.indexOf('=') let key = tag.substring(0, firstEqualIndex) let value = tag.substring(firstEqualIndex + 1) encodeTags += `${key}=${encodeURIComponent(value)}` // 构建对象 encodeTags += index === headerTags.length - 1 ? '' : '&' // 最后一个不加& }) return encodeTags } /** * 获取object的元数据 * @param {*} reponse - 返回的reponse */ function getObjectMetadata(reponse) { const res = {} if (!reponse) return res const headers = reponse.headers // 不展示的信息 const exclude = [ 'connection', 'x-application-context', 'x-kss-request-id', 'server', ] for (let key in headers) { if (exclude.indexOf(key) < 0) { res[key] = headers[key] } } return res } function isStream(stream) { return ( stream !== null && typeof stream === 'object' && typeof stream.pipe === 'function' ) } // 将当前时间转成 20220414T093758Z function getStsDate() { let d = new Date().toISOString() // "2022-04-14T09:48:03.586Z" d = d.replace(/-/g, '').replace(/:/g, '') d = d.split('.')[0] return d + 'Z' } /** * 将obj中的空置去掉 * @param {*} obj */ function clearEmpty(obj) { let res = {} for (let k in obj) { const v = obj[k] if (v || v == 0) { res[k] = v } } return res } /** * 验证是否符合服务端加密方式。目前合法值:AES256 * @param {*} encrypt */ function verifyServerEncryption(encrypt) { return encrypt == 'AES256' } /** * * @param {*} reposeError 返回的error信息 * @param {*} headers header信息 * @param {*} opt 附加信息 * @returns * { * requestId: '', // 请求id * statusCode: '', // http返回的状态码 404 * code: '', // 错误码 eg: 'NoSuchBucket' * headers: {}, // http返回的header信息 * responseError: {} // http返回的结果 * } */ function error(responseError, headers, opt) { var rerr = {} if (responseError && opt && opt.error) { rerr.resolveError = responseError } else { rerr.error = responseError || {} } rerr.headers = headers || {} for (const key in opt) { rerr[key] = opt[key] } //兼容老版本,response中没有statusCode赋值为-1 if (opt && !opt.statusCode) { rerr.code = -1 } return rerr } /** * 通过文件大小获取分块信息 * @param {*} fileSize 文件大小 * @param {*} chunkSize 5MB≤PartSize<500MB之间 * @param {*} maxNum 最大分块数量 (最大10000个文件块) * NOTICE: 如果超过10000个文件块 则启动调节分块大小 */ function getSliceChunks(fileSize, chunkSize, maxNum = 10000) { if (chunkSize > 500 * 1024 * 1024 || chunkSize < 5 * 1024 * 1024) { throw new Error('请检查分块大小是否符合要求~') } const chunkList = [] if (fileSize < chunkSize) { return [ { PartNumber: 1, Range: `bytes=0-${fileSize}`, }, ] } let chunkNum = fileSize / chunkSize if (chunkNum > maxNum) { console.warn('分块数大于10000,将调整分块大小~') // 根据文件大小及最大分块数量动态调节分块大小 const SIZE = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] const unit = 1024 * 1024 // MB const index = SIZE.findIndex((item) => item * unit > chunkSize) if (index < SIZE.length) { const temp = SIZE.slice(index) let autoSize = unit let i = 0 for (; i < temp.length; i++) { autoSize = temp[i] * unit if (fileSize / autoSize <= maxNum) break } chunkSize = autoSize chunkNum = fileSize / chunkSize } } for (var i = 0; i < chunkNum; i++) { const start = i * chunkSize var end = start + chunkSize - 1 if (end > fileSize) end = fileSize chunkList.push({ PartNumber: i + 1, Range: `bytes=${start}-${end}`, ChunkStart: start, ChunkEnd: end, }) } return chunkList } /** * 动态计算进度 * @param {*} total * @param {*} onProgress * @param {*} interval */ function throttleOnProgress(total, onProgress, interval = 1000) { var size1 = 0 var size2 = 0 var time1 = Date.now() var time2 var timer = null // 计算进度 function update() { timer = 0 if (onProgress && typeof onProgress == 'function') { time2 = Date.now() // 变化量 = dsize / dtime * 100 /100 var diff = Math.round(((size2 - size1) / ((time2 - time1) / 1000)) * 100) / 100 var speed = Math.max(0, diff) || 0 var percent if (size2 == 0 && total == 0) percent = 1 else percent = Math.floor((size2 / total) * 100) / 100 || 0 time1 = time2 size1 = size2 onProgress({ loaded: size2, total, speed, percent, }) } } var progress = function (currentData, isRefresh) { if (currentData) { size2 = currentData.loaded total = currentData.total } if (isRefresh) { clearTimeout(timer) update() } else { if (timer) return timer = setTimeout(update, interval) } } return progress } function apiWrapper(belongModule, apiFn, args) { var core = this var apiName = `${belongModule}.${apiFn.name}` var isSync = belongModule != 'util' || apiName != 'object.generatePresignedUrl' || apiFn.indexOf('Stream') < 0 // 如果传入2个参数 则使用回调形式 否则为promise形式 if (Promise && isSync && args.length < 1) { return new Promise((resolve, reject) => { let tmp = function (err, data, res, body) { if (err) { reject(err) } else { resolve(res) } } apiFn.call(core, args, tmp) }) } else { return apiFn.call(core, ...args) } } function toUpperCaseParmas(params) { const newParams = {} for (const key in params) { if (params.hasOwnProperty(key) && key !== 'onProgress') { const capitalizedKey = key.charAt(0).toUpperCase() + key.slice(1) newParams[capitalizedKey] = params[key] } else { newParams[key] = params[key] } } return newParams } function convertToASCII(hostname) { try { const url = new URL(`http://${hostname}`) return url.hostname } catch (e) { console.error(`Error converting hostname to ASCII: ${e.message}`) return hostname } } function isValidIPAddress(ip) { // 正则表达式用来匹配有效 IP 地址 const ipRegex = /^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/ return ipRegex.test(ip) } module.exports = { isArray, isObject, verifyBucket: verifyBucket, verifyAcl: verifyAcl, verifyBucketType: verifyBucketType, verifyStorageClass: verifyStorageClass, encodeKey: encodeKey, walkFile: walkFile, walkDir: walkDir, xml2json: xml2json, getDate: function () { return new Date().toUTCString() }, getStsDate, getExpiresTime, getTotalChunk, getChunk: getChunk, generateCompleteXML, getContentTypeByKey, getETagsFromXml, extractSubResourceAndQuery, paramsEncode, getFileMd5, getStringMd5, convertToCORSXML, convertToReplicationXML, objToXml, checkLifecycleRule, checkTag, splitTag, getObjectMetadata, isStream, clearEmpty, encodePresignedUrlKey, verifyServerEncryption, getSliceChunks: getSliceChunks, throttleOnProgress, apiWrapper, ignoreLowerCase, capitalize, lowerFirst, error, deepClone, toUpperCaseParmas, convertToASCII, isValidIPAddress, }