UNPKG

@shencom/oss-upload

Version:
921 lines (744 loc) 25.3 kB
import { basename, join, normalize } from 'path'; import log from '@shencom/npmlog'; import type { ListObjectResult, PutObjectOptions } from 'ali-oss'; import AliOSS from 'ali-oss'; import chalk from 'chalk'; import pLimit from 'p-limit'; import { CheckEnvPath, _argv, confirm, isFile, spinner, throwErr, versionExec, versionSort, versionValid, } from './utils'; import type { FileData } from './helpers'; import { fileStreamHandle, filterOssName, getDirFilesPath, isOssPath, listIgnoreHandle, } from './helpers'; import { ParseAccessKeyEnv, TipLog } from './env'; interface FileMap extends FileData { type: 'delete' | 'upload'; } export interface OssOptions { /** 上传路径 如: plugins/test/scloud/xxx */ ossPath: string; /** bucket名 */ bucket?: string; /** 过滤上传删除文件,同 glob.ignore */ ignore?: string[]; /** 过滤上传oss文件,合并 ignore */ uploadIgnore?: string[]; /** 过滤删除oss文件,合并 ignore */ deleteIgnore?: string[]; /** 超时时间 */ timeout?: number; /** 开启debug,开启后不进行上传和删除操作 */ debug?: boolean; /** oss 上传配置 */ uploadOptions?: | ((filePath: string, defaultOptions: Partial<PutObjectOptions>) => Partial<PutObjectOptions>) | Partial<PutObjectOptions>; } export interface UploadOptionsBase extends Partial<Pick<OssOptions, 'ossPath'>> { /** 目录路径 */ dirPath: string; /** * 是否开启本地文件和oss文件进行对比 * * ```text * 对比规则: * 本地存在,oss不存在: 上传 * 本地存在,oss存在: 覆盖 * 本地不存在,oss存在: 删除 * ``` */ diff?: boolean; /** 开启debug,开启后不进行上传和删除操作 */ debug?: boolean; /** 是否关闭确认提示,默认为: false */ isCloseConfirm?: boolean; /** 处理获取本地文件列表钩子 */ localFileHook?: (p: string[]) => string[]; /** 处理oss路径列表钩子 */ ossPathHook?: (paths: string[]) => string[]; /** 处理获取oss文件列表钩子 */ ossFileHook?: (p: AliOSS.ObjectMeta[]) => AliOSS.ObjectMeta[]; /** 上传前钩子 */ beforeUploadHook?: (p: string[]) => string[]; /** 删除前钩子 */ beforeDeleteHook?: (p: string[]) => string[]; } interface CloseVersion { /** 是否开启清除版本号目录,默认: false */ isClearVersion?: false; /** 当前版本号 */ version?: string; } interface OpenVersion { /** 是否开启清除版本号目录,默认: false */ isClearVersion: true; /** 当前版本号 */ version: string; } export type UploadOptionsVersion = (OpenVersion | CloseVersion) & { /** 保留版本号目录个数,默认: 10 */ versionLimit?: number; }; export type OssPutOptions = Pick<UploadOptionsBase, 'isCloseConfirm'>; export type UploadOptions = UploadOptionsBase & UploadOptionsVersion; interface OssUploadItem { ossPath: string; filePath: string; } interface OssDownloadItem { ossPath: string; localPath: string; } const TIME = 1000 * 60 * 10; if (_argv.debug) { log.level = 'verbose'; } function debugLog(...args: [string, ...any[]]) { const [message, ...arg] = args; log.verbose('debug', message, ...arg); return confirm('任意键进行下一步'); } /** * 给本地文件添加添加MD5标识 * * @param {string[]} list * @return {FileData[]} */ const handleLocalFileMD5 = (list: string[]): FileData[] => { const objects = list.map((v) => fileStreamHandle(v)).filter(Boolean) as FileData[]; return objects; }; /** 处理掉重复的 // */ function handlePathSlash(path: string) { return path.replace(/\/\//g, '/'); } /** 处理 / or \ 的平台差异 */ function handlePathSep(path: string) { return path.replace(/\\/g, '/'); } /** 纠正oss路径 */ function handleOssBasePath(path: string) { if (path === '') return path; if (path[0] === '/') path = path.slice(1); if (path.slice(-1) !== '/') path += '/'; path = handlePathSep(path); path = handlePathSlash(path); return path; } /** * 本地文件路径将其转化为oss文件路径 * * @param {string[]} list * @param {string} ossPath * @param {string} dirPath * @return {string[]} */ function localPathToOssPath(list: string[], ossPath: string, dirPath: string): string[]; function localPathToOssPath(list: string, ossPath: string, dirPath: string): string; function localPathToOssPath(list: string[] | string, ossPath: string, dirPath: string) { const paths = Array.isArray(list) ? list : [list]; const objects = paths.map((v) => { let str = ossPath + normalize(v).replace(dirPath, ''); str = handlePathSep(str); str = handlePathSlash(str); return str; }); return Array.isArray(list) ? objects : objects[0]; } /** * 只保留oss的 name(oss存储路径) 字段 * * @param {AliOSS.ObjectMeta[]} objects * @return {string[]} */ function handleOssObjects(objects: AliOSS.ObjectMeta[]): FileData[] { const list = objects .map((v) => { const paths = v.name.split('/'); // 过滤 oss 文件夹 if (!paths[paths.length - 1]) return null; return { url: v.name, md5: v.etag && v.etag.toLocaleUpperCase().replace(/['"]/g, '') }; }) .filter(Boolean) as FileData[]; return list; } /** * 拆分文件 * * @param {Map<string, FileMap>} list * @returns */ function splitFile(list: Map<string, FileMap>) { const uploadFiles: string[] = []; const deleteFiles: string[] = []; list.forEach((item) => { if (item.type === 'upload') { uploadFiles.push(item.url); } else if (item.type === 'delete') { deleteFiles.push(item.url); } }); return { uploadFiles, deleteFiles }; } /** * 获取oss上传、删除文件路径 * * @static * @param {FileData[]} localList 本地路径列表 * @param {FileData[]} ossList oss路径列表 * @returns * @memberof OSS */ function diffFileData(localList: FileData[], ossList: FileData[], options: UploadOptions) { const { dirPath, ossPath = '' } = options; const map = new Map<string, FileMap>(); for (let i = 0; i < localList.length; i++) { const e = localList[i]; const path = e.url.replace(new RegExp(`${dirPath}\/?`), ''); map.set(path, { ...e, type: 'upload' }); } for (let i = 0; i < ossList.length; i++) { const e = ossList[i]; const path = e.url.replace(ossPath, ''); const f = map.get(path); // 目录一样, 内容一样 => 不传不删 // 目录一样, 内容不一样 => 传(覆盖) // 目录不一样, 内容一样 => 传、删 // 目录不一样, 内容不一样 => 删 if (f) { if (f.md5 === e.md5) map.delete(path); } else { map.set(path, { ...e, type: 'delete' }); } } return splitFile(map); } /** * 处理过滤规则 * * @param {string} dirPath * @param {string[]} ignores * @return {string[]} */ function handleIgnores(dirPath: string, ignores: string[]) { ignores = ignores.map((v) => { const e = join(v); const val = e.replace(dirPath, '**/'); return join(val); }); return ignores; } /** * 处理oss根目录文件路径 * * @param {(Pick<AliOSS.ListObjectResult, 'objects' | 'prefixes'>[])} root * @return {string[]} */ function handelOssRootFilesPath(root: Pick<AliOSS.ListObjectResult, 'objects' | 'prefixes'>[]) { const rootPaths = root .map((item) => { const objects = filterOssName(item?.objects); const prefixes = item?.prefixes; return objects.concat(prefixes); }) .flat(); return rootPaths.filter(Boolean); } class OSSBase { protected config = <OssOptions>{}; protected CLIENT?: AliOSS; protected uploadIgnore: string[] = ['**/.DS_Store']; protected deleteIgnore: string[] = []; constructor(options: OssOptions) { const { timeout, bucket, debug } = options; const { accessKeySecret, accessKeyId } = ParseAccessKeyEnv(); if (!(accessKeyId || accessKeySecret)) { spinner.fail(chalk.red(`未配置oss秘钥`)); TipLog(); process.exit(1); } if (debug) { log.level = 'verbose'; } options.ossPath = handleOssBasePath(options.ossPath); this.config = options; if (options.ossPath === undefined || options.ossPath === null) { spinner.fail('oss 上传路径不能为空'); process.exit(1); } this.CLIENT = new AliOSS({ region: 'oss-cn-shenzhen', accessKeyId, accessKeySecret, bucket: bucket || 'scplugins', timeout: timeout || TIME, }); this.setIgnore(); } private async setIgnore() { const { ignore = [], uploadIgnore = [], deleteIgnore = [] } = this.config; this.uploadIgnore.push(...ignore, ...uploadIgnore); this.deleteIgnore.push(...ignore, ...deleteIgnore); } protected async _list(prefix = this.config.ossPath, options?: AliOSS.ListV2ObjectsQuery) { try { if (!this.CLIENT) { spinner.fail(chalk.red('alioss 初始化失败')); process.exit(1); } const list: Pick<ListObjectResult, 'objects' | 'prefixes'>[] = []; let continuationToken; do { const res = (await this.CLIENT.listV2( { ...options, 'continuation-token': continuationToken, 'max-keys': '1000', prefix, }, { timeout: 1000 * 60 }, )) as ListObjectResult & { nextContinuationToken: string }; const { objects = [], prefixes = [] } = res; continuationToken = res.nextContinuationToken; list.push({ objects, prefixes }); } while (continuationToken); return list as ListObjectResult[]; } catch (error) { spinner.fail(chalk.red('获取 OSS 失败!')); throw error; } } /** * 获取文件对象 * * @protected * @param {string} [refix=this.config.ossPath] * @return {Promise<AliOSS.ObjectMeta[]>} * @memberof OSSBase */ protected async _listObjects(refix = this.config.ossPath) { const list = await this._list(refix); return list.flatMap((v) => v.objects); } /** * 获取目录 * * @protected * @param {string} [refix=this.config.ossPath] * @return {Promise<string[]>} * @memberof OSSBase */ protected async _listPrefixes(refix = this.config.ossPath) { const list = await this._list(refix, { delimiter: '/' }); return list.flatMap((v) => v.prefixes); } protected async _head(path: string) { if (!this.CLIENT) { spinner.fail(chalk.red('alioss 初始化失败')); process.exit(1); } try { spinner.start(chalk.cyan(`🚀 获取 ${basename(path)} MD5中...`)); const head = await this.CLIENT.head(path); spinner.stop(); const md5 = (head.res.headers as Record<string, string>)['content-md5']; return md5; } catch (error) { spinner.fail(chalk.red(`获取 ${basename(path)}MD5 失败!`)); return ''; } } protected _put(options: OssUploadItem) { if (!this.CLIENT) { spinner.fail(chalk.red('alioss 初始化失败')); process.exit(1); } if (!options) return Promise.resolve(); const { filePath, ossPath } = options; if (!isFile(filePath)) { spinner.fail(chalk.red(`${filePath} 文件路径不存在`)); process.exit(1); } if (!isOssPath(ossPath, this.config.ossPath)) { spinner.fail(chalk.red(`${ossPath} oss路径出错`)); process.exit(1); } spinner.start(chalk.cyan(`上传 ${basename(filePath)} 文件中...`)); const { uploadOptions } = this.config; let ossOptions: Partial<PutObjectOptions> = { headers: { disabledMD5: false }, }; if (typeof uploadOptions === 'function') { ossOptions = uploadOptions(filePath, ossOptions); } else if (Object.prototype.toString.call(uploadOptions) === '[object Object]') { ossOptions = Object.assign({}, ossOptions, uploadOptions); } log.verbose('debug', `上传配置: ${JSON.stringify(ossOptions)}`); return this.CLIENT.put(ossPath, filePath, ossOptions); } protected async _get(paths: OssDownloadItem, options: AliOSS.GetObjectOptions = {}) { if (!this.CLIENT) { spinner.fail(chalk.red('alioss 初始化失败')); return Promise.reject(new Error('alioss 初始化失败')); } if (!paths) { return Promise.reject(new Error('未配置下载路径 paths')); } const { ossPath, localPath } = paths; try { spinner.start(chalk.cyan(`🚀 下载 ${ossPath} 中...`)); const res = await this.CLIENT.get(ossPath, localPath, { timeout: TIME, ...options, }); spinner.stop(); return res; } catch (error) { spinner.fail(chalk.red(`下载 ${ossPath} 失败!`)); return Promise.reject(error); } } protected async _delete(paths: string[]) { if (!this.CLIENT) { spinner.fail(chalk.red('alioss 初始化失败')); process.exit(1); } try { spinner.start(chalk.green('🗑 正在删除多余文件中...')); await this.CLIENT.deleteMulti(paths, { quiet: true }); spinner.succeed(chalk.green(`删除${paths.length}个文件成功!`)); } catch (error) { spinner.fail(chalk.red('删除失败,请手动删除!')); } } } export class OSS extends OSSBase { constructor(ops: OssOptions) { super(ops); } /** * 获取oss文件列表 * * @param {string[]} ossPaths oss路径 * @param {string} returnType 返回类型 * @return {Promise<(AliOSS.ObjectMeta|string)[]>} * @memberof OSS */ public async list(ossPaths: string[] | string, returnType?: 'file'): Promise<AliOSS.ObjectMeta[]>; public async list(ossPaths: string[] | string, returnType?: 'dir'): Promise<string[]>; public async list( ossPaths: string[] | string, returnType?: 'all', options?: AliOSS.ListV2ObjectsQuery, ): Promise<Pick<AliOSS.ListObjectResult, 'objects' | 'prefixes'>[]>; public async list( ossPaths: string[] | string, returnType: 'file' | 'dir' | 'all' = 'all', options?: AliOSS.ListV2ObjectsQuery, ): Promise<any[]> { spinner.start('🚀 获取 OSS 文件中...'); if (!Array.isArray(ossPaths)) ossPaths = [ossPaths]; const filesPath: (ListObjectResult | string | AliOSS.ObjectMeta)[] = []; for await (const file of ossPaths) { let objects: (ListObjectResult | string | AliOSS.ObjectMeta)[] = []; if (returnType === 'file') { objects = await this._listObjects(file); } else if (returnType === 'dir') { objects = await this._listPrefixes(file); } else if (returnType === 'all') { objects = await this._list(file, options); } filesPath.push(...objects); } log.verbose('debug', `OSS路径: ${ossPaths}`); spinner.succeed(chalk.green(`获取 OSS ${filesPath.length}个文件!`)); return filesPath; } /** * 批量获取oss文件的md5 * * @param {string[]} ossFilePaths oss文件路径 * @returns {Promise<FileData[]>} * @memberof OSS */ public async head(ossFilePaths: string[]): Promise<FileData[]> { const heads = []; for await (const path of ossFilePaths) { const md5 = await this._head(path); heads.push({ url: path, md5 }); } spinner.succeed(chalk.green(`获取 OSS ${heads.length} 个文件!`)); return heads; } /** * 删除oss文件 * * @param {string[]} ossPaths oss文件路径 * @returns * @memberof OSS */ public async delete(ossPaths: string[]) { if (!ossPaths.length) return; ossPaths.forEach((item) => { spinner.info(chalk.cyan(`🗑 删除 ${basename(item)} 文件`)); }); await this._delete(ossPaths); } /** * 上传文件 * * @param {OssUploadItem[]} paths 上传路径对象 * @returns * @memberof OSS */ public async put(paths: OssUploadItem[], options?: OssPutOptions) { if (!paths.length) return; /** 上传错误文件列表 */ const failUploadFiles: string[] = []; /** 上传成功文件列表 */ const successUploadFiles: string[] = []; if (!options?.isCloseConfirm) { const flag = await confirm('确认上传吗?'); if (!flag) process.exit(); } for await (const item of paths) { try { const res = await this._put(item); if (!res) continue; spinner.succeed(chalk.green(`${basename(res.url)} 上传成功`)); successUploadFiles.push(basename(res.url)); } catch (error) { spinner.fail(chalk.red(`${basename(item.filePath)} 上传失败`)); failUploadFiles.push(basename(item.filePath)); throw error; } } if (successUploadFiles.length) { spinner.info(chalk.cyan(`🚀 上传成功${successUploadFiles.length}个文件`)); } if (failUploadFiles.length) { spinner.fail(chalk.red(`上传失败${failUploadFiles.length}个文件`)); } } /** * 下载文件 * * @param {OssDownloadItem[]} paths 下载路径对象 * @param {AliOSS.GetObjectOptions} options 下载参数 * @returns * @memberof OSS */ public download(paths: OssDownloadItem[], options?: AliOSS.GetObjectOptions) { return new Promise((resolve) => { // 限制最大并发数 const limit = pLimit(100); const requests = paths.map((p) => limit(() => this._get(p, options))); Promise.allSettled(requests).then((res) => { const success: AliOSS.GetObjectResult[] = []; const fail: PromiseRejectedResult['reason'][] = []; res.forEach((item) => { if (item.status === 'fulfilled') { success.push(item.value); } else { fail.push(item.reason); } }); if (success.length) { spinner.info(chalk.cyan(`🚀 下载成功 ${success.length} 个文件`)); } if (fail.length) { spinner.info(chalk.cyan(`❌ 下载失败 ${fail.length} 个文件`)); } resolve({ success, fail }); }); }); } /** * 代码上传 * * @param {UploadOptions} ops 上传配置 * @memberof OSS */ public async upload(ops: UploadOptions) { try { ops.dirPath = join(ops.dirPath || 'dist'); ops.debug = ops.debug ?? _argv.debug; ops.diff = ops.diff ?? _argv.diff ?? true; ops.ossPath = ops.ossPath ?? this.config.ossPath; const { isClearVersion, version } = ops; if (ops.debug) log.level = 'verbose'; this.setOssBasePath(ops.ossPath); if (!CheckEnvPath(ops.dirPath)) { throwErr('oss', 'dirPath 不存在'); } if (ops.ossPath === undefined || ops.ossPath === null) { throwErr('oss', 'ossPath 为空'); } if (isClearVersion && !version) throwErr('oss', 'isClearVersion 为 true 时,version 不能为空'); const localFileData = await this._handleLocalPath(ops); const ossFileData = await this._handleOssPath(ops.ossPath, ops); const { uploadFiles, deleteFiles } = diffFileData(localFileData, ossFileData, ops); if (!ops.isCloseConfirm) { const flag = await confirm('确认上传吗?'); if (!flag) process.exit(); } await this._handlePutFile(uploadFiles, ops); await this._handleDeleteFile(deleteFiles, ops); if (ops.isClearVersion) await this._handleClearVersionDir(ops); } catch (error) { let message; if (error instanceof Error) { message = error.message; } else { message = '错误'; } spinner.fail(message); throw error; } } /** * 清除版本号文件夹 * * @param {string[]} paths 文件路径 * @param {number} [limit] 保留版本号目录个数 * @return * @memberof OSS */ public async clearVersionDir(paths: string[], opt: UploadOptions) { const { versionLimit, version } = opt; const { ossPath } = this.config; const limit = versionLimit ?? 10; if (!ossPath) throwErr('oss', 'ossPath 为空'); let versions = paths .map((v) => { const p = versionExec(v) || []; return p[1]; }) .filter((v) => versionValid(v)) .filter((v) => v !== version); versions = versionSort(versions); if (_argv.debug) await debugLog('版本号: ', `[${chalk.green(versions.join('、'))}]`); if (versions.length <= limit) return; const delDirs = versions.slice(0, versions.length - limit); const delPath = delDirs.map((v) => handleOssBasePath(ossPath + v)); const flag = await confirm(`是否删除下面版本目录?\n${chalk.red(delPath.join('\n'))}\n`); if (!flag) process.exit(); const delFiles = await this.list(delPath, 'file'); if (!_argv.debug) await this.delete(delFiles.map((v) => v.name)); } private setOssBasePath(path: string) { this.config.ossPath = handleOssBasePath(path); } private async _handleClearVersionDir(ops: UploadOptions) { const list = await this._listPrefixes(this.config.ossPath); return this.clearVersionDir(list, ops); } private async _handleOssPath(ossPath: string | string[], ops: UploadOptions) { const { ossFileHook, ossPathHook, diff, debug, version, isClearVersion } = ops; if (!diff) return []; let ossObjectMeta: AliOSS.ObjectMeta[] = []; let paths: string[] = []; if (isClearVersion && !version) throwErr('oss', 'isClearVersion 为 true 时,version 不能为空'); if (version) { const root = await this.list(ossPath, 'all', { delimiter: '/' }); const rootPaths = handelOssRootFilesPath(root); rootPaths.forEach((v) => { const item = versionExec(v); if (!item || version === item[1]) { paths.push(v); } }); } else { paths = Array.isArray(ossPath) ? ossPath : [ossPath]; } if (typeof ossPathHook === 'function') { paths = ossPathHook(paths) || []; } ossObjectMeta = await this.list(paths, 'file'); if (typeof ossFileHook === 'function') { ossObjectMeta = ossFileHook(ossObjectMeta) || []; } const ossFileData = handleOssObjects(ossObjectMeta); if (debug) await debugLog('oss文件信息: ', ossFileData); return ossFileData; } private async _handleLocalPath(ops: UploadOptions) { const { localFileHook: hook, dirPath, debug } = ops; // 获取本地文件路径 let localPath = getDirFilesPath(dirPath, { ignore: this.uploadIgnore, }); if (typeof hook === 'function') { localPath = hook(localPath) || []; } localPath = localPath.filter(Boolean); if (debug) await debugLog('本地文件路径: ', localPath); spinner.succeed(chalk.green(`成功获取本地${localPath.length}个文件!`)); // 本地文件添加MD5 const localFileData = handleLocalFileMD5(localPath); if (debug) await debugLog('本地文件添加MD5: ', localFileData); return localFileData; } private async _handlePutFile(putFiles: string[], ops: UploadOptions) { const { beforeUploadHook: hook, dirPath, debug } = ops; const baseOssPath = this.config.ossPath; let files = putFiles; if (typeof hook === 'function') { files = hook(putFiles) || []; } if (debug) await debugLog('上传oss文件列表: ', files); const ossPutFileObjects = files.map((item) => ({ ossPath: localPathToOssPath(item, baseOssPath, dirPath), filePath: item, })); if (debug) await debugLog('上传oss文件信息列表: ', ossPutFileObjects); log.info('info', chalk.cyan(`⚠️ oss地址: ${baseOssPath}`)); const errorFiles = ossPutFileObjects.filter((item) => !item.ossPath.includes(baseOssPath)); if (errorFiles.length) { debugLog( '上传路径错误文件: ', errorFiles.map((v) => v.ossPath), ); process.exit(1); } if (!ossPutFileObjects.length) return; if (!debug) await this.put(ossPutFileObjects, { ...ops, isCloseConfirm: false }); } private async _handleDeleteFile(deleteFiles: string[], ops: UploadOptions) { const { beforeDeleteHook: hook, dirPath, debug, isClearVersion, version } = ops; let ossDeleteFileObjects = deleteFiles; if (this.deleteIgnore.length) { const patterns = handleIgnores(dirPath, this.deleteIgnore); if (debug) debugLog('删除过滤规则: ', patterns); ossDeleteFileObjects = listIgnoreHandle(ossDeleteFileObjects, patterns); } if (typeof hook === 'function') { ossDeleteFileObjects = hook(ossDeleteFileObjects) || []; } // 相同版本号下面的文件需要删除 if (version) ossDeleteFileObjects = ossDeleteFileObjects.filter((v) => v.includes(`/${version}/`)); // 如果存在清除版本号的形式 if (isClearVersion) { // 过滤列表里面的含有版本号的路径 const versionFiles = ossDeleteFileObjects.filter((v) => !versionExec(v)); ossDeleteFileObjects = ossDeleteFileObjects.concat(versionFiles); } ossDeleteFileObjects = [...new Set(ossDeleteFileObjects)]; if (debug) await debugLog('删除oss文件列表: ', ossDeleteFileObjects); if (!debug) await this.delete(ossDeleteFileObjects); } }