s3-cli-js
Version:
A TypeScript-based npm package that replaces AWS CLI for S3 operations using presigned URLs
227 lines • 8.37 kB
JavaScript
;
/**
* Concurrent operations utility for S3 file transfers
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.executeConcurrentOperations = executeConcurrentOperations;
exports.createUploadOperations = createUploadOperations;
exports.createDownloadOperations = createDownloadOperations;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const chalk_1 = __importDefault(require("chalk"));
const progress_1 = require("./progress");
/**
* Execute file operations concurrently with controlled concurrency
*/
async function executeConcurrentOperations(client, operations, options) {
const { concurrency, onProgress, dryRun = false } = options;
if (operations.length === 0) {
return { completed: [], failed: [], totalBytes: 0 };
}
// Calculate total bytes for progress tracking
const totalBytes = operations.reduce((sum, op) => {
if (op.type === 'upload' && fs.existsSync(op.localPath)) {
const stats = fs.statSync(op.localPath);
return sum + stats.size;
}
return sum + (op.size || 0);
}, 0);
const progress = {
completed: 0,
total: operations.length,
failed: 0,
inProgress: 0,
totalBytes,
completedBytes: 0
};
const completed = [];
const failed = [];
const inProgress = new Set();
// Create a semaphore to limit concurrency
const semaphore = new Semaphore(concurrency);
// Process all operations
const allPromises = operations.map(async (operation) => {
await semaphore.acquire();
try {
progress.inProgress++;
onProgress?.(progress);
if (dryRun) {
await simulateOperation(operation);
}
else {
await executeOperation(client, operation, (fileProgress) => {
// Update progress for this specific file
// Note: This is a simplified progress update
onProgress?.(progress);
});
}
completed.push(operation);
progress.completed++;
progress.inProgress--;
// Update completed bytes (simplified - in real implementation would track per-file progress)
if (operation.type === 'upload' && fs.existsSync(operation.localPath)) {
const stats = fs.statSync(operation.localPath);
progress.completedBytes += stats.size;
}
else {
progress.completedBytes += operation.size || 0;
}
onProgress?.(progress);
}
catch (error) {
failed.push({ operation, error: error });
progress.failed++;
progress.inProgress--;
onProgress?.(progress);
console.error(chalk_1.default.red(`✗ Failed ${operation.type}: ${operation.localPath} - ${error}`));
}
finally {
semaphore.release();
}
});
await Promise.all(allPromises);
return { completed, failed, totalBytes };
}
/**
* Execute a single file operation
*/
async function executeOperation(client, operation, onProgress) {
const { type, localPath, bucket, key } = operation;
if (type === 'upload') {
if (!fs.existsSync(localPath)) {
throw new Error(`Source file does not exist: ${localPath}`);
}
const stats = fs.statSync(localPath);
const filename = path.basename(localPath);
console.log(chalk_1.default.blue(`upload: ${localPath} to s3://${bucket}/${key}`));
await client.uploadFile(localPath, bucket, key, onProgress);
console.log(chalk_1.default.green(`✓ Upload completed: ${(0, progress_1.formatBytes)(stats.size)}`));
}
else if (type === 'download') {
// Ensure directory exists
const dir = path.dirname(localPath);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
}
console.log(chalk_1.default.blue(`download: s3://${bucket}/${key} to ${localPath}`));
await client.downloadFile(bucket, key, localPath, onProgress);
const stats = fs.existsSync(localPath) ? fs.statSync(localPath) : null;
const size = stats ? stats.size : operation.size || 0;
console.log(chalk_1.default.green(`✓ Download completed: ${(0, progress_1.formatBytes)(size)}`));
}
}
/**
* Simulate operation for dry run
*/
async function simulateOperation(operation) {
const { type, localPath, bucket, key } = operation;
// Simulate some processing time
await new Promise(resolve => setTimeout(resolve, Math.random() * 100));
if (type === 'upload') {
console.log(chalk_1.default.blue(`[DRY RUN] upload: ${localPath} to s3://${bucket}/${key}`));
}
else {
console.log(chalk_1.default.blue(`[DRY RUN] download: s3://${bucket}/${key} to ${localPath}`));
}
}
/**
* Simple semaphore implementation for controlling concurrency
*/
class Semaphore {
permits;
waiting = [];
constructor(permits) {
this.permits = permits;
}
async acquire() {
if (this.permits > 0) {
this.permits--;
return Promise.resolve();
}
return new Promise((resolve) => {
this.waiting.push(resolve);
});
}
release() {
this.permits++;
if (this.waiting.length > 0) {
const resolve = this.waiting.shift();
this.permits--;
resolve();
}
}
}
/**
* Create file operations for upload directory
*/
function createUploadOperations(sourceDir, bucket, keyPrefix, files) {
return files.map(file => {
const relativePath = path.relative(sourceDir, file);
// Use manual path joining for S3 keys to ensure forward slashes
const normalizedRelativePath = relativePath.replace(/\\/g, '/');
const s3Key = keyPrefix.endsWith('/')
? keyPrefix + normalizedRelativePath
: keyPrefix + '/' + normalizedRelativePath;
return {
type: 'upload',
localPath: file,
bucket,
key: s3Key.replace(/\/+/g, '/') // Remove duplicate slashes
};
});
}
/**
* Create file operations for download directory
*/
function createDownloadOperations(bucket, objects, keyPrefix, destination) {
return objects.map(object => {
const relativePath = object.key.startsWith(keyPrefix)
? object.key.slice(keyPrefix.length)
: object.key;
const localPath = path.join(destination, relativePath);
return {
type: 'download',
localPath,
bucket,
key: object.key,
size: object.size
};
});
}
//# sourceMappingURL=concurrent.js.map