cfn-forge/src/lib/core/aws/S3Client.js

CloudFormation deployment automation tool with git-based workflows

/**
 * S3 Client Implementation
 * Cross-platform AWS S3 operations using AWS SDK v3
 */

const {
  S3Client: AWSS3Client,
  PutObjectCommand,
  GetObjectCommand,
  DeleteObjectCommand,
  HeadObjectCommand,
  ListObjectsV2Command,
  CreateMultipartUploadCommand,
  UploadPartCommand,
  CompleteMultipartUploadCommand,
  AbortMultipartUploadCommand,
  GetObjectCommand: GetObjectMetadataCommand,
} = require('@aws-sdk/client-s3');
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
const archiver = require('archiver');
const fs = require('fs').promises;
const path = require('path');
const crypto = require('crypto');
const { createReadStream, createWriteStream } = require('fs');
const { pipeline } = require('stream/promises');

const IS3Client = require('../interfaces/IS3Client');

class S3Client extends IS3Client {
  constructor(config = {}) {
    super();
    this.client = new AWSS3Client({
      region: config.region || process.env.AWS_REGION || 'us-east-1',
      ...config,
    });
    this.config = config;
    this.multipartThreshold = config.multipartThreshold || 5 * 1024 * 1024; // 5MB
    this.partSize = config.partSize || 5 * 1024 * 1024; // 5MB parts
  }

  /**
   * Upload a file to S3
   * @param {Object} params - Upload parameters
   * @returns {Promise<Object>} Upload result
   */
  async uploadFile(params) {
    const {
      bucket, key, body, metadata = {}, tags, serverSideEncryption,
    } = params;

    try {
      // Determine if we need multipart upload
      let fileSize = 0;
      if (typeof body === 'string' && await this._fileExists(body)) {
        const stats = await fs.stat(body);
        fileSize = stats.size;
      } else if (Buffer.isBuffer(body)) {
        fileSize = body.length;
      }

      if (fileSize > this.multipartThreshold && typeof body === 'string') {
        return await this._multipartUploadFile({
          bucket, key, filePath: body, metadata, tags,
        });
      }

      // Regular upload
      const uploadBody = typeof body === 'string' && await this._fileExists(body)
        ? createReadStream(body)
        : body;

      const command = new PutObjectCommand({
        Bucket: bucket,
        Key: key,
        Body: uploadBody,
        Metadata: metadata,
        Tagging: tags ? this._formatTags(tags) : undefined,
        ServerSideEncryption: serverSideEncryption,
      });

      const result = await this.client.send(command);

      return {
        bucket,
        key,
        etag: result.ETag,
        versionId: result.VersionId,
        serverSideEncryption: result.ServerSideEncryption,
      };
    } catch (error) {
      throw new Error(`Failed to upload file to s3://${bucket}/${key}: ${error.message}`);
    }
  }

  /**
   * Download a file from S3
   * @param {Object} params - Download parameters
   * @returns {Promise<Buffer>} File content
   */
  async downloadFile(params) {
    const { bucket, key, versionId } = params;

    try {
      const command = new GetObjectCommand({
        Bucket: bucket,
        Key: key,
        VersionId: versionId,
      });

      const result = await this.client.send(command);

      // Convert stream to buffer
      const chunks = [];
      for await (const chunk of result.Body) {
        chunks.push(chunk);
      }

      return Buffer.concat(chunks);
    } catch (error) {
      throw new Error(`Failed to download file from s3://${bucket}/${key}: ${error.message}`);
    }
  }

  /**
   * Delete a file from S3
   * @param {Object} params - Delete parameters
   * @returns {Promise<void>}
   */
  async deleteFile(params) {
    const { bucket, key, versionId } = params;

    try {
      const command = new DeleteObjectCommand({
        Bucket: bucket,
        Key: key,
        VersionId: versionId,
      });

      await this.client.send(command);
    } catch (error) {
      throw new Error(`Failed to delete file from s3://${bucket}/${key}: ${error.message}`);
    }
  }

  /**
   * Check if an object exists in S3
   * @param {Object} params - Check parameters
   * @returns {Promise<boolean>} True if object exists
   */
  async objectExists(params) {
    const { bucket, key } = params;

    try {
      const command = new HeadObjectCommand({
        Bucket: bucket,
        Key: key,
      });

      await this.client.send(command);
      return true;
    } catch (error) {
      if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) {
        return false;
      }
      throw new Error(`Failed to check object existence at s3://${bucket}/${key}: ${error.message}`);
    }
  }

  /**
   * List objects in S3 bucket
   * @param {Object} params - List parameters
   * @returns {Promise<Array>} Array of objects
   */
  async listObjects(params) {
    const {
      bucket, prefix, maxKeys = 1000, continuationToken,
    } = params;

    try {
      const command = new ListObjectsV2Command({
        Bucket: bucket,
        Prefix: prefix,
        MaxKeys: maxKeys,
        ContinuationToken: continuationToken,
      });

      const result = await this.client.send(command);

      return {
        objects: (result.Contents || []).map((obj) => ({
          key: obj.Key,
          size: obj.Size,
          lastModified: obj.LastModified,
          etag: obj.ETag,
          storageClass: obj.StorageClass,
        })),
        isTruncated: result.IsTruncated,
        nextContinuationToken: result.NextContinuationToken,
      };
    } catch (error) {
      throw new Error(`Failed to list objects in s3://${bucket}/${prefix || ''}: ${error.message}`);
    }
  }

  /**
   * Get object metadata
   * @param {Object} params - Metadata parameters
   * @returns {Promise<Object>} Object metadata
   */
  async getObjectMetadata(params) {
    const { bucket, key, versionId } = params;

    try {
      const command = new HeadObjectCommand({
        Bucket: bucket,
        Key: key,
        VersionId: versionId,
      });

      const result = await this.client.send(command);

      return {
        contentLength: result.ContentLength,
        contentType: result.ContentType,
        etag: result.ETag,
        lastModified: result.LastModified,
        metadata: result.Metadata || {},
        versionId: result.VersionId,
        serverSideEncryption: result.ServerSideEncryption,
        storageClass: result.StorageClass,
      };
    } catch (error) {
      throw new Error(`Failed to get object metadata for s3://${bucket}/${key}: ${error.message}`);
    }
  }

  /**
   * Generate presigned URL for object
   * @param {Object} params - URL parameters
   * @returns {Promise<string>} Presigned URL
   */
  async generatePresignedUrl(params) {
    const {
      bucket, key, expiresIn = 3600, operation = 'getObject',
    } = params;

    try {
      let command;
      switch (operation.toLowerCase()) {
        case 'getobject':
          command = new GetObjectCommand({ Bucket: bucket, Key: key });
          break;
        case 'putobject':
          command = new PutObjectCommand({ Bucket: bucket, Key: key });
          break;
        default:
          throw new Error(`Unsupported operation: ${operation}`);
      }

      const url = await getSignedUrl(this.client, command, { expiresIn });
      return url;
    } catch (error) {
      throw new Error(`Failed to generate presigned URL for s3://${bucket}/${key}: ${error.message}`);
    }
  }

  /**
   * Upload directory to S3
   * @param {Object} params - Directory upload parameters
   * @returns {Promise<Array>} Array of upload results
   */
  async uploadDirectory(params) {
    const {
      localPath, bucket, prefix = '', progressCallback, excludePatterns = [],
    } = params;

    try {
      const results = [];
      const files = await this._getAllFiles(localPath, excludePatterns);
      const totalFiles = files.length;
      let uploadedFiles = 0;

      for (const file of files) {
        const relativePath = path.relative(localPath, file);
        const key = prefix ? path.posix.join(prefix, relativePath) : relativePath;
        const s3Key = key.replace(/\\/g, '/'); // Ensure forward slashes for S3

        const result = await this.uploadFile({
          bucket,
          key: s3Key,
          body: file,
        });

        results.push(result);
        uploadedFiles++;

        if (progressCallback) {
          progressCallback({
            current: uploadedFiles,
            total: totalFiles,
            percentage: Math.round((uploadedFiles / totalFiles) * 100),
            currentFile: relativePath,
          });
        }
      }

      return results;
    } catch (error) {
      throw new Error(`Failed to upload directory ${localPath}: ${error.message}`);
    }
  }

  /**
   * Create ZIP package and upload to S3
   * @param {Object} params - Package parameters
   * @returns {Promise<Object>} Upload result with package info
   */
  async uploadPackage(params) {
    const {
      sourcePath, bucket, key, excludePatterns = [], compression = 'DEFLATE',
    } = params;

    try {
      // Create temporary zip file
      const tempDir = await fs.mkdtemp(path.join(require('os').tmpdir(), 'cfn-forge-'));
      const zipPath = path.join(tempDir, 'package.zip');

      // Create zip archive
      const output = createWriteStream(zipPath);
      const archive = archiver('zip', {
        zlib: { level: compression === 'STORE' ? 0 : 9 },
      });

      archive.pipe(output);

      // Add files to archive
      const stats = await fs.stat(sourcePath);
      if (stats.isDirectory()) {
        archive.directory(sourcePath, false, {
          ignore: (filePath) => {
            const relativePath = path.relative(sourcePath, filePath);
            return excludePatterns.some((pattern) => this._matchPattern(relativePath, pattern));
          },
        });
      } else {
        archive.file(sourcePath, { name: path.basename(sourcePath) });
      }

      await archive.finalize();

      // Wait for archive to finish
      await new Promise((resolve, reject) => {
        output.on('close', resolve);
        archive.on('error', reject);
      });

      // Get package info
      const packageStats = await fs.stat(zipPath);
      const hash = await this._calculateFileHash(zipPath);

      // Upload zip to S3
      const uploadResult = await this.uploadFile({
        bucket,
        key,
        body: zipPath,
        metadata: {
          'package-size': String(packageStats.size),
          'package-hash': hash,
          'source-path': path.basename(sourcePath),
        },
      });

      // Cleanup temp file
      await fs.unlink(zipPath);
      await fs.rmdir(tempDir);

      return {
        ...uploadResult,
        packageSize: packageStats.size,
        packageHash: hash,
        compressionType: compression,
      };
    } catch (error) {
      throw new Error(`Failed to create and upload package: ${error.message}`);
    }
  }

  /**
   * Multipart upload implementation
   * @private
   */
  async _multipartUploadFile(params) {
    const {
      bucket, key, filePath, metadata, tags,
    } = params;
    const fileStats = await fs.stat(filePath);
    const fileSize = fileStats.size;

    try {
      // Initiate multipart upload
      const createCommand = new CreateMultipartUploadCommand({
        Bucket: bucket,
        Key: key,
        Metadata: metadata,
        Tagging: tags ? this._formatTags(tags) : undefined,
      });

      const { UploadId } = await this.client.send(createCommand);

      // Calculate parts
      const numParts = Math.ceil(fileSize / this.partSize);
      const uploadPromises = [];
      const partETags = [];

      // Upload parts
      for (let partNumber = 1; partNumber <= numParts; partNumber++) {
        const start = (partNumber - 1) * this.partSize;
        const end = Math.min(start + this.partSize, fileSize);

        uploadPromises.push(
          this._uploadPart({
            bucket,
            key,
            uploadId: UploadId,
            partNumber,
            filePath,
            start,
            end,
          }).then((etag) => {
            partETags[partNumber - 1] = { PartNumber: partNumber, ETag: etag };
          }),
        );

        // Limit concurrent uploads
        if (uploadPromises.length >= 5) {
          await Promise.race(uploadPromises);
        }
      }

      // Wait for all parts to complete
      await Promise.all(uploadPromises);

      // Complete multipart upload
      const completeCommand = new CompleteMultipartUploadCommand({
        Bucket: bucket,
        Key: key,
        UploadId,
        MultipartUpload: { Parts: partETags },
      });

      const result = await this.client.send(completeCommand);

      return {
        bucket,
        key,
        etag: result.ETag,
        location: result.Location,
        versionId: result.VersionId,
      };
    } catch (error) {
      // Abort multipart upload on error
      if (error.UploadId) {
        await this.abortMultipartUpload({ bucket, key, uploadId: error.UploadId });
      }
      throw error;
    }
  }

  /**
   * Upload a single part
   * @private
   */
  async _uploadPart(params) {
    const {
      bucket, key, uploadId, partNumber, filePath, start, end,
    } = params;

    const stream = createReadStream(filePath, { start, end: end - 1 });

    const command = new UploadPartCommand({
      Bucket: bucket,
      Key: key,
      UploadId: uploadId,
      PartNumber: partNumber,
      Body: stream,
    });

    const result = await this.client.send(command);
    return result.ETag;
  }

  /**
   * Abort multipart upload
   * @param {Object} params - Abort parameters
   * @returns {Promise<void>}
   */
  async abortMultipartUpload(params) {
    const { bucket, key, uploadId } = params;

    try {
      const command = new AbortMultipartUploadCommand({
        Bucket: bucket,
        Key: key,
        UploadId: uploadId,
      });

      await this.client.send(command);
    } catch (error) {
      // Ignore errors when aborting
    }
  }

  /**
   * Get all files in directory recursively
   * @private
   */
  async _getAllFiles(dirPath, excludePatterns = []) {
    const files = [];
    const entries = await fs.readdir(dirPath, { withFileTypes: true });

    for (const entry of entries) {
      const fullPath = path.join(dirPath, entry.name);
      const relativePath = entry.name;

      // Check exclude patterns
      if (excludePatterns.some((pattern) => this._matchPattern(relativePath, pattern))) {
        continue;
      }

      if (entry.isDirectory()) {
        const subFiles = await this._getAllFiles(fullPath, excludePatterns);
        files.push(...subFiles);
      } else {
        files.push(fullPath);
      }
    }

    return files;
  }

  /**
   * Simple pattern matching
   * @private
   */
  _matchPattern(filePath, pattern) {
    // Convert glob-like pattern to regex
    const regexPattern = pattern
      .replace(/\./g, '\\.')
      .replace(/\*/g, '.*')
      .replace(/\?/g, '.');

    return new RegExp(`^${regexPattern}$`).test(filePath);
  }

  /**
   * Calculate file hash
   * @private
   */
  async _calculateFileHash(filePath) {
    const hash = crypto.createHash('sha256');
    const stream = createReadStream(filePath);

    for await (const chunk of stream) {
      hash.update(chunk);
    }

    return hash.digest('hex');
  }

  /**
   * Check if file exists
   * @private
   */
  async _fileExists(filePath) {
    try {
      await fs.access(filePath);
      return true;
    } catch {
      return false;
    }
  }

  /**
   * Format tags for S3 API
   * @private
   */
  _formatTags(tags) {
    return Object.entries(tags)
      .map(([key, value]) => `${key}=${value}`)
      .join('&');
  }
}

module.exports = S3Client;