publish-to-s3/dist/index.mjs

Upload your dist folder to S3 with a single command

#!/usr/bin/env node
import consola, { consola as consola$1 } from 'consola';
import path from 'node:path';
import pLimit from 'p-limit';
import dotenv from 'dotenv';
import fs from 'node:fs';
import crypto from 'node:crypto';
import { S3Client, HeadObjectCommand } from '@aws-sdk/client-s3';
import { Upload } from '@aws-sdk/lib-storage';
import mime from 'mime';

dotenv.config();
const config = {
  s3Region: process.env.S3_REGION,
  s3AccessKeyId: process.env.S3_ACCESS_KEY_ID,
  s3SecretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
  s3Endpoint: process.env.S3_ENDPOINT,
  s3BucketName: process.env.S3_BUCKET_NAME,
  localFolderPath: process.env.LOCAL_FOLDER_PATH || path.join(process.cwd(), "dist"),
  s3DestinationPath: process.env.S3_DESTINATION_PATH || "",
  s3UploadConcurrencyLimit: parseInt(process.env.S3_UPLOAD_CONCURRENCY_LIMIT || "16")
};
if (!config.s3Region || !config.s3AccessKeyId || !config.s3SecretAccessKey || !config.s3Endpoint || !config.s3BucketName) {
  throw new Error(
    "You did not provide one or more environment variables. Please define them or place a .env file. You can get help by reading the README.md file."
  );
}

const calculateMD5 = (filePath) => {
  return new Promise((resolve, reject) => {
    const hash = crypto.createHash("md5");
    const stream = fs.createReadStream(filePath);
    stream.on("data", (data) => hash.update(data));
    stream.on("end", () => resolve(hash.digest("hex")));
    stream.on("error", (error) => {
      reject(
        new Error(`Failed to calculate MD5 for ${filePath}: ${error.message}`)
      );
    });
  });
};
const collectFiles = (folderPath, basePath = "") => {
  const files = fs.readdirSync(folderPath);
  let fileList = [];
  for (const file of files) {
    const filePath = path.join(folderPath, file);
    const fileStat = fs.statSync(filePath);
    if (fileStat.isFile()) {
      const key = path.join(basePath, file).replace(/\\/g, "/");
      fileList.push({ key, filePath });
    } else if (fileStat.isDirectory()) {
      fileList = fileList.concat(
        collectFiles(filePath, path.join(basePath, file))
      );
    }
  }
  return fileList;
};

const s3Client = new S3Client({
  region: config.s3Region,
  credentials: {
    accessKeyId: config.s3AccessKeyId,
    secretAccessKey: config.s3SecretAccessKey
  },
  endpoint: config.s3Endpoint
});

const isFileInS3 = async (bucketName, key, localFilePath) => {
  try {
    const headParams = {
      Bucket: bucketName,
      Key: key.replace(/\\/g, "/")
    };
    const headData = await s3Client.send(new HeadObjectCommand(headParams));
    const localFileMD5 = await calculateMD5(localFilePath);
    return headData.ETag === `"${localFileMD5}"`;
  } catch (error) {
    if (error.name === "NotFound") {
      return false;
    }
    throw error;
  }
};
const uploadFileToS3 = async ({
  bucketName,
  key,
  filePath,
  maxRetries = 3
}) => {
  key = key.replace(/\\/g, "/");
  if (await isFileInS3(bucketName, key, filePath)) {
    consola.info(
      `File ${key} already exists in ${bucketName} and is identical. Skipping upload`
    );
    return;
  }
  const fileStream = fs.createReadStream(filePath);
  const contentType = mime.getType(filePath) || "application/octet-stream";
  const uploadParams = {
    Bucket: bucketName,
    Key: key,
    Body: fileStream,
    ContentType: contentType
  };
  let retries = 0;
  while (retries < maxRetries) {
    try {
      const start = Date.now();
      const upload = new Upload({
        client: s3Client,
        params: uploadParams
      });
      await upload.done();
      consola.success(
        `Uploaded ${key} to ${bucketName} in ${Date.now() - start}ms`
      );
      return;
    } catch (error) {
      retries++;
      consola.warn(
        `Error uploading ${key}, attempt ${retries} of ${maxRetries}:`,
        error
      );
      if (retries === maxRetries) {
        consola.error(`Failed to upload ${key} after ${maxRetries} attempts`);
        throw error;
      }
    }
  }
};

const main = async () => {
  const startTime = Date.now();
  consola$1.start(
    `Preparing to upload files to the S3 bucket "${config.s3BucketName}"`
  );
  const localPath = path.join(config.localFolderPath);
  const s3Path = path.join(config.s3BucketName, config.s3DestinationPath).replace(/\\/g, "/");
  consola$1.info(`All files in "${localPath}" will be uploaded to "${s3Path}"`);
  const filesToUpload = collectFiles(config.localFolderPath);
  consola$1.info(`Found ${filesToUpload.length} files to upload`);
  const limit = pLimit(config.s3UploadConcurrencyLimit);
  const uploadTasks = filesToUpload.map(
    (file) => limit(
      () => uploadFileToS3({
        bucketName: config.s3BucketName,
        key: path.join(config.s3DestinationPath, file.key),
        filePath: file.filePath
      })
    )
  );
  await Promise.all(uploadTasks);
  const duration = Date.now() - startTime;
  consola$1.success(`Uploading completed in ${duration}ms.`);
};
main().catch((error) => {
  consola$1.error("Upload failed:", error);
  process.exit(1);
});