@pradyumn-el/pollycli/src/s3-node-client-wrapper.js

pollycli lets users access the functionalities of Polly over a command line interface

const cliProgress = require('cli-progress')
const s3 = require('@elucidatainc/s3-node-client');
const helper = require('./helper-functions');
const fs = require('fs');

const pollymsg = require('./message');
const pollyEnv = require('./env.json');


const ETA_BUFFER = 10000
const PROGRESS_FORMAT = 'progress [{bar}] {percentage}% | {fileSizeProgress} | ETA: {eta}s | time elapsed: {duration}s';


export function uploadFile(params) {
    /*
      params = {
        localFile: localPath,
        s3Params: {
            Bucket: projectCreds.bucket,
            Key: s3Key,
            MetaData: metadata
        },
        creds: projectCreds,
      };
            
    */

    const client = s3.createClient({
      s3Options: {
        accessKeyId: params.creds.AccessKeyId,
        secretAccessKey: params.creds.SecretAccessKey,
        sessionToken: params.creds.SessionToken,
        region: process.env.POLLY_ENV ? pollyEnv.region[process.env.POLLY_ENV] : pollyEnv.region["prod"]
      },
    });

    // TODO: Need to check if Metadata key is working
    const uploadSync = client.uploadFile(params);

    let uploadBar = new cliProgress.SingleBar({
        format: PROGRESS_FORMAT,
        etaBuffer: ETA_BUFFER
        }, 
        cliProgress.Presets.shades_classic
    );

    let totalData = 0;
    let sizeProgress = 'NA';
    let doneOnce = true;
    
    uploadSync.on('error', function(err) {
    uploadBar.stop();
    pollymsg.pollyError(`Trouble in uploading to path ${err.path}`);
    });

    uploadSync.on('progress', function(data) {
    if (uploadSync.progressTotal > 0) {
        if (totalData != uploadSync.progressTotal && uploadSync.progressTotal > 0) {
        if (doneOnce) {
            uploadBar.start(uploadSync.progressTotal, uploadSync.progressAmount, {fileSizeProgress: sizeProgress});
            doneOnce = false;
        }
        uploadBar.setTotal(uploadSync.progressTotal)
        totalData = uploadSync.progressTotal;
        }
        sizeProgress = `${helper.formatBytes(uploadSync.progressAmount)}/${helper.formatBytes(uploadSync.progressTotal)}`
        uploadBar.update(uploadSync.progressAmount, {fileSizeProgress: sizeProgress});
    }
    });
    
    uploadSync.on('end', function() {
    uploadBar.stop();
    pollymsg.pollySuccess("Upload complete");
    process.exit(0);
    });
}


export function downloadFile(params) {
    /*
      params = {
        localFile: localPath,
        s3Params: {
            Bucket: projectCreds.bucket,
            Key: s3Key,
            MetaData: metadata
        },
        creds: projectCreds,
      };
    */

    const client = s3.createClient({
      s3Options: {
        accessKeyId: params.creds.AccessKeyId,
        secretAccessKey: params.creds.SecretAccessKey,
        sessionToken: params.creds.SessionToken,
        region: process.env.POLLY_ENV ? pollyEnv.region[process.env.POLLY_ENV] : pollyEnv.region["prod"]
      },
    });

    client.s3.headObject({
      Bucket: params.s3Params.Bucket,
      Key: params.s3Params.Key,
    }, function(err, data) {
      if (err) {
        pollymsg.pollyError("File not found on Polly")
        return;
      }
      
      const downloadSync = client.downloadFile(params);

      let downloadBar = new cliProgress.SingleBar({
          format: PROGRESS_FORMAT,
          etaBuffer: ETA_BUFFER
        }, 
        cliProgress.Presets.shades_classic
      );

      let totalData = 0;
      let sizeProgress = 'NA';
      let doneOnce = true;

      downloadSync.on('error', function(err) {
        downloadBar.stop();
        pollymsg.pollyError(`Trouble in downloading to path ${err.path}`);
      });

      downloadSync.on('progress', function() {
        if (downloadSync.progressTotal > 0) {
          if (totalData != downloadSync.progressTotal && downloadSync.progressTotal > 0) {
            if (doneOnce) {
              downloadBar.start(downloadSync.progressTotal, downloadSync.progressAmount, {fileSizeProgress: sizeProgress});
              doneOnce = false;
            }
            downloadBar.setTotal(downloadSync.progressTotal);
            totalData = downloadSync.progressTotal;
          }
          sizeProgress = `${helper.formatBytes(downloadSync.progressAmount)}/${helper.formatBytes(downloadSync.progressTotal)}`
          downloadBar.update(downloadSync.progressAmount, {fileSizeProgress: sizeProgress});
        }
      });
      
      downloadSync.on('end', function() {
        downloadBar.stop();
        pollymsg.pollySuccess("Download complete");
        process.exit(0);
      });
    });
}


export function syncS3(params, verbose=false) {

    var logFileStream = null
    var logFilePath = './sync.log'
    if(verbose) {
      logFileStream = fs.createWriteStream(logFilePath, { flags: 'w'})
    }

    const client = s3.createClient({
      s3Options: {
        accessKeyId: params.creds.AccessKeyId,
        secretAccessKey: params.creds.SecretAccessKey,
        sessionToken: params.creds.SessionToken,
        region: process.env.POLLY_ENV ? pollyEnv.region[process.env.POLLY_ENV] : pollyEnv.region["prod"]
      },
    });
    
    const uploadSync = client.uploadDir(params);
    let uploadBar = new cliProgress.SingleBar({
      format: PROGRESS_FORMAT,
      etaBuffer: ETA_BUFFER
    }, cliProgress.Presets.shades_classic);
    let totalData = 0;
    let sizeProgress = 'NA';
    let doneOnce = true;
    let skippedFileSize = 0;
    let progressAmount = 0;


    uploadSync.on('error', function(err) {
      uploadBar.stop();
      let msg = `Trouble in syncing path ${err.path} - ${err}`
      if(logFileStream) {
        logFileStream.write(msg, function(fErr) {
          if(fErr) {
            console.log(`Error writing to log file : ${fErr}`)
          }
        });
        logFileStream.close();
      }
      pollymsg.pollyError(msg);
    });

    uploadSync.on('fileSkipped', function(size, name) {
      skippedFileSize += size;
      if(logFileStream) {
        let msg = `File ${name} already present on S3\n`
        logFileStream.write(msg, function(fErr) {
          if(fErr) {
            console.log(`Error writing to log file : ${fErr}`)
          }
        });
      }
    });


    uploadSync.on('progress', function(data) {
      if (uploadSync.progressMd5Total > 0) {
        if (totalData !== uploadSync.progressMd5Total && uploadSync.progressMd5Total > 0) {
          if (doneOnce) {
            uploadBar.start(uploadSync.progressMd5Total, uploadSync.progressAmount, {fileSizeProgress: sizeProgress});
            doneOnce = false;
          }
          uploadBar.setTotal(uploadSync.progressMd5Total)
          totalData = uploadSync.progressMd5Total;
        }
        sizeProgress = `${helper.formatBytes(uploadSync.progressAmount)}/${helper.formatBytes(uploadSync.progressMd5Total)}`
        uploadBar.update(uploadSync.progressAmount, {fileSizeProgress: sizeProgress});
        progressAmount = uploadSync.progressAmount;
      }
    });
    
    uploadSync.on('fileUploadEnd', function(path, key) {
      var message = `File uploaded : ${path} -> ${key}\n`
      if(logFileStream) {
        logFileStream.write(message, function(err) {
          if(err) {
            console.log(`Error writing to log file : ${err}`)
          }
        });
      }
    });

    uploadSync.on('end', function() {
      // Cases of skipped files - It's possible for some files to be already present at the destination
      // path, possibly because of previously failed sync operation or when user decided to upload some
      // files manually. In both the cases, a few or maybe all files might already be present on s3.
      // For such cases, we keep a track of files that are not uploaded/skipped. We also keep a note
      // of size of all such files using 'skippedFileSize' variable
      // To update the progress bar accurately, size of all the skipped files is added to 'progressAmount',
      // which keeps track of size of files that were uploaded in the current operation only, to make sure
      // uploaded amount, i.e 'totalUploaded', always matches the total amount that had to be uploaded,
      // i.e 'progressMd5Total'.
      let totalUploaded = progressAmount + skippedFileSize;
      sizeProgress = `${helper.formatBytes(totalUploaded)}/${helper.formatBytes(uploadSync.progressMd5Total)}`
      uploadBar.update(totalUploaded, {fileSizeProgress: sizeProgress});

      uploadBar.stop();
      pollymsg.pollySuccess("Sync complete");
      if(logFileStream) {
        pollymsg.pollyMessage(`Please check ${logFilePath} for complete logs`)
        logFileStream.close();
      }
      process.exit(0);
    });
}


export function syncLocal(params, verbose=false) {

    var logFileStream = null
    var logFilePath = './sync.log'
    if(verbose) {
      logFileStream = fs.createWriteStream(logFilePath, { flags: 'w'})
    }


    // TODO: put this is inside a re-usable function
    const client = s3.createClient({
      s3Options: {
        accessKeyId: params.creds.AccessKeyId,
        secretAccessKey: params.creds.SecretAccessKey,
        sessionToken: params.creds.SessionToken,
        region: process.env.POLLY_ENV ? pollyEnv.region[process.env.POLLY_ENV] : pollyEnv.region["prod"]
      },
    });

    const downloadSync = client.downloadDir(params);

    let downloadBar = new cliProgress.SingleBar({
      format: PROGRESS_FORMAT,
      etaBuffer: ETA_BUFFER
    }, cliProgress.Presets.shades_classic);

    let totalData = 0;
    let sizeProgress = 'NA';
    let doneOnce = true;

    downloadSync.on('error', function(err) {
      downloadBar.stop();
      msg = `Trouble in syncing path ${err.path}`
      if(logFileStream) {
        logFileStream.write(msg, function(fErr) {
          if(fErr) {
            console.log(`Error writing to log file : ${fErr}`)
          }
        });
        logFileStream.close();
      }
      pollymsg.pollyError(msg);
    });
    
    downloadSync.on('progress', function() {
      if (downloadSync.progressTotal > 0) {
        if (totalData != downloadSync.progressTotal && downloadSync.progressTotal > 0) {
          if (doneOnce) {
            downloadBar.start(downloadSync.progressTotal, downloadSync.progressAmount, {fileSizeProgress: sizeProgress});
            doneOnce = false;
          }
          downloadBar.setTotal(downloadSync.progressTotal)
          totalData = downloadSync.progressTotal;
        }
        sizeProgress = `${helper.formatBytes(downloadSync.progressAmount)}/${helper.formatBytes(downloadSync.progressTotal)}`
        downloadBar.update(downloadSync.progressAmount, {fileSizeProgress: sizeProgress});
      }
    });
    
    downloadSync.on('fileDownloadEnd', function(path, key) {
      var message = `File downloaded : ${key} -> ${path}\n`
      if(logFileStream) {
        logFileStream.write(message, function(err) {
          if(err) {
            console.log(`Error writing to log file : ${err}`)
          }
        });
      }
    });

    downloadSync.on('end', function() {
      downloadBar.stop();
      pollymsg.pollySuccess("Sync complete");
      if(logFileStream) {
        pollymsg.pollyMessage(`Please check ${logFilePath} for complete logs`)
        logFileStream.close();
      }
      process.exit(0);
    });
}