@pradyumn-el/pollycli/src/files.js

pollycli lets users access the functionalities of Polly over a command line interface

const fs = require('fs');
const pollyEnv = require('./env.json');
const pollymsg = require('./message');
const helper = require('./helper-functions');
const { handleApiError } = require('./helper-functions');
const { pollyApi } = require('./api-client');
const s3 = require('@elucidatainc/s3-node-client');
const path = require('path');
const cliProgress = require('cli-progress')
const { v4: uuidv4 }  = require("uuid")
import { stdout } from 'process'
import { downloadFromS3, uploadToS3 } from './aws_cli_wrapper'
import { copyHelper } from './aws_cli_wrapper'
const s3NodeClientWrapper = require('./s3-node-client-wrapper')
const awsCliWrapper = require('./aws_cli_wrapper')


const ETA_BUFFER = 10000
const PROGRESS_FORMAT = 'progress [{bar}] {percentage}% | {fileSizeProgress} | ETA: {eta}s | time elapsed: {duration}s';

const loadUserInfo = async () => {
  const baseUrl = "/users/me";
  const userInfo = ((await pollyApi.get(baseUrl)).data).data;
  return userInfo;
};


async function getPollyProjectsCreds(workspaceId) {
  if (!workspaceId) {
    pollymsg.pollyError("No workspace ID given.")
  }
  const projectFilesCredsUrl = `/projects/${workspaceId}/credentials/files`;
  try {
    const credsResult = await pollyApi.get(projectFilesCredsUrl);
    const projectCreds = credsResult.data.data[0].attributes.credentials;
    return projectCreds;
  } catch(error) {
    const err_msg = "Not able to list the files";
    if(error.isAxiosError) {
      handleApiError(error, err_msg);
    }

    pollymsg.pollyError(err_msg);
  }
}


export async function syncFiles(workspaceId, sourcePath, destinationPath, verbose, extraArgs) {
  
  // check if the inputs are valid or throw error
  if (!workspaceId) {
    pollymsg.pollyError("No workspace ID given.")
  }
  else if (!sourcePath) {
    pollymsg.pollyError("No source path given.")
  }
  else if (!destinationPath) {
    pollymsg.pollyError("No destination path given.")
  }

  try {

    const projectCreds = await getPollyProjectsCreds(workspaceId);

    if (destinationPath.includes("polly://")) {
      syncS3(workspaceId, sourcePath, destinationPath, verbose, extraArgs, projectCreds);
    } 
    else if (sourcePath.includes("polly://")) {
      syncLocal(workspaceId, sourcePath, destinationPath, verbose, extraArgs, projectCreds);
    } 
    else {
      pollymsg.pollyError("File paths not in the required format")
    }

  } 
  catch (error) {
    pollymsg.pollyError("Not able to sync file")
  }
}


export async function listFiles(workspaceId, pollyPath) {

  // Check if the inputs are valid or throw error
if (!workspaceId) {
    pollymsg.pollyError("No workspace ID given.")
  }
  else if (!pollyPath) {
    pollymsg.pollyError("No Polly Workspace Path given.")
  }

  try {
    const projectCreds = await getPollyProjectsCreds(workspaceId);
    const client = s3.createClient({
      s3Options: {
        accessKeyId: projectCreds.AccessKeyId,
        secretAccessKey: projectCreds.SecretAccessKey,
        sessionToken: projectCreds.SessionToken,
        region: process.env.POLLY_ENV ? pollyEnv.region[process.env.POLLY_ENV] : pollyEnv.region["prod"]
      },
    });

    if (!pollyPath.endsWith("/")) {
      pollyPath = `${pollyPath}/`
    }

    var params = {
      s3Params: {
        Bucket: projectCreds.bucket,
        Prefix:`${workspaceId}/${pollyPath.split("polly://")[1]}`,
        Delimiter: "/"
      },
    };
    const ListObjects = client.listObjects(params)
    ListObjects.on('error', function(err) {
      pollymsg.pollyError("Not able to list the files");
    });
    
    ListObjects.on('data', function(pollyFolderdata) {
      for (const pollyFolders of pollyFolderdata['CommonPrefixes']) {
        console.log(pollyFolders['Prefix'].split(pollyFolderdata['Prefix'])[1]);
      }
      for (const pollyFiles of pollyFolderdata['Contents']) {
        console.log(pollyFiles['Key'].split(pollyFolderdata['Prefix'])[1]);
      }
      if (!pollyFolderdata.IsTruncated) {
        process.exit(0);
      }
    });
  } catch(error) {
    pollymsg.pollyError("Not able to list the files");
  }
}


export async function copyFile(workspaceId, sourcePath, destinationPath, extraArgs) {
  
    // check if the inputs are valid or throw error
    if (!workspaceId) {
      pollymsg.pollyError("No workspace ID given.")
    }
    else if (!sourcePath) {
      pollymsg.pollyError("No source path given.")
    }
    else if (!destinationPath) {
      pollymsg.pollyError("No destination path given.")
    }

  try {
    const projectCreds = await getPollyProjectsCreds(workspaceId);

    if (destinationPath.includes("polly://")) {
      copyToS3(workspaceId, sourcePath, destinationPath, extraArgs, projectCreds);
    }
    else if(sourcePath.includes("polly://")) {
      copyFromS3(workspaceId, sourcePath, destinationPath, extraArgs, projectCreds);
    }
    else {
        pollymsg.pollyError("File paths not in the required format")
    }

  } catch (error) {
    pollymsg.pollyError(`Failed to copy file - ${error}`)
  }
}


async function copyToS3(workspaceId, sourcePath, destinationPath, extraArgs, projectCreds) {

    let localPath = path.resolve(sourcePath);

    if (!fs.existsSync(localPath)) {
      pollymsg.pollyError(`Local path ${localPath} does not exist`);
    }

    let ext = path.extname(sourcePath);
    if (!destinationPath.split("polly://")[1].endsWith(ext)) {
      pollymsg.pollyError("Remote destination should not be a directory for copy operation");
    }

    if(destinationPath.split("polly://")[1].trim() == '') {
      pollymsg.pollyError("Remote destination should not be a directory for copy operation");
    }

    // Add report-id and creator-id tags
    const userInfo = await loadUserInfo();
    let Metadata = { "creator_id": userInfo.id };
    if(extraArgs.is_report) {
      Metadata["report-id"] = uuidv4()
    };

    var params = {
      localFile: localPath,   
      s3Params: {
        Bucket: projectCreds.bucket,
        Key:`${workspaceId}/${destinationPath.split("polly://")[1]}`,
        Metadata: Metadata
      },
      creds: projectCreds
    };

    awsCliWrapper.uploadToS3(params);

    awsCliWrapper.copyHelper.on('error', (err) => {
      /* 
        This will be emitted in cases when aws-cli is not installed on system. In 
        such a scenario, fallback to use s3-node-client library
      */
      if(err.code === 'ENOENT') {
        console.error('Extra dependencies not found. Uploading using S3 node client lib');
        s3NodeClientWrapper.uploadFile(params);
      }      

    })
    
    awsCliWrapper.copyHelper.on('stdError', (err) => {
      console.error(`Error while Uploading file - ${err}`);
    })

}


function copyFromS3(workspaceId, sourcePath, destinationPath, extraArgs, projectCreds) {

    let localPath = path.resolve(destinationPath);

    const parentDir = path.dirname(localPath);

    if (!fs.existsSync(parentDir)) {
      fs.mkdirSync(parentDir, {recursive: true});
    }

    try {
      if (fs.lstatSync(localPath).isDirectory()) {
        if (!localPath.endsWith("/")) {
          localPath = `${localPath}/`
        }
        localPath = `${localPath}${sourcePath.split(path.sep).pop()}`
      }    
    } catch (error) {
      if (error.code != 'ENOENT') {
        pollymsg.pollyError('Local path invalid');
      }
    }

    const s3Key = `${workspaceId}/${sourcePath.split("polly://")[1]}`; 

    var params = {
      localFile: localPath,
      s3Params: {
        Bucket: projectCreds.bucket,
        Key: s3Key,
      },
      creds: projectCreds
    };

    awsCliWrapper.downloadFromS3(params);

    awsCliWrapper.copyHelper.on('error', (err) => {
      // Use s3-node-client lib as fallback option
      // If aws-cli is not found in the path, ENOENT("No such file or directory") error will be raised
      if(err.code === 'ENOENT') {
        console.error('Extra dependencies not found. Downloading using S3 node client lib');
        s3NodeClientWrapper.downloadFile(params)
      }
    })
    
    awsCliWrapper.copyHelper.on('stdError', (err) => {
      console.error(`Error while downloading file - ${err}`);
    })

}


async function syncS3(workspaceId, sourcePath, destinationPath, verbose, extraArgs, projectCreds) {

    let localPath = path.resolve(sourcePath);
    
    if (!fs.existsSync(localPath)) {
      pollymsg.pollyError("Local path does not exist");
    }

    if (!fs.lstatSync(localPath).isDirectory()) {
      pollymsg.pollyError("Local path is not a directory");
    }

    const userInfo = await loadUserInfo();
    let Metadata = { "creator_id": userInfo.id };
    if(extraArgs.is_report) {
      Metadata["report-id"] = uuidv4()
    };

    var params = {
      localDir: localPath,   
      s3Params: {
        Bucket: projectCreds.bucket,
        Prefix:`${workspaceId}/${destinationPath.split("polly://")[1]}`,
        Metadata: Metadata
      },
      creds: projectCreds
    };
    

    awsCliWrapper.syncS3(params);

    awsCliWrapper.copyHelper.on('error', (err) => {
      if(err.code === 'ENOENT') {
        console.error('Extra dependencies not found. Syncing files via S3 node client library.');
        s3NodeClientWrapper.syncS3(params, verbose)
      }
    })

    awsCliWrapper.copyHelper.on('stdError', (err) => {
      console.error(`Error while syncing file - ${err}`);
    })
}


async function syncLocal(workspaceId, sourcePath, destinationPath, verbose, extraArgs, projectCreds) {

    let localPath = path.resolve(destinationPath);

    if (!fs.existsSync(localPath)) {
      fs.mkdirSync(localPath, {recursive: true});
    }

    if (!fs.lstatSync(localPath).isDirectory()) {
      pollymsg.pollyError("Local path is not a directory");
    }

    var params = {
      localDir: localPath,     
      s3Params: {
        Bucket: projectCreds.bucket,
        Prefix:`${workspaceId}/${sourcePath.split("polly://")[1]}`,
      },
      creds: projectCreds
    };

    awsCliWrapper.syncLocal(params);

    awsCliWrapper.copyHelper.on('error', (err) => {
      if(err.code === 'ENOENT') {
        console.error('Extra dependencies not found. Syncing files via S3 node client library.');
        s3NodeClientWrapper.syncLocal(params, verbose)
      }
    });

    awsCliWrapper.copyHelper.on('stdError', (err) => {
      console.error(`Error while syncing file - ${err}`);
    })
}