@pradyumn-el/pollycli/src/studio.js

pollycli lets users access the functionalities of Polly over a command line interface

const axios = require('axios');
const pollyEnv = require('./env.json');
const pollyHeaders = require('./pollyheaders');
const pollymsg = require('./message');
const chalk = require('chalk');
const fs = require('fs');
const path = require('path')
const tar = require('tar');
const s3 = require('@elucidatainc/s3-node-client');
const cliProgress = require('cli-progress');
const git = require('simple-git');
import {
  submitJob,
  jobStatus
} from './job';
import {
  getJobLogs
} from './logs';
import {
  organizationDetails
} from './organization';
import {
  getAllDockerRepos,
  getAllDockerCommits,
  createADockerRepo
} from './docker';
const { pollyApi } = require('./api-client');

const rmdir = require('rimraf');

const ETA_BUFFER = 10000
const PROGRESS_FORMAT = 'progress [{bar}] {percentage}% | {fileSizeProgress} | ETA: {eta}s | time elapsed: {duration}s';
const helper = require('./helper-functions');

export async function generateStudioComponentTemplate(templateType, componentName) {
  const orgDetails = await organizationDetails();
  const orgSlug = orgDetails.slug;
  const orgOrgId = orgDetails.org_id;
  const ownDockers = await getAllDockerRepos(orgOrgId, 'own');
  for (const ownDocker of ownDockers) {
    if (ownDocker.attributes.resource_name == `${orgSlug}/${componentName}`) {
      pollymsg.pollyError(`studio component with the name ${componentName} already exists`);
    }
  }

  let gitBranches = await axios.get('https://api.github.com/repos/ElucidataInc/studio-templates/branches');
  const allowedTemplate = []
  for (const eachBranches of gitBranches.data) {
    if (eachBranches.name != 'master') {
      allowedTemplate.push(eachBranches.name)
    }
  }

  if (allowedTemplate.indexOf(templateType) < 0) {
    pollymsg.pollyError('We not support this type. If you would like to add this please do get in touch with out customer success team');
  }

  try {
    await git().silent(true)
      .clone('https://github.com/ElucidataInc/studio-templates.git', componentName, ['--single-branch', '--branch', templateType]);
  } catch (error) {
    pollymsg.pollyError('Not able to get the template');
  }
  const removedGit = await (async () => {
    return new Promise((resolve, reject) => {
      rmdir(`${path.resolve(".")}/${componentName}/.git`, function (error) {
        if (error) {
          resolve(false);
        } else {
          resolve(true);
        }
      });
    });
  })();
  const studioConfig = JSON.stringify({
    dockerName: `${pollyEnv.dockerDomain}/${orgSlug}/${componentName}`,
    componentName: `${componentName}`
  },null, '  ');
  fs.writeFileSync(`${path.resolve(".")}/${componentName}/.studioConfig.json`, studioConfig);

  const createStatus = await createADockerRepo(componentName, `This is a docker for studio component ${componentName}`, false, true);
  if (createStatus) {
    pollymsg.pollyMessage(`Studio component ${componentName} generated`);
  } else {
    if (fs.existsSync(`${path.resolve(".")}/${componentName}`)) {
      await (async () => {
        return new Promise((resolve, reject) => {
          rmdir(`${path.resolve(".")}/${componentName}`, function (error) {
            if (error) {
              resolve(false);
            } else {
              resolve(true);
            }
          });
        });
      })();
    }
    pollymsg.pollyError("Not able to create component please contact the customer success team");
  }
}

export async function componentBuildStatus(buildID) {
  let buff = Buffer.from(buildID, 'base64');
  let extractedSecret = buff.toString('ascii');
  const jsonData = JSON.parse(extractedSecret);
  const statusData = await jobStatus(jsonData.data.project_id, jsonData.data.job_id, true);
  let dataArray = [
    ['Build state'],
    [statusData[0].attributes.state]
  ]
  pollymsg.pollyTable(dataArray);
  await getJobLogs(jsonData.data.project_id, jsonData.data.job_id);
}

export async function buildStudioComponent(localComponentPath = '.') {
  const organizationInfo = await organizationDetails();
  const orgSlug = organizationInfo.slug;
  const orgId = organizationInfo.org_id;
  const configFilePath = `${path.resolve(".")}/.studioConfig.json`;
  if (!fs.existsSync(configFilePath)) {
    pollymsg.pollyError("Seems to be not building from the studio component directory");
  }
  const dockerNameWithOutTag  = JSON.parse(fs.readFileSync(configFilePath)).dockerName;
  const dockerName = dockerNameWithOutTag.split(`${pollyEnv.dockerDomain}/${orgSlug}/`)[1];
  let foundDocker = false;
  const allDockerImages = await getAllDockerRepos(orgId);

  let dockerCommits = [];
  for (const dockers of allDockerImages) {
    if (`${pollyEnv.dockerDomain}/${dockers.attributes.resource_name}` == `${pollyEnv.dockerDomain}/${orgSlug}/${dockerName}`) {
      foundDocker = true;
      dockerCommits = await getAllDockerCommits(orgId, dockers.attributes.resource_id,'public' == dockers.attributes.resource_access, true);
      break;
    }
  }
  if (!foundDocker) {
    pollymsg.pollyError("Seems like new component creation was not successful or the component is deleted");
  }
  let dockerTag = "V1";
  if (dockerCommits.length != 0) {
    const dockerTagAppender = "V";
    let allTags = [];
    for (const dockerCommit of dockerCommits) {
      allTags.push(parseInt(dockerCommit.attributes.tag_name.split(dockerTagAppender)[1]));
    }
    dockerTag = `V${Math.max(...allTags) + 1}`;
  }
  const dockerNameWithTag = `${dockerName}:${dockerTag}`

  let localPath = `${path.resolve(localComponentPath)}`;
  if (!localPath.endsWith('/')) {
    localPath = `${localPath}/`;
  }
  if (!fs.existsSync(`${localPath}Dockerfile`)) {
    pollymsg.pollyError('Not able to find the Dockerfile in the given path');
  }
  const compressedFileName = `/tmp/${Math.random().toString(36).slice(2)}.tar.gz`
  const isCompressed = await compressStudioCode(compressedFileName);
  if (!isCompressed) {
    pollymsg.pollyError('Not able to package the folder');
  }
  const workspaceId = await getWorkSpaceId();
  if (!workspaceId) {
    pollymsg.pollyError('Not able to get the workspaceId to bundle the component');
  }
  let fileName = dockerNameWithTag.replace(/\//g, '_');
  fileName = fileName.replace(/\:/g, '-');
  fileName = fileName.replace(/\./g, '-');
  fileName = `polly://${fileName}.tar.gz`;
  if (!fs.existsSync(compressedFileName)) {
    pollymsg.pollyError('Packaging of code did not happen');
  }
  const fileUploadStatus = await uploadFiles(workspaceId, compressedFileName, fileName, true);
  if (!fileUploadStatus) {
    pollymsg.pollyError('Error in uploading the packaged file');
  } else {
    pollymsg.pollyMessage('Studio component is uploaded. The build process will start soon.')
  }

  const jobTemplate = {
    "cpu": "1",
    "memory": "2Gi",
    "image": `${pollyEnv.dockerDomain}/elucidata/polly-docker-builder`,
    "tag": "latest",
    "name": `Docker build for ${dockerNameWithTag}`,
    "env": {
      "POLLY_PATH_TO_DOCKER_TAR_GZ": fileName,
      "POLLY_DOMAIN_DOCKER_NAME_WITH_TAG": `${pollyEnv.dockerDomain}/${orgSlug}/${dockerNameWithTag}`,
      "STUDIO_BUILD": "TRUE"
    },
    "secret_env": {
      "POLLY_ENV": pollyEnv.dockerDomain,
    }
  }
  const jobFile = `/tmp/${Math.random().toString(36).slice(2)}.json`
  const jobFileData = JSON.stringify(jobTemplate, null, 2);
  fs.writeFileSync(jobFile, jobFileData);

  if (!fs.existsSync(jobFile)) {
    pollymsg.pollyError('Not able to start the component build');
  }
  const submittedData = await submitJob(workspaceId.toString(), jobFile, true);
  const buff = new Buffer(JSON.stringify(submittedData));
  const base64data = buff.toString('base64');
  pollymsg.pollyMessage('\nStudio component building have started. \n \n');
  let dataArray = [
    ['Build ID'],
    [base64data]
  ]
  // pollymsg.pollyTable(dataArray);
  pollymsg.pollyMessage(chalk.white(`To get the status and logs run:
${chalk.italic('polly studio component-build-status --id ')} ${chalk.italic(base64data)}`))
}

async function getWorkSpaceId() {
  const workspaceName = 'studio_component_builder_default'
  const workspaceDesc = 'A workspace in which all the build code for component is kept'
  const workspaceUrl = `${pollyEnv.baseV2Api}`
  const workspaceDataComplete = [];
  let requestUrl = ''
  let workspaceData = null;
  let workSpaceFound = false;
  let workSpaceId = null;
  requestUrl = "/workspaces?page%5Bsize%5D=100";
  do {
    workspaceData = await pollyApi.get(requestUrl);
    workspaceDataComplete.push(...workspaceData.data.data)

    if (workspaceData.data.links.next) {
      requestUrl = `${workspaceData.data.links.next}`
    }
  } while (workspaceData.data.links.next);

  for (const eachWorkspace of workspaceDataComplete) {
    if (eachWorkspace.attributes.name == workspaceName) {
      workSpaceFound = true;
      workSpaceId = eachWorkspace.attributes.id
      break;
    }
  }
  if (!workSpaceFound) {
    const workSpacePayload = {
      "data": {
        "type": "workspaces",
        "attributes": {
          "name": workspaceName,
          "description": workspaceDesc,
          "project_property": {
            "type": "workspaces",
            "labels": ""
          }
        }
      }
    }
    try {
      const createdData = await pollyApi.post("/workspaces", workSpacePayload, await pollyHeaders.getV2Headers());
      workSpaceId = createdData.data.data.attributes.id;
    } catch (error) {
      pollymsg.pollyError('Not able to create workspaces for build purpose');
    }
  }
  return workSpaceId;
}

async function compressStudioCode(compressedFileName) {
  return new Promise((resolve, rejects) => {
    tar.c({
        gzip: true,
        file: compressedFileName
      },
      ['.']
    ).then(_ => {
      resolve(true);
    }).catch((err) => {
      resolve(false);
    })
  });
}

async function getPollyProjectsCreds(workspaceId) {
  if (!workspaceId) {
    pollymsg.pollyError("No workspace ID given.")
  }
  const projectFilesCredsUrl = `/projects/${workspaceId}/credentials/files`;
  try {
    const credsResult = await pollyApi.get(projectFilesCredsUrl, await pollyHeaders.getV2Headers());
    const projectCreds = credsResult.data.data[0].attributes.credentials;
    return projectCreds;
  } catch (error) {
    pollymsg.pollyError(`Not enough permission over the workspace ${workspaceId}`)
  }
}

async function uploadFiles(workspaceId, sourcePath, destinationPath, internalCall = false) {
  return new Promise(async (resolve, reject) => {
    const projectCreds = await getPollyProjectsCreds(workspaceId);
    const client = s3.createClient({
      s3Options: {
        accessKeyId: projectCreds.AccessKeyId,
        secretAccessKey: projectCreds.SecretAccessKey,
        sessionToken: projectCreds.SessionToken,
        region: process.env.POLLY_ENV ? pollyEnv.region[process.env.POLLY_ENV] : pollyEnv.region["prod"]
      },
    });

    let localPath = path.resolve(sourcePath);

    if (!fs.existsSync(localPath)) {
      pollymsg.pollyError("Local path does not exist");
    }

    var params = {
      localFile: localPath,
      s3Params: {
        Bucket: projectCreds.bucket,
        Key: `${workspaceId}/${destinationPath.split("polly://")[1]}`,
      },
    };

    let ext = path.extname(sourcePath);
    if (!destinationPath.split("polly://")[1].endsWith(ext)) {
      pollymsg.pollyError("remote destination should not be a directory for copy operation");
    }

    if (destinationPath.split("polly://")[1].trim() == '') {
      pollymsg.pollyError("remote destination should not be a directory for copy operation");
    }
    const uploadSync = client.uploadFile(params);
    if (!internalCall) {
      var uploadBar = new cliProgress.SingleBar({
        format: PROGRESS_FORMAT,
        etaBuffer: ETA_BUFFER
      }, cliProgress.Presets.shades_classic);
    }
    let totalData = 0;
    let sizeProgress = 'NA';
    let doneOnce = true;
    uploadSync.on('error', function (err) {
      if (!internalCall) {
        uploadBar.stop();
      }
      resolve(false);
    });

    uploadSync.on('progress', function (data) {
      if (!internalCall && uploadSync.progressTotal > 0) {
        if (totalData != uploadSync.progressTotal && uploadSync.progressTotal > 0) {
          if (doneOnce) {
            uploadBar.start(uploadSync.progressTotal, uploadSync.progressAmount, {
              fileSizeProgress: sizeProgress
            });
            doneOnce = false;
          }
          uploadBar.setTotal(uploadSync.progressTotal)
          totalData = uploadSync.progressTotal;
        }
        sizeProgress = `${helper.formatBytes(uploadSync.progressAmount)}/${helper.formatBytes(uploadSync.progressTotal)}`
        uploadBar.update(uploadSync.progressAmount, {
          fileSizeProgress: sizeProgress
        });
      }
    });

    uploadSync.on('end', function () {
      if (!internalCall) {
        uploadBar.stop();
      }
      resolve(true);
    });
  });
}