UNPKG

@pradyumn-el/pollycli

Version:

pollycli lets users access the functionalities of Polly over a command line interface

680 lines (620 loc) 24.4 kB
const pollyEnv = require('./env.json'); const pollyHeaders = require('./pollyheaders'); const pollymsg = require('./message'); const chalk = require('chalk'); const pollyHelpers = require('./polly-helpers'); const { handleApiError } = require('./helper-functions'); const path = require('path'); const fs = require('fs'); const tar = require('tar'); const dockerLint = require("dockerfilelint") const dockerLintReport = require("dockerfilelint/lib/reporter/json_reporter"); const s3 = require('@elucidatainc/s3-node-client'); import { organizationDetails } from './organization'; import { submitJob, jobStatus } from './job'; import { getJobLogs } from './logs'; import CliReporter from 'dockerfilelint/lib/reporter/cli_reporter'; const { pollyApi } = require('./api-client'); export async function getAllDockerRepos(orgId, accessType = 'all') { const dockerResponse = []; let allTypes = {}; if (accessType == 'own') { allTypes = { 'own': '' }; } else if (accessType == 'public') { allTypes = { 'public': '?filter[resource_access]=public&filter[scope]=global' }; } else { allTypes = { 'own': '', 'public': '?filter[resource_access]=public&filter[scope]=global' }; } try { for (const typeIn of Object.keys(allTypes)) { let dockerRepoMakeUrl = `/organizations/${orgId}/resources/dockers${allTypes[typeIn]}` let dockerCommitsResponseInter = null; do { try { dockerCommitsResponseInter = await pollyApi.get(dockerRepoMakeUrl); for (const eachDocker of dockerCommitsResponseInter.data.data) { if (accessType == 'own') { dockerResponse.push(eachDocker); } else if (accessType == 'public' && eachDocker.attributes.resource_access == "public") { dockerResponse.push(eachDocker); } else if (accessType == 'all' && !(typeIn == 'public' && eachDocker.attributes.org_id == orgId)) { dockerResponse.push(eachDocker); } } dockerCommitsResponseInter = await pollyApi.get(dockerRepoMakeUrl); if (dockerCommitsResponseInter.data.links.next) { dockerRepoMakeUrl = `${dockerCommitsResponseInter.data.links.next}` } } catch(e) { // ignore if any one of the api calls returns error } } while (dockerCommitsResponseInter && dockerCommitsResponseInter.data.links.next); } } catch (error) { if (dockerResponse.length < 1) { try { if (error.response.data.errors[0].code == 'resource_not_found') { pollymsg.pollyError("No docker repositories present. Create docker repository with polly dockers create"); } } catch (error) { pollymsg.pollyError(`Not able to get the docker repository`); } pollymsg.pollyError(`Not able to get the docker repository`); } } if (dockerResponse.length < 1) { pollymsg.pollyMessage(`No docker repositories present`); process.exit(0); } return dockerResponse; } export async function getAllDockerCommits(orgId, dockerId, isPublic = false, internalCall = false) { const dockerCommitBase = `/organizations/${orgId}/resources/dockers/${dockerId}/commits`; let dockerRepoMakeUrl; const dockerCommitsResponse = [] let dockerCommitsResponseInter = null; let urlAppend = null; try { if (isPublic) { urlAppend = `resource_access=public` } else { urlAppend = `sort=-last_modified_at` } dockerRepoMakeUrl = `${dockerCommitBase}?${urlAppend}` do { dockerCommitsResponseInter = await pollyApi.get(dockerRepoMakeUrl); if (dockerCommitsResponseInter.data.data.length > 0) { dockerCommitsResponse.push(...dockerCommitsResponseInter.data.data); } if (dockerCommitsResponseInter.data.data.length > 0 && dockerCommitsResponseInter.data.links.next) { urlAppend = dockerCommitsResponseInter.data.links.next.split('commits?')[1]; dockerRepoMakeUrl = `${dockerCommitBase}?${urlAppend}` } } while (dockerCommitsResponseInter.data.data.length > 0 && dockerCommitsResponseInter.data.links.next); } catch (error) { try { if (error.response.data.errors[0].code == 'resource_not_found') { let dockerRepoMakeUrl = `${pollyEnv.baseV2Api}/organizations/${orgId}/resources/dockers/${dockerId}` const dockerInfo = await pollyApi.get(dockerRepoMakeUrl, await pollyHeaders.getV2Headers()); pollymsg.pollyError(`No commits pushed to the docker repository ${pollyEnv.dockerDomain}/${dockerInfo.data.data.attributes.resource_name}`); } } catch (error) { pollymsg.pollyError(`Not able to get the docker commits`); } pollymsg.pollyError(`Not able to get the docker commits`); } if (dockerCommitsResponse.length < 1 && !internalCall) { pollymsg.pollyMessage(`No docker commits present for this docker repository`); process.exit(0); } return dockerCommitsResponse; } export async function listAllDockerRepos(own = true, get_public = false, all = false) { let accessType = 'own'; if (get_public) { accessType = 'public'; } else if (all) { accessType = 'all'; } const organizationInfo = await organizationDetails(); const orgId = organizationInfo.org_id; const dockerResponse = await getAllDockerRepos(orgId, accessType); try { const colConfig = { 0: { width: 70, } }; let dockerTable = chalk.green.bold(['docker repository', 'created by', 'created at', 'docker access']).split(","); let stream = null; stream = pollymsg.pollyStreamTable(stream, dockerTable, colConfig) dockerResponse.sort((a, b) => { return b.attributes.created_at - a.attributes.created_at }); for (const docker of dockerResponse) { const creatorEmail = docker.attributes.creator_email; const accessType = Object.keys(docker.attributes).indexOf('resource_access') > -1 && docker.attributes.resource_access == 'public' ? "public" : "private"; stream = pollymsg.pollyStreamTable(stream, [`${pollyEnv.dockerDomain}/${docker.attributes.resource_name}`, creatorEmail, pollyHelpers.dateFormat(docker.attributes.created_at), accessType]) } console.log('\n'); } catch (error) { pollymsg.pollyError(`Not able to display the docker repositories`); } } export async function listAllDockerCommits(dockerName) { const organizationInfo = await organizationDetails(); const orgId = organizationInfo.org_id; const orgSlug = organizationInfo.slug; let dockerListType = 'own'; if (dockerName.split('/')[0] != orgSlug) { dockerListType = 'public'; } const dockerRepoList = await getAllDockerRepos(orgId, dockerListType); let searchRepo = null; let foundRepo = false; for (const dockerRepo of dockerRepoList) { if (dockerRepo.attributes.resource_name == dockerName) { searchRepo = dockerRepo; foundRepo = true; break; } } if (!foundRepo) { pollymsg.pollyError(`Not able to find the docker repository ${dockerName}`) } const dockerId = searchRepo.id; const dockerCommitsResponse = await getAllDockerCommits(orgId, dockerId, dockerListType == 'public'); try { console.log(`Commits for the docker repository ${chalk.green.bold(pollyEnv.dockerDomain + '/' + dockerName)}`) let dockerTable = chalk.green.bold(['tag', 'pushed by', 'pushed at', 'commit_hash', 'tag status']).split(","); let stream = null; const colConfig = { 3: { width: 75, }, 4: { width: 10, }, }; stream = pollymsg.pollyStreamTable(stream, dockerTable, colConfig); for (const dockerCommits of dockerCommitsResponse) { const arryToSort = Object.keys(dockerCommits.attributes.commit_info); arryToSort.sort((a, b) => { return dockerCommits.attributes.commit_info[b].pushed_at - dockerCommits.attributes.commit_info[a].pushed_at }); for (const version of arryToSort) { const versionData = dockerCommits.attributes.commit_info[version]; const tagName = dockerCommits.attributes.last_modified_at == versionData.pushed_at ? chalk.greenBright(dockerCommits.attributes.tag_name) : chalk.redBright(dockerCommits.attributes.tag_name); const tagStatus = dockerCommits.attributes.last_modified_at == versionData.pushed_at ? chalk.greenBright('active') : chalk.redBright('inactive'); stream = pollymsg.pollyStreamTable(stream, [tagName, versionData.pushed_by_email, pollyHelpers.dateFormat(versionData.pushed_at), version, tagStatus]) } } console.log(`\nTo pull the docker for: ${chalk.greenBright('active tag')}: docker pull ${pollyEnv.dockerDomain}/${dockerName}:tag ${chalk.redBright('inactive tag')}: docker pull ${pollyEnv.dockerDomain}/${dockerName}@commit_hash`); } catch (error) { pollymsg.pollyError(`Not able to display docker commits`); } } export async function createADockerRepo(dockerName, dockerDescription, isPublic = false, internalCall = false) { const organizationInfo = await organizationDetails(); const orgId = organizationInfo.org_id; const resourceAccess = isPublic ? "public" : "private" const dockerPayload = { "data": { "type": "dockers", "attributes": { "resource_name": dockerName, "resource_info": { "description": dockerDescription }, "resource_access": resourceAccess } } } try { const dockerRepoMakeUrl = `/organizations/${orgId}/resources/dockers` const dockerResponse = await pollyApi.post(dockerRepoMakeUrl, dockerPayload, await pollyHeaders.getV2Headers()); if (internalCall) { return true; } pollymsg.pollySuccess("docker repository generation"); let dockerTable = [ ['docker name', 'description'], [`${pollyEnv.dockerDomain}/${dockerResponse.data.data.attributes.resource_name}`, dockerResponse.data.data.attributes.resource_info.description] ] pollymsg.pollyTable(dockerTable) } catch (error) { const err_msg = `Not able to create the docker repository named ${dockerName}`; if (internalCall) { return false; } try { if (error.response.data.errors[0].title == 'Organization slug missing') { pollymsg.pollyError(`${error.response.data.errors[0].title}. Please contact the customer support at polly@elucidata.io`) } } catch (error) { pollymsg.pollyError(err_msg); } if(error.isAxiosError) { handleApiError(error, err_msg); } pollymsg.pollyError(err_msg); } } export async function dockerLoginCommand() { const dockerCredsData = await dockerLoginFormat(); const dockerPassword = Buffer.from(dockerCredsData).toString('base64'); console.log(`docker login -u POLLY -p ${dockerPassword} ${pollyEnv.dockerDomain}`); } export async function dockerLogoutCommand() { console.log(`docker logout ${pollyEnv.dockerDomain}`); } async function dockerLoginFormat() { const email = pollystore.get('pollyUser').pollyemail; const otherCreds = pollystore.get(email); const organizationInfo = await organizationDetails(); const orgId = organizationInfo.org_id; let returnValue = { "pollyUser": { "pollyemail": email, "organizationId": orgId }, "pollyIdToken": { "key": `CognitoIdentityServiceProvider.${otherCreds.pollyAud}.${otherCreds.pollySub}.idToken`, "value": otherCreds.pollyIdToken }, "pollyRefreshToken": { "key": `CognitoIdentityServiceProvider.${otherCreds.pollyAud}.${otherCreds.pollySub}.refreshToken`, "value": otherCreds.pollyrefreshToken } } return JSON.stringify(returnValue, null, ' '); } export async function updateDockerRepo(dockerName, dockerStatus) { const organizationInfo = await organizationDetails(); const orgId = organizationInfo.org_id; const resourceAccess = dockerStatus ? "public" : "private"; const dockerRepoList = await getAllDockerRepos(organizationInfo.org_id, 'own'); let modifiedDocker = null; for (const eachDocker of dockerRepoList) { if (eachDocker.attributes.resource_name == `${organizationInfo.slug}/${dockerName}`) { modifiedDocker = eachDocker; break; } } const dockerPayload = { "data": { "type": "dockers", "id": modifiedDocker.id, "attributes": { "resource_access": resourceAccess } } } try { const dockerRepoMakeUrl = `/organizations/${orgId}/resources/dockers/${modifiedDocker.id}`; await pollyApi.patch(dockerRepoMakeUrl, dockerPayload); pollymsg.pollySuccess('Docker information updated') } catch (error) { pollymsg.pollyError('Not able to update the docker information'); } } function checkDockerValid(dockerFilePath) { const configPath = path.dirname(dockerFilePath) const dockerContent = fs.readFileSync(dockerFilePath).toString(); const validationResults = dockerLint.run(configPath, dockerContent); const finalResultsError = []; const finalResultsOthers = []; const reporter = new CliReporter(); for (const vRes of validationResults) { if (vRes.category.indexOf('Possible Bug') > -1) { finalResultsError.push(vRes); } else if (global.checkData) { finalResultsOthers.push(vRes); } } reporter.addFile(dockerFilePath, dockerContent, [...finalResultsError, ...finalResultsOthers]); var report = reporter.buildReport(); console.log(report.toString()); if (finalResultsError.length > 0) { process.exit(1); } else if (finalResultsOthers.length > 0) { process.exit(0); } } export async function dockerBuild(dockerNameWithTag, localComponentPath = '.') { localComponentPath = path.resolve(localComponentPath); if (fs.lstatSync(localComponentPath).isDirectory()) { if (!localComponentPath.endsWith('/')) { localComponentPath = `${localComponentPath}/`; } if (!fs.existsSync(`${localComponentPath}Dockerfile`)) { pollymsg.pollyError('Not able to find the Dockerfile in the given path'); } checkDockerValid(`${localComponentPath}Dockerfile`); } else { checkDockerValid(localComponentPath); } if (fs.lstatSync(localComponentPath).isFile()) { if (!fs.existsSync(localComponentPath)) { pollymsg.pollyError('Not able to find the Dockerfile in the given path'); } } const organizationInfo = await organizationDetails(); const orgSlug = organizationInfo.slug; const orgId = organizationInfo.org_id; const partialDockernameWithTag = dockerNameWithTag.split(`${pollyEnv.dockerDomain}/${orgSlug}/`)[1]; if(!partialDockernameWithTag) { pollymsg.pollyError('Not enough permission to build/push the Docker'); } if (partialDockernameWithTag.split(":").length < 2) { pollymsg.pollyError('Tag not present in the docker name'); } let foundDocker = false; const allDockerImages = await getAllDockerRepos(orgId); for (const dockers of allDockerImages) { if (`${pollyEnv.dockerDomain}/${dockers.attributes.resource_name}` == `${pollyEnv.dockerDomain}/${orgSlug}/${partialDockernameWithTag.split(":")[0]}`) { foundDocker = true; break; } } if (!foundDocker) { pollymsg.pollyError("Seems like new component creation was not successful or the component is deleted"); } const compressedFileName = `/tmp/${Math.random().toString(36).slice(2)}.tar.gz` let tarWorkingDir = localComponentPath; if (fs.lstatSync(localComponentPath).isFile()) { if (`${localComponentPath}` !== `${path.dirname(localComponentPath)}/Dockerfile`) { fs.copyFileSync(localComponentPath, `${path.dirname(localComponentPath)}/Dockerfile`) } tarWorkingDir = path.dirname(localComponentPath); } const isCompressed = await compressDockerCode(compressedFileName, tarWorkingDir); if (!isCompressed) { pollymsg.pollyError('Not able to package the folder'); } if (fs.lstatSync(localComponentPath).isFile()) { if (`${localComponentPath}` !== `${path.dirname(localComponentPath)}/Dockerfile`) { fs.unlinkSync(`${path.dirname(localComponentPath)}/Dockerfile`) } } const workspaceId = await getWorkSpaceId('polly_automated_docker_builder'); if (!workspaceId) { pollymsg.pollyError('Not able to get the workspaceId to bundle the component'); } let fileName = partialDockernameWithTag.replace(/\//g, '_'); fileName = fileName.replace(/\:/g, '-'); fileName = fileName.replace(/\./g, '-'); fileName = `polly://${fileName}.tar.gz`; if (!fs.existsSync(compressedFileName)) { pollymsg.pollyError('Packaging of code did not happen'); } const fileUploadStatus = await uploadFiles(workspaceId, compressedFileName, fileName, true); if (!fileUploadStatus) { pollymsg.pollyError('Error in uploading the packaged file'); } else { pollymsg.pollyMessage('Dockerfile and its dependencies is getting uploaded. The build process will start soon.') } const jobTemplate = { "machineType": "gp", "image": `${pollyEnv.dockerDomain}/elucidata/polly-docker-builder`, "tag": "latest", "name": `Docker build for ${partialDockernameWithTag}`, "env": { "POLLY_PATH_TO_DOCKER_TAR_GZ": fileName, "POLLY_DOMAIN_DOCKER_NAME_WITH_TAG": `${pollyEnv.dockerDomain}/${orgSlug}/${partialDockernameWithTag}`, }, "secret_env": { "POLLY_ENV": pollyEnv.dockerDomain, } } const jobFile = `/tmp/${Math.random().toString(36).slice(2)}.json` const jobFileData = JSON.stringify(jobTemplate, null, 2); fs.writeFileSync(jobFile, jobFileData); if (!fs.existsSync(jobFile)) { pollymsg.pollyError('Not able to start the component build'); } const submittedData = await submitJob(workspaceId.toString(), jobFile, true); const buff = Buffer.from(JSON.stringify(submittedData)); const base64data = buff.toString('base64'); pollymsg.pollyMessage('\nDocker building has started.\n'); pollymsg.pollyMessage(chalk.white(`To get the status and logs run: ${chalk.italic('polly dockers build-status --id ')} ${chalk.italic(base64data)}`)) } export async function getWorkSpaceId(workspaceName) { const workspaceDesc = 'A workspace in which all the build code for component is kept' const workspaceDataComplete = []; let requestUrl = '' let workspaceData = null; let workSpaceFound = false; let workSpaceId = null; requestUrl = `/workspaces?page%5Bsize%5D=100` do { workspaceData = await pollyApi.get(requestUrl); workspaceDataComplete.push(...workspaceData.data.data) if (workspaceData.data.links.next) { requestUrl = `${workspaceData.data.links.next}` } } while (workspaceData.data.links.next); for (const eachWorkspace of workspaceDataComplete) { if (eachWorkspace.attributes.name == workspaceName) { workSpaceFound = true; workSpaceId = eachWorkspace.attributes.id break; } } if (!workSpaceFound) { const workSpacePayload = { "data": { "type": "workspaces", "attributes": { "name": workspaceName, "description": workspaceDesc, "project_property": { "type": "workspaces", "labels": "" } } } } try { const createdData = await pollyApi.post("/workspaces", workSpacePayload); workSpaceId = createdData.data.data.attributes.id; } catch (error) { pollymsg.pollyError('Not able to create workspaces for build purpose'); } } return workSpaceId; } async function compressDockerCode(compressedFileName, workingDir) { return new Promise((resolve, rejects) => { tar.c({ gzip: true, file: compressedFileName, cwd: workingDir }, ['.'] ).then(_ => { resolve(true); }).catch((err) => { resolve(false); }) }); } async function getPollyProjectsCreds(workspaceId) { if (!workspaceId) { pollymsg.pollyError("No workspace ID given.") } const projectFilesCredsUrl = `/projects/${workspaceId}/credentials/files`; try { const credsResult = await pollyApi.get(projectFilesCredsUrl); const projectCreds = credsResult.data.data[0].attributes.credentials; return projectCreds; } catch (error) { const err_msg = `Not able to get workspace information`; if(error.isAxiosError) { handleApiError(error, err_msg); } pollymsg.pollyError(err_msg); } } async function uploadFiles(workspaceId, sourcePath, destinationPath, internalCall = false) { return new Promise(async (resolve, reject) => { const projectCreds = await getPollyProjectsCreds(workspaceId); const client = s3.createClient({ s3Options: { accessKeyId: projectCreds.AccessKeyId, secretAccessKey: projectCreds.SecretAccessKey, sessionToken: projectCreds.SessionToken, region: process.env.POLLY_ENV ? pollyEnv.region[process.env.POLLY_ENV] : pollyEnv.region["prod"] }, }); let localPath = path.resolve(sourcePath); if (!fs.existsSync(localPath)) { pollymsg.pollyError("Local path does not exist"); } var params = { localFile: localPath, s3Params: { Bucket: projectCreds.bucket, Key: `${workspaceId}/${destinationPath.split("polly://")[1]}`, }, }; let ext = path.extname(sourcePath); if (!destinationPath.split("polly://")[1].endsWith(ext)) { pollymsg.pollyError("remote destination should not be a directory for copy operation"); } if (destinationPath.split("polly://")[1].trim() == '') { pollymsg.pollyError("remote destination should not be a directory for copy operation"); } const uploadSync = client.uploadFile(params); if (!internalCall) { var uploadBar = new cliProgress.SingleBar({ format: PROGRESS_FORMAT, etaBuffer: ETA_BUFFER }, cliProgress.Presets.shades_classic); } let totalData = 0; let sizeProgress = 'NA'; let doneOnce = true; uploadSync.on('error', function (err) { if (!internalCall) { uploadBar.stop(); } resolve(false); }); uploadSync.on('progress', function (data) { if (!internalCall && uploadSync.progressTotal > 0) { if (totalData != uploadSync.progressTotal && uploadSync.progressTotal > 0) { if (doneOnce) { uploadBar.start(uploadSync.progressTotal, uploadSync.progressAmount, { fileSizeProgress: sizeProgress }); doneOnce = false; } uploadBar.setTotal(uploadSync.progressTotal) totalData = uploadSync.progressTotal; } sizeProgress = `${helper.formatBytes(uploadSync.progressAmount)}/${helper.formatBytes(uploadSync.progressTotal)}` uploadBar.update(uploadSync.progressAmount, { fileSizeProgress: sizeProgress }); } }); uploadSync.on('end', function () { if (!internalCall) { uploadBar.stop(); } resolve(true); }); }); } export async function dockerBuildStatus(buildID) { let workspaceId = null; let jobId = null; if (!buildID) { workspaceId = await getWorkSpaceId('polly_automated_docker_builder'); } else { let buff = Buffer.from(buildID, 'base64'); let extractedSecret = buff.toString('ascii'); const jsonData = JSON.parse(extractedSecret); workspaceId = jsonData.data.project_id jobId = jsonData.data.job_id } const statusData = await jobStatus(workspaceId, jobId, true); let dataArray = [ ['Docker', 'Build state', 'Build start time', 'Build ID'], ] for (const eachStatus of statusData) { let conf = eachStatus.attributes.config_json if (typeof conf == 'string') { conf = JSON.parse(eachStatus.attributes.config_json) } const buff = Buffer.from(JSON.stringify({ "data": { "project_id": eachStatus.attributes.project_id, "job_id": eachStatus.attributes.job_id } })); const base64data = buff.toString('base64'); dataArray.push([conf.env.POLLY_DOMAIN_DOCKER_NAME_WITH_TAG, eachStatus.attributes.state, pollyHelpers.dateFormat(eachStatus.attributes.created_ts), base64data]); } const tableConfig = { columns: { 0: { width: 35 }, 2: { width: 17 }, 3: { width: 60 } } }; pollymsg.pollyTable(dataArray, 1, dataArray.length, tableConfig); if (buildID) { await getJobLogs(workspaceId, jobId, global.getTailData ? 'latest':'all'); } }