@pradyumn-el/pollycli
Version:
pollycli lets users access the functionalities of Polly over a command line interface
680 lines (620 loc) • 24.4 kB
JavaScript
const pollyEnv = require('./env.json');
const pollyHeaders = require('./pollyheaders');
const pollymsg = require('./message');
const chalk = require('chalk');
const pollyHelpers = require('./polly-helpers');
const { handleApiError } = require('./helper-functions');
const path = require('path');
const fs = require('fs');
const tar = require('tar');
const dockerLint = require("dockerfilelint")
const dockerLintReport = require("dockerfilelint/lib/reporter/json_reporter");
const s3 = require('@elucidatainc/s3-node-client');
import { organizationDetails } from './organization';
import {
submitJob,
jobStatus
} from './job';
import {
getJobLogs
} from './logs';
import CliReporter from 'dockerfilelint/lib/reporter/cli_reporter';
const { pollyApi } = require('./api-client');
export async function getAllDockerRepos(orgId, accessType = 'all') {
const dockerResponse = [];
let allTypes = {};
if (accessType == 'own') {
allTypes = { 'own': '' };
} else if (accessType == 'public') {
allTypes = { 'public': '?filter[resource_access]=public&filter[scope]=global' };
} else {
allTypes = { 'own': '', 'public': '?filter[resource_access]=public&filter[scope]=global' };
}
try {
for (const typeIn of Object.keys(allTypes)) {
let dockerRepoMakeUrl = `/organizations/${orgId}/resources/dockers${allTypes[typeIn]}`
let dockerCommitsResponseInter = null;
do {
try {
dockerCommitsResponseInter = await pollyApi.get(dockerRepoMakeUrl);
for (const eachDocker of dockerCommitsResponseInter.data.data) {
if (accessType == 'own') {
dockerResponse.push(eachDocker);
} else if (accessType == 'public' && eachDocker.attributes.resource_access == "public") {
dockerResponse.push(eachDocker);
} else if (accessType == 'all' && !(typeIn == 'public' && eachDocker.attributes.org_id == orgId)) {
dockerResponse.push(eachDocker);
}
}
dockerCommitsResponseInter = await pollyApi.get(dockerRepoMakeUrl);
if (dockerCommitsResponseInter.data.links.next) {
dockerRepoMakeUrl = `${dockerCommitsResponseInter.data.links.next}`
}
} catch(e) {
// ignore if any one of the api calls returns error
}
} while (dockerCommitsResponseInter && dockerCommitsResponseInter.data.links.next);
}
} catch (error) {
if (dockerResponse.length < 1) {
try {
if (error.response.data.errors[0].code == 'resource_not_found') {
pollymsg.pollyError("No docker repositories present. Create docker repository with polly dockers create");
}
} catch (error) {
pollymsg.pollyError(`Not able to get the docker repository`);
}
pollymsg.pollyError(`Not able to get the docker repository`);
}
}
if (dockerResponse.length < 1) {
pollymsg.pollyMessage(`No docker repositories present`);
process.exit(0);
}
return dockerResponse;
}
export async function getAllDockerCommits(orgId, dockerId, isPublic = false, internalCall = false) {
const dockerCommitBase = `/organizations/${orgId}/resources/dockers/${dockerId}/commits`;
let dockerRepoMakeUrl;
const dockerCommitsResponse = []
let dockerCommitsResponseInter = null;
let urlAppend = null;
try {
if (isPublic) {
urlAppend = `resource_access=public`
} else {
urlAppend = `sort=-last_modified_at`
}
dockerRepoMakeUrl = `${dockerCommitBase}?${urlAppend}`
do {
dockerCommitsResponseInter = await pollyApi.get(dockerRepoMakeUrl);
if (dockerCommitsResponseInter.data.data.length > 0) {
dockerCommitsResponse.push(...dockerCommitsResponseInter.data.data);
}
if (dockerCommitsResponseInter.data.data.length > 0 && dockerCommitsResponseInter.data.links.next) {
urlAppend = dockerCommitsResponseInter.data.links.next.split('commits?')[1];
dockerRepoMakeUrl = `${dockerCommitBase}?${urlAppend}`
}
} while (dockerCommitsResponseInter.data.data.length > 0 && dockerCommitsResponseInter.data.links.next);
} catch (error) {
try {
if (error.response.data.errors[0].code == 'resource_not_found') {
let dockerRepoMakeUrl = `${pollyEnv.baseV2Api}/organizations/${orgId}/resources/dockers/${dockerId}`
const dockerInfo = await pollyApi.get(dockerRepoMakeUrl, await pollyHeaders.getV2Headers());
pollymsg.pollyError(`No commits pushed to the docker repository ${pollyEnv.dockerDomain}/${dockerInfo.data.data.attributes.resource_name}`);
}
} catch (error) {
pollymsg.pollyError(`Not able to get the docker commits`);
}
pollymsg.pollyError(`Not able to get the docker commits`);
}
if (dockerCommitsResponse.length < 1 && !internalCall) {
pollymsg.pollyMessage(`No docker commits present for this docker repository`);
process.exit(0);
}
return dockerCommitsResponse;
}
export async function listAllDockerRepos(own = true, get_public = false, all = false) {
let accessType = 'own';
if (get_public) {
accessType = 'public';
} else if (all) {
accessType = 'all';
}
const organizationInfo = await organizationDetails();
const orgId = organizationInfo.org_id;
const dockerResponse = await getAllDockerRepos(orgId, accessType);
try {
const colConfig = {
0: {
width: 70,
}
};
let dockerTable = chalk.green.bold(['docker repository', 'created by', 'created at', 'docker access']).split(",");
let stream = null;
stream = pollymsg.pollyStreamTable(stream, dockerTable, colConfig)
dockerResponse.sort((a, b) => {
return b.attributes.created_at - a.attributes.created_at
});
for (const docker of dockerResponse) {
const creatorEmail = docker.attributes.creator_email;
const accessType = Object.keys(docker.attributes).indexOf('resource_access') > -1 && docker.attributes.resource_access == 'public' ? "public" : "private";
stream = pollymsg.pollyStreamTable(stream, [`${pollyEnv.dockerDomain}/${docker.attributes.resource_name}`, creatorEmail, pollyHelpers.dateFormat(docker.attributes.created_at), accessType])
}
console.log('\n');
} catch (error) {
pollymsg.pollyError(`Not able to display the docker repositories`);
}
}
export async function listAllDockerCommits(dockerName) {
const organizationInfo = await organizationDetails();
const orgId = organizationInfo.org_id;
const orgSlug = organizationInfo.slug;
let dockerListType = 'own';
if (dockerName.split('/')[0] != orgSlug) {
dockerListType = 'public';
}
const dockerRepoList = await getAllDockerRepos(orgId, dockerListType);
let searchRepo = null;
let foundRepo = false;
for (const dockerRepo of dockerRepoList) {
if (dockerRepo.attributes.resource_name == dockerName) {
searchRepo = dockerRepo;
foundRepo = true;
break;
}
}
if (!foundRepo) {
pollymsg.pollyError(`Not able to find the docker repository ${dockerName}`)
}
const dockerId = searchRepo.id;
const dockerCommitsResponse = await getAllDockerCommits(orgId, dockerId, dockerListType == 'public');
try {
console.log(`Commits for the docker repository ${chalk.green.bold(pollyEnv.dockerDomain + '/' + dockerName)}`)
let dockerTable = chalk.green.bold(['tag', 'pushed by', 'pushed at', 'commit_hash', 'tag status']).split(",");
let stream = null;
const colConfig = {
3: {
width: 75,
},
4: {
width: 10,
},
};
stream = pollymsg.pollyStreamTable(stream, dockerTable, colConfig);
for (const dockerCommits of dockerCommitsResponse) {
const arryToSort = Object.keys(dockerCommits.attributes.commit_info);
arryToSort.sort((a, b) => {
return dockerCommits.attributes.commit_info[b].pushed_at - dockerCommits.attributes.commit_info[a].pushed_at
});
for (const version of arryToSort) {
const versionData = dockerCommits.attributes.commit_info[version];
const tagName = dockerCommits.attributes.last_modified_at == versionData.pushed_at ? chalk.greenBright(dockerCommits.attributes.tag_name) : chalk.redBright(dockerCommits.attributes.tag_name);
const tagStatus = dockerCommits.attributes.last_modified_at == versionData.pushed_at ? chalk.greenBright('active') : chalk.redBright('inactive');
stream = pollymsg.pollyStreamTable(stream, [tagName, versionData.pushed_by_email, pollyHelpers.dateFormat(versionData.pushed_at), version, tagStatus])
}
}
console.log(`\nTo pull the docker for:
${chalk.greenBright('active tag')}: docker pull ${pollyEnv.dockerDomain}/${dockerName}:tag
${chalk.redBright('inactive tag')}: docker pull ${pollyEnv.dockerDomain}/${dockerName}@commit_hash`);
} catch (error) {
pollymsg.pollyError(`Not able to display docker commits`);
}
}
export async function createADockerRepo(dockerName, dockerDescription, isPublic = false, internalCall = false) {
const organizationInfo = await organizationDetails();
const orgId = organizationInfo.org_id;
const resourceAccess = isPublic ? "public" : "private"
const dockerPayload = {
"data": {
"type": "dockers",
"attributes": {
"resource_name": dockerName,
"resource_info": {
"description": dockerDescription
},
"resource_access": resourceAccess
}
}
}
try {
const dockerRepoMakeUrl = `/organizations/${orgId}/resources/dockers`
const dockerResponse = await pollyApi.post(dockerRepoMakeUrl, dockerPayload, await pollyHeaders.getV2Headers());
if (internalCall) {
return true;
}
pollymsg.pollySuccess("docker repository generation");
let dockerTable = [
['docker name', 'description'],
[`${pollyEnv.dockerDomain}/${dockerResponse.data.data.attributes.resource_name}`, dockerResponse.data.data.attributes.resource_info.description]
]
pollymsg.pollyTable(dockerTable)
} catch (error) {
const err_msg = `Not able to create the docker repository named ${dockerName}`;
if (internalCall) {
return false;
}
try {
if (error.response.data.errors[0].title == 'Organization slug missing') {
pollymsg.pollyError(`${error.response.data.errors[0].title}. Please contact the customer support at polly@elucidata.io`)
}
} catch (error) {
pollymsg.pollyError(err_msg);
}
if(error.isAxiosError) {
handleApiError(error, err_msg);
}
pollymsg.pollyError(err_msg);
}
}
export async function dockerLoginCommand() {
const dockerCredsData = await dockerLoginFormat();
const dockerPassword = Buffer.from(dockerCredsData).toString('base64');
console.log(`docker login -u POLLY -p ${dockerPassword} ${pollyEnv.dockerDomain}`);
}
export async function dockerLogoutCommand() {
console.log(`docker logout ${pollyEnv.dockerDomain}`);
}
async function dockerLoginFormat() {
const email = pollystore.get('pollyUser').pollyemail;
const otherCreds = pollystore.get(email);
const organizationInfo = await organizationDetails();
const orgId = organizationInfo.org_id;
let returnValue = {
"pollyUser": {
"pollyemail": email,
"organizationId": orgId
},
"pollyIdToken": {
"key": `CognitoIdentityServiceProvider.${otherCreds.pollyAud}.${otherCreds.pollySub}.idToken`,
"value": otherCreds.pollyIdToken
},
"pollyRefreshToken": {
"key": `CognitoIdentityServiceProvider.${otherCreds.pollyAud}.${otherCreds.pollySub}.refreshToken`,
"value": otherCreds.pollyrefreshToken
}
}
return JSON.stringify(returnValue, null, ' ');
}
export async function updateDockerRepo(dockerName, dockerStatus) {
const organizationInfo = await organizationDetails();
const orgId = organizationInfo.org_id;
const resourceAccess = dockerStatus ? "public" : "private";
const dockerRepoList = await getAllDockerRepos(organizationInfo.org_id, 'own');
let modifiedDocker = null;
for (const eachDocker of dockerRepoList) {
if (eachDocker.attributes.resource_name == `${organizationInfo.slug}/${dockerName}`) {
modifiedDocker = eachDocker;
break;
}
}
const dockerPayload = {
"data": {
"type": "dockers",
"id": modifiedDocker.id,
"attributes": {
"resource_access": resourceAccess
}
}
}
try {
const dockerRepoMakeUrl = `/organizations/${orgId}/resources/dockers/${modifiedDocker.id}`;
await pollyApi.patch(dockerRepoMakeUrl, dockerPayload);
pollymsg.pollySuccess('Docker information updated')
} catch (error) {
pollymsg.pollyError('Not able to update the docker information');
}
}
function checkDockerValid(dockerFilePath) {
const configPath = path.dirname(dockerFilePath)
const dockerContent = fs.readFileSync(dockerFilePath).toString();
const validationResults = dockerLint.run(configPath, dockerContent);
const finalResultsError = [];
const finalResultsOthers = [];
const reporter = new CliReporter();
for (const vRes of validationResults) {
if (vRes.category.indexOf('Possible Bug') > -1) {
finalResultsError.push(vRes);
} else if (global.checkData) {
finalResultsOthers.push(vRes);
}
}
reporter.addFile(dockerFilePath, dockerContent, [...finalResultsError, ...finalResultsOthers]);
var report = reporter.buildReport();
console.log(report.toString());
if (finalResultsError.length > 0) {
process.exit(1);
} else if (finalResultsOthers.length > 0) {
process.exit(0);
}
}
export async function dockerBuild(dockerNameWithTag, localComponentPath = '.') {
localComponentPath = path.resolve(localComponentPath);
if (fs.lstatSync(localComponentPath).isDirectory()) {
if (!localComponentPath.endsWith('/')) {
localComponentPath = `${localComponentPath}/`;
}
if (!fs.existsSync(`${localComponentPath}Dockerfile`)) {
pollymsg.pollyError('Not able to find the Dockerfile in the given path');
}
checkDockerValid(`${localComponentPath}Dockerfile`);
} else {
checkDockerValid(localComponentPath);
}
if (fs.lstatSync(localComponentPath).isFile()) {
if (!fs.existsSync(localComponentPath)) {
pollymsg.pollyError('Not able to find the Dockerfile in the given path');
}
}
const organizationInfo = await organizationDetails();
const orgSlug = organizationInfo.slug;
const orgId = organizationInfo.org_id;
const partialDockernameWithTag = dockerNameWithTag.split(`${pollyEnv.dockerDomain}/${orgSlug}/`)[1];
if(!partialDockernameWithTag) {
pollymsg.pollyError('Not enough permission to build/push the Docker');
}
if (partialDockernameWithTag.split(":").length < 2) {
pollymsg.pollyError('Tag not present in the docker name');
}
let foundDocker = false;
const allDockerImages = await getAllDockerRepos(orgId);
for (const dockers of allDockerImages) {
if (`${pollyEnv.dockerDomain}/${dockers.attributes.resource_name}` == `${pollyEnv.dockerDomain}/${orgSlug}/${partialDockernameWithTag.split(":")[0]}`) {
foundDocker = true;
break;
}
}
if (!foundDocker) {
pollymsg.pollyError("Seems like new component creation was not successful or the component is deleted");
}
const compressedFileName = `/tmp/${Math.random().toString(36).slice(2)}.tar.gz`
let tarWorkingDir = localComponentPath;
if (fs.lstatSync(localComponentPath).isFile()) {
if (`${localComponentPath}` !== `${path.dirname(localComponentPath)}/Dockerfile`) {
fs.copyFileSync(localComponentPath, `${path.dirname(localComponentPath)}/Dockerfile`)
}
tarWorkingDir = path.dirname(localComponentPath);
}
const isCompressed = await compressDockerCode(compressedFileName, tarWorkingDir);
if (!isCompressed) {
pollymsg.pollyError('Not able to package the folder');
}
if (fs.lstatSync(localComponentPath).isFile()) {
if (`${localComponentPath}` !== `${path.dirname(localComponentPath)}/Dockerfile`) {
fs.unlinkSync(`${path.dirname(localComponentPath)}/Dockerfile`)
}
}
const workspaceId = await getWorkSpaceId('polly_automated_docker_builder');
if (!workspaceId) {
pollymsg.pollyError('Not able to get the workspaceId to bundle the component');
}
let fileName = partialDockernameWithTag.replace(/\//g, '_');
fileName = fileName.replace(/\:/g, '-');
fileName = fileName.replace(/\./g, '-');
fileName = `polly://${fileName}.tar.gz`;
if (!fs.existsSync(compressedFileName)) {
pollymsg.pollyError('Packaging of code did not happen');
}
const fileUploadStatus = await uploadFiles(workspaceId, compressedFileName, fileName, true);
if (!fileUploadStatus) {
pollymsg.pollyError('Error in uploading the packaged file');
} else {
pollymsg.pollyMessage('Dockerfile and its dependencies is getting uploaded. The build process will start soon.')
}
const jobTemplate = {
"machineType": "gp",
"image": `${pollyEnv.dockerDomain}/elucidata/polly-docker-builder`,
"tag": "latest",
"name": `Docker build for ${partialDockernameWithTag}`,
"env": {
"POLLY_PATH_TO_DOCKER_TAR_GZ": fileName,
"POLLY_DOMAIN_DOCKER_NAME_WITH_TAG": `${pollyEnv.dockerDomain}/${orgSlug}/${partialDockernameWithTag}`,
},
"secret_env": {
"POLLY_ENV": pollyEnv.dockerDomain,
}
}
const jobFile = `/tmp/${Math.random().toString(36).slice(2)}.json`
const jobFileData = JSON.stringify(jobTemplate, null, 2);
fs.writeFileSync(jobFile, jobFileData);
if (!fs.existsSync(jobFile)) {
pollymsg.pollyError('Not able to start the component build');
}
const submittedData = await submitJob(workspaceId.toString(), jobFile, true);
const buff = Buffer.from(JSON.stringify(submittedData));
const base64data = buff.toString('base64');
pollymsg.pollyMessage('\nDocker building has started.\n');
pollymsg.pollyMessage(chalk.white(`To get the status and logs run:
${chalk.italic('polly dockers build-status --id ')} ${chalk.italic(base64data)}`))
}
export async function getWorkSpaceId(workspaceName) {
const workspaceDesc = 'A workspace in which all the build code for component is kept'
const workspaceDataComplete = [];
let requestUrl = ''
let workspaceData = null;
let workSpaceFound = false;
let workSpaceId = null;
requestUrl = `/workspaces?page%5Bsize%5D=100`
do {
workspaceData = await pollyApi.get(requestUrl);
workspaceDataComplete.push(...workspaceData.data.data)
if (workspaceData.data.links.next) {
requestUrl = `${workspaceData.data.links.next}`
}
} while (workspaceData.data.links.next);
for (const eachWorkspace of workspaceDataComplete) {
if (eachWorkspace.attributes.name == workspaceName) {
workSpaceFound = true;
workSpaceId = eachWorkspace.attributes.id
break;
}
}
if (!workSpaceFound) {
const workSpacePayload = {
"data": {
"type": "workspaces",
"attributes": {
"name": workspaceName,
"description": workspaceDesc,
"project_property": {
"type": "workspaces",
"labels": ""
}
}
}
}
try {
const createdData = await pollyApi.post("/workspaces", workSpacePayload);
workSpaceId = createdData.data.data.attributes.id;
} catch (error) {
pollymsg.pollyError('Not able to create workspaces for build purpose');
}
}
return workSpaceId;
}
async function compressDockerCode(compressedFileName, workingDir) {
return new Promise((resolve, rejects) => {
tar.c({
gzip: true,
file: compressedFileName,
cwd: workingDir
},
['.']
).then(_ => {
resolve(true);
}).catch((err) => {
resolve(false);
})
});
}
async function getPollyProjectsCreds(workspaceId) {
if (!workspaceId) {
pollymsg.pollyError("No workspace ID given.")
}
const projectFilesCredsUrl = `/projects/${workspaceId}/credentials/files`;
try {
const credsResult = await pollyApi.get(projectFilesCredsUrl);
const projectCreds = credsResult.data.data[0].attributes.credentials;
return projectCreds;
} catch (error) {
const err_msg = `Not able to get workspace information`;
if(error.isAxiosError) {
handleApiError(error, err_msg);
}
pollymsg.pollyError(err_msg);
}
}
async function uploadFiles(workspaceId, sourcePath, destinationPath, internalCall = false) {
return new Promise(async (resolve, reject) => {
const projectCreds = await getPollyProjectsCreds(workspaceId);
const client = s3.createClient({
s3Options: {
accessKeyId: projectCreds.AccessKeyId,
secretAccessKey: projectCreds.SecretAccessKey,
sessionToken: projectCreds.SessionToken,
region: process.env.POLLY_ENV ? pollyEnv.region[process.env.POLLY_ENV] : pollyEnv.region["prod"]
},
});
let localPath = path.resolve(sourcePath);
if (!fs.existsSync(localPath)) {
pollymsg.pollyError("Local path does not exist");
}
var params = {
localFile: localPath,
s3Params: {
Bucket: projectCreds.bucket,
Key: `${workspaceId}/${destinationPath.split("polly://")[1]}`,
},
};
let ext = path.extname(sourcePath);
if (!destinationPath.split("polly://")[1].endsWith(ext)) {
pollymsg.pollyError("remote destination should not be a directory for copy operation");
}
if (destinationPath.split("polly://")[1].trim() == '') {
pollymsg.pollyError("remote destination should not be a directory for copy operation");
}
const uploadSync = client.uploadFile(params);
if (!internalCall) {
var uploadBar = new cliProgress.SingleBar({
format: PROGRESS_FORMAT,
etaBuffer: ETA_BUFFER
}, cliProgress.Presets.shades_classic);
}
let totalData = 0;
let sizeProgress = 'NA';
let doneOnce = true;
uploadSync.on('error', function (err) {
if (!internalCall) {
uploadBar.stop();
}
resolve(false);
});
uploadSync.on('progress', function (data) {
if (!internalCall && uploadSync.progressTotal > 0) {
if (totalData != uploadSync.progressTotal && uploadSync.progressTotal > 0) {
if (doneOnce) {
uploadBar.start(uploadSync.progressTotal, uploadSync.progressAmount, {
fileSizeProgress: sizeProgress
});
doneOnce = false;
}
uploadBar.setTotal(uploadSync.progressTotal)
totalData = uploadSync.progressTotal;
}
sizeProgress = `${helper.formatBytes(uploadSync.progressAmount)}/${helper.formatBytes(uploadSync.progressTotal)}`
uploadBar.update(uploadSync.progressAmount, {
fileSizeProgress: sizeProgress
});
}
});
uploadSync.on('end', function () {
if (!internalCall) {
uploadBar.stop();
}
resolve(true);
});
});
}
export async function dockerBuildStatus(buildID) {
let workspaceId = null;
let jobId = null;
if (!buildID) {
workspaceId = await getWorkSpaceId('polly_automated_docker_builder');
} else {
let buff = Buffer.from(buildID, 'base64');
let extractedSecret = buff.toString('ascii');
const jsonData = JSON.parse(extractedSecret);
workspaceId = jsonData.data.project_id
jobId = jsonData.data.job_id
}
const statusData = await jobStatus(workspaceId, jobId, true);
let dataArray = [
['Docker', 'Build state', 'Build start time', 'Build ID'],
]
for (const eachStatus of statusData) {
let conf = eachStatus.attributes.config_json
if (typeof conf == 'string') {
conf = JSON.parse(eachStatus.attributes.config_json)
}
const buff = Buffer.from(JSON.stringify({ "data": { "project_id": eachStatus.attributes.project_id, "job_id": eachStatus.attributes.job_id } }));
const base64data = buff.toString('base64');
dataArray.push([conf.env.POLLY_DOMAIN_DOCKER_NAME_WITH_TAG, eachStatus.attributes.state, pollyHelpers.dateFormat(eachStatus.attributes.created_ts), base64data]);
}
const tableConfig = {
columns: {
0: {
width: 35
},
2: {
width: 17
},
3: {
width: 60
}
}
};
pollymsg.pollyTable(dataArray, 1, dataArray.length, tableConfig);
if (buildID) {
await getJobLogs(workspaceId, jobId, global.getTailData ? 'latest':'all');
}
}