@topgroup/diginext
Version:
A BUILD SERVER & CLI to deploy apps to any Kubernetes clusters.
513 lines (512 loc) • 25.1 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.startBuild = exports.stopBuild = exports.saveLogs = exports.testBuild = exports.queue = void 0;
const chalk_1 = __importDefault(require("chalk"));
const dayjs_1 = __importDefault(require("dayjs"));
const log_1 = require("diginext-utils/dist/xconsole/log");
const humanize_duration_1 = __importDefault(require("humanize-duration"));
const lodash_1 = require("lodash");
const mongoose_1 = require("mongoose");
const p_queue_1 = __importDefault(require("p-queue"));
const path_1 = __importDefault(require("path"));
const app_config_1 = require("../../app.config");
const const_1 = require("../../config/const");
const entities_1 = require("../../entities");
const plugins_1 = require("../../plugins");
const array_1 = require("../../plugins/array");
const mongodb_1 = require("../../plugins/mongodb");
const server_1 = require("../../server");
const services_1 = require("../../services");
const builder_1 = __importDefault(require("../builder"));
const docker_1 = require("../builder/docker");
const create_build_slug_1 = require("../deploy/create-build-slug");
const git_utils_1 = require("../git/git-utils");
const connect_registry_1 = require("../registry/connect-registry");
const send_log_message_1 = require("./send-log-message");
const update_build_status_1 = require("./update-build-status");
exports.queue = new p_queue_1.default({ concurrency: 1 });
async function testBuild() {
let socketServer = (0, server_1.getIO)();
(0, log_1.log)("socketServer:", socketServer);
}
exports.testBuild = testBuild;
/**
* Save build log content to database
*/
async function saveLogs(buildSlug, logs) {
const { DB } = await Promise.resolve().then(() => __importStar(require("../api/DB")));
if (!buildSlug)
throw new Error(`Build's slug is required, it's empty now.`);
const [build] = await DB.update("build", { slug: buildSlug }, { logs });
return build;
}
exports.saveLogs = saveLogs;
/**
* Stop the build process.
*/
const stopBuild = async (projectSlug, appSlug, buildSlug, status = "failed", deployStatus = "pending") => {
let error;
// Validate...
if (!appSlug) {
error = `App "${appSlug}" not found.`;
(0, log_1.logError)(`[STOP_BUILD]`, error);
return { error };
}
// Stop the f*cking buildx driver...
const builderName = `${projectSlug.toLowerCase()}_${appSlug.toLowerCase()}`;
await builder_1.default[(0, lodash_1.upperFirst)(app_config_1.Config.BUILDER)].stopBuild(builderName);
// Update the status in the database
const stoppedBuild = await (0, update_build_status_1.updateBuildStatusByAppSlug)(appSlug, buildSlug, status, deployStatus);
(0, log_1.logSuccess)(`Build process of "${buildSlug}" has been stopped.`);
return stoppedBuild;
};
exports.stopBuild = stopBuild;
async function startBuild(params, options) {
var _a, _b, _c;
const { DB } = await Promise.resolve().then(() => __importStar(require("../../modules/api/DB")));
// parse variables
const startTime = (0, dayjs_1.default)();
const {
// require
buildTag, buildNumber, message: buildMessage, gitBranch, registrySlug, appSlug,
// optional
userId, args: buildArgs, user, env, buildWatch = true, shouldDeploy = false, isDebugging = false, cliVersion, serverVersion = (0, plugins_1.currentVersion)(), serverLocation = app_config_1.Config.LOCATION, } = params;
// validate
if (!buildTag)
throw new Error(`Unable to start building, "buildTag" is required.`);
if (!gitBranch)
throw new Error(`Unable to start building, "gitBranch" is required.`);
if (!registrySlug)
throw new Error(`Unable to start building, "registrySlug" is required.`);
if (!appSlug)
throw new Error(`Unable to start building, "appSlug" is required.`);
if (!user && !userId)
throw new Error(`Unable to start building, "user" or "userId" is required.`);
const owner = user || (await DB.findOne("user", { _id: userId }, { populate: ["workspaces", "activeWorkspaces"] }));
if (isDebugging)
console.log("owner :>> ", owner);
// get app info
const app = await DB.findOne("app", { slug: appSlug }, { populate: ["owner", "workspace", "project"] });
// project info
if (!app.project || app.project instanceof mongoose_1.Types.ObjectId || typeof app.project === "string")
throw new Error(`Invalid "app.project": "${app.project}", should be an instance of {IProject}.`);
const { project } = app;
const { slug: projectSlug } = project;
// get workspace info
const { activeWorkspace, slug: username } = owner;
const workspace = activeWorkspace;
// socket & logs
const SOCKET_ROOM = (0, create_build_slug_1.createBuildSlug)({ projectSlug, appSlug, buildTag });
const logger = new plugins_1.Logger(SOCKET_ROOM);
// Emit socket message to request the BUILD SERVER to start building...
server_1.socketIO === null || server_1.socketIO === void 0 ? void 0 : server_1.socketIO.to(SOCKET_ROOM).emit("message", { action: "start" });
// Validating...
if ((0, lodash_1.isEmpty)(app)) {
(0, send_log_message_1.sendLog)({ SOCKET_ROOM, type: "error", action: "end", message: `[START BUILD] App "${appSlug}" not found.` });
if (options === null || options === void 0 ? void 0 : options.onError)
options === null || options === void 0 ? void 0 : options.onError(`[START BUILD] App "${appSlug}" not found.`);
return;
}
// the container registry to store this build image
const regModel = (0, mongoose_1.model)("container_registries", entities_1.containerRegistrySchema, "container_registries");
const registry = await regModel.findOne({ slug: registrySlug });
if ((0, lodash_1.isEmpty)(registry)) {
(0, send_log_message_1.sendLog)({ SOCKET_ROOM, type: "error", action: "end", message: `[START BUILD] Container registry "${registrySlug}" not found.` });
if (options === null || options === void 0 ? void 0 : options.onError)
options === null || options === void 0 ? void 0 : options.onError(`[START BUILD] Container registry "${registrySlug}" not found.`);
return;
}
// Git repo of this app
if ((0, lodash_1.isEmpty)(app.git) || (0, lodash_1.isEmpty)((_a = app.git) === null || _a === void 0 ? void 0 : _a.repoSSH)) {
(0, send_log_message_1.sendLog)({
SOCKET_ROOM,
type: "error",
action: "end",
message: `[START BUILD] App "${appSlug}" doesn't have any git repository data (probably deleted?).`,
});
if (options === null || options === void 0 ? void 0 : options.onError)
options === null || options === void 0 ? void 0 : options.onError(`[START BUILD] App "${appSlug}" doesn't have any git repository data (probably deleted?).`);
return;
}
// get latest build of this app to utilize the cache for this build process
const latestBuild = await DB.findOne("build", { appSlug, projectSlug, status: "success" }, { order: { createdAt: -1 }, ignorable: true });
// get app's repository data:
const { git: { repoSSH }, } = app;
if (isDebugging)
(0, log_1.log)("[START BUILD] Input params :>>", params);
/**
* ===============================================
* Specify BUILD DIRECTORY to pull source code to:
* ===============================================
*/
const SOURCE_CODE_DIR = `cache/${projectSlug}/${appSlug}/${gitBranch}`;
let buildDir = app_config_1.isServerMode ? path_1.default.resolve(const_1.CLI_CONFIG_DIR, SOURCE_CODE_DIR) : params.buildDir;
// detect "gitProvider" from git repo SSH URI:
const gitProvider = (0, plugins_1.getGitProviderFromRepoSSH)(repoSSH);
/**
* Generate build number & update build image data
*/
const { image: imageURL = `${registry.imageBaseURL}/${projectSlug}-${app.slug}` } = app;
const buildImage = `${imageURL}:${buildTag}`;
if (params.isDebugging)
console.log("startBuild > imageURL :>> ", imageURL);
/**
* Create new build in database
*/
const buildData = {
name: buildImage,
slug: SOCKET_ROOM,
env,
message: buildMessage,
tag: buildTag,
num: buildNumber,
image: imageURL,
status: "building",
deployStatus: "pending",
startTime: startTime.toDate(),
createdBy: username,
branch: gitBranch,
logs: logger === null || logger === void 0 ? void 0 : logger.content,
registry: registry._id,
app: app._id,
appSlug,
project: project._id,
projectSlug,
owner: owner._id,
ownerSlug: owner.slug,
workspace: workspace._id,
workspaceSlug: workspace.slug,
// versions
cliVersion,
serverVersion,
serverLocation,
};
const newBuild = await DB.create("build", buildData);
if (!newBuild) {
console.log("buildData :>> ", buildData);
(0, send_log_message_1.sendLog)({ SOCKET_ROOM, message: "[START BUILD] Failed to create new build on server." });
if (options === null || options === void 0 ? void 0 : options.onError)
options === null || options === void 0 ? void 0 : options.onError("[START BUILD] Failed to create new build on server.");
return;
}
(0, send_log_message_1.sendLog)({ SOCKET_ROOM, message: "[START BUILD] Created new build on server!" });
// create a webhook
// TODO: check user notification settings -> subscribe to webhook
let webhook;
const webhookSvc = new services_1.WebhookService();
webhookSvc.ownership = { owner, workspace };
if (app_config_1.isServerMode) {
const projectOwner = await DB.findOne("user", { _id: project.owner });
const appOwner = app.owner;
const consumers = (0, array_1.filterUniqueItems)([projectOwner === null || projectOwner === void 0 ? void 0 : projectOwner._id, appOwner === null || appOwner === void 0 ? void 0 : appOwner._id, owner === null || owner === void 0 ? void 0 : owner._id])
.filter((uid) => typeof uid !== "undefined")
.map((uid) => mongodb_1.MongoDB.toString(uid));
webhook = await webhookSvc.create({
events: ["build_status"],
channels: ["email"],
consumers,
workspace: mongodb_1.MongoDB.toString(workspace._id),
project: mongodb_1.MongoDB.toString(project._id),
app: mongodb_1.MongoDB.toString(app._id),
build: mongodb_1.MongoDB.toString(newBuild._id),
});
}
/**
* Verify SSH before cloning/pulling files from a git repository.
*/
// const gitAuth = await verifySSH({ gitProvider });
// if (!gitAuth) {
// // print the logs to client (Dashboard & CLI)
// sendLog({
// SOCKET_ROOM,
// action: "end",
// type: "error",
// message: `[START BUILD] "${buildDir}" -> Failed to verify "${gitProvider}" git SSH key.`,
// });
// if (options?.onError) options?.onError(`[START BUILD] "${buildDir}" -> Failed to verify "${gitProvider}" git SSH key.`);
// // update build status
// await updateBuildStatus(newBuild, "failed");
// // dispatch/trigger webhook
// if (webhook) webhookSvc.trigger(MongoDB.toString(webhook._id), "failed");
// return;
// }
async function notifyClientGitPullFailure(e) {
// print the logs to client (Dashboard & CLI)
(0, send_log_message_1.sendLog)({ SOCKET_ROOM, type: "error", action: "end", message: `[GIT] Failed to pull: "${e}"` });
if (options === null || options === void 0 ? void 0 : options.onError)
options === null || options === void 0 ? void 0 : options.onError(`Failed to pull: "${e}"`);
// update build status
await (0, update_build_status_1.updateBuildStatus)(newBuild, "failed");
// dispatch/trigger webhook
if (webhook)
webhookSvc.trigger(mongodb_1.MongoDB.toString(webhook._id), "failed");
}
// Clone/pull with repoSSH first, if failed, try repoURL...
// try {
// await pullOrCloneGitRepo(repoSSH, buildDir, gitBranch, {
// onUpdate: (message) => sendLog({ SOCKET_ROOM, message }),
// });
// } catch (e) {
// // give another try with HTTPS and access token
// if (app.gitProvider) {
// const git = await DB.findOne("git", { _id: app.gitProvider });
// const repoURL = repoSshToRepoURL(repoSSH);
// try {
// await pullOrCloneGitRepoHTTP(repoURL, buildDir, gitBranch, {
// useAccessToken: {
// type: git.method === "basic" ? "Basic" : "Bearer",
// value: git.access_token,
// },
// onUpdate: (message) => sendLog({ SOCKET_ROOM, message }),
// });
// } catch (e2) {
// notifyClientGitPullFailure(e2);
// }
// } else {
// notifyClientGitPullFailure(e);
// }
// }
// Clone or pull repository with HTTPS + access token:
if (app.gitProvider) {
// find the git provider of this app:
let git = await DB.findOne("git", { _id: app.gitProvider });
if (!git) {
if (!((_b = app.git) === null || _b === void 0 ? void 0 : _b.provider)) {
await notifyClientGitPullFailure(`Git provider not found (${app.gitProvider}).`);
return;
}
// try with any similar provider
git = await DB.findOne("git", { type: (_c = app.git) === null || _c === void 0 ? void 0 : _c.provider });
if (!git) {
await notifyClientGitPullFailure(`Git provider not found (${app.gitProvider}).`);
return;
}
}
// console.log("git :>> ", git);
// parse repo URL from repo SSH
const repoURL = (0, git_utils_1.repoSshToRepoURL)(repoSSH);
// notify client...
(0, send_log_message_1.sendLog)({ SOCKET_ROOM, message: `[START BUILD] Pulling latest source code from "${repoURL}" at "${gitBranch}" branch...` });
try {
await (0, git_utils_1.pullOrCloneGitRepoHTTP)(repoURL, buildDir, gitBranch, {
// isDebugging: true,
useAccessToken: {
type: git.method === "basic" ? "Basic" : "Bearer",
value: git.access_token,
},
onUpdate: (message) => (0, send_log_message_1.sendLog)({ SOCKET_ROOM, message }),
});
}
catch (err) {
await notifyClientGitPullFailure(`${repoURL} :>> ${err}`);
return;
}
}
else {
await notifyClientGitPullFailure(`This app doesn't attach to any git provider.`);
return;
}
// emit socket message to "digirelease" app:
(0, send_log_message_1.sendLog)({ SOCKET_ROOM, message: `[START BUILD] Finished pulling latest files of "${gitBranch}"...` });
/**
* Check if Dockerfile existed
*/
let dockerFile = (0, plugins_1.resolveDockerfilePath)({ targetDirectory: buildDir, env });
if (isDebugging)
console.log("dockerFile :>> ", dockerFile);
if (!dockerFile) {
if (options === null || options === void 0 ? void 0 : options.onError)
options === null || options === void 0 ? void 0 : options.onError(`No "Dockerfile" found in the repository.`);
(0, send_log_message_1.sendLog)({
SOCKET_ROOM,
type: "error",
action: "end",
message: `[START BUILD] Missing "Dockerfile" to build the application, please create your "Dockerfile" in the root directory of the source code.`,
});
// update build status
await (0, update_build_status_1.updateBuildStatus)(newBuild, "failed");
// dispatch/trigger webhook
if (webhook)
webhookSvc.trigger(mongodb_1.MongoDB.toString(webhook._id), "failed");
return;
}
// Update app so it can be sorted on top!
const updatedAppData = { lastUpdatedBy: username };
let updatedApp;
try {
updatedApp = await DB.updateOne("app", { slug: appSlug }, updatedAppData, { ownership: { owner, workspace } });
}
catch (e) {
if (options === null || options === void 0 ? void 0 : options.onError)
options === null || options === void 0 ? void 0 : options.onError(`Server network error, unable to perform data updating.`);
(0, send_log_message_1.sendLog)({
SOCKET_ROOM,
type: "error",
action: "end",
message: `Server network error, unable to perform data updating.`,
});
// update build status
await (0, update_build_status_1.updateBuildStatus)(newBuild, "failed");
// dispatch/trigger webhook
if (webhook)
webhookSvc.trigger(mongodb_1.MongoDB.toString(webhook._id), "failed");
return;
}
(0, send_log_message_1.sendLog)({ SOCKET_ROOM, message: `[START BUILD] Generated the deployment files successfully!` });
/**
* =====================================================
* Build the app with BUILDER ENGINE (Docker or Podman):
* =====================================================
*/
(0, send_log_message_1.sendLog)({ SOCKET_ROOM, message: `[START BUILD] Start building the Docker image...` });
const notifyClientBuildSuccess = async (finishedBuild) => {
const humanDuration = (0, humanize_duration_1.default)(finishedBuild.duration);
(0, send_log_message_1.sendLog)({
SOCKET_ROOM: finishedBuild.slug,
message: chalk_1.default.green(`✓ FINISHED BUILDING IMAGE AFTER ${humanDuration}`),
type: shouldDeploy ? "log" : "success",
action: shouldDeploy ? "log" : "end",
});
if (shouldDeploy) {
(0, send_log_message_1.sendLog)({
SOCKET_ROOM,
message: chalk_1.default.green(`⏳ Preparing to deploy this build...`),
type: "log",
});
return;
}
// dispatch/trigger webhook
if (webhook)
await webhookSvc.trigger(mongodb_1.MongoDB.toString(webhook._id), "success");
};
// authenticate build engine with container registry before building & pushing image
try {
await (0, connect_registry_1.connectRegistry)(registry, { userId, workspaceId: workspace._id });
}
catch (e) {
// notify dashboard client
(0, send_log_message_1.sendLog)({
SOCKET_ROOM,
message: chalk_1.default.green(`Unable to authenticate with "${registry.name}" registry: ${e}`),
type: "error",
});
await (0, update_build_status_1.updateBuildStatus)(newBuild, "failed");
// dispatch/trigger webhook
if (webhook)
await webhookSvc.trigger(mongodb_1.MongoDB.toString(webhook._id), "failed");
// callback
if (options === null || options === void 0 ? void 0 : options.onError)
options === null || options === void 0 ? void 0 : options.onError(`Unable to authenticate with "${registry.name}" registry: ${e}`);
return;
}
// initialize build engine
const buildEngineName = process.env.BUILDER || "podman";
const buildEngine = buildEngineName === "docker" ? builder_1.default.Docker : builder_1.default.Podman;
if (buildWatch === true) {
try {
await buildEngine.build(buildImage, {
args: buildArgs,
platforms: ["linux/amd64"],
builder: `${projectSlug.toLowerCase()}_${appSlug.toLowerCase()}`,
cacheFroms: latestBuild ? [{ type: "registry", value: latestBuild.image }] : [],
dockerFile: dockerFile,
buildDirectory: buildDir,
shouldPush: true,
onBuilding: (message) => (0, send_log_message_1.sendLog)({ SOCKET_ROOM, message }),
});
// send notification message to dashboard client
(0, send_log_message_1.sendLog)({
SOCKET_ROOM,
message: `✓ Pushed "${buildImage}" to container registry (${registrySlug}) successfully!`,
});
// update build status as "success"
await (0, update_build_status_1.updateBuildStatus)(newBuild, "success", { env });
await notifyClientBuildSuccess(newBuild);
if (options === null || options === void 0 ? void 0 : options.onSucceed)
options === null || options === void 0 ? void 0 : options.onSucceed(newBuild);
return { SOCKET_ROOM, build: newBuild, imageURL, buildImage, startTime, builder: buildEngineName };
}
catch (e) {
// send notification message to dashboard client
(0, send_log_message_1.sendLog)({ SOCKET_ROOM, message: e.message, type: "error", action: "end" });
await (0, update_build_status_1.updateBuildStatus)(newBuild, "failed");
if (options === null || options === void 0 ? void 0 : options.onError)
options === null || options === void 0 ? void 0 : options.onError(`Build failed: ${e}`);
// dispatch/trigger webhook
if (webhook)
webhookSvc.trigger(mongodb_1.MongoDB.toString(webhook._id), "failed");
return;
}
}
else {
buildEngine
.build(buildImage, {
args: buildArgs,
platforms: ["linux/amd64"],
builder: `${projectSlug.toLowerCase()}_${appSlug.toLowerCase()}`,
cacheFroms: latestBuild ? [{ type: "registry", value: latestBuild.image }] : [],
dockerFile: dockerFile,
buildDirectory: buildDir,
shouldPush: true,
onBuilding: (message) => (0, send_log_message_1.sendLog)({ SOCKET_ROOM, message }),
})
.then(async (_imageURL) => {
// send notification message to dashboard client
(0, send_log_message_1.sendLog)({
SOCKET_ROOM,
message: `✓ Pushed "${buildImage}" to container registry (${registrySlug}) successfully!`,
});
// update build status as "success"
const finishedBuild = await DB.findOne("build", { name: _imageURL });
await (0, update_build_status_1.updateBuildStatus)(finishedBuild, "success", { env });
await notifyClientBuildSuccess(finishedBuild);
if (options === null || options === void 0 ? void 0 : options.onSucceed)
options === null || options === void 0 ? void 0 : options.onSucceed(finishedBuild);
})
.catch(async (error) => {
if (error instanceof docker_1.BuildContainerError) {
console.error("Error data:", error.data);
const finishedBuild = await DB.findOne("build", { name: error.data.imageName });
(0, send_log_message_1.sendLog)({ SOCKET_ROOM, message: error.message, type: "error", action: "end" });
await (0, update_build_status_1.updateBuildStatus)(finishedBuild, "failed");
if (options === null || options === void 0 ? void 0 : options.onError)
options === null || options === void 0 ? void 0 : options.onError(`Build failed: ${error}`);
// dispatch/trigger webhook
webhook = await webhookSvc.findOne({ build: finishedBuild._id });
if (webhook)
webhookSvc.trigger(mongodb_1.MongoDB.toString(webhook._id), "failed");
}
else {
console.error("startBuild() > Error:", error);
}
});
}
return { SOCKET_ROOM, build: newBuild, imageURL, buildImage, startTime, builder: buildEngineName };
}
exports.startBuild = startBuild;