gitlab-ci-local
Version:
Tired of pushing to test your .gitlab-ci.yml?
1,022 lines (1,021 loc) • 247 kB
JavaScript
import chalk from "chalk";
import * as dotenv from "dotenv";
import fs from "fs-extra";
import prettyHrtime from "pretty-hrtime";
import split2 from "split2";
import { Utils } from "./utils.js";
import assert, { AssertionError } from "assert";
import { Mutex } from "./mutex.js";
import execa from "execa";
import { GitlabRunnerCPUsPresetValue, GitlabRunnerMemoryPresetValue, GitlabRunnerPresetValues } from "./gitlab-preset.js";
import { handler } from "./handler.js";
import * as yaml from "js-yaml";
import { Parser } from "./parser.js";
import { resolveIncludeLocal, validateIncludeLocal } from "./parser-includes.js";
import globby from "globby";
import terminalLink from "terminal-link";
const GCL_SHELL_PROMPT_PLACEHOLDER = "<gclShellPromptPlaceholder>";
const dateFormatter = new Intl.DateTimeFormat(undefined, {
year: undefined,
month: undefined,
hour: "numeric",
minute: "numeric",
second: "numeric",
hour12: false,
});
export class Job {
generateJobId() {
return Math.floor(Math.random() * 1000000);
}
static illegalJobNames = new Set([
"include", "local_configuration", "image", "services",
"stages", "before_script", "default",
"after_script", "variables", "cache", "workflow", "page:deploy",
]);
argv;
name;
baseName;
dependencies;
environment;
jobId;
rules;
allowFailure;
when;
exists;
pipelineIid;
gitData;
inherit;
_dotenvVariables = {};
_prescriptsExitCode = null;
_afterScriptsExitCode = 0;
_coveragePercent = null;
_running = false;
_containerId = null;
_serviceNetworkId = null;
_longRunningSilentTimeout = -1;
_producers = null;
_jobNamePad = null;
_ciProjectDir = null;
_startTime;
_endTime;
_filesToRm = [];
_globalVariables = {};
_variables = {};
_containersToClean = [];
_containerVolumeNames = [];
jobData;
writeStreams;
constructor(opt) {
const jobData = opt.data;
const jobVariables = jobData.variables ?? {};
const variablesFromFiles = opt.variablesFromFiles;
const argv = opt.argv;
const cwd = argv.cwd;
const argvVariables = argv.variable;
const expandVariables = opt.expandVariables ?? true;
this.argv = argv;
this.writeStreams = opt.writeStreams;
this.gitData = opt.gitData;
this.name = opt.name;
this.baseName = opt.baseName;
this.jobId = this.generateJobId();
this.jobData = opt.data;
this.pipelineIid = opt.pipelineIid;
this._globalVariables = opt.globalVariables;
this.inherit = {};
this.inherit.variables = this.jobData.inherit?.variables ?? true;
this.when = jobData.when || "on_success";
this.exists = jobData.exists || [];
this.allowFailure = jobData.allow_failure ?? false;
this.dependencies = jobData.dependencies || null;
this.rules = jobData.rules || null;
this.environment = typeof jobData.environment === "string" ? { name: jobData.environment } : jobData.environment;
const matrixVariables = opt.matrixVariables ?? {};
const fileVariables = Utils.findEnvMatchedVariables(variablesFromFiles, this.fileVariablesDir);
const predefinedVariables = this._predefinedVariables(opt);
this._variables = { ...predefinedVariables, ...this.globalVariables, ...jobVariables, ...matrixVariables, ...fileVariables, ...argvVariables };
if (this.rules && expandVariables) {
const expanded = Utils.expandVariables(this._variables);
// Expand variables in rules:changes
this.rules.forEach((rule, ruleIdx, rules) => {
const changes = Array.isArray(rule.changes) ? rule.changes : rule.changes?.paths;
if (!changes) {
return;
}
changes.forEach((change, changeIdx, changes) => {
changes[changeIdx] = Utils.expandText(change, expanded);
});
rules[ruleIdx].changes = changes;
});
// Expand variables in rules:exists
this.rules.forEach((rule, ruleIdx, rules) => {
const exists = Array.isArray(rule.exists) ? rule.exists : null;
if (!exists) {
return;
}
exists.forEach((exist, existId, exists) => {
exists[existId] = Utils.expandText(exist, expanded);
});
rules[ruleIdx].exists = exists;
});
}
let ruleVariables;
// Set {when, allowFailure} based on rules result
if (this.rules) {
const ruleResult = Utils.getRulesResult({ argv, cwd, rules: this.rules, variables: this._variables }, this.gitData, this.when, this.allowFailure);
this.when = ruleResult.when;
this.allowFailure = ruleResult.allowFailure;
ruleVariables = ruleResult.variables;
this._variables = { ...this._variables, ...ruleVariables, ...argvVariables };
}
// Find environment matched variables
if (this.environment && expandVariables) {
const expanded = Utils.expandVariables(this._variables);
this.environment.name = Utils.expandText(this.environment.name, expanded);
this.environment.url = Utils.expandText(this.environment.url, expanded);
}
const envMatchedVariables = Utils.findEnvMatchedVariables(variablesFromFiles, this.fileVariablesDir, this.environment);
const userDefinedVariables = { ...this.globalVariables, ...jobVariables, ...matrixVariables, ...ruleVariables, ...envMatchedVariables, ...argvVariables };
this.discourageOverridingOfPredefinedVariables(predefinedVariables, userDefinedVariables);
// Merge and expand after finding env matched variables
this._variables = { ...predefinedVariables, ...userDefinedVariables };
// Delete variables the user intentionally wants unset
for (const unsetVariable of argv.unsetVariables) {
delete this._variables[unsetVariable];
}
// Set GCL_PROJECT_DIR_ON_HOST if docker image
if (this.imageName(this._variables)) {
this._variables = { ...this._variables, ...{ GCL_PROJECT_DIR_ON_HOST: cwd } };
}
assert(this.scripts || this.trigger, chalk `{blueBright ${this.name}} must have script specified`);
assert(!(this.interactive && !this.argv.shellExecutorNoImage), chalk `${this.formattedJobName} @Interactive decorator cannot be used with --no-shell-executor-no-image`);
if (this.interactive && (this.when !== "manual" || this.imageName(this._variables) !== null)) {
throw new AssertionError({ message: `${this.formattedJobName} @Interactive decorator cannot have image: and must be when:manual` });
}
if (this.injectSSHAgent && this.imageName(this._variables) === null) {
throw new AssertionError({ message: `${this.formattedJobName} @InjectSSHAgent can only be used with image:` });
}
const expanded = Utils.expandVariables(this._variables);
for (const [i, c] of Object.entries(this.cache)) {
c.policy = Utils.expandText(c.policy, expanded);
assert(["pull", "push", "pull-push"].includes(c.policy), chalk `{blue ${this.name}} cache[${i}].policy is not 'pull', 'push' or 'pull-push'`);
assert(["on_success", "on_failure", "always"].includes(c.when), chalk `{blue ${this.name}} cache[${i}].when is not 'on_success', 'on_failure' or 'always'`);
assert(Array.isArray(c.paths), chalk `{blue ${this.name}} cache[${i}].paths must be array`);
}
for (const [i, s] of Object.entries(this.services)) {
assert(s.name, chalk `{blue ${this.name}} services[${i}].name is undefined`);
assert(!s.command || Array.isArray(s.command), chalk `{blue ${this.name}} services[${i}].command must be an array`);
assert(!s.entrypoint || Array.isArray(s.entrypoint), chalk `{blue ${this.name}} services[${i}].entrypoint must be an array`);
}
assert(!this.artifacts?.paths || Array.isArray(this.artifacts.paths), chalk `{blue ${this.name}} artifacts.paths must be an array`);
if (this.imageName(this._variables) && argv.mountCache) {
for (const c of this.cache) {
c.paths.forEach((p) => {
const path = Utils.expandText(p, expanded);
assert(!path.includes("*"), chalk `{blue ${this.name}} cannot have * in cache paths, when --mount-cache is enabled`);
});
}
}
}
/**
* Warn when overriding of predefined variables is detected
*/
discourageOverridingOfPredefinedVariables(predefinedVariables, userDefinedVariables) {
const predefinedVariablesKeys = Object.keys(predefinedVariables);
const userDefinedVariablesKeys = Object.keys(userDefinedVariables);
const overridingOfPredefinedVariables = userDefinedVariablesKeys.filter(ele => predefinedVariablesKeys.includes(ele));
if (overridingOfPredefinedVariables.length == 0) {
return;
}
const linkToGitlab = "https://gitlab.com/gitlab-org/gitlab/-/blob/v17.7.1-ee/doc/ci/variables/predefined_variables.md?plain=1&ref_type=tags#L15-16";
this.writeStreams.memoStdout(chalk `{bgYellowBright WARN } ${terminalLink("Avoid overriding predefined variables", linkToGitlab)} [{bold ${overridingOfPredefinedVariables}}] as it can cause the pipeline to behave unexpectedly.\n`);
}
/**
* Get the predefinedVariables that's enriched with the additional info that's only available when constructing
*/
_predefinedVariables(opt) {
const argv = this.argv;
const cwd = argv.cwd;
const stateDir = this.argv.stateDir;
const gitData = this.gitData;
let ciBuildsDir;
if (this.jobData["image"]) {
ciBuildsDir = "/builds";
this.ciProjectDir = `${ciBuildsDir}/${gitData.remote.group}/${gitData.remote.project}`;
}
else if (argv.shellIsolation) {
ciBuildsDir = `${cwd}/${stateDir}/builds/${this.safeJobName}`;
this.ciProjectDir = ciBuildsDir;
}
else {
ciBuildsDir = cwd;
this.ciProjectDir = ciBuildsDir;
}
const predefinedVariables = opt.predefinedVariables;
predefinedVariables["CI_JOB_ID"] = `${this.jobId}`;
predefinedVariables["CI_PIPELINE_ID"] = `${this.pipelineIid + 1000}`;
predefinedVariables["CI_PIPELINE_IID"] = `${this.pipelineIid}`;
predefinedVariables["CI_JOB_NAME"] = `${this.name}`;
predefinedVariables["CI_JOB_NAME_SLUG"] = `${this.name.replace(/[^a-z\d]+/ig, "-").replace(/^-/, "").slice(0, 63).replace(/-$/, "").toLowerCase()}`;
predefinedVariables["CI_JOB_STAGE"] = `${this.stage}`;
predefinedVariables["CI_BUILDS_DIR"] = ciBuildsDir;
predefinedVariables["CI_PROJECT_DIR"] = this.ciProjectDir;
predefinedVariables["CI_JOB_URL"] = `${predefinedVariables["CI_SERVER_URL"]}/${gitData.remote.group}/${gitData.remote.project}/-/jobs/${this.jobId}`; // Changes on rerun.
predefinedVariables["CI_PIPELINE_URL"] = `${predefinedVariables["CI_SERVER_URL"]}/${gitData.remote.group}/${gitData.remote.project}/pipelines/${this.pipelineIid}`;
predefinedVariables["CI_ENVIRONMENT_NAME"] = this.environment?.name ?? "";
predefinedVariables["CI_ENVIRONMENT_SLUG"] = this.environment?.name?.replace(/[^a-z\d]+/ig, "-").replace(/^-/, "").slice(0, 23).replace(/-$/, "").toLowerCase() ?? "";
predefinedVariables["CI_ENVIRONMENT_URL"] = this.environment?.url ?? "";
predefinedVariables["CI_ENVIRONMENT_TIER"] = this.environment?.deployment_tier ?? "";
predefinedVariables["CI_ENVIRONMENT_ACTION"] = this.environment?.action ?? "";
if (opt.nodeIndex !== null) {
predefinedVariables["CI_NODE_INDEX"] = `${opt.nodeIndex}`;
}
predefinedVariables["CI_NODE_TOTAL"] = `${opt.nodesTotal}`;
predefinedVariables["CI_REGISTRY"] = `local-registry.${this.gitData.remote.host}`;
predefinedVariables["CI_REGISTRY_IMAGE"] = `$CI_REGISTRY/${predefinedVariables["CI_PROJECT_PATH"].toLowerCase()}`;
return predefinedVariables;
}
get jobStatus() {
if (this.preScriptsExitCode == null)
return "pending";
if (this.preScriptsExitCode == 0)
return "success";
let allowedExitCodes = [0];
const allowFailure = this.allowFailure;
switch (typeof allowFailure) {
case "boolean":
if (allowFailure) {
allowedExitCodes = [this.preScriptsExitCode];
}
break;
case "object":
if (!Array.isArray(allowFailure.exit_codes)) {
allowedExitCodes = [allowFailure.exit_codes];
}
else {
allowedExitCodes = allowFailure.exit_codes;
}
break;
default:
throw new Error(`Unexpected type: ${typeof allowFailure}`);
}
return allowedExitCodes.includes(this.preScriptsExitCode) ? "warning" : "failed";
}
get artifactsToSource() {
if (this.jobData["gclArtifactsToSource"] != null)
return this.jobData["gclArtifactsToSource"];
return this.argv.artifactsToSource;
}
get prettyDuration() {
if (this._endTime) {
return prettyHrtime(this._endTime);
}
return this._startTime ?
prettyHrtime(process.hrtime(this._startTime)) :
"0 ms";
}
get formattedJobName() {
let prefix = "";
if (this.argv.childPipelineDepth > 0)
prefix = "\t".repeat(this.argv.childPipelineDepth) + `[${this.argv.variable.GCL_TRIGGERER}] -> `;
const timestampPrefix = this.argv.showTimestamps ?
`[${dateFormatter.format(new Date())} ${this.prettyDuration.padStart(7)}] ` :
"";
// [16:33:19 1.37 min] my-job > hello world
return chalk `${timestampPrefix}{blueBright ${prefix}${this.name.padEnd(this.jobNamePad)}}`;
}
get safeJobName() {
return Utils.safeDockerString(this.name);
}
get needs() {
return this.jobData["needs"] ?? null;
}
get buildVolumeName() {
return `gcl-${this.safeJobName}-${this.jobId}-build`;
}
get tmpVolumeName() {
return `gcl-${this.safeJobName}-${this.jobId}-tmp`;
}
get services() {
const services = [];
if (!this.jobData["services"])
return [];
for (const service of Object.values(this.jobData["services"])) {
const expanded = Utils.expandVariables({ ...this._variables, ...this._dotenvVariables, ...service["variables"] });
let serviceName = Utils.expandText(service["name"], expanded);
if (serviceName == "")
continue;
serviceName = serviceName.includes(":") ? serviceName : `${serviceName}:latest`;
services.push({
name: serviceName,
entrypoint: service["entrypoint"] ?? null,
command: service["command"] ?? null,
variables: expanded,
alias: Utils.expandText(service["alias"], expanded) ?? null,
});
}
return services;
}
set ciProjectDir(ciProjectDir) {
assert(this._ciProjectDir == null, "this._ciProjectDir can only be set once");
this._ciProjectDir = ciProjectDir;
}
get ciProjectDir() {
assert(this._ciProjectDir, "attempted to access this._ciProjectDir before it is initialized");
return this._ciProjectDir;
}
set jobNamePad(jobNamePad) {
assert(this._jobNamePad == null, "this._jobNamePad can only be set once");
this._jobNamePad = jobNamePad;
}
get jobNamePad() {
return this._jobNamePad ?? 0;
}
get producers() {
return this._producers;
}
set producers(producers) {
assert(this._producers == null, "this._producers can only be set once");
this._producers = producers;
}
get stage() {
return this.jobData["stage"] || "test";
}
get interactive() {
return this.jobData["gclInteractive"] || false;
}
get injectSSHAgent() {
return this.jobData["gclInjectSSHAgent"] || false;
}
get description() {
return this.jobData["gclDescription"] ?? "";
}
get artifacts() {
return this.jobData["artifacts"];
}
deleteArtifacts() {
delete this.jobData["artifacts"];
}
get cache() {
return this.jobData["cache"] || [];
}
async getUniqueCacheName(cwd, expanded, cacheIndex) {
const getCachePrefix = (index) => {
const prefix = this.jobData["cache"][index]["key"]["prefix"];
if (prefix) {
return `${index}_${Utils.expandText(prefix, expanded)}-`;
}
const filenames = this.jobData["cache"][index]["key"]["files"].map((p) => {
const expandP = Utils.expandText(p, expanded);
return expandP.split(".")[0];
}).join("_");
return `${index}_${filenames}-`;
};
const key = this.jobData["cache"][cacheIndex].key;
if (typeof key === "string" || key == null) {
return Utils.expandText(key ?? "default", expanded);
}
const files = key["files"].map((f) => {
let path = Utils.expandText(f, expanded);
if (path.startsWith(`${this.ciProjectDir}/`)) {
path = path.slice(`${this.ciProjectDir}/`.length);
}
return `${cwd}/${path}`;
});
return getCachePrefix(cacheIndex) + await Utils.checksumFiles(cwd, files);
}
get beforeScripts() {
const beforeScripts = this.jobData["before_script"] || [];
return typeof beforeScripts === "string" ? [beforeScripts] : beforeScripts;
}
get afterScripts() {
const afterScripts = this.jobData["after_script"] || [];
return typeof afterScripts === "string" ? [afterScripts] : afterScripts;
}
get scripts() {
const script = this.jobData["script"];
return typeof script === "string" ? [script] : script;
}
get trigger() {
return this.jobData["trigger"];
}
async startTriggerPipeline() {
this.writeStreams.memoStdout(chalk `{bgYellowBright WARN } downstream pipeline is experimental in gitlab-ci-local\n`);
await this.fetchTriggerInclude();
const variablesForDownstreamPipeline = Object.entries({ ...this.globalVariables, ...this.jobData.variables }).map(([key, value]) => `${key}=${value}`);
const gclTriggerer = this.argv.variable["GCL_TRIGGERER"] ?
`${this.argv.variable["GCL_TRIGGERER"]} -> ${this.name}` :
this.name;
await handler({
...Object.fromEntries(this.argv.map),
file: `${this.argv.stateDir}/includes/triggers/${this.name}.yml`,
variable: [
chalk `GCL_TRIGGERER=${gclTriggerer}`,
"CI_PIPELINE_SOURCE=parent_pipeline",
].concat(variablesForDownstreamPipeline),
}, this.writeStreams, [], this.argv.childPipelineDepth + 1);
}
get preScriptsExitCode() {
return this._prescriptsExitCode;
}
get afterScriptsExitCode() {
return this._afterScriptsExitCode;
}
get started() {
return this._running || this._prescriptsExitCode !== null;
}
get finished() {
return !this._running && this._prescriptsExitCode !== null;
}
get coveragePercent() {
return this._coveragePercent;
}
get fileVariablesDir() {
return `/tmp/gitlab-ci-local-file-variables-${this._variables["CI_PROJECT_PATH_SLUG"]}-${this.jobId}`;
}
get globalVariables() {
if (this.inherit.variables === false) {
return {};
}
else if (Array.isArray(this.inherit.variables)) {
const inheritVariables = this.inherit.variables;
return Object.fromEntries(Object.entries(this._globalVariables).filter(([k]) => inheritVariables.includes(k)));
}
return this._globalVariables;
}
async start() {
if (this.trigger) {
await this.startTriggerPipeline();
this._prescriptsExitCode = 0; // NOTE: so that `this.finished` will implicitly be set to true
return;
}
this._running = true;
const argv = this.argv;
this._startTime = process.hrtime();
this._variables["CI_JOB_STARTED_AT"] = new Date().toISOString().split(".")[0] + "Z";
const writeStreams = this.writeStreams;
this._dotenvVariables = await this.initProducerReportsDotenvVariables(writeStreams, Utils.expandVariables(this._variables));
const expanded = Utils.unscape$$Variables(Utils.expandVariables({ ...this._variables, ...this._dotenvVariables }));
const imageName = this.imageName(expanded);
const helperImageName = argv.helperImage;
const safeJobName = this.safeJobName;
const outputLogFilePath = `${argv.cwd}/${argv.stateDir}/output/${safeJobName}.log`;
await fs.ensureFile(outputLogFilePath);
await fs.truncate(outputLogFilePath);
if (!this.interactive) {
writeStreams.stdout(chalk `${this.formattedJobName} {magentaBright starting} ${imageName ?? "shell"} ({yellow ${this.stage}})\n`);
}
if (imageName) {
await this.pullImage(writeStreams, imageName);
const buildVolumeName = this.buildVolumeName;
const tmpVolumeName = this.tmpVolumeName;
const fileVariablesDir = this.fileVariablesDir;
const volumePromises = [];
volumePromises.push(Utils.spawn([this.argv.containerExecutable, "volume", "create", `${buildVolumeName}`], argv.cwd));
volumePromises.push(Utils.spawn([this.argv.containerExecutable, "volume", "create", `${tmpVolumeName}`], argv.cwd));
this._containerVolumeNames.push(buildVolumeName);
this._containerVolumeNames.push(tmpVolumeName);
await Promise.all(volumePromises);
const time = process.hrtime();
this.refreshLongRunningSilentTimeout(writeStreams);
let chownOpt = "0:0";
let chmodOpt = "a+rw";
if (!this.argv.umask) {
const { stdout } = await Utils.spawn([this.argv.containerExecutable, "run", "--rm", "--entrypoint", "sh", imageName, "-c", "echo \"$(id -u):$(id -g)\""]);
chownOpt = stdout;
if (chownOpt == "0:0") {
chmodOpt = "g-w";
}
}
if (helperImageName) {
await this.pullImage(writeStreams, helperImageName);
}
const { stdout: containerId } = await Utils.spawn([
this.argv.containerExecutable, "create", "--user=0:0", `--volume=${buildVolumeName}:${this.ciProjectDir}`, `--volume=${tmpVolumeName}:${this.fileVariablesDir}`, `${helperImageName}`,
...["sh", "-c", `chown ${chownOpt} -R ${this.ciProjectDir} && chmod ${chmodOpt} -R ${this.ciProjectDir} && chown ${chownOpt} -R /tmp/ && chmod ${chmodOpt} -R /tmp/`],
], argv.cwd);
this._containersToClean.push(containerId);
if (await fs.pathExists(fileVariablesDir)) {
await Utils.spawn([this.argv.containerExecutable, "cp", `${fileVariablesDir}/.`, `${containerId}:${fileVariablesDir}`], argv.cwd);
this.refreshLongRunningSilentTimeout(writeStreams);
}
await Utils.spawn([this.argv.containerExecutable, "cp", `${argv.stateDir}/builds/.docker/.`, `${containerId}:${this.ciProjectDir}`], argv.cwd);
await Utils.spawn([this.argv.containerExecutable, "start", "--attach", containerId], argv.cwd);
await Utils.spawn([this.argv.containerExecutable, "rm", "-vf", containerId], argv.cwd);
const endTime = process.hrtime(time);
writeStreams.stdout(chalk `${this.formattedJobName} {magentaBright copied to ${this.argv.containerExecutable} volumes} in {magenta ${prettyHrtime(endTime)}}\n`);
}
if (this.services?.length) {
// `host` and `none` networks do not work with services because aliases only work for
// user defined networks.
for (const network of this.argv.network) {
if (["host", "none"].includes(network)) {
throw new AssertionError({ message: `Cannot add service network alias with network mode '${network}'` });
}
}
await this.createDockerNetwork(`gitlab-ci-local-${this.jobId}`);
await Promise.all(this.services.map(async (service, serviceIndex) => {
const serviceName = service.name;
await this.pullImage(writeStreams, serviceName);
const serviceContainerId = await this.startService(writeStreams, Utils.expandVariables({ ...expanded, ...service.variables }), service);
const serviceContainerLogFile = `${argv.cwd}/${argv.stateDir}/services-output/${this.safeJobName}/${serviceName}-${serviceIndex}.log`;
await this.serviceHealthCheck(writeStreams, service, serviceIndex, serviceContainerLogFile);
const { stdout, stderr } = await Utils.spawn([this.argv.containerExecutable, "logs", serviceContainerId]);
await fs.ensureFile(serviceContainerLogFile);
await fs.promises.writeFile(serviceContainerLogFile, `### stdout ###\n${stdout}\n### stderr ###\n${stderr}\n`);
}));
}
await this.execPreScripts(expanded);
if (this._prescriptsExitCode == null)
throw Error("this._prescriptsExitCode must be defined!");
await this.execAfterScripts(expanded);
this._running = false;
this._endTime = this._endTime ?? process.hrtime(this._startTime);
this.printFinishedString();
await this.copyCacheOut(this.writeStreams, expanded);
await this.copyArtifactsOut(this.writeStreams, expanded);
if (this.jobData["coverage"]) {
this._coveragePercent = await Utils.getCoveragePercent(argv.cwd, argv.stateDir, this.jobData["coverage"], safeJobName);
}
}
async cleanupResources() {
clearTimeout(this._longRunningSilentTimeout);
if (!this.argv.cleanup)
return;
if (this._containersToClean.length > 0) {
try {
await Utils.spawn([this.argv.containerExecutable, "rm", "-vf", ...this._containersToClean]);
}
catch (e) {
assert(e instanceof Error, "e is not instanceof Error");
}
}
if (this._serviceNetworkId) {
try {
await Utils.spawn([this.argv.containerExecutable, "network", "rm", `${this._serviceNetworkId}`]);
}
catch (e) {
assert(e instanceof Error, "e is not instanceof Error");
}
}
if (this._containerVolumeNames.length > 0) {
try {
await Utils.spawn([this.argv.containerExecutable, "volume", "rm", ...this._containerVolumeNames]);
}
catch (e) {
assert(e instanceof Error, "e is not instanceof Error");
}
}
const rmPromises = [];
for (const file of this._filesToRm) {
rmPromises.push(fs.rm(file, { recursive: true, force: true }));
}
await Promise.all(rmPromises);
const fileVariablesDir = this.fileVariablesDir;
try {
await fs.rm(fileVariablesDir, { recursive: true, force: true });
}
catch (e) {
assert(e instanceof Error, "e is not instanceof Error");
}
}
generateInjectSSHAgentOptions() {
if (!this.injectSSHAgent) {
return "";
}
if (process.platform === "darwin" || /^darwin/.exec(process.env.OSTYPE ?? "")) {
return "--env SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock -v /run/host-services/ssh-auth.sock:/run/host-services/ssh-auth.sock";
}
return `--env SSH_AUTH_SOCK=${process.env.SSH_AUTH_SOCK} -v ${process.env.SSH_AUTH_SOCK}:${process.env.SSH_AUTH_SOCK}`;
}
generateScriptCommands(scripts) {
let cmd = "";
scripts.forEach((script) => {
const split = script.split(/\r?\n/);
const multilineText = split.length > 1 ? " # collapsed multi-line command" : "";
const text = split[0]?.replace(/\\/g, "\\\\").replace(/"/g, "\\\"").replace(/[$]/g, "\\$");
if (this.interactive) {
cmd += chalk `echo "{green $ ${text}${multilineText}}"\n`;
}
else {
// Print command echo'ed with $GCL_SHELL_PROMPT_PLACEHOLDER
cmd += `echo "${GCL_SHELL_PROMPT_PLACEHOLDER} ${text}${multilineText}"\n`;
}
// Execute actual script
cmd += `${script}\n`;
});
return cmd;
}
async mountCacheCmd(writeStreams, expanded) {
if (this.imageName(expanded) && !this.argv.mountCache)
return [];
const cmd = [];
for (const [index, c] of this.cache.entries()) {
const uniqueCacheName = await this.getUniqueCacheName(this.argv.cwd, expanded, index);
c.paths.forEach((p) => {
const path = Utils.expandText(p, expanded);
writeStreams.stdout(chalk `${this.formattedJobName} {magentaBright mounting cache} for path ${path}\n`);
const cacheMount = Utils.safeDockerString(`gcl-${expanded.CI_PROJECT_PATH_SLUG}-${uniqueCacheName}`);
cmd.push("-v", `${cacheMount}:${this.ciProjectDir}/${path}`);
});
}
return cmd;
}
async execPreScripts(expanded) {
const prescripts = this.beforeScripts.concat(this.scripts);
expanded["CI_JOB_STATUS"] = "running";
this._prescriptsExitCode = await this.execScripts(prescripts, expanded, "");
expanded["CI_JOB_STATUS"] = this._prescriptsExitCode === 0 ? "success" : "failed";
this.printExitedString(this._prescriptsExitCode);
}
async execAfterScripts(expanded) {
const message = "Running after script...";
this._afterScriptsExitCode = await this.execScripts(this.afterScripts, expanded, message);
this.printAfterScriptExitedString(this._afterScriptsExitCode);
}
async execScripts(scripts, expanded, message) {
const cwd = this.argv.cwd;
const stateDir = this.argv.stateDir;
const safeJobName = this.safeJobName;
const outputFilesPath = `${cwd}/${stateDir}/output/${safeJobName}.log`;
const buildVolumeName = this.buildVolumeName;
const tmpVolumeName = this.tmpVolumeName;
const imageName = this.imageName(expanded);
const writeStreams = this.writeStreams;
if (scripts.length === 0 || scripts[0] == null)
return 0;
if (message.length > 0)
writeStreams.stdout(chalk `${this.formattedJobName} {magentaBright ${message}}\n`);
await fs.remove(`${cwd}/${stateDir}/artifacts/${safeJobName}`);
// Copy git tracked files to build folder if shell isolation enabled.
if (!imageName && this.argv.shellIsolation) {
await Utils.rsyncTrackedFiles(cwd, stateDir, `${safeJobName}`);
}
if (this.interactive) {
let iCmd = "set -eo pipefail\n";
iCmd += this.generateScriptCommands(scripts);
const interactiveCp = execa(iCmd, {
cwd,
shell: "bash",
stdio: ["inherit", "inherit", "inherit"],
env: { ...expanded, ...process.env },
});
return new Promise((resolve, reject) => {
void interactiveCp.on("exit", (code) => resolve(code ?? 0));
void interactiveCp.on("error", (err) => reject(err));
});
}
this.refreshLongRunningSilentTimeout(writeStreams);
if (imageName && !this._containerId) {
let dockerCmd = `${this.argv.containerExecutable} create --interactive ${this.generateInjectSSHAgentOptions()} `;
if (this.argv.privileged) {
dockerCmd += "--privileged ";
}
for (const device of this.argv.device) {
dockerCmd += `--device ${device} `;
}
if (this.argv.ulimit !== null) {
dockerCmd += `--ulimit nofile=${this.argv.ulimit} `;
}
if (this.argv.umask) {
dockerCmd += "--user 0:0 ";
}
if (this.argv.userns != undefined) {
dockerCmd += `--userns=${this.argv.userns} `;
}
if (this.argv.containerMacAddress) {
dockerCmd += `--mac-address "${this.argv.containerMacAddress}" `;
}
const imageUser = this.imageUser(expanded);
if (imageUser) {
dockerCmd += `--user ${imageUser} `;
}
if (this.argv.containerEmulate) {
const runnerName = this.argv.containerEmulate;
if (!GitlabRunnerPresetValues.includes(runnerName)) {
throw new Error("Invalid gitlab runner to emulate.");
}
const memoryConfig = GitlabRunnerMemoryPresetValue[runnerName];
const cpuConfig = GitlabRunnerCPUsPresetValue[runnerName];
dockerCmd += `--memory=${memoryConfig}m `;
dockerCmd += `--kernel-memory=${memoryConfig}m `;
dockerCmd += `--cpus=${cpuConfig} `;
}
// host and none networks have to be specified using --network, since they cannot be used with
// `docker network connect`.
for (const network of this.argv.network) {
if (["host", "none"].includes(network)) {
dockerCmd += `--network ${network} `;
}
}
// The default podman network mode is not `bridge`, which means a `podman network connect` call will fail
// when connecting user defined networks. The workaround is to use a user defined network on container
// creation.
//
// See https://github.com/containers/podman/issues/19577
//
// This should not clash with the `host` and `none` networks above, since service creation should have
// failed when using `host` or `none` networks.
if (this._serviceNetworkId) {
// `build` alias: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27060
dockerCmd += `--network ${this._serviceNetworkId} --network-alias build `;
}
dockerCmd += `--volume ${buildVolumeName}:${this.ciProjectDir} `;
dockerCmd += `--volume ${tmpVolumeName}:${this.fileVariablesDir} `;
dockerCmd += `--workdir ${this.ciProjectDir} `;
for (const volume of this.argv.volume) {
dockerCmd += `--volume ${volume} `;
}
for (const extraHost of this.argv.extraHost) {
dockerCmd += `--add-host=${extraHost} `;
}
for (const [key, val] of Object.entries(expanded)) {
// Replacing `'` with `'\''` to correctly handle single quotes(if `val` contains `'`) in shell commands
dockerCmd += ` -e '${key}=${val.toString().replace(/'/g, "'\\''")}' \\\n`;
}
if (this.imageEntrypoint) {
dockerCmd += `--entrypoint ${Utils.safeBashString(this.imageEntrypoint[0])} `;
}
dockerCmd += `${(await this.mountCacheCmd(writeStreams, expanded)).join(" ")} `;
dockerCmd += `${imageName} `;
if (this.imageEntrypoint?.length ?? 0 > 1) {
this.imageEntrypoint?.slice(1).forEach((e) => {
dockerCmd += `${Utils.safeBashString(e)} `;
});
}
dockerCmd += "sh -c \"\n";
dockerCmd += "if [ -x /usr/local/bin/bash ]; then\n";
dockerCmd += "\texec /usr/local/bin/bash \n";
dockerCmd += "elif [ -x /usr/bin/bash ]; then\n";
dockerCmd += "\texec /usr/bin/bash \n";
dockerCmd += "elif [ -x /bin/bash ]; then\n";
dockerCmd += "\texec /bin/bash \n";
dockerCmd += "elif [ -x /usr/local/bin/sh ]; then\n";
dockerCmd += "\texec /usr/local/bin/sh \n";
dockerCmd += "elif [ -x /usr/bin/sh ]; then\n";
dockerCmd += "\texec /usr/bin/sh \n";
dockerCmd += "elif [ -x /bin/sh ]; then\n";
dockerCmd += "\texec /bin/sh \n";
dockerCmd += "elif [ -x /busybox/sh ]; then\n";
dockerCmd += "\texec /busybox/sh \n";
dockerCmd += "else\n";
dockerCmd += "\techo shell not found\n";
dockerCmd += "\texit 1\n";
dockerCmd += "fi\n\"";
const { stdout: containerId } = await Utils.bash(dockerCmd, cwd);
for (const network of this.argv.network) {
// Special network names that do not work with `docker network connect`
if (["host", "none"].includes(network)) {
continue;
}
await Utils.spawn([this.argv.containerExecutable, "network", "connect", network, `${containerId}`]);
}
this._containerId = containerId;
this._containersToClean.push(this._containerId);
}
await this.copyCacheIn(writeStreams, expanded);
await this.copyArtifactsIn(writeStreams);
let cmd = "set -eo pipefail\n";
cmd += "exec 0< /dev/null\n";
if (!imageName && this.argv.shellIsolation) {
cmd += `cd ${stateDir}/builds/${safeJobName}/\n`;
}
if (imageName) {
cmd += `cd ${this.ciProjectDir} \n`;
if (expanded["CI_JOB_STATUS"] != "running") {
// Ensures the env `CI_JOB_STATUS` is passed to the after_script context
cmd += `export CI_JOB_STATUS=${expanded["CI_JOB_STATUS"]}\n`;
}
}
cmd += this.generateScriptCommands(scripts);
cmd += "exit 0\n";
const jobScriptFile = `${cwd}/${stateDir}/scripts/${safeJobName}_${this.jobId}`;
await fs.outputFile(jobScriptFile, cmd, "utf-8");
await fs.chmod(jobScriptFile, "0755");
this._filesToRm.push(jobScriptFile);
if (imageName) {
await Utils.spawn([this.argv.containerExecutable, "cp", `${stateDir}/scripts/${safeJobName}_${this.jobId}`, `${this._containerId}:/gcl-cmd`], cwd);
}
const cp = execa(this._containerId ? `${this.argv.containerExecutable} start --attach -i ${this._containerId}` : "bash", {
cwd,
shell: "bash",
env: imageName ? process.env : expanded,
});
const outFunc = (line, stream, colorize) => {
this.refreshLongRunningSilentTimeout(writeStreams);
stream(`${this.formattedJobName} `);
if (line.startsWith(GCL_SHELL_PROMPT_PLACEHOLDER)) {
// replace the GCL_SHELL_PROMPT_PLACEHOLDER with `$` and make the SHELL_PROMPT line green color
line = line.slice(GCL_SHELL_PROMPT_PLACEHOLDER.length);
line = chalk `{green $${line}}`;
}
else {
stream(`${colorize(">")} `);
}
stream(`${line}\n`);
fs.appendFileSync(outputFilesPath, `${line}\n`);
};
const quiet = this.argv.quiet;
return await new Promise((resolve, reject) => {
if (!quiet) {
cp.stdout?.pipe(split2()).on("data", (e) => outFunc(e, writeStreams.stdout.bind(writeStreams), (s) => chalk `{greenBright ${s}}`));
cp.stderr?.pipe(split2()).on("data", (e) => outFunc(e, writeStreams.stderr.bind(writeStreams), (s) => chalk `{redBright ${s}}`));
}
void cp.on("exit", (code) => {
clearTimeout(this._longRunningSilentTimeout);
return resolve(code ?? 0);
});
void cp.on("error", (err) => {
clearTimeout(this._longRunningSilentTimeout);
return reject(err);
});
if (imageName) {
cp.stdin?.end(". /gcl-cmd");
}
else {
cp.stdin?.end(`./${stateDir}/scripts/${safeJobName}_${this.jobId}`);
}
});
}
imageName(vars = {}) {
if (this.argv.forceShellExecutor) {
return null;
}
const image = this.jobData["image"];
if (!image) {
if (this.argv.shellExecutorNoImage) {
return null;
}
else {
// https://docs.gitlab.com/ee/ci/runners/hosted_runners/linux.html#container-images
return this.argv.defaultImage;
}
}
const expanded = Utils.expandVariables(vars);
const imageName = Utils.expandText(image.name, expanded);
return imageName.includes(":") ? imageName : `${imageName}:latest`;
}
imageUser(vars = {}) {
const image = this.jobData["image"];
if (!image)
return null;
if (!image["docker"])
return null;
return Utils.expandText(image["docker"]["user"], vars);
}
get imageEntrypoint() {
const image = this.jobData["image"];
if (!image?.entrypoint) {
return null;
}
assert(Array.isArray(image.entrypoint), "image:entrypoint must be an array");
return image.entrypoint;
}
async validateCiDependencyProxyServerAuthentication(imageName) {
const CI_DEPENDENCY_PROXY_SERVER = this._variables["CI_DEPENDENCY_PROXY_SERVER"];
if (!imageName.startsWith(CI_DEPENDENCY_PROXY_SERVER)) {
return;
}
try {
await Utils.spawn([this.argv.containerExecutable, "login", CI_DEPENDENCY_PROXY_SERVER]);
}
catch (e) {
const errMsg = `Please authenticate to the Dependency Proxy (${CI_DEPENDENCY_PROXY_SERVER}) https://docs.gitlab.com/ee/user/packages/dependency_proxy/#authenticate-with-the-dependency-proxy`;
if (this.argv.containerExecutable == "docker") {
assert(!e.stderr.includes("Cannot perform an interactive login"), errMsg);
}
else if (this.argv.containerExecutable == "podman") {
assert(!e.stderr.includes("Username: Error: getting username and password: reading username: EOF"), errMsg);
}
throw e;
}
}
async pullImage(writeStreams, imageToPull) {
const pullPolicy = this.argv.pullPolicy;
const actualPull = async () => {
await this.validateCiDependencyProxyServerAuthentication(imageToPull);
const time = process.hrtime();
await Utils.spawn([this.argv.containerExecutable, "pull", imageToPull]);
const endTime = process.hrtime(time);
writeStreams.stdout(chalk `${this.formattedJobName} {magentaBright pulled} ${imageToPull} in {magenta ${prettyHrtime(endTime)}}\n`);
this.refreshLongRunningSilentTimeout(writeStreams);
};
if (pullPolicy === "always") {
await actualPull();
return;
}
try {
await Utils.spawn([this.argv.containerExecutable, "image", "inspect", imageToPull]);
}
catch {
await actualPull();
}
}
async initProducerReportsDotenvVariables(writeStreams, expanded) {
const cwd = this.argv.cwd;
const stateDir = this.argv.stateDir;
const producers = this.producers;
let producerReportsEnvs = {};
for (const producer of producers ?? []) {
const producerDotenv = Utils.expandText(producer.dotenv, expanded);
if (producerDotenv === null)
continue;
const safeProducerName = Utils.safeDockerString(producer.name);
const dotenvFolder = `${cwd}/${stateDir}/artifacts/${safeProducerName}/.gitlab-ci-reports/dotenv/`;
if (await fs.pathExists(dotenvFolder)) {
const dotenvFiles = (await Utils.spawn(["find", ".", "-type", "f"], dotenvFolder)).stdout.split("\n");
for (const dotenvFile of dotenvFiles) {
if (dotenvFile == "")
continue;
const producerReportEnv = dotenv.parse(await fs.readFile(`${dotenvFolder}/${dotenvFile}`));
producerReportsEnvs = { ...producerReportsEnvs, ...producerReportEnv };
}
}
else {
writeStreams.stderr(chalk `${this.formattedJobName} {yellow reports.dotenv produced by '${producer.name}' could not be found}\n`);
}
}
return producerReportsEnvs;
}
async copyCacheIn(writeStreams, expanded) {
if (this.argv.mountCache && this.imageName(expanded))
return;
if ((!this.imageName(expanded) && !this.argv.shellIsolation) || this.cache.length === 0)
return;
const cwd = this.argv.cwd;
const stateDir = this.argv.stateDir;
for (const [index, c] of this.cache.entries()) {
if (!["pull", "pull-push"].includes(c.policy))
return;
const time = process.hrtime();
const cacheName = await this.getUniqueCacheName(cwd, expanded, index);
const cacheFolder = `${cwd}/${stateDir}/cache/${cacheName}`;
if (!await fs.pathExists(cacheFolder)) {
continue;
}
await Mutex.exclusive(cacheName, async () => {
await this.copyIn(cacheFolder);
});
const endTime = process.hrtime(time);
writeStreams.stdout(chalk `${this.formattedJobName} {magentaBright imported cache '${cacheName}'} in {magenta ${prettyHrtime(endTime)}}\n`);
}
}
async copyArtifactsIn(writeStreams) {
if ((!this.imageName(this._variables) && !this.argv.shellIsolation) || (this.producers ?? []).length === 0)
return;
const cwd = this.argv.cwd;
const stateDir = this.argv.stateDir;
const time = process.hrtime();
const promises = [];
for (const producer of this.producers ?? []) {
const producerSafeName = Utils.safeDockerString(producer.name);
const artifactFolder = `${cwd}/${stateDir}/artifacts/${producerSafeName}`;
if (!await fs.pathExists(artifactFolder)) {
await fs.mkdirp(artifactFolder);
}
const readdir = await fs.readdir(artifactFolder);
if (readdir.length === 0) {
writeStreams.stderr(chalk `${this.formattedJobName} {yellow artifacts from {blueBright ${producerSafeName}} was empty}\n`);
}
promises.push(this.copyIn(artifactFolder));
}
await Promise.all(promises);
const endTime = process.hrtime(time);
writeStreams.stdout(chalk `${this.formattedJobName} {magentaBright imported artifacts} in {magenta ${prettyHrtime(endTime)}}\n`);
}
copyIn(source) {
const safeJobName = this.safeJobName;
if (!this.imageName(this._variables) && this.argv.shellIsolation) {
return Utils.spawn(["rsync", "-a", `${source}/.`, `${this.argv.cwd}/${this.argv.stateDir}/builds/${safeJobName}`]);
}
return Utils.spawn([this.argv.containerExecutable, "cp", `${source}/.`, `${this._containerId}:${this.ciProjectDir}`]);
}
async copyCacheOut(writeStreams, expanded) {
if (this.argv.mountCache && this.imageName(expanded))
return;
if ((!this.imageName(expanded) && !this.argv.shellIsolation) || this.cache.length === 0)
return;
const cwd = this.argv.cwd;
const stateDir = this.argv.stateDir;
const cachePath = this.imageName(expanded) ? "/cache" : "../../cache";
let time, endTime;
for (const [index, c] of this.cache.entries()) {
if (!["push", "pull-push"].includes(c.policy))
return;
if ("on_success" === c.when && this.jobStatus !== "success")
return;
if ("on_failure" === c.when && this.jobStatus === "success")
return;
const cacheName = await this.getUniqueCacheName(cwd, expanded, index);
let paths = "";
for (const path of c.paths) {
if (!Utils.isSubpath(path, this.argv.cwd, this.argv.cwd))
continue;
paths += " " + Utils.expandText(path, expanded).replace(`${expanded.CI_PROJECT_DIR}/`, "");
}
time = process.hrtime();
let cmd = "shopt -s globstar nullglob dotglob\n";
cmd += `mkdir -p ${Utils.safeBashString(cachePath + "/" + cacheName)}\n`;
cmd += `rsync -Ra ${paths} ${Utils.safeBashString(cachePath + "/" + cacheName + "/.")} || true\n`;
await Mutex.exclusive(cacheName, async () => {
await this.copyOut(cmd, stateDir, "cache", []);