@catladder/pipeline/dist/pipeline/gitlab/createGitlabJobs.js

Panter workflow for cloud CI/CD and DevOps

"use strict";

var __assign = this && this.__assign || function () {
  __assign = Object.assign || function (t) {
    for (var s, i = 1, n = arguments.length; i < n; i++) {
      s = arguments[i];
      for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p];
    }
    return t;
  };
  return __assign.apply(this, arguments);
};
var __awaiter = this && this.__awaiter || function (thisArg, _arguments, P, generator) {
  function adopt(value) {
    return value instanceof P ? value : new P(function (resolve) {
      resolve(value);
    });
  }
  return new (P || (P = Promise))(function (resolve, reject) {
    function fulfilled(value) {
      try {
        step(generator.next(value));
      } catch (e) {
        reject(e);
      }
    }
    function rejected(value) {
      try {
        step(generator["throw"](value));
      } catch (e) {
        reject(e);
      }
    }
    function step(result) {
      result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
    }
    step((generator = generator.apply(thisArg, _arguments || [])).next());
  });
};
var __generator = this && this.__generator || function (thisArg, body) {
  var _ = {
      label: 0,
      sent: function () {
        if (t[0] & 1) throw t[1];
        return t[1];
      },
      trys: [],
      ops: []
    },
    f,
    y,
    t,
    g;
  return g = {
    next: verb(0),
    "throw": verb(1),
    "return": verb(2)
  }, typeof Symbol === "function" && (g[Symbol.iterator] = function () {
    return this;
  }), g;
  function verb(n) {
    return function (v) {
      return step([n, v]);
    };
  }
  function step(op) {
    if (f) throw new TypeError("Generator is already executing.");
    while (g && (g = 0, op[0] && (_ = 0)), _) try {
      if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
      if (y = 0, t) op = [op[0] & 2, t.value];
      switch (op[0]) {
        case 0:
        case 1:
          t = op;
          break;
        case 4:
          _.label++;
          return {
            value: op[1],
            done: false
          };
        case 5:
          _.label++;
          y = op[1];
          op = [0];
          continue;
        case 7:
          op = _.ops.pop();
          _.trys.pop();
          continue;
        default:
          if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
            _ = 0;
            continue;
          }
          if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
            _.label = op[1];
            break;
          }
          if (op[0] === 6 && _.label < t[1]) {
            _.label = t[1];
            t = op;
            break;
          }
          if (t && _.label < t[2]) {
            _.label = t[2];
            _.ops.push(op);
            break;
          }
          if (t[2]) _.ops.pop();
          _.trys.pop();
          continue;
      }
      op = body.call(thisArg, _);
    } catch (e) {
      op = [6, e];
      y = 0;
    } finally {
      f = t = 0;
    }
    if (op[0] & 5) throw op[1];
    return {
      value: op[0] ? op[1] : void 0,
      done: true
    };
  }
};
var __rest = this && this.__rest || function (s, e) {
  var t = {};
  for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0) t[p] = s[p];
  if (s != null && typeof Object.getOwnPropertySymbols === "function") for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
    if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i])) t[p[i]] = s[p[i]];
  }
  return t;
};
var __read = this && this.__read || function (o, n) {
  var m = typeof Symbol === "function" && o[Symbol.iterator];
  if (!m) return o;
  var i = m.call(o),
    r,
    ar = [],
    e;
  try {
    while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
  } catch (error) {
    e = {
      error: error
    };
  } finally {
    try {
      if (r && !r.done && (m = i["return"])) m.call(i);
    } finally {
      if (e) throw e.error;
    }
  }
  return ar;
};
var __spreadArray = this && this.__spreadArray || function (to, from, pack) {
  if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
    if (ar || !(i in from)) {
      if (!ar) ar = Array.prototype.slice.call(from, 0, i);
      ar[i] = from[i];
    }
  }
  return to.concat(ar || Array.prototype.slice.call(from));
};
Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.createGitlabJobs = exports.makeGitlabJob = exports.GITLAB_ENVIRONMENT_URL_VARIABLE = void 0;
var lodash_1 = require("lodash");
var getInjectVarsScript_1 = require("../../bash/getInjectVarsScript");
var defaults_1 = require("../../defaults");
var utils_1 = require("../../utils");
var gitlab_1 = require("../../utils/gitlab");
var removeUndefined_1 = require("../../utils/removeUndefined");
var BashExpression_1 = require("../../bash/BashExpression");
var cache_1 = require("./cache");
exports.GITLAB_ENVIRONMENT_URL_VARIABLE = "CL_GITLAB_ENVIRONMENT_URL";
var getFullJobName = function (_a) {
  var type = _a.type,
    name = _a.name,
    baseName = _a.baseName,
    allJobs = _a.allJobs,
    env = _a.env;
  var shouldAddIcon = allJobs.workspaces.length > 0;
  var icon = type === "component" ? "🔹" : type === "agent" ? "🤖" : "🔸";
  var prefix = shouldAddIcon ? icon + " " : "";
  if (env) {
    return "".concat(prefix).concat(baseName, " ").concat(name, " | ").concat(env, " ");
  }
  return "".concat(prefix).concat(baseName, " ").concat(name);
};
var getFullReferencedJobNameFromComponent = function (referencedJobName, componentName, env, allJobs) {
  var _a, _b;
  var referencedJob = (_b = (_a = allJobs.components.find(function (j) {
    return j.context.name === componentName && j.context.env === env;
  })) === null || _a === void 0 ? void 0 : _a.jobs) === null || _b === void 0 ? void 0 : _b.find(function (j) {
    return j.name === referencedJobName;
  });
  if (!referencedJob) {
    throw new Error("unknown job referenced: '".concat(referencedJobName, "' from '").concat(env, ":").concat(componentName, "'"));
  }
  return getFullJobName({
    type: "component",
    name: referencedJobName,
    baseName: componentName,
    env: env,
    allJobs: allJobs
  });
};
var getFullReferencedJobNameFromWorkspace = function (referencedJobName, workspaceName, env, allJobs) {
  var _a, _b;
  var referencedJob = (_b = (_a = allJobs.workspaces.find(function (w) {
    return w.context.name === workspaceName;
  })) === null || _a === void 0 ? void 0 : _a.jobs) === null || _b === void 0 ? void 0 : _b.find(function (j) {
    return j.name === referencedJobName;
  });
  if (!referencedJob) {
    throw new Error("unknown job referenced: '".concat(referencedJobName, "' from workspace ").concat(env, ":").concat(workspaceName, "'"));
  }
  return getFullJobName({
    type: "workspace",
    name: referencedJobName,
    baseName: workspaceName,
    env: env,
    allJobs: allJobs
  });
};
var getJobName = function (need) {
  return (0, lodash_1.isObject)(need) ? need.job : need;
};
var makeGitlabJob = function (context, job, allJobs, baseRules) {
  var _a, _b;
  var environment = job.environment,
    envMode = job.envMode,
    needsStages = job.needsStages,
    name = job.name,
    needs = job.needs,
    jobTags = job.jobTags,
    script = job.script,
    variables = job.variables,
    runnerVariables = job.runnerVariables,
    when = job.when,
    cache = job.cache,
    rest = __rest(job, ["environment", "envMode", "needsStages", "name", "needs", "jobTags", "script", "variables", "runnerVariables", "when", "cache"]);
  var stage = envMode === "stagePerEnv" && context.type !== "agent" ? "".concat(job.stage, " ").concat(context.env) : job.stage;
  var deduplicatedGitlabNeeds = getGitlabNeeds(context, job, allJobs);
  var fullJobName = getFullJobName({
    type: context.type,
    name: name,
    baseName: context.name,
    env: context.type !== "agent" ? context.env : undefined,
    allJobs: allJobs
  });
  // backwards compatibility, some may still use KUBERNETES_CPU_REQUEST, KUBERNETES_MEMORY_REQUEST, etc. in variables.
  // those should now be set in the runnerVariables as they don't work in the variables key of the catladder job, becuase those get injected
  var PIPELINE_RUNNER_VARIABLES = ["KUBERNETES_CPU_REQUEST", "KUBERNETES_MEMORY_REQUEST", "KUBERNETES_CPU_LIMIT", "KUBERNETES_MEMORY_LIMIT"];
  // remove those from variables and add them to runnerVariables
  var varsInjectScripts = (0, gitlab_1.collapseableSection)("injectvars", "Injecting variables")(__spreadArray([], __read((0, getInjectVarsScript_1.getInjectVarsScript)(
  // remove legacy variables
  Object.fromEntries(Object.entries(variables !== null && variables !== void 0 ? variables : {}).filter(function (_a) {
    var _b = __read(_a, 1),
      key = _b[0];
    return !PIPELINE_RUNNER_VARIABLES.includes(key);
  })))), false));
  var legacyRunnerVariables = Object.fromEntries(Object.entries(variables !== null && variables !== void 0 ? variables : {}).filter(function (_a) {
    var _b = __read(_a, 1),
      key = _b[0];
    return PIPELINE_RUNNER_VARIABLES.includes(key);
  }));
  if (Object.keys(legacyRunnerVariables).length > 0) {
    console.warn("Legacy variables detected in ".concat(fullJobName, ": ").concat(Object.keys(legacyRunnerVariables).join(", "), ". Please move them to the runnerVariables key."));
  }
  var rules = __spreadArray(__spreadArray([], __read((_a = job.rules) !== null && _a !== void 0 ? _a : []), false), __read(baseRules ? baseRules.map(function (rule) {
    return __assign({
      when: when
    }, rule);
  }) : when ? [{
    when: when
  }] : []), false);
  var gitlabJob = __assign(__assign({
    retry: defaults_1.BASE_RETRY,
    interruptible: true
  }, rest), {
    cache: cache ? (0, cache_1.addCacheFallback)(cache, context) : undefined,
    rules: rules.length > 0 ? rules : undefined,
    variables: __assign(__assign({}, legacyRunnerVariables), runnerVariables),
    script: __spreadArray(__spreadArray([], __read(varsInjectScripts), false), __read((_b = script === null || script === void 0 ? void 0 : script.filter(utils_1.notNil)) !== null && _b !== void 0 ? _b : []), false),
    tags: jobTags,
    stage: stage,
    // sort in a predictable manner for snapshot tests
    needs: deduplicatedGitlabNeeds
  });
  var modified = addGitlabEnvironment(context, environment, gitlabJob, allJobs);
  return [fullJobName, (0, removeUndefined_1.removeUndefined)(modified)];
};
exports.makeGitlabJob = makeGitlabJob;
var addGitlabEnvironment = function (context, catladderJobEnvironment, job, allJobs) {
  var _a, _b;
  if (!catladderJobEnvironment) {
    return job;
  }
  if (context.type !== "component") {
    // don't add enviornment for workspace and agent jobs atm.
    return job;
  }
  var env = context.env,
    name = context.name,
    environment = context.environment;
  var envVars = environment.envVars,
    envType = environment.envType;
  var on_stop = catladderJobEnvironment.on_stop,
    restEnvironment = __rest(catladderJobEnvironment, ["on_stop"]);
  // those can be dynamic, so we therefore have to do this: https://docs.gitlab.com/ee/ci/environments/#set-a-dynamic-environment-url
  var dotEnvFile = "gitlab_environment.env";
  var createsJobEnv = !catladderJobEnvironment.action || catladderJobEnvironment.action === "start";
  var artifacts = (0, lodash_1.merge)((_a = job.artifacts) !== null && _a !== void 0 ? _a : {}, createsJobEnv ? {
    reports: {
      dotenv: dotEnvFile
    }
  } : {});
  var scriptToAdd = ["echo \"".concat(exports.GITLAB_ENVIRONMENT_URL_VARIABLE, "=").concat((0, BashExpression_1.getBashVariable)("ROOT_URL"), "\" >> ").concat(dotEnvFile)];
  // this is NOT a bashVariable since it NEEDS to be used as a string in gitlab
  var gitlabEnvironmentName = envType === "review" ? "".concat(env, "/$CI_COMMIT_REF_NAME/").concat(name) // FIXME: should be replaced with mr name as well
  : "".concat(env, "/").concat(name);
  return __assign(__assign(__assign(__assign({}, job), {
    environment: __assign(__assign(__assign({
      name: gitlabEnvironmentName
    }, createsJobEnv ? {
      url: "$".concat(exports.GITLAB_ENVIRONMENT_URL_VARIABLE)
    } : {}), on_stop ? {
      on_stop: getFullReferencedJobNameFromComponent(on_stop, name, env, allJobs)
    } : {}), restEnvironment)
  }), !(0, lodash_1.isEmpty)(artifacts) ? {
    artifacts: artifacts
  } : {}), {
    script: __spreadArray(__spreadArray([], __read((_b = job.script) !== null && _b !== void 0 ? _b : []), false), __read(createsJobEnv ? scriptToAdd : []), false)
  });
};
var createGitlabJobs = function (allJobs, baseRules) {
  return __awaiter(void 0, void 0, void 0, function () {
    return __generator(this, function (_a) {
      return [2 /*return*/, __spreadArray(__spreadArray(__spreadArray([], __read(allJobs.workspaces), false), __read(allJobs.components), false), __read(allJobs.agents), false).flatMap(function (_a) {
        var context = _a.context,
          jobs = _a.jobs;
        return jobs.map(function (job) {
          var _a = __read((0, exports.makeGitlabJob)(context, job, allJobs, baseRules), 2),
            fullJobName = _a[0],
            gitlabJob = _a[1];
          return {
            name: fullJobName,
            gitlabJob: gitlabJob,
            context: context
          };
        });
      })];
    });
  });
};
exports.createGitlabJobs = createGitlabJobs;
function getGitlabNeeds(context, job, allJobs) {
  var _a;
  var needs = context.type === "workspace" ? getGitlabNeedsForWorkspaceJob(context, job, allJobs) : context.type === "agent" ? (_a = job.needs) !== null && _a !== void 0 ? _a : null : getGitlabNeedsForComponentJob(context, job, allJobs);
  return needs ? deduplicateNeeds(needs) : undefined;
}
function deduplicateNeeds(needs) {
  return needs ? __spreadArray([], __read(new Map(needs.map(function (n) {
    return [(0, lodash_1.isObject)(n) ? n.job : n, n];
  })).values()), false) : undefined;
}
function getGitlabNeedsForComponentJob(context, _a, allJobs) {
  var needsStages = _a.needsStages,
    needs = _a.needs;
  var needsFromStages = needsStages === null || needsStages === void 0 ? void 0 : needsStages.flatMap(function (n) {
    var _a, _b, _c, _d, _e, _f, _g, _h;
    var componentName = context.name;
    if (!n.workspaceName) {
      var allJobNamesFromThatStage = (_c = (_b = (_a = allJobs.components.filter(function (j) {
        return j.context.name === componentName && j.context.env === context.env;
      }).flatMap(function (j) {
        return j.jobs;
      })) === null || _a === void 0 ? void 0 : _a.filter(function (j) {
        return j.stage === n.stage;
      })) === null || _b === void 0 ? void 0 : _b.map(function (j) {
        return j.name;
      })) !== null && _c !== void 0 ? _c : [];
      return allJobNamesFromThatStage.map(function (job) {
        var _a;
        return {
          job: job,
          artifacts: (_a = n.artifacts) !== null && _a !== void 0 ? _a : false,
          componentName: componentName
        };
      });
    } else {
      var allJobNamesFromThatStage = (_h = (_g = (_f = (_e = (_d = allJobs.workspaces.find(function (w) {
        return w.context.name === n.workspaceName && w.context.env === context.env;
      })) === null || _d === void 0 ? void 0 : _d.jobs) === null || _e === void 0 ? void 0 : _e.flatMap(function (j) {
        return j;
      })) === null || _f === void 0 ? void 0 : _f.filter(function (j) {
        return j.stage === n.stage;
      })) === null || _g === void 0 ? void 0 : _g.map(function (j) {
        return j.name;
      })) !== null && _h !== void 0 ? _h : [];
      return allJobNamesFromThatStage.map(function (job) {
        var _a;
        return {
          job: job,
          artifacts: (_a = n.artifacts) !== null && _a !== void 0 ? _a : false,
          workspaceName: n.workspaceName
        };
      });
    }
  });
  var cleanedNeeds = __spreadArray(__spreadArray([], __read(needsFromStages !== null && needsFromStages !== void 0 ? needsFromStages : []), false), __read(needs !== null && needs !== void 0 ? needs : []), false);
  return cleanedNeeds === null || cleanedNeeds === void 0 ? void 0 : cleanedNeeds.map(function (n) {
    var _a;
    return (0, lodash_1.isObject)(n) ? "workspaceName" in n ? {
      job: getFullReferencedJobNameFromWorkspace(n.job, n.workspaceName, context.env, allJobs),
      artifacts: n.artifacts
    } : {
      job: getFullReferencedJobNameFromComponent(n.job, (_a = n.componentName) !== null && _a !== void 0 ? _a : context.name, context.env, allJobs),
      artifacts: n.artifacts
    } : getFullReferencedJobNameFromComponent(n, context.name, context.env, allJobs);
  }).sort(function (a, b) {
    return getJobName(a).localeCompare(getJobName(b));
  });
}
/**
 *
 *unclear whether we actually need this. So far jobs in a workspace don't have needs to other jobs from the same workspace
 */
function getGitlabNeedsForWorkspaceJob(context, _a, allJobs) {
  var needsStages = _a.needsStages,
    needs = _a.needs;
  var needsFromStages = needsStages === null || needsStages === void 0 ? void 0 : needsStages.flatMap(function (n) {
    var _a, _b, _c, _d;
    var workspaceName = (_a = n.workspaceName) !== null && _a !== void 0 ? _a : context.name;
    var allJobNamesFromThatStage = (_d = (_c = (_b = allJobs.workspaces.filter(function (j) {
      return j.context.name === workspaceName && j.context.env === context.env;
    }).flatMap(function (j) {
      return j.jobs;
    })) === null || _b === void 0 ? void 0 : _b.filter(function (j) {
      return j.stage === n.stage;
    })) === null || _c === void 0 ? void 0 : _c.map(function (j) {
      return j.name;
    })) !== null && _d !== void 0 ? _d : [];
    return allJobNamesFromThatStage.map(function (job) {
      var _a;
      return {
        job: job,
        artifacts: (_a = n.artifacts) !== null && _a !== void 0 ? _a : false,
        workspaceName: workspaceName
      };
    });
  });
  var cleanedNeeds = __spreadArray(__spreadArray([], __read(needsFromStages !== null && needsFromStages !== void 0 ? needsFromStages : []), false), __read(needs !== null && needs !== void 0 ? needs : []), false);
  return cleanedNeeds === null || cleanedNeeds === void 0 ? void 0 : cleanedNeeds.map(function (n) {
    return (0, lodash_1.isObject)(n) ? {
      job: getFullReferencedJobNameFromWorkspace(n.job, "workspaceName" in n && n.workspaceName ? n.workspaceName : context.name, context.env, allJobs),
      artifacts: n.artifacts
    } : getFullReferencedJobNameFromWorkspace(n, context.name, context.env, allJobs);
  }).sort(function (a, b) {
    return getJobName(a).localeCompare(getJobName(b));
  });
}