@atomist/sdm-core
Version:
Atomist Software Delivery Machine - Implementation
555 lines • 23 kB
JavaScript
;
/*
* Copyright © 2019 Atomist, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
// tslint:disable:max-file-line-count
const automation_client_1 = require("@atomist/automation-client");
const sdm_1 = require("@atomist/sdm");
const k8s = require("@kubernetes/client-node");
const cluster = require("cluster");
const fs = require("fs-extra");
const stringify = require("json-stringify-safe");
const _ = require("lodash");
const os = require("os");
const array_1 = require("../../util/misc/array");
const config_1 = require("./config");
const service_1 = require("./service");
/**
* GoalScheduler implementation that schedules SDM goals inside k8s jobs.
*
* It reuses the podSpec of the deployed SDM to create a new jobSpec from.
* Subclasses may change the spec and job creation behavior by overwriting beforeCreation
* and/or afterCreation methods.
*/
class KubernetesGoalScheduler {
constructor(options = { isolateAll: false }) {
this.options = options;
}
supports(gi) {
return __awaiter(this, void 0, void 0, function* () {
return !process.env.ATOMIST_ISOLATED_GOAL &&
(
// Goal is marked as isolated and SDM is configured to use k8s jobs
(gi.goal.definition.isolated && isConfiguredInEnv("kubernetes")) ||
// Force all goals to run isolated via env var
isConfiguredInEnv("kubernetes-all") ||
// Force all goals to run isolated via explicit option
(this.options.isolateAll && isConfiguredInEnv("kubernetes")) ||
// Force all goals to run isolated via explicit configuration
_.get(gi.configuration, "sdm.k8s.isolateAll", false) === true);
});
}
schedule(gi) {
return __awaiter(this, void 0, void 0, function* () {
const { goalEvent } = gi;
const podNs = yield readNamespace();
const kc = config_1.loadKubeConfig();
const batch = kc.makeApiClient(k8s.BatchV1Api);
const jobSpec = createJobSpec(_.cloneDeep(this.podSpec), podNs, gi);
const jobDesc = `k8s job '${jobSpec.metadata.namespace}:${jobSpec.metadata.name}' for goal '${goalEvent.uniqueName}'`;
yield this.beforeCreation(gi, jobSpec);
gi.progressLog.write(`/--`);
gi.progressLog.write(`Scheduling k8s job '${jobSpec.metadata.namespace}:${jobSpec.metadata.name}' for goal '${goalEvent.name} (${goalEvent.uniqueName})'`);
gi.progressLog.write("\\--");
try {
// Check if this job was previously launched
yield batch.readNamespacedJob(jobSpec.metadata.name, jobSpec.metadata.namespace);
automation_client_1.logger.debug(`${jobDesc} already exists. Deleting...`);
try {
yield batch.deleteNamespacedJob(jobSpec.metadata.name, jobSpec.metadata.namespace, {});
automation_client_1.logger.debug(`${jobDesc} deleted`);
}
catch (e) {
automation_client_1.logger.error(`Failed to delete ${jobDesc}: ${stringify(e.body)}`);
return {
code: 1,
message: `Failed to delete ${jobDesc}: ${prettyPrintError(e)}`,
};
}
}
catch (e) {
// This is ok to ignore as it just means the job doesn't exist
}
try {
automation_client_1.logger.debug(`Job spec for ${jobDesc}: ${JSON.stringify(jobSpec)}`);
// Previous deletion might not have completed; hence the retry here
const jobResult = (yield automation_client_1.doWithRetry(() => batch.createNamespacedJob(jobSpec.metadata.namespace, jobSpec), `Scheduling ${jobDesc}`)).body;
yield this.afterCreation(gi, jobResult);
automation_client_1.logger.info(`Scheduled ${jobDesc} with result: ${stringify(jobResult.status)}`);
automation_client_1.logger.log("silly", stringify(jobResult));
}
catch (e) {
automation_client_1.logger.error(`Failed to schedule ${jobDesc}: ${stringify(e.body)}`);
return {
code: 1,
message: `Failed to schedule ${jobDesc}: ${prettyPrintError(e)}`,
};
}
yield gi.progressLog.flush();
return {
code: 0,
message: `Scheduled ${jobDesc}`,
};
});
}
/**
* Extension point for sub classes to modify k8s resources or provided jobSpec before the
* Job gets created in k8s.
* Note: A potentially existing job with the same name has already been deleted at this point.
* @param gi
* @param jobSpec
*/
beforeCreation(gi, jobSpec) {
return __awaiter(this, void 0, void 0, function* () {
// Intentionally left empty
});
}
/**
* Extension point for sub classes to modify k8s resources after the job has been created.
* The provided jobSpec contains the result of the job creation API call.
* @param gi
* @param jobSpec
*/
afterCreation(gi, jobSpec) {
return __awaiter(this, void 0, void 0, function* () {
// Intentionally left empty
});
}
initialize(configuration) {
return __awaiter(this, void 0, void 0, function* () {
const podName = process.env.ATOMIST_POD_NAME || os.hostname();
const podNs = yield readNamespace();
try {
const kc = config_1.loadKubeClusterConfig();
const core = kc.makeApiClient(k8s.CoreV1Api);
this.podSpec = (yield core.readNamespacedPod(podName, podNs)).body;
}
catch (e) {
automation_client_1.logger.error(`Failed to obtain parent pod spec from k8s: ${prettyPrintError(e)}`);
if (!!this.options.podSpec) {
this.podSpec = this.options.podSpec;
}
else {
throw new Error(`Failed to obtain parent pod spec from k8s: ${prettyPrintError(e)}`);
}
}
if (configuration.cluster.enabled === false || cluster.isMaster) {
setInterval(() => {
return this.cleanUp()
.then(() => {
automation_client_1.logger.debug("Finished cleaning scheduled goal jobs");
});
}, _.get(configuration, "sdm.k8s.job.cleanupInterval", 1000 * 60 * 60 * 2)).unref();
}
});
}
/**
* Extension point to allow for custom clean up logic.
*/
cleanUp() {
return __awaiter(this, void 0, void 0, function* () {
return cleanCompletedJobs();
});
}
}
exports.KubernetesGoalScheduler = KubernetesGoalScheduler;
/**
* Cleanup scheduled k8s goal jobs
* @returns {Promise<void>}
*/
function cleanCompletedJobs() {
return __awaiter(this, void 0, void 0, function* () {
const selector = `atomist.com/creator=${sanitizeName(automation_client_1.configurationValue("name"))}`;
const jobs = yield listJobs(selector);
const completedJobs = jobs.filter(j => j.status && j.status.completionTime && j.status.succeeded && j.status.succeeded > 0);
if (completedJobs.length > 0) {
automation_client_1.logger.debug(`Deleting the following k8s jobs: ${completedJobs.map(j => `${j.metadata.namespace}:${j.metadata.name}`).join(", ")}`);
for (const completedSdmJob of completedJobs) {
const job = { name: completedSdmJob.metadata.name, namespace: completedSdmJob.metadata.namespace };
automation_client_1.logger.debug(`Deleting k8s job '${job.namespace}:${job.name}'`);
yield deleteJob(job);
automation_client_1.logger.debug(`Deleting k8s pods for job '${job.namespace}:${job.name}'`);
yield deletePods(job);
}
}
});
}
exports.cleanCompletedJobs = cleanCompletedJobs;
/** Unique name for goal to use in k8s job spec. */
function k8sJobGoalName(goalEvent) {
return goalEvent.uniqueName.split("#")[0].toLowerCase();
}
/** Unique name for job to use in k8s job spec. */
function k8sJobName(podSpec, goalEvent) {
const goalName = k8sJobGoalName(goalEvent);
return `${podSpec.spec.containers[0].name}-job-${goalEvent.goalSetId.slice(0, 7)}-${goalName}`
.slice(0, 63).replace(/[^a-z0-9]*$/, "");
}
exports.k8sJobName = k8sJobName;
/**
* Kubernetes container spec environment variables that specify an SDM
* running in single-goal mode.
*/
function k8sJobEnv(podSpec, goalEvent, context) {
const goalName = k8sJobGoalName(goalEvent);
const jobName = k8sJobName(podSpec, goalEvent);
const envVars = [
{
name: "ATOMIST_JOB_NAME",
value: jobName,
},
{
name: "ATOMIST_REGISTRATION_NAME",
value: `${automation_client_1.automationClientInstance().configuration.name}-job-${goalEvent.goalSetId.slice(0, 7)}-${goalName}`,
},
{
name: "ATOMIST_GOAL_TEAM",
value: context.workspaceId,
},
{
name: "ATOMIST_GOAL_TEAM_NAME",
value: context.context.workspaceName,
},
{
name: "ATOMIST_GOAL_ID",
value: goalEvent.id,
},
{
name: "ATOMIST_GOAL_SET_ID",
value: goalEvent.goalSetId,
},
{
name: "ATOMIST_GOAL_UNIQUE_NAME",
value: goalEvent.uniqueName,
},
{
name: "ATOMIST_CORRELATION_ID",
value: context.correlationId,
},
{
name: "ATOMIST_ISOLATED_GOAL",
value: "true",
},
];
return envVars;
}
exports.k8sJobEnv = k8sJobEnv;
/**
* Create a jobSpec by modifying the provided podSpec
* @param podSpec
* @param podNs
* @param gi
*/
function createJobSpec(podSpec, podNs, gi) {
const { goalEvent, context } = gi;
const jobSpec = createJobSpecWithAffinity(podSpec, gi);
jobSpec.metadata.name = k8sJobName(podSpec, goalEvent);
jobSpec.metadata.namespace = podNs;
jobSpec.spec.backoffLimit = 1;
jobSpec.spec.template.spec.restartPolicy = "Never";
jobSpec.spec.template.spec.containers[0].name = jobSpec.metadata.name;
jobSpec.spec.template.spec.containers[0].env.push(...k8sJobEnv(podSpec, goalEvent, context));
rewriteCachePath(jobSpec, context.workspaceId);
// Add additional specs from registered services to the job spec
if (_.get(gi.configuration, "sdm.k8s.service.enabled", true)) {
if (!!goalEvent.data) {
let data = {};
try {
data = JSON.parse(goalEvent.data);
}
catch (e) {
automation_client_1.logger.warn(`Failed to parse goal data on '${goalEvent.uniqueName}'`);
}
if (!!data[sdm_1.ServiceRegistrationGoalDataKey]) {
_.forEach(data[sdm_1.ServiceRegistrationGoalDataKey], (v, k) => {
automation_client_1.logger.debug(`Service with name '${k}' and type '${v.type}' found for goal '${goalEvent.uniqueName}'`);
if (v.type === service_1.K8sServiceRegistrationType.K8sService) {
const spec = v.spec;
if (!!spec.container) {
const c = array_1.toArray(spec.container);
jobSpec.spec.template.spec.containers.push(...c);
}
if (!!spec.initContainer) {
const ic = array_1.toArray(spec.initContainer);
jobSpec.spec.template.spec.initContainers = [
...(jobSpec.spec.template.spec.initContainers || []),
...ic,
];
}
if (!!spec.volume) {
const vo = array_1.toArray(spec.volume);
jobSpec.spec.template.spec.volumes = [
...(jobSpec.spec.template.spec.volumes || []),
...vo,
];
}
if (!!spec.volumeMount) {
const vm = array_1.toArray(spec.volumeMount);
[...jobSpec.spec.template.spec.containers, ...jobSpec.spec.template.spec.initContainers].forEach(c => {
c.volumeMounts = [
...(c.volumeMounts || []),
...vm,
];
});
}
if (!!spec.imagePullSecret) {
const ips = array_1.toArray(spec.imagePullSecret);
jobSpec.spec.template.spec.imagePullSecrets = [
...(jobSpec.spec.template.spec.imagePullSecrets || []),
...ips,
];
}
}
});
}
}
}
return jobSpec;
}
exports.createJobSpec = createJobSpec;
/**
* Create a k8s Job spec with affinity to jobs for the same goal set
* @param goalSetId
*/
function createJobSpecWithAffinity(podSpec, gi) {
const { goalEvent, configuration, context } = gi;
_.defaultsDeep(podSpec.spec.affinity, {
podAffinity: {
preferredDuringSchedulingIgnoredDuringExecution: [
{
weight: 100,
podAffinityTerm: {
labelSelector: {
matchExpressions: [
{
key: "atomist.com/goal-set-id",
operator: "In",
values: [
goalEvent.goalSetId,
],
},
],
},
topologyKey: "kubernetes.io/hostname",
},
},
],
},
});
// Clean up podSpec
// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.13/#pod-v1-core note on nodeName
delete podSpec.spec.nodeName;
const labels = {
"atomist.com/goal-set-id": goalEvent.goalSetId,
"atomist.com/goal-id": goalEvent.id,
"atomist.com/creator": sanitizeName(configuration.name),
"atomist.com/workspace-id": context.workspaceId,
};
const detail = {
sdm: {
name: configuration.name,
version: configuration.version,
},
goal: {
goalId: goalEvent.id,
goalSetId: goalEvent.goalSetId,
uniqueName: goalEvent.uniqueName,
},
};
const annotations = {
"atomist.com/sdm": JSON.stringify(detail),
};
return {
kind: "Job",
apiVersion: "batch/v1",
metadata: {
labels,
annotations,
},
spec: {
template: {
metadata: {
labels,
},
spec: podSpec.spec,
},
},
};
}
/**
* Rewrite the volume host path to include the workspace id to prevent cross workspace content ending
* up in the same directory.
* @param jobSpec
* @param workspaceId
*/
function rewriteCachePath(jobSpec, workspaceId) {
const cachePath = automation_client_1.configurationValue("sdm.cache.path", "/opt/data");
const containers = _.get(jobSpec, "spec.template.spec.containers", []);
const cacheVolumeNames = [];
containers.forEach(c => {
cacheVolumeNames.push(...c.volumeMounts.filter(vm => vm.mountPath === cachePath).map(cm => cm.name));
});
_.uniq(cacheVolumeNames).forEach(vn => {
const volume = _.get(jobSpec, "spec.template.spec.volumes", []).find(v => v.name === vn);
if (!!volume && !!volume.hostPath && !!volume.hostPath.path) {
const path = volume.hostPath.path;
if (!path.endsWith(workspaceId) || !path.endsWith(`${workspaceId}/`)) {
if (path.endsWith("/")) {
volume.hostPath.path = `${path}${workspaceId}`;
}
else {
volume.hostPath.path = `${path}/${workspaceId}`;
}
}
}
});
}
/**
* Checks if one of the provided values is configured in ATOMIST_GOAL_SCHEDULER or -
* for backwards compatibility reasons - ATOMIST_GOAL_LAUNCHER.
* @param values
*/
function isConfiguredInEnv(...values) {
const value = process.env.ATOMIST_GOAL_SCHEDULER || process.env.ATOMIST_GOAL_LAUNCHER;
if (!!value) {
try {
const json = JSON.parse(value);
if (Array.isArray(json)) {
return json.some(v => values.includes(v));
}
else {
return values.includes(json);
}
}
catch (e) {
if (typeof value === "string") {
return values.includes(value);
}
}
}
return false;
}
exports.isConfiguredInEnv = isConfiguredInEnv;
/**
* Strip out any characters that aren't allowed a k8s label value
* @param name
*/
function sanitizeName(name) {
return name.replace(/@/g, "").replace(/\//g, ".");
}
exports.sanitizeName = sanitizeName;
/**
* List k8s jobs for a single namespace or cluster-wide depending on evn configuration
* @param labelSelector
*/
function listJobs(labelSelector) {
return __awaiter(this, void 0, void 0, function* () {
const kc = config_1.loadKubeConfig();
const batch = kc.makeApiClient(k8s.BatchV1Api);
if (automation_client_1.configurationValue("sdm.k8s.job.singleNamespace", true)) {
const podNs = yield readNamespace();
return (yield batch.listNamespacedJob(podNs, undefined, undefined, undefined, undefined, labelSelector)).body.items;
}
else {
return (yield batch.listJobForAllNamespaces(undefined, undefined, undefined, labelSelector)).body.items;
}
});
}
exports.listJobs = listJobs;
exports.K8sNamespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace";
/**
* Read the namespace of the deployment from environment and k8s service account files.
* Falls back to the default namespace and no other configuration can be found.
*/
function readNamespace() {
return __awaiter(this, void 0, void 0, function* () {
let podNs = process.env.ATOMIST_POD_NAMESPACE || process.env.ATOMIST_DEPLOYMENT_NAMESPACE;
if (!!podNs) {
return podNs;
}
if (yield fs.pathExists(exports.K8sNamespaceFile)) {
podNs = (yield fs.readFile(exports.K8sNamespaceFile)).toString().trim();
}
if (!!podNs) {
return podNs;
}
return "default";
});
}
exports.readNamespace = readNamespace;
function prettyPrintError(e) {
if (!!e.body) {
return e.body.message;
}
else {
return e.message;
}
}
exports.prettyPrintError = prettyPrintError;
function deleteJob(job) {
return __awaiter(this, void 0, void 0, function* () {
try {
const kc = config_1.loadKubeConfig();
const batch = kc.makeApiClient(k8s.BatchV1Api);
yield batch.readNamespacedJob(job.name, job.namespace);
try {
yield batch.deleteNamespacedJob(job.name, job.namespace, { propagationPolicy: "Foreground" });
}
catch (e) {
automation_client_1.logger.warn(`Failed to delete k8s jobs '${job.namespace}:${job.name}': ${prettyPrintError(e)}`);
}
}
catch (e) {
// This is ok to ignore because the job doesn't exist any more
}
});
}
exports.deleteJob = deleteJob;
function deletePods(job) {
return __awaiter(this, void 0, void 0, function* () {
try {
const kc = config_1.loadKubeConfig();
const core = kc.makeApiClient(k8s.CoreV1Api);
const selector = `job-name=${job.name}`;
const pods = yield core.listNamespacedPod(job.namespace, undefined, undefined, undefined, undefined, selector);
if (pods.body && pods.body.items) {
for (const pod of pods.body.items) {
try {
yield core.deleteNamespacedPod(pod.metadata.name, pod.metadata.namespace, {});
}
catch (e) {
// Probably ok because pod might be gone already
automation_client_1.logger.debug(`Failed to delete k8s pod '${pod.metadata.namespace}:${pod.metadata.name}': ${prettyPrintError(e)}`);
}
}
}
}
catch (e) {
automation_client_1.logger.warn(`Failed to list pods for k8s job '${job.namespace}:${job.name}': ${prettyPrintError(e)}`);
}
});
}
exports.deletePods = deletePods;
//# sourceMappingURL=KubernetesGoalScheduler.js.map