@atomist/sdm-core
Version:
Atomist Software Delivery Machine - Implementation
339 lines • 15.2 kB
JavaScript
;
/*
* Copyright © 2019 Atomist, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
const automation_client_1 = require("@atomist/automation-client");
const poll_1 = require("@atomist/automation-client/lib/internal/util/poll");
const sdm_1 = require("@atomist/sdm");
const k8s = require("@kubernetes/client-node");
const stringify = require("json-stringify-safe");
const _ = require("lodash");
const os = require("os");
const stream_1 = require("stream");
const config_1 = require("../../pack/k8s/config");
const KubernetesGoalScheduler_1 = require("../../pack/k8s/KubernetesGoalScheduler");
const service_1 = require("../../pack/k8s/service");
const array_1 = require("../../util/misc/array");
const container_1 = require("./container");
const util_1 = require("./util");
exports.k8sContainerScheduler = (goal, registration) => {
goal.addFulfillment(Object.assign({ goalExecutor: executeK8sJob(goal, registration) }, registration));
goal.addFulfillmentCallback({
goal,
callback: k8sFulfillmentCallback(goal, registration),
});
};
/**
* Add Kubernetes job scheduling information to SDM goal event data
* for use by the [[KubernetesGoalScheduler]].
*/
function k8sFulfillmentCallback(goal, registration) {
return (goalEvent, repoContext) => __awaiter(this, void 0, void 0, function* () {
const spec = _.merge({}, { containers: registration.containers, volumes: registration.volumes });
if (registration.callback) {
const project = yield automation_client_1.GitCommandGitProject.cloned(repoContext.credentials, repoContext.id);
_.merge(spec, yield registration.callback(registration, project, goal, goalEvent, repoContext.context));
}
if (!spec.containers || spec.containers.length < 1) {
throw new Error("No containers defined in K8sGoalContainerSpec");
}
if (spec.containers[0].workingDir === "") {
delete spec.containers[0].workingDir;
}
else if (!spec.containers[0].workingDir) {
spec.containers[0].workingDir = container_1.ContainerProjectHome;
}
const containerEnvs = yield util_1.containerEnvVars(goalEvent, repoContext);
spec.containers.forEach(c => {
c.env = [
...containerEnvs,
...(c.env || []),
];
});
const goalSchedulers = array_1.toArray(repoContext.configuration.sdm.goalScheduler) || [];
const k8sScheduler = goalSchedulers.find(gs => gs instanceof KubernetesGoalScheduler_1.KubernetesGoalScheduler);
if (!k8sScheduler) {
throw new Error("Failed to find KubernetesGoalScheduler in goal schedulers");
}
if (!k8sScheduler.podSpec) {
throw new Error("KubernetesGoalScheduler has no podSpec defined");
}
const initContainer = _.cloneDeep(k8sScheduler.podSpec.spec.containers[0]);
delete initContainer.lifecycle;
delete initContainer.livenessProbe;
delete initContainer.readinessProbe;
initContainer.name = `container-goal-init-${automation_client_1.guid().split("-")[0]}`;
initContainer.env = [
...(initContainer.env || []),
...KubernetesGoalScheduler_1.k8sJobEnv(k8sScheduler.podSpec, goalEvent, repoContext.context),
{
name: "ATOMIST_ISOLATED_GOAL_INIT",
value: "true",
},
];
const projectVolume = `project-${automation_client_1.guid().split("-")[0]}`;
initContainer.volumeMounts = [
...(initContainer.volumeMounts || []),
{
mountPath: container_1.ContainerProjectHome,
name: projectVolume,
},
];
initContainer.workingDir = container_1.ContainerProjectHome;
const serviceSpec = {
type: service_1.K8sServiceRegistrationType.K8sService,
spec: {
container: spec.containers,
initContainer,
volume: [
{
name: projectVolume,
emptyDir: {},
},
],
volumeMount: [
{
mountPath: container_1.ContainerProjectHome,
name: projectVolume,
},
],
},
};
const data = JSON.parse(goalEvent.data || "{}");
const servicesData = {};
_.set(servicesData, `${sdm_1.ServiceRegistrationGoalDataKey}.${registration.name}`, serviceSpec);
goalEvent.data = JSON.stringify(_.merge(data, servicesData));
return goalEvent;
});
}
exports.k8sFulfillmentCallback = k8sFulfillmentCallback;
/**
* Wait for first container to exit and stream its logs to the
* progress log.
*/
function executeK8sJob(goal, registration) {
return sdm_1.doWithProject((gi) => __awaiter(this, void 0, void 0, function* () {
const { context, goalEvent, progressLog, project } = gi;
if (process.env.ATOMIST_ISOLATED_GOAL_INIT === "true") {
try {
yield util_1.copyProject(project.baseDir, process.cwd());
}
catch (e) {
const message = `Failed to copy project for goal execution: ${e.message}`;
util_1.loglog(message, automation_client_1.logger.error, progressLog);
return { code: 1, message };
}
goalEvent.state = sdm_1.SdmGoalState.in_process;
return goalEvent;
}
const spec = _.merge({}, { containers: registration.containers, volumes: registration.volumes }, (registration.callback) ? yield registration.callback(registration, project, goal, goalEvent, context) : {});
let containerName = _.get(spec, "containers[0].name");
if (!containerName) {
const msg = `Failed to get main container name from goal registration: ${stringify(spec)}`;
util_1.loglog(msg, automation_client_1.logger.warn, progressLog);
let svcSpec;
try {
const data = JSON.parse(goalEvent.data || "{}");
svcSpec = _.get(data, `${sdm_1.ServiceRegistrationGoalDataKey}.${registration.name}.spec`);
}
catch (e) {
const message = `Failed to parse Kubernetes spec from goal data '${goalEvent.data}': ${e.message}`;
util_1.loglog(message, automation_client_1.logger.error, progressLog);
return { code: 1, message };
}
containerName = _.get(svcSpec, "container[1].name");
if (!containerName) {
const message = `Failed to get main container name from either goal registration or data: '${goalEvent.data}'`;
util_1.loglog(message, automation_client_1.logger.error, progressLog);
return { code: 1, message };
}
}
const ns = yield KubernetesGoalScheduler_1.readNamespace();
const podName = os.hostname();
let kc;
try {
kc = config_1.loadKubeConfig();
}
catch (e) {
const message = `Failed to load Kubernetes configuration: ${e.message}`;
util_1.loglog(message, automation_client_1.logger.error, progressLog);
return { code: 1, message };
}
const container = {
config: kc,
name: containerName,
pod: podName,
ns,
log: progressLog,
};
try {
yield containerStarted(container);
}
catch (e) {
util_1.loglog(e.message, automation_client_1.logger.error, progressLog);
return { code: 1, message: e.message };
}
const log = followK8sLog(container);
const status = { code: 0, message: `Container '${containerName}' completed successfully` };
try {
const podStatus = yield containerWatch(container);
util_1.loglog(`Container '${containerName}' exited: ${stringify(podStatus)}`, automation_client_1.logger.debug, progressLog);
}
catch (e) {
const message = `Container '${containerName}' failed: ${e.message}`;
util_1.loglog(message, automation_client_1.logger.error, progressLog);
status.code++;
status.message = message;
}
finally {
// Give the logs some time to be delivered
yield poll_1.sleep(1000);
log.abort();
}
try {
yield util_1.copyProject(process.cwd(), project.baseDir);
}
catch (e) {
const message = `Failed to update project after goal execution: ${e.message}`;
util_1.loglog(message, automation_client_1.logger.error, progressLog);
status.code++;
status.message += ` but f${message.slice(1)}`;
}
return status;
}), { readOnly: false });
}
exports.executeK8sJob = executeK8sJob;
/**
* Wait for container in pod to start, return when it does.
*
* @param container Information about container to check
* @param attempts Maximum number of attempts, waiting 500 ms between
*/
function containerStarted(container, attempts = 120) {
return __awaiter(this, void 0, void 0, function* () {
let core;
try {
core = container.config.makeApiClient(k8s.CoreV1Api);
}
catch (e) {
e.message = `Failed to create Kubernetes core API client: ${e.message}`;
util_1.loglog(e.message, automation_client_1.logger.error, container.log);
throw e;
}
const sleepTime = 500; // ms
for (let i = 0; i < attempts; i++) {
yield poll_1.sleep(500);
const pod = (yield core.readNamespacedPod(container.pod, container.ns)).body;
const containerStatus = pod.status.containerStatuses.find(c => c.name === container.name);
if (containerStatus && (!!_.get(containerStatus, "state.running.startedAt") || !!_.get(containerStatus, "state.terminated"))) {
const message = `Container '${container.name}' started`;
util_1.loglog(message, automation_client_1.logger.debug, container.log);
return;
}
}
const errMsg = `Container '${container.name}' failed to start within ${attempts * sleepTime} ms`;
util_1.loglog(errMsg, automation_client_1.logger.error, container.log);
throw new Error(errMsg);
});
}
/**
* Watch pod until container `container.name` exits. Resolve promise
* with status if container `container.name` exits with status 0.
* Reject promise otherwise, including pod status in the `podStatus`
* property of the error.
*
* @param container Information about container to watch
* @return Status of pod after container terminates
*/
function containerWatch(container) {
return new Promise((resolve, reject) => {
let watch;
try {
watch = new k8s.Watch(container.config);
}
catch (e) {
e.message = `Failed to create Kubernetes watch client: ${e.message}`;
util_1.loglog(e.message, automation_client_1.logger.error, container.log);
reject(e);
}
const watchPath = `/api/v1/watch/namespaces/${container.ns}/pods/${container.pod}`;
let watcher;
watcher = watch.watch(watchPath, {}, (phase, obj) => {
const pod = obj;
if (pod && pod.status && pod.status.containerStatuses) {
const containerStatus = pod.status.containerStatuses.find(c => c.name === container.name);
if (containerStatus && containerStatus.state && containerStatus.state.terminated) {
const exitCode = _.get(containerStatus, "state.terminated.exitCode");
if (exitCode === 0) {
const msg = `Container '${container.name}' exited with status 0`;
util_1.loglog(msg, automation_client_1.logger.debug, container.log);
resolve(pod.status);
}
else {
const msg = `Container '${container.name}' exited with status ${exitCode}`;
util_1.loglog(msg, automation_client_1.logger.error, container.log);
const err = new Error(msg);
err.podStatus = pod.status;
reject(err);
}
if (watcher) {
watcher.abort();
}
return;
}
}
util_1.loglog(`Container '${container.name}' still running`, automation_client_1.logger.debug, container.log);
}, err => {
err.message = `Container watcher failed: ${err.message}`;
util_1.loglog(err.message, automation_client_1.logger.error, container.log);
reject(err);
});
});
}
/**
* Set up log follower for container.
*/
function followK8sLog(container) {
const k8sLog = new k8s.Log(container.config);
const logStream = new stream_1.Writable({
write: (chunk, encoding, callback) => {
container.log.write(chunk.toString());
callback();
},
});
const doneCallback = e => {
if (e) {
if (e.message) {
util_1.loglog(e.message, automation_client_1.logger.error, container.log);
}
else {
util_1.loglog(stringify(e), automation_client_1.logger.error, container.log);
}
}
};
const logOptions = { follow: true };
return k8sLog.log(container.ns, container.pod, container.name, logStream, doneCallback, logOptions);
}
//# sourceMappingURL=k8s.js.map