@atomist/sdm-core/src/lib/goal/container/container.ts

Atomist Software Delivery Machine - Implementation

/*
 * Copyright © 2019 Atomist, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import { GitProject } from "@atomist/automation-client";
import {
    DefaultGoalNameGenerator,
    FulfillableGoal,
    FulfillableGoalDetails,
    FulfillableGoalWithRegistrations,
    Fulfillment,
    getGoalDefinitionFrom,
    Goal,
    GoalFulfillmentCallback,
    ImplementationRegistration,
    RepoContext,
    SdmGoalEvent,
    SoftwareDeliveryMachine,
    testProgressReporter,
} from "@atomist/sdm";
import {
    KubernetesFulfillmentGoalScheduler,
    KubernetesFulfillmentOptions,
} from "../../pack/k8s/KubernetesFulfillmentGoalScheduler";
import {
    isConfiguredInEnv,
    KubernetesGoalScheduler,
} from "../../pack/k8s/KubernetesGoalScheduler";
import { KubernetesJobDeletingGoalCompletionListenerFactory } from "../../pack/k8s/KubernetesJobDeletingGoalCompletionListener";
import { toArray } from "../../util/misc/array";
import {
    CacheEntry,
    cachePut,
    cacheRestore,
} from "../cache/goalCaching";
import { dockerContainerScheduler } from "./docker";
import { k8sContainerScheduler } from "./k8s";
import {
    runningAsGoogleCloudFunction,
    runningInK8s,
} from "./util";

export const ContainerRegistrationGoalDataKey = "@atomist/sdm/container";

/**
 * Create and return a container goal with the provided container
 * specification.
 *
 * @param displayName Goal display name
 * @param registration Goal containers, volumes, cache details, etc.
 * @return SDM container goal
 */
export function container<T extends ContainerRegistration>(displayName: string, registration: T): FulfillableGoal {
    return new Container({ displayName }).with(registration);
}

export const ContainerProgressReporter = testProgressReporter({
    test: /docker 'network' 'create'/i,
    phase: "starting up",
}, {
    test: /docker 'network' 'rm'/i,
    phase: "shutting down",
}, {
    test: /docker 'run' .* '--workdir=[a-zA-Z\/]*' .* '--network-alias=([a-zA-Z \-_]*)'/i,
    phase: "running $1",
}, {
    test: /atm:phase=(.*)/i,
    phase: "$1",
});

/**
 * Ports to expose from container.
 */
export interface ContainerPort {
    /**
     * Number of port to expose from the container. This must be
     * a valid port number, 0 < x < 65536.
     */
    containerPort: number;
}

/**
 * Volumes to mount in container.
 */
export interface ContainerVolumeMount {
    /** Path to mount point of volume. */
    mountPath: string;
    /** Name of volume from [[GoalContainer.volumes]]. */
    name: string;
}

export interface ContainerSecrets {
    env?: Array<{ name: string } & GoalContainerSecret>;
    fileMounts?: Array<{ mountPath: string } & GoalContainerSecret>;
}

/**
 * Simplified container abstraction for goals.
 */
export interface GoalContainer {
    /** Unique name for this container within this goal. */
    name: string;
    /** Full Docker image name, i.e., `registry/name:tag`. */
    image: string;
    /**
     * Docker command and arguments.  We call this `args` rather than
     * `command` because we think k8s got it right.
     */
    args?: string[];
    /**
     * Docker image entrypoint.  We call this `command` rather than
     * `entrypoint` because we think k8s got it right.
     */
    command?: string[];
    /**
     * Environment variables to set in Docker container.
     */
    env?: Array<{ name: string, value: string }>;
    /**
     * Ports to expose from container.
     */
    ports?: ContainerPort[];
    /**
     * Volumes to mount in container.
     */
    volumeMounts?: ContainerVolumeMount[];
    /**
     * Provider secrets that should be made available to the container
     */
    secrets?: ContainerSecrets;
}

export interface GoalContainerProviderSecret {
    provider: {
        type: "docker" | "npm" | "maven2" | "scm" | "atomist";
        names?: string[];
    };
}

export interface GoalContainerEncryptedSecret {
    encrypted: string;
}

export interface GoalContainerSecret {
    value: GoalContainerProviderSecret | GoalContainerEncryptedSecret;
}

/**
 * Volumes that containers in goal can mount.
 */
export interface GoalContainerVolume {
    /** Volume to be created from local host file system location. */
    hostPath: {
        /** Absolute path on host to volume. */
        path: string;
    };
    /** Unique name of volume, referenced by [[ContainerVolumeMount.name]]. */
    name: string;
}

/**
 * File system location of goal project in containers.
 */
export const ContainerProjectHome = "/atm/home";

/**
 * File system location for goal container input.
 */
export const ContainerInput = "/atm/input";

/**
 * File system location for goal container output.
 */
export const ContainerOutput = "/atm/output";

/**
 * Goal execution result file
 */
export const ContainerResult = `${ContainerOutput}/result.json`;

/**
 * Specification of containers and volumes for a container goal.
 */
export interface GoalContainerSpec {
    /**
     * Containers to run for this goal.  The goal result is based on
     * the exit status of the first element of the `containers` array.
     * The other containers are considered "sidecar" containers
     * provided functionality that the main container needs to
     * function.  The working directory of the first container is set
     * to [[ContainerProjectHome]], which contains the project upon
     * which the goal should operate.
     */
    containers: GoalContainer[];
    /**
     * Volumes available to mount in containers.
     */
    volumes?: GoalContainerVolume[];
}

/**
 * Function signature for callback that can modify and return the
 * [[ContainerRegistration]] object.
 */
export type ContainerSpecCallback =
    (r: ContainerRegistration, p: GitProject, g: Container, e: SdmGoalEvent, c: RepoContext) => Promise<GoalContainerSpec>;

/**
 * Container goal artifacts and implementations information.
 */
export interface ContainerRegistration extends Partial<ImplementationRegistration>, GoalContainerSpec {
    /**
     * Callback function to dynamically modify the goal container spec
     * when goal is executed.
     */
    callback?: ContainerSpecCallback;
    /**
     * Cache classifiers to retrieve from cache before starting goal
     * execution.  The values must correspond to output classifiers
     * from previously executed container goals in the same goal set.
     */
    input?: Array<{ classifier: string }>;
    /**
     * File path globs to store in cache after goal execution.
     * They values should be glob paths relative to the root of
     * the project directory.
     */
    output?: CacheEntry[];
}

/**
 * Container goal scheduler implementation.  The goal execution is
 * handled as part of the execution of the container.
 */
export type ContainerScheduler = (goal: Container, registration: ContainerRegistration) => void;

export interface ContainerGoalDetails extends FulfillableGoalDetails {
    /**
     * Container goal scheduler.  If no scheduler is provided, the k8s
     * scheduler is used if the SDM is running in a Kubernetes
     * cluster, otherwise the Docker scheduler is used.
     */
    scheduler?: ContainerScheduler;
}

/**
 * Goal run as a container, as seen on TV.
 */
export class Container extends FulfillableGoalWithRegistrations<ContainerRegistration> {

    public readonly details: ContainerGoalDetails;

    constructor(details: ContainerGoalDetails = {}, ...dependsOn: Goal[]) {
        const prefix = "container" + (details.displayName ? `-${details.displayName}` : "");
        const detailsToUse = { ...details, isolate: true };
        super(getGoalDefinitionFrom(detailsToUse, DefaultGoalNameGenerator.generateName(prefix)), ...dependsOn);
        this.details = detailsToUse;
    }

    public register(sdm: SoftwareDeliveryMachine): void {
        super.register(sdm);

        const goalSchedulers = toArray(sdm.configuration.sdm.goalScheduler) || [];
        if (runningInK8s()) {
            // Make sure that the KubernetesGoalScheduler gets added if needed
            if (!goalSchedulers.some(gs => gs instanceof KubernetesGoalScheduler)) {
                if (!process.env.ATOMIST_ISOLATED_GOAL && isConfiguredInEnv("kubernetes", "kubernetes-all")) {
                    sdm.configuration.sdm.goalScheduler = [...goalSchedulers, new KubernetesGoalScheduler()];
                    sdm.addGoalCompletionListener(new KubernetesJobDeletingGoalCompletionListenerFactory(sdm).create());
                }
            }
        } else if (runningAsGoogleCloudFunction()) {
            const options: KubernetesFulfillmentOptions = sdm.configuration.sdm?.k8s?.fulfillment;
            if (!goalSchedulers.some(gs => gs instanceof KubernetesFulfillmentGoalScheduler)) {
                sdm.configuration.sdm.goalScheduler = [
                    ...goalSchedulers,
                    new KubernetesFulfillmentGoalScheduler(options),
                ];
            }
        }
    }

    public with(registration: ContainerRegistration): this {
        super.with(registration);

        registration.name = (registration.name || `container-${this.definition.displayName}`).replace(/\.+/g, "-");
        if (!this.details.scheduler) {
            if (runningInK8s() || runningAsGoogleCloudFunction()) {
                this.details.scheduler = k8sContainerScheduler;
            } else {
                this.details.scheduler = dockerContainerScheduler;
            }
        }
        this.details.scheduler(this, registration);
        if (registration.input && registration.input.length > 0) {
            this.withProjectListener(cacheRestore({ entries: registration.input }));
        }
        if (registration.output && registration.output.length > 0) {
            this.withProjectListener(cachePut({ entries: registration.output }));
        }
        return this;
    }

    public addFulfillment(fulfillment: Fulfillment): this {
        return super.addFulfillment(fulfillment);
    }

    public addFulfillmentCallback(cb: GoalFulfillmentCallback): this {
        return super.addFulfillmentCallback(cb);
    }
}