@adpt/cloud
Version:
AdaptJS cloud component library
541 lines • 18.3 kB
JavaScript
;
/*
* Copyright 2019-2020 Unbounded Systems, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
Object.defineProperty(exports, "__esModule", { value: true });
const tslib_1 = require("tslib");
const utils_1 = require("@adpt/utils");
const debug_1 = tslib_1.__importDefault(require("debug"));
const execa_1 = tslib_1.__importDefault(require("execa"));
const fs_extra_1 = tslib_1.__importDefault(require("fs-extra"));
const lodash_1 = tslib_1.__importDefault(require("lodash"));
const path = tslib_1.__importStar(require("path"));
const randomstring_1 = tslib_1.__importDefault(require("randomstring"));
const shellwords_ts_1 = tslib_1.__importDefault(require("shellwords-ts"));
const common_1 = require("../common");
const env_1 = require("../env");
const labels_1 = require("./labels");
const debug = debug_1.default("adapt:cloud:docker");
// Enable with DEBUG=adapt:cloud:docker:out*
const debugOut = debug_1.default("adapt:cloud:docker:out");
let cmdId = 0;
// Should move to utils
function streamToDebug(s, d, prefix) {
prefix = prefix ? `[${prefix}] ` : "";
s.on("data", (chunk) => d(prefix + chunk.toString()));
s.on("error", (err) => debug(prefix, err));
}
exports.pickGlobals = (opts) => lodash_1.default.pick(opts, "dockerHost");
/**
* Common version of busybox to use internally.
* @internal
*/
exports.busyboxImage = "busybox:1";
/*
* Staged build utilities
*/
async function writeFiles(pwd, files) {
// Strip any leading slash
files = files.map((f) => {
return f.path.startsWith("/") ?
{ path: f.path.slice(1), contents: f.contents } :
f;
});
// Make any directories required
const dirs = lodash_1.default.uniq(files
.map((f) => path.dirname(f.path))
.filter((d) => d !== "."));
await Promise.all(dirs.map(async (d) => fs_extra_1.default.mkdirp(path.resolve(pwd, d))));
await Promise.all(files.map(async (f) => {
const contents = lodash_1.default.isString(f.contents) ? Buffer.from(f.contents) : f.contents;
return fs_extra_1.default.writeFile(path.resolve(pwd, f.path), contents);
}));
}
async function buildFilesImage(files, opts) {
const dockerfile = `
FROM scratch
COPY . /
`;
return utils_1.withTmpDir(async (dir) => {
await writeFiles(dir, files);
return dockerBuild("-", dir, Object.assign({}, exports.pickGlobals(opts), { forceRm: true, imageName: "adapt-tmp-files", uniqueTag: true, stdin: dockerfile, deployID: opts.deployID }));
}, { prefix: "adapt-docker-build" });
}
exports.buildFilesImage = buildFilesImage;
async function withFilesImage(files, opts, fn) {
if (!files || files.length === 0)
return fn(undefined);
const image = await buildFilesImage(files, opts);
try {
return await fn(image);
}
finally {
const { deployID } = opts, rmOpts = tslib_1.__rest(opts, ["deployID"]);
// Only remove what we built. If we tagged the image, just remove
// the tag and let Docker decide if the actual image is unused.
// If there's no tag, try delete by ID, but don't warn if that ID
// has been tagged by someone else.
const nameOrId = image.nameTag || image.id;
try {
await dockerRemoveImage(Object.assign({ nameOrId }, rmOpts));
}
catch (err) {
err = utils_1.ensureError(err);
if (!/image is referenced in multiple repositories/.test(err.message)) {
// tslint:disable-next-line: no-console
console.warn(`Unable to delete temporary Docker image: `, err.message);
}
}
}
}
exports.withFilesImage = withFilesImage;
/** @internal */
async function execDocker(args, options) {
const globalArgs = [];
if (options.dockerHost)
globalArgs.push("-H", options.dockerHost);
args = globalArgs.concat(args);
const execaOpts = {
all: true,
input: options.stdin,
env: env_1.mergeEnvSimple(options.env),
};
const cmdDebug = debugOut.enabled ? debugOut.extend((++cmdId).toString()) :
debug.enabled ? debug :
null;
if (cmdDebug)
cmdDebug(`Running: ${"docker " + args.join(" ")}`);
try {
const ret = execa_1.default("docker", args, execaOpts);
if (debugOut.enabled && cmdDebug) {
streamToDebug(ret.stdout, cmdDebug);
streamToDebug(ret.stderr, cmdDebug);
}
return await ret;
}
catch (e) {
if (e.all)
e.message += "\n" + e.all;
throw e;
}
}
exports.execDocker = execDocker;
exports.defaultDockerBuildOptions = {
forceRm: true,
uniqueTag: false,
};
function collectBuildArgs(opts) {
if (!opts.buildArgs)
return [];
const buildArgs = env_1.mergeEnvPairs(opts.buildArgs);
if (!buildArgs)
return [];
const expanded = buildArgs.map((e) => ["--build-arg", `${e.name}=${e.value}`]);
return lodash_1.default.flatten(expanded);
}
async function dockerBuild(dockerfile, contextPath, options = {}) {
const opts = Object.assign({}, exports.defaultDockerBuildOptions, options);
let nameTag;
const args = ["build", "-f", dockerfile];
if (dockerfile === "-" && !opts.stdin) {
throw new Error(`dockerBuild: stdin option must be set if dockerfile is "-"`);
}
if (opts.forceRm)
args.push("--force-rm");
if (opts.uniqueTag && !opts.imageName) {
throw new Error(`dockerBuild: imageName must be set if uniqueTag is true`);
}
if (opts.imageName) {
const tag = createTag(opts.imageTag, opts.uniqueTag);
nameTag = tag ? `${opts.imageName}:${tag}` : opts.imageName;
if (!opts.uniqueTag)
args.push("-t", nameTag);
}
if (opts.deployID) {
args.push("--label", `${labels_1.adaptDockerDeployIDKey}=${opts.deployID}`);
}
const buildArgs = collectBuildArgs(opts);
args.push(...buildArgs);
args.push(contextPath);
const cmdRet = await execDocker(args, opts);
const { stdout, stderr } = cmdRet;
if (debug.enabled)
debugBuild(cmdRet);
const match = /^Successfully built ([0-9a-zA-Z]+)$/mg.exec(stdout);
if (!match || !match[1])
throw new Error("Could not extract image sha\n" + stdout + "\n\n" + stderr);
const id = await dockerImageId(match[1], opts);
if (id == null)
throw new Error(`Built image ID not found`);
if (opts.uniqueTag) {
const prevId = opts.prevUniqueTag && await dockerImageId(opts.prevUniqueTag, opts);
if (prevId === id)
nameTag = opts.prevUniqueTag; // prev points to current id
else {
if (!nameTag)
throw new utils_1.InternalError(`nameTag not set`);
await dockerTag(Object.assign({ existing: id, newTag: nameTag }, exports.pickGlobals(opts)));
}
}
const ret = { id };
if (nameTag)
ret.nameTag = nameTag;
return ret;
}
exports.dockerBuild = dockerBuild;
function debugBuild(cmdRet) {
const steps = [];
let cur = "";
cmdRet.stdout.split("\n").forEach((l) => {
if (l.startsWith("Step")) {
if (cur)
steps.push(cur);
cur = l;
}
else if (l.startsWith(" ---> ")) {
cur += l;
}
});
if (cur)
steps.push(cur);
const cached = cur.includes("Using cache");
debug(`docker ${cmdRet.command}:\n Cached: ${cached}\n ${steps.join("\n ")}`);
}
/**
* Fetch the image id for a Docker image
*
* @internal
*/
async function dockerImageId(name, opts = {}) {
try {
const inspect = await dockerInspect([name], Object.assign({ type: "image" }, opts));
if (inspect.length > 1)
throw new Error(`Multiple images found`);
if (inspect.length === 0)
return undefined;
return inspect[0].Id;
}
catch (err) {
throw new Error(`Error getting image id for ${name}: ${err.message}`);
}
}
exports.dockerImageId = dockerImageId;
async function dockerTag(options) {
const { existing, newTag } = options;
await execDocker(["tag", existing, newTag], options);
}
exports.dockerTag = dockerTag;
const dockerRemoveImageDefaults = {
force: false,
};
async function dockerRemoveImage(options) {
const opts = Object.assign({}, dockerRemoveImageDefaults, options);
const args = ["rmi"];
if (opts.force)
args.push("--force");
args.push(opts.nameOrId);
await execDocker(args, opts);
}
exports.dockerRemoveImage = dockerRemoveImage;
function createTag(baseTag, appendUnique) {
if (!baseTag && !appendUnique)
return undefined;
let tag = baseTag || "";
if (baseTag && appendUnique)
tag += "-";
if (appendUnique) {
tag += randomstring_1.default.generate({
length: 8,
charset: "alphabetic",
readable: true,
capitalization: "lowercase",
});
}
return tag;
}
async function dockerInspect(namesOrIds, opts = {}) {
const execArgs = ["inspect"];
if (opts.type)
execArgs.push(`--type=${opts.type}`);
let inspectRet;
try {
inspectRet = await execDocker([...execArgs, ...namesOrIds], opts);
}
catch (e) {
if (common_1.isExecaError(e) && e.stderr.startsWith("Error: No such")) {
inspectRet = e;
}
else
throw e;
}
try {
const inspect = JSON.parse(inspectRet.stdout);
if (!Array.isArray(inspect))
throw new Error(`docker inspect result is not an array`);
return inspect;
}
catch (err) {
throw new Error(`Error inspecting docker objects ${namesOrIds}: ${err.message}`);
}
}
exports.dockerInspect = dockerInspect;
/**
* Return a list of all network names
*
* @internal
*/
async function dockerNetworkLs(opts) {
const result = await execDocker(["network", "ls", "--format", "{{json .Name}}"], opts);
const ret = [];
for (const line of result.stdout.split("\n")) {
ret.push(JSON.parse(line));
}
return ret;
}
exports.dockerNetworkLs = dockerNetworkLs;
/**
* Return all networks and their inspect reports
*
* @internal
*/
async function dockerNetworks(opts) {
const networks = await dockerNetworkLs(opts);
return dockerInspect(networks, Object.assign({}, opts, { type: "network" }));
}
exports.dockerNetworks = dockerNetworks;
/**
* Run docker stop
*
* @internal
*/
async function dockerStop(namesOrIds, opts) {
const args = ["stop", ...namesOrIds];
await execDocker(args, opts);
}
exports.dockerStop = dockerStop;
/**
* Run docker rm
*
* @internal
*/
async function dockerRm(namesOrIds, opts) {
const args = ["rm", ...namesOrIds];
await execDocker(args, opts);
}
exports.dockerRm = dockerRm;
const defaultDockerRunOptions = {
background: true,
privileged: false,
};
/**
* Run a container via docker run
*
* @internal
*/
async function dockerRun(options) {
const opts = Object.assign({}, defaultDockerRunOptions, options);
const { background, labels, mounts, name, portBindings, ports, privileged, restartPolicy, } = opts;
const args = ["run"];
if (privileged)
args.push("--privileged");
if (background)
args.push("-d");
if (name)
args.push("--name", name);
if (labels) {
for (const l of Object.keys(labels)) {
args.push("--label", `${l}=${labels[l]}`); //FIXME(manishv) better quoting/format checking here
}
}
if (opts.autoRemove)
args.push("--rm");
if (portBindings) {
const portArgs = Object.keys(portBindings).map((k) => `-p${portBindings[k]}:${k}`);
args.push(...portArgs);
}
if (opts.stopSignal)
args.push("--stop-signal", opts.stopSignal);
if (opts.network !== undefined) {
args.push("--network", opts.network);
}
if (opts.environment !== undefined) {
const envPairs = env_1.mergeEnvPairs(opts.environment);
if (envPairs) {
for (const evar of envPairs) {
args.push("-e", `${evar.name}=${evar.value}`);
}
}
}
if (ports)
args.push(...ports.map((p) => `--expose=${p}`));
args.push(...restartPolicyArgs(restartPolicy));
if (mounts)
args.push(...lodash_1.default.flatten(mounts.map(mountArgs)));
args.push(opts.image);
if (typeof opts.command === "string")
args.push(...shellwords_ts_1.default.split(opts.command));
if (Array.isArray(opts.command))
args.push(...opts.command);
return execDocker(args, opts);
}
exports.dockerRun = dockerRun;
function restartPolicyArgs(policy) {
if (!policy)
return [];
switch (policy.name) {
case undefined:
case "":
case "Never":
return [];
case "Always":
return ["--restart=always"];
case "UnlessStopped":
return ["--restart=unless-stopped"];
case "OnFailure":
const max = policy.maximumRetryCount ? ":" + policy.maximumRetryCount : "";
return [`--restart=on-failure${max}`];
default:
throw new Error(`Invalid RestartPolicy name '${policy.name}'`);
}
}
const stringVal = (key) => (val) => `${key}=${val}`;
const mountArgTransform = {
type: stringVal("type"),
source: stringVal("source"),
destination: stringVal("destination"),
readonly: (val) => val ? "readonly" : "",
propagation: stringVal("propagation"),
};
function mountArgs(mount) {
const items = [];
for (const [k, v] of Object.entries(mount)) {
const xform = mountArgTransform[k];
if (!xform) {
throw new Error(`Invalid mount property '${k}'`);
}
items.push(xform(v));
}
return ["--mount", items.join(",")];
}
/**
* Attach containers to given networks
*
* @internal
*/
async function dockerNetworkConnect(containerNameOrId, networks, opts) {
const optsWithDefs = Object.assign({ alreadyConnectedError: true }, opts);
const { alreadyConnectedError } = optsWithDefs, execOpts = tslib_1.__rest(optsWithDefs, ["alreadyConnectedError"]);
const alreadyConnectedRegex = new RegExp(`^Error response from daemon: endpoint with name ${containerNameOrId} already exists in network`);
for (const net of networks) {
try {
await execDocker(["network", "connect", net, containerNameOrId], execOpts);
}
catch (e) {
if (!alreadyConnectedError && common_1.isExecaError(e)) {
if (alreadyConnectedRegex.test(e.stderr))
continue;
}
throw e;
}
}
}
exports.dockerNetworkConnect = dockerNetworkConnect;
/**
* Detach containers from given networks
*
* @internal
*/
async function dockerNetworkDisconnect(containerNameOrId, networks, opts) {
const optsWithDefs = Object.assign({ alreadyDisconnectedError: true }, opts);
const { alreadyDisconnectedError } = optsWithDefs, execOpts = tslib_1.__rest(optsWithDefs, ["alreadyDisconnectedError"]);
for (const net of networks) {
try {
await execDocker(["network", "disconnect", net, containerNameOrId], execOpts);
}
catch (e) {
if (!alreadyDisconnectedError && common_1.isExecaError(e)) {
if (/^Error response from daemon: container [0-9a-fA-F]+ is not connected to network/.test(e.stderr) ||
(new RegExp(`^Error response from daemon: network ${net} not found`).test(e.stderr))) {
continue;
}
}
throw e;
}
}
}
exports.dockerNetworkDisconnect = dockerNetworkDisconnect;
/**
* Push an image to a registry
*
* @internal
*/
async function dockerPush(opts) {
const args = ["push", opts.nameTag];
await execDocker(args, opts);
}
exports.dockerPush = dockerPush;
/**
* Push an image to a registry
*
* @internal
*/
async function dockerPull(opts) {
const args = ["pull", opts.imageName];
const repo = removeTag(opts.imageName);
const { stdout } = await execDocker(args, opts);
const m = stdout.match(/Digest:\s+(\S+)/);
if (!m)
throw new Error(`Output from docker pull did not contain Digest. Output:\n${stdout}`);
const repoDigest = `${repo}@${m[1]}`;
const info = await dockerInspect([repoDigest], Object.assign({ type: "image" }, exports.pickGlobals(opts)));
if (info.length !== 1) {
throw new Error(`Unexpected number of images (${info.length}) match ${repoDigest}`);
}
return {
id: info[0].Id,
repoDigest,
};
}
exports.dockerPull = dockerPull;
/**
* Given a *valid* ImageNameString, removes the optional tag and returns only the
* `[registry/]repo` portion.
* NOTE(mark): This does not attempt to be a generic parser for all Docker
* image name strings because there's ambiguity in how to parse that requires
* context of where it came from or which argument of which CLI it is.
*/
function removeTag(imageName) {
const parts = imageName.split(":");
switch (parts.length) {
case 1:
// 0 colons - no tag present
break;
case 2:
// 1 colon - Could be either from hostname:port or :tag
// If it's hostname:port, then parts[1] *must* include a slash
// else it's a tag, so dump it.
if (!parts[1].includes("/"))
parts.pop();
break;
case 3:
// 2 colons - last part is the tag
parts.pop();
break;
default:
throw new Error(`Invalid docker image name '${imageName}'`);
}
return parts.join(":");
}
//# sourceMappingURL=cli.js.map