snyk-docker-plugin
Version:
Snyk CLI docker plugin
316 lines (279 loc) • 9.53 kB
text/typescript
import * as Debug from "debug";
import { createReadStream } from "fs";
import * as gunzip from "gunzip-maybe";
import { normalize as normalizePath, sep as pathSeparator } from "path";
import { PassThrough } from "stream";
import { extract, Extract } from "tar-stream";
import { getPlatformFromConfig, InvalidArchiveError } from "..";
import { streamToJson } from "../../stream-utils";
import { PluginOptions } from "../../types";
import { extractImageLayer } from "../layer";
import {
ExtractAction,
ExtractedLayers,
ExtractedLayersAndManifest,
ImageConfig,
OciArchiveManifest,
OciImageIndex,
OciManifestInfo,
OciPlatformInfo,
} from "../types";
const debug = Debug("snyk");
const MEDIATYPE_DOCKER_MANIFEST_V2 =
"application/vnd.docker.distribution.manifest.v2+json";
const MEDIATYPE_DOCKER_MANIFEST_LIST_V2 =
"application/vnd.docker.distribution.manifest.list.v2+json";
const MEDIATYPE_OCI_MANIFEST_V1 = "application/vnd.oci.image.manifest.v1+json";
const MEDIATYPE_OCI_MANIFEST_LIST_V1 =
"application/vnd.oci.image.index.v1+json";
/**
* Retrieve the products of files content from the specified oci-archive.
* @param ociArchiveFilesystemPath Path to image file saved in oci-archive format.
* @param extractActions Array of pattern-callbacks pairs.
* @param options PluginOptions
* @returns Array of extracted files products sorted by the reverse order of the layers from last to first.
*/
export async function extractArchive(
ociArchiveFilesystemPath: string,
extractActions: ExtractAction[],
options: PluginOptions,
): Promise<ExtractedLayersAndManifest> {
return new Promise((resolve, reject) => {
const tarExtractor: Extract = extract();
const layers: Record<string, ExtractedLayers> = {};
const manifests: Record<string, OciArchiveManifest> = {};
const configs: ImageConfig[] = [];
let mainIndexFile: OciImageIndex;
const indexFiles: Record<string, OciImageIndex> = {};
tarExtractor.on("entry", async (header, stream, next) => {
if (header.type === "file") {
const normalizedHeaderName = normalizePath(header.name);
if (isMainIndexFile(normalizedHeaderName)) {
mainIndexFile = await streamToJson<OciImageIndex>(stream);
} else {
const jsonStream = new PassThrough();
const layerStream = new PassThrough();
stream.pipe(jsonStream);
stream.pipe(layerStream);
const promises = [
streamToJson(jsonStream).catch(() => undefined),
extractImageLayer(layerStream, extractActions).catch(
() => undefined,
),
];
const [manifest, layer] = await Promise.all(promises);
// header format is /blobs/hash_name/hash_value
// we're extracting hash_name:hash_value format to match manifest digest
const headerParts = normalizedHeaderName.split(pathSeparator);
const hashName = headerParts[1];
const hashValue = headerParts[headerParts.length - 1];
const digest = `${hashName}:${hashValue}`;
if (isArchiveManifest(manifest)) {
manifests[digest] = manifest;
} else if (isImageIndexFile(manifest)) {
indexFiles[digest] = manifest as OciImageIndex;
} else if (isImageConfigFile(manifest)) {
configs.push(manifest);
}
if (layer !== undefined) {
layers[digest] = layer as ExtractedLayers;
}
}
}
stream.resume(); // auto drain the stream
next(); // ready for next entry
});
tarExtractor.on("finish", () => {
try {
resolve(
getLayersContentAndArchiveManifest(
mainIndexFile,
manifests,
indexFiles,
configs,
layers,
options,
),
);
} catch (error) {
debug(
`Error getting layers and manifest content from oci archive: '${error.message}'`,
);
reject(new InvalidArchiveError("Invalid OCI archive"));
}
});
tarExtractor.on("error", (error) => {
reject(error);
});
createReadStream(ociArchiveFilesystemPath)
.pipe(gunzip())
.pipe(tarExtractor);
});
}
function getLayersContentAndArchiveManifest(
imageIndex: OciImageIndex | undefined,
manifestCollection: Record<string, OciArchiveManifest>,
indexFiles: Record<string, OciImageIndex>,
configs: ImageConfig[],
layers: Record<string, ExtractedLayers>,
options: Partial<PluginOptions>,
): {
layers: ExtractedLayers[];
manifest: OciArchiveManifest;
imageConfig: ImageConfig;
} {
const platform =
options?.platform ||
(configs.length === 1 ? getPlatformFromConfig(configs[0]) : "linux/amd64");
const platformInfo = getOciPlatformInfoFromOptionString(platform as string);
// get manifest file first
const manifest = getManifest(
imageIndex,
manifestCollection,
indexFiles,
platformInfo,
);
// filter empty layers
// get the layers content without the name
// reverse layers order from last to first
const filteredLayers = manifest.layers
.filter((layer) => layers[layer.digest])
.map((layer) => layers[layer.digest])
.reverse();
if (filteredLayers.length === 0) {
throw new Error("We found no layers in the provided image");
}
const imageConfig = getImageConfig(configs, platformInfo);
if (imageConfig === undefined) {
throw new Error("Could not find the image config in the provided image");
}
return {
layers: filteredLayers,
manifest,
imageConfig,
};
}
function getManifest(
imageIndex: OciImageIndex | undefined,
manifestCollection: Record<string, OciArchiveManifest>,
indexFiles: Record<string, OciImageIndex>,
platformInfo: OciPlatformInfo,
): OciArchiveManifest {
if (!imageIndex) {
return manifestCollection[Object.keys(manifestCollection)[0]];
}
const allManifests = getAllManifestsIndexItems(imageIndex, indexFiles);
const manifestInfo = getImageManifestInfo(allManifests, platformInfo);
if (manifestInfo === undefined) {
throw new Error(
"Image does not support type of CPU architecture or operating system",
);
}
return manifestCollection[manifestInfo.digest];
}
function getAllManifestsIndexItems(
imageIndex: OciImageIndex,
indexFiles: Record<string, OciImageIndex>,
): OciManifestInfo[] {
const allManifestsInfo: OciManifestInfo[] = [];
for (const manifest of imageIndex.manifests) {
if (
manifest.mediaType === MEDIATYPE_OCI_MANIFEST_V1 ||
manifest.mediaType === MEDIATYPE_DOCKER_MANIFEST_V2
) {
// an archive manifest file
allManifestsInfo.push(manifest);
} else if (
manifest.mediaType === MEDIATYPE_OCI_MANIFEST_LIST_V1 ||
manifest.mediaType === MEDIATYPE_DOCKER_MANIFEST_LIST_V2
) {
// nested index
const index = indexFiles[manifest.digest];
allManifestsInfo.push(...getAllManifestsIndexItems(index, indexFiles));
}
}
return allManifestsInfo;
}
function isArchiveManifest(manifest: any): manifest is OciArchiveManifest {
return (
manifest !== undefined && manifest.layers && Array.isArray(manifest.layers)
);
}
function isImageConfigFile(json: any): json is ImageConfig {
return json !== undefined && json.architecture && json.rootfs;
}
function isImageIndexFile(json: any): boolean {
return (
(json?.mediaType === MEDIATYPE_OCI_MANIFEST_LIST_V1 ||
json?.mediaType === MEDIATYPE_DOCKER_MANIFEST_LIST_V2) &&
Array.isArray(json?.manifests)
);
}
function isMainIndexFile(name: string): boolean {
return name === "index.json";
}
function getOciPlatformInfoFromOptionString(platform: string): OciPlatformInfo {
const [os, architecture, variant] = platform.split("/") as [
os: string,
architecture: string,
variant: string | undefined,
];
return {
os,
architecture,
variant,
};
}
function getImageManifestInfo(
manifests: OciManifestInfo[],
platformInfo: OciPlatformInfo,
): OciManifestInfo | undefined {
// manifests do not always have a plaform, this is the case for OCI
// images built with Docker when no platform is specified
if (manifests.length === 1 && !manifests[0].platform) {
return manifests[0];
}
return getBestMatchForPlatform(
manifests,
platformInfo,
(target: OciManifestInfo): OciPlatformInfo => {
return {
os: target.platform?.os,
architecture: target.platform?.architecture,
variant: target.platform?.variant,
};
},
);
}
function getImageConfig(
manifests: ImageConfig[],
platformInfo: OciPlatformInfo,
): ImageConfig | undefined {
return getBestMatchForPlatform(
manifests,
platformInfo,
(target: ImageConfig): OciPlatformInfo => {
return {
os: target.os,
architecture: target.architecture,
};
},
);
}
function getBestMatchForPlatform<T>(
manifests: T[],
platformInfo: OciPlatformInfo,
extractPlatformInfoFromManifest: (target: T) => OciPlatformInfo,
): T | undefined {
const matches = manifests.filter((item) => {
const { os, architecture } = extractPlatformInfoFromManifest(item);
return os === platformInfo.os && architecture === platformInfo.architecture;
});
if (matches.length > 1) {
return matches.find((item) => {
const { variant } = extractPlatformInfoFromManifest(item);
return variant === platformInfo.variant;
});
}
return matches[0] || undefined;
}