@cyclonedx/cdxgen
Version:
Creates CycloneDX Software Bill-of-Materials (SBOM) from source or container image
1,588 lines (1,548 loc) • 142 kB
JavaScript
const parsePackageJsonName = require("parse-packagejson-name");
const os = require("os");
const pathLib = require("path");
const ssri = require("ssri");
const fs = require("fs");
const got = require("got");
const { v4: uuidv4 } = require("uuid");
const { PackageURL } = require("packageurl-js");
const builder = require("xmlbuilder");
const utils = require("./utils");
const { spawnSync } = require("child_process");
const selfPjson = require("./package.json");
const { findJSImports } = require("./analyzer");
const semver = require("semver");
const dockerLib = require("./docker");
const binaryLib = require("./binary");
const osQueries = require("./data/queries.json");
const isWin = require("os").platform() === "win32";
const { table } = require("table");
// Construct gradle cache directory
let GRADLE_CACHE_DIR =
process.env.GRADLE_CACHE_DIR ||
pathLib.join(os.homedir(), ".gradle", "caches", "modules-2", "files-2.1");
if (process.env.GRADLE_USER_HOME) {
GRADLE_CACHE_DIR =
process.env.GRADLE_USER_HOME + "/caches/modules-2/files-2.1";
}
// Clojure CLI
let CLJ_CMD = "clj";
if (process.env.CLJ_CMD) {
CLJ_CMD = process.env.CLJ_CMD;
}
let LEIN_CMD = "lein";
if (process.env.LEIN_CMD) {
LEIN_CMD = process.env.LEIN_CMD;
}
let SWIFT_CMD = "swift";
if (process.env.SWIFT_CMD) {
SWIFT_CMD = process.env.SWIFT_CMD;
}
// Construct sbt cache directory
let SBT_CACHE_DIR =
process.env.SBT_CACHE_DIR || pathLib.join(os.homedir(), ".ivy2", "cache");
// Debug mode flag
const DEBUG_MODE =
process.env.CDXGEN_DEBUG_MODE === "debug" ||
process.env.SCAN_DEBUG_MODE === "debug" ||
process.env.SHIFTLEFT_LOGGING_LEVEL === "debug" ||
process.env.NODE_ENV === "development";
// CycloneDX Hash pattern
const HASH_PATTERN =
"^([a-fA-F0-9]{32}|[a-fA-F0-9]{40}|[a-fA-F0-9]{64}|[a-fA-F0-9]{96}|[a-fA-F0-9]{128})$";
// Timeout milliseconds. Default 10 mins
const TIMEOUT_MS = parseInt(process.env.CDXGEN_TIMEOUT_MS) || 10 * 60 * 1000;
const createDefaultParentComponent = (path) => {
// Create a parent component based on the directory name
let dirName = pathLib.dirname(path);
const tmpA = dirName.split(pathLib.sep);
dirName = tmpA[tmpA.length - 1];
const parentComponent = {
group: "",
name: dirName,
type: "application"
};
const ppurl = new PackageURL(
"application",
parentComponent.group,
parentComponent.name,
parentComponent.version,
null,
null
).toString();
parentComponent["bom-ref"] = ppurl;
parentComponent["purl"] = ppurl;
return parentComponent;
};
const determineParentComponent = (options) => {
let parentComponent = undefined;
if (options.projectName && options.projectVersion) {
parentComponent = {
group: options.projectGroup || "",
name: options.projectName,
version: "" + options.projectVersion || "",
type: "application"
};
} else if (
options.parentComponent &&
Object.keys(options.parentComponent).length
) {
return options.parentComponent;
}
return parentComponent;
};
/**
* Method to create global external references
*
* @param pkg
* @returns {Array}
*/
function addGlobalReferences(src, filename, format = "xml") {
let externalReferences = [];
if (format === "json") {
externalReferences.push({
type: "other",
url: src,
comment: "Base path"
});
} else {
externalReferences.push({
reference: { "@type": "other", url: src, comment: "Base path" }
});
}
let packageFileMeta = filename;
if (!filename.includes(src)) {
packageFileMeta = pathLib.join(src, filename);
}
if (format === "json") {
externalReferences.push({
type: "other",
url: packageFileMeta,
comment: "Package file"
});
} else {
externalReferences.push({
reference: {
"@type": "other",
url: packageFileMeta,
comment: "Package file"
}
});
}
return externalReferences;
}
/**
* Function to create the services block
*/
function addServices(services, format = "xml") {
let serv_list = [];
for (const aserv of services) {
if (format === "xml") {
let service = {
"@bom-ref": aserv["bom-ref"],
group: aserv.group || "",
name: aserv.name,
version: aserv.version | "latest"
};
delete service["bom-ref"];
const aentry = {
service
};
serv_list.push(aentry);
} else {
serv_list.push(aserv);
}
}
return serv_list;
}
/**
* Function to create the dependency block
*/
function addDependencies(dependencies) {
let deps_list = [];
for (const adep of dependencies) {
let dependsOnList = adep.dependsOn.map((v) => ({
"@ref": v
}));
const aentry = {
dependency: { "@ref": adep.ref }
};
if (dependsOnList.length) {
aentry.dependency.dependency = dependsOnList;
}
deps_list.push(aentry);
}
return deps_list;
}
/**
* Function to create metadata block
*
*/
function addMetadata(parentComponent = {}, format = "xml", options = {}) {
// DO NOT fork this project to just change the vendor or author's name
// Try to contribute to this project by sending PR or filing issues
let metadata = {
timestamp: new Date().toISOString(),
tools: [
{
tool: {
vendor: "cyclonedx",
name: "cdxgen",
version: selfPjson.version
}
}
],
authors: [
{
author: { name: "Prabhu Subramanian", email: "prabhu@appthreat.com" }
}
],
supplier: undefined
};
if (format === "json") {
metadata.tools = [
{
vendor: "cyclonedx",
name: "cdxgen",
version: selfPjson.version
}
];
metadata.authors = [
{ name: "Prabhu Subramanian", email: "prabhu@appthreat.com" }
];
}
if (
parentComponent &&
Object.keys(parentComponent) &&
Object.keys(parentComponent).length
) {
const allPComponents = listComponents(
{},
{},
parentComponent,
parentComponent.type,
format
);
if (allPComponents.length) {
const firstPComp = allPComponents[0];
if (format == "xml" && firstPComp.component) {
metadata.component = firstPComp.component;
} else {
// Retain the components of parent component
// Bug #317 fix
if (parentComponent && parentComponent.components) {
firstPComp.components = parentComponent.components;
}
metadata.component = firstPComp;
}
} else {
// As a fallback, retain the parent component
if (format === "json") {
metadata.component = parentComponent;
}
}
}
if (options) {
const mproperties = [];
if (options.exportData) {
const inspectData = options.exportData.inspectData;
if (inspectData) {
if (inspectData.Id) {
mproperties.push({
name: "oci:image:Id",
value: inspectData.Id
});
}
if (
inspectData.RepoTags &&
Array.isArray(inspectData.RepoTags) &&
inspectData.RepoTags.length
) {
mproperties.push({
name: "oci:image:RepoTag",
value: inspectData.RepoTags[0]
});
}
if (
inspectData.RepoDigests &&
Array.isArray(inspectData.RepoDigests) &&
inspectData.RepoDigests.length
) {
mproperties.push({
name: "oci:image:RepoDigest",
value: inspectData.RepoDigests[0]
});
}
if (inspectData.Created) {
mproperties.push({
name: "oci:image:Created",
value: inspectData.Created
});
}
if (inspectData.Architecture) {
mproperties.push({
name: "oci:image:Architecture",
value: inspectData.Architecture
});
}
if (inspectData.Os) {
mproperties.push({
name: "oci:image:Os",
value: inspectData.Os
});
}
}
const manifestList = options.exportData.manifest;
if (manifestList && Array.isArray(manifestList) && manifestList.length) {
const manifest = manifestList[0] || {};
if (manifest.Config) {
mproperties.push({
name: "oci:image:manifest:Config",
value: manifest.Config
});
}
if (
manifest.Layers &&
Array.isArray(manifest.Layers) &&
manifest.Layers.length
) {
mproperties.push({
name: "oci:image:manifest:Layers",
value: manifest.Layers.join("\\n")
});
}
}
const lastLayerConfig = options.exportData.lastLayerConfig;
if (lastLayerConfig) {
if (lastLayerConfig.id) {
mproperties.push({
name: "oci:image:lastLayer:Id",
value: lastLayerConfig.id
});
}
if (lastLayerConfig.parent) {
mproperties.push({
name: "oci:image:lastLayer:ParentId",
value: lastLayerConfig.parent
});
}
if (lastLayerConfig.created) {
mproperties.push({
name: "oci:image:lastLayer:Created",
value: lastLayerConfig.created
});
}
if (lastLayerConfig.config) {
const env = lastLayerConfig.config.Env;
if (env && Array.isArray(env) && env.length) {
mproperties.push({
name: "oci:image:lastLayer:Env",
value: env.join("\\n")
});
}
const ccmd = lastLayerConfig.config.Cmd;
if (ccmd && Array.isArray(ccmd) && ccmd.length) {
mproperties.push({
name: "oci:image:lastLayer:Cmd",
value: ccmd.join(" ")
});
}
}
}
}
if (options.allOSComponentTypes && options.allOSComponentTypes.length) {
mproperties.push({
name: "oci:image:componentTypes",
value: options.allOSComponentTypes.join("\\n")
});
}
if (mproperties.length) {
if (format === "json") {
metadata.properties = mproperties;
} else {
metadata.properties = mproperties.map((v) => {
return {
property: {
"@name": v.name,
"#text": v.value
}
};
});
}
}
}
return metadata;
}
/**
* Method to create external references
*
* @param pkg
* @returns {Array}
*/
function addExternalReferences(opkg, format = "xml") {
let externalReferences = [];
let pkgList = [];
if (Array.isArray(opkg)) {
pkgList = opkg;
} else {
pkgList = [opkg];
}
for (const pkg of pkgList) {
if (pkg.externalReferences) {
if (format === "xml") {
for (const ref of pkg.externalReferences) {
// If the value already comes from json format
if (ref.type && ref.url) {
externalReferences.push({
reference: { "@type": ref.type, url: ref.url }
});
}
}
} else {
externalReferences.concat(pkg.externalReferences);
}
} else {
if (format === "xml") {
if (pkg.homepage && pkg.homepage.url) {
externalReferences.push({
reference: { "@type": "website", url: pkg.homepage.url }
});
}
if (pkg.bugs && pkg.bugs.url) {
externalReferences.push({
reference: { "@type": "issue-tracker", url: pkg.bugs.url }
});
}
if (pkg.repository && pkg.repository.url) {
externalReferences.push({
reference: { "@type": "vcs", url: pkg.repository.url }
});
}
} else {
if (pkg.homepage && pkg.homepage.url) {
externalReferences.push({
type: "website",
url: pkg.homepage.url
});
}
if (pkg.bugs && pkg.bugs.url) {
externalReferences.push({
type: "issue-tracker",
url: pkg.bugs.url
});
}
if (pkg.repository && pkg.repository.url) {
externalReferences.push({
type: "vcs",
url: pkg.repository.url
});
}
}
}
}
return externalReferences;
}
/**
* For all modules in the specified package, creates a list of
* component objects from each one.
*/
exports.listComponents = listComponents;
function listComponents(
options,
allImports,
pkg,
ptype = "npm",
format = "xml"
) {
let compMap = {};
let isRootPkg = ptype === "npm";
if (Array.isArray(pkg)) {
pkg.forEach((p) => {
addComponent(options, allImports, p, ptype, compMap, false, format);
});
} else {
addComponent(options, allImports, pkg, ptype, compMap, isRootPkg, format);
}
if (format === "xml") {
return Object.keys(compMap).map((k) => ({ component: compMap[k] }));
} else {
return Object.keys(compMap).map((k) => compMap[k]);
}
}
/**
* Given the specified package, create a CycloneDX component and add it to the list.
*/
function addComponent(
options,
allImports,
pkg,
ptype,
compMap,
isRootPkg = false,
format = "xml"
) {
if (!pkg || pkg.extraneous) {
return;
}
if (!isRootPkg) {
let pkgIdentifier = parsePackageJsonName(pkg.name);
let author = pkg.author || "";
let publisher = pkg.publisher || "";
let group = pkg.group || pkgIdentifier.scope;
// Create empty group
group = group || "";
let name = pkgIdentifier.fullName || pkg.name || "";
// name is mandatory
if (!name) {
return;
}
if (!ptype && pkg.qualifiers && pkg.qualifiers.type === "jar") {
ptype = "maven";
}
// Skip @types package for npm
if (
ptype == "npm" &&
(group === "types" ||
group === "@types" ||
!name ||
name.startsWith("@types"))
) {
return;
}
let version = pkg.version;
if (!version || ["dummy", "ignore"].includes(version)) {
return;
}
let licenses = pkg.licenses || utils.getLicenses(pkg, format);
let purl =
pkg.purl ||
new PackageURL(
ptype,
utils.encodeForPurl(group),
utils.encodeForPurl(name),
version,
pkg.qualifiers,
utils.encodeForPurl(pkg.subpath)
);
let purlString = purl.toString();
purlString = decodeURIComponent(purlString);
let description = { "#cdata": pkg.description };
if (format === "json") {
description = pkg.description || "";
}
let compScope = pkg.scope;
if (allImports) {
const impPkgs = Object.keys(allImports);
if (
impPkgs.includes(name) ||
impPkgs.includes(group + "/" + name) ||
impPkgs.includes("@" + group + "/" + name) ||
impPkgs.includes(group) ||
impPkgs.includes("@" + group)
) {
compScope = "required";
} else if (impPkgs.length) {
compScope = "optional";
}
}
if (options.requiredOnly && ["optional", "excluded"].includes(compScope)) {
return;
}
let component = {
author,
publisher,
group,
name,
version,
description,
scope: compScope,
hashes: [],
licenses,
purl: purlString,
externalReferences: addExternalReferences(pkg, format)
};
if (format === "xml") {
component["@type"] = determinePackageType(pkg);
component["@bom-ref"] = purlString;
} else {
component["type"] = determinePackageType(pkg);
component["bom-ref"] = purlString;
}
if (
component.externalReferences === undefined ||
component.externalReferences.length === 0
) {
delete component.externalReferences;
}
processHashes(pkg, component, format);
// Retain any component properties
if (format === "json" && pkg.properties && pkg.properties.length) {
component.properties = pkg.properties;
}
if (compMap[component.purl]) return; //remove cycles
compMap[component.purl] = component;
}
if (pkg.dependencies) {
Object.keys(pkg.dependencies)
.map((x) => pkg.dependencies[x])
.filter((x) => typeof x !== "string") //remove cycles
.map((x) =>
addComponent(options, allImports, x, ptype, compMap, false, format)
);
}
}
/**
* If the author has described the module as a 'framework', the take their
* word for it, otherwise, identify the module as a 'library'.
*/
function determinePackageType(pkg) {
if (pkg.type === "application") {
return "application";
}
if (pkg.purl) {
try {
let purl = PackageURL.fromString(pkg.purl);
if (purl.type) {
if (["docker", "oci", "container"].includes(purl.type)) {
return "container";
}
if (["github"].includes(purl.type)) {
return "application";
}
}
if (purl.namespace) {
for (const cf of [
"System.Web",
"System.ServiceModel",
"System.Data",
"spring",
"flask",
"django",
"beego",
"chi",
"echo",
"gin",
"gorilla",
"rye",
"httprouter",
"akka",
"dropwizard",
"vertx",
"gwt",
"jax-rs",
"jax-ws",
"jsf",
"play",
"spark",
"struts",
"angular",
"react",
"next",
"ember",
"express",
"knex",
"vue",
"aiohttp",
"bottle",
"cherrypy",
"drt",
"falcon",
"hug",
"pyramid",
"sanic",
"tornado",
"vibora"
]) {
if (purl.namespace.includes(cf)) {
return "framework";
}
}
}
} catch (e) {
// continue regardless of error
}
} else if (pkg.group) {
if (["actions"].includes(pkg.group)) {
return "application";
}
}
if (Object.prototype.hasOwnProperty.call(pkg, "keywords")) {
for (let keyword of pkg.keywords) {
if (keyword.toLowerCase() === "framework") {
return "framework";
}
}
}
return "library";
}
/**
* Uses the SHA1 shasum (if present) otherwise utilizes Subresource Integrity
* of the package with support for multiple hashing algorithms.
*/
function processHashes(pkg, component, format = "xml") {
if (pkg.hashes) {
// This attribute would be available when we read a bom json directly
// Eg: cyclonedx-maven-plugin. See: Bugs: #172, #175
for (const ahash of pkg.hashes) {
addComponentHash(ahash.alg, ahash.content, component, format);
}
} else if (pkg._shasum) {
let ahash = { "@alg": "SHA-1", "#text": pkg._shasum };
if (format === "json") {
ahash = { alg: "SHA-1", content: pkg._shasum };
component.hashes.push(ahash);
} else {
component.hashes.push({
hash: ahash
});
}
} else if (pkg._integrity) {
let integrity = ssri.parse(pkg._integrity) || {};
// Components may have multiple hashes with various lengths. Check each one
// that is supported by the CycloneDX specification.
if (Object.prototype.hasOwnProperty.call(integrity, "sha512")) {
addComponentHash(
"SHA-512",
integrity.sha512[0].digest,
component,
format
);
}
if (Object.prototype.hasOwnProperty.call(integrity, "sha384")) {
addComponentHash(
"SHA-384",
integrity.sha384[0].digest,
component,
format
);
}
if (Object.prototype.hasOwnProperty.call(integrity, "sha256")) {
addComponentHash(
"SHA-256",
integrity.sha256[0].digest,
component,
format
);
}
if (Object.prototype.hasOwnProperty.call(integrity, "sha1")) {
addComponentHash("SHA-1", integrity.sha1[0].digest, component, format);
}
}
if (component.hashes.length === 0) {
delete component.hashes; // If no hashes exist, delete the hashes node (it's optional)
}
}
/**
* Adds a hash to component.
*/
function addComponentHash(alg, digest, component, format = "xml") {
let hash = "";
// If it is a valid hash simply use it
if (new RegExp(HASH_PATTERN).test(digest)) {
hash = digest;
} else {
// Check if base64 encoded
const isBase64Encoded =
Buffer.from(digest, "base64").toString("base64") === digest;
hash = isBase64Encoded
? Buffer.from(digest, "base64").toString("hex")
: digest;
}
let ahash = { "@alg": alg, "#text": hash };
if (format === "json") {
ahash = { alg: alg, content: hash };
component.hashes.push(ahash);
} else {
component.hashes.push({ hash: ahash });
}
}
/**
* Return Bom in xml format
*
* @param {String} Serial number
* @param {Object} parentComponent Parent component object
* @param {Array} components Bom components
* @param {Object} context Context object
* @returns bom xml string
*/
const buildBomXml = (
serialNum,
parentComponent,
components,
context,
options = {}
) => {
const bom = builder
.create("bom", { encoding: "utf-8", separateArrayItems: true })
.att("xmlns", "http://cyclonedx.org/schema/bom/1.4");
bom.att("serialNumber", serialNum);
bom.att("version", 1);
const metadata = addMetadata(parentComponent, "xml", options);
bom.ele("metadata").ele(metadata);
if (components && components.length) {
bom.ele("components").ele(components);
if (context && context.src && context.filename) {
bom
.ele("externalReferences")
.ele(addGlobalReferences(context.src, context.filename, "xml"));
}
if (context) {
if (context.services && context.services.length) {
bom.ele("services").ele(addServices(context.services, "xml"));
}
if (context.dependencies && context.dependencies.length) {
bom.ele("dependencies").ele(addDependencies(context.dependencies));
}
}
const bomString = bom.end({
pretty: true,
indent: " ",
newline: "\n",
width: 0,
allowEmpty: false,
spacebeforeslash: ""
});
return bomString;
}
return "";
};
/**
* Return the BOM in xml, json format including any namespace mapping
*/
const buildBomNSData = (options, pkgInfo, ptype, context) => {
const bomNSData = {
bomXml: undefined,
bomXmlFiles: undefined,
bomJson: undefined,
bomJsonFiles: undefined,
nsMapping: undefined,
dependencies: undefined,
parentComponent: undefined
};
const serialNum = "urn:uuid:" + uuidv4();
let allImports = {};
if (context && context.allImports) {
allImports = context.allImports;
}
const nsMapping = context.nsMapping || {};
const dependencies = context.dependencies || [];
const parentComponent =
determineParentComponent(options) || context.parentComponent;
const metadata = addMetadata(parentComponent, "json", options);
const components = listComponents(options, allImports, pkgInfo, ptype, "xml");
if (components && (components.length || parentComponent)) {
const bomString = buildBomXml(
serialNum,
parentComponent,
components,
context,
options
);
// CycloneDX 1.4 Json Template
const jsonTpl = {
bomFormat: "CycloneDX",
specVersion: "1.4",
serialNumber: serialNum,
version: 1,
metadata: metadata,
components: listComponents(options, allImports, pkgInfo, ptype, "json"),
dependencies
};
if (context && context.src && context.filename) {
jsonTpl.externalReferences = addGlobalReferences(
context.src,
context.filename,
"json"
);
}
bomNSData.bomXml = bomString;
bomNSData.bomJson = jsonTpl;
bomNSData.nsMapping = nsMapping;
bomNSData.dependencies = dependencies;
bomNSData.parentComponent = parentComponent;
}
return bomNSData;
};
/**
* Function to create bom string for Java jars
*
* @param path to the project
* @param options Parse options from the cli
*/
const createJarBom = (path, options) => {
console.log(
`About to create SBoM for all jar files under ${path}. This would take a while ...`
);
let pkgList = [];
let jarFiles = utils.getAllFiles(
path,
(options.multiProject ? "**/" : "") + "*.[jw]ar"
);
// Jenkins plugins
const hpiFiles = utils.getAllFiles(
path,
(options.multiProject ? "**/" : "") + "*.hpi"
);
if (hpiFiles.length) {
jarFiles = jarFiles.concat(hpiFiles);
}
let tempDir = fs.mkdtempSync(pathLib.join(os.tmpdir(), "jar-deps-"));
for (let jar of jarFiles) {
if (DEBUG_MODE) {
console.log(`Parsing ${jar}`);
}
const dlist = utils.extractJarArchive(jar, tempDir);
if (dlist && dlist.length) {
pkgList = pkgList.concat(dlist);
}
}
// Clean up
if (tempDir && tempDir.startsWith(os.tmpdir()) && fs.rmSync) {
console.log(`Cleaning up ${tempDir}`);
fs.rmSync(tempDir, { recursive: true, force: true });
}
return buildBomNSData(options, pkgList, "maven", {
src: path,
filename: jarFiles.join(", "),
nsMapping: {}
});
};
/**
* Function to create bom string for Java projects
*
* @param path to the project
* @param options Parse options from the cli
*/
const createJavaBom = async (path, options) => {
let jarNSMapping = {};
let pkgList = [];
let dependencies = [];
// cyclone-dx-maven plugin creates a component for the app under metadata
// This is subsequently referred to in the dependencies list
let parentComponent = {};
// war/ear mode
if (path.endsWith(".war")) {
// Check if the file exists
if (fs.existsSync(path)) {
if (DEBUG_MODE) {
console.log(`Retrieving packages from ${path}`);
}
let tempDir = fs.mkdtempSync(pathLib.join(os.tmpdir(), "war-deps-"));
pkgList = utils.extractJarArchive(path, tempDir);
if (pkgList.length) {
pkgList = await utils.getMvnMetadata(pkgList);
}
// Should we attempt to resolve class names
if (options.resolveClass) {
console.log(
"Creating class names list based on available jars. This might take a few mins ..."
);
jarNSMapping = utils.collectJarNS(tempDir);
}
// Clean up
if (tempDir && tempDir.startsWith(os.tmpdir()) && fs.rmSync) {
console.log(`Cleaning up ${tempDir}`);
fs.rmSync(tempDir, { recursive: true, force: true });
}
} else {
console.log(`${path} doesn't exist`);
}
return buildBomNSData(options, pkgList, "maven", {
src: pathLib.dirname(path),
filename: path,
nsMapping: jarNSMapping,
dependencies,
parentComponent
});
} else {
// maven - pom.xml
const pomFiles = utils.getAllFiles(
path,
(options.multiProject ? "**/" : "") + "pom.xml"
);
if (pomFiles && pomFiles.length) {
const cdxMavenPlugin =
process.env.CDX_MAVEN_PLUGIN ||
"org.cyclonedx:cyclonedx-maven-plugin:2.7.9";
const cdxMavenGoal = process.env.CDX_MAVEN_GOAL || "makeAggregateBom";
let mvnArgs = [`${cdxMavenPlugin}:${cdxMavenGoal}`, "-DoutputName=bom"];
if (utils.includeMavenTestScope) {
mvnArgs.push("-DincludeTestScope=true");
}
// By using quiet mode we can reduce the maxBuffer used and avoid crashes
if (!DEBUG_MODE) {
mvnArgs.push("-q");
}
// Support for passing additional settings and profile to maven
if (process.env.MVN_ARGS) {
const addArgs = process.env.MVN_ARGS.split(" ");
mvnArgs = mvnArgs.concat(addArgs);
}
for (let f of pomFiles) {
const basePath = pathLib.dirname(f);
const settingsXml = pathLib.join(basePath, "settings.xml");
if (fs.existsSync(settingsXml)) {
console.log(
`maven settings.xml found in ${basePath}. Please set the MVN_ARGS environment variable based on the full mvn build command used for this project.\nExample: MVN_ARGS='--settings ${settingsXml}'`
);
}
let mavenCmd = utils.getMavenCommand(basePath, path);
// Should we attempt to resolve class names
if (options.resolveClass) {
console.log(
"Creating class names list based on available jars. This might take a few mins ..."
);
jarNSMapping = utils.collectMvnDependencies(mavenCmd, basePath);
}
console.log(
`Executing '${mavenCmd} ${mvnArgs.join(" ")}' in`,
basePath
);
let result = spawnSync(mavenCmd, mvnArgs, {
cwd: basePath,
shell: true,
encoding: "utf-8",
timeout: TIMEOUT_MS
});
// Check if the cyclonedx plugin created the required bom.xml file
// Sometimes the plugin fails silently for complex maven projects
const bomJsonFiles = utils.getAllFiles(path, "**/target/*.json");
const bomGenerated = bomJsonFiles.length;
if (!bomGenerated || result.status !== 0 || result.error) {
let tempDir = fs.mkdtempSync(pathLib.join(os.tmpdir(), "cdxmvn-"));
let tempMvnTree = pathLib.join(tempDir, "mvn-tree.txt");
let mvnTreeArgs = ["dependency:tree", "-DoutputFile=" + tempMvnTree];
if (process.env.MVN_ARGS) {
const addArgs = process.env.MVN_ARGS.split(" ");
mvnTreeArgs = mvnTreeArgs.concat(addArgs);
}
console.log(
`Fallback to executing ${mavenCmd} ${mvnTreeArgs.join(" ")}`
);
result = spawnSync(mavenCmd, mvnTreeArgs, {
cwd: basePath,
shell: true,
encoding: "utf-8",
timeout: TIMEOUT_MS
});
if (result.status !== 0 || result.error) {
console.error(result.stdout, result.stderr);
console.log(
"Resolve the above maven error. This could be due to the following:\n"
);
if (
result.stderr &&
result.stderr.includes(
"Could not resolve dependencies" ||
result.stderr.includes("no dependency information available")
)
) {
console.log(
"1. Try building the project with 'mvn package -Dmaven.test.skip=true' using the correct version of Java and maven before invoking cdxgen."
);
} else {
console.log(
"1. Java version requirement: cdxgen container image bundles Java 19 with maven 3.9 which might be incompatible."
);
}
console.log(
"2. Private dependencies cannot be downloaded: Check if any additional arguments must be passed to maven and set them via MVN_ARGS environment variable."
);
console.log(
"3. Check if all required environment variables including any maven profile arguments are passed correctly to this tool."
);
// Do not fall back to methods that can produce incomplete results when failOnError is set
options.failOnError && process.exit(1);
console.log(
"\nFalling back to manual pom.xml parsing. The result would be incomplete!"
);
const dlist = utils.parsePom(f);
if (dlist && dlist.length) {
pkgList = pkgList.concat(dlist);
}
} else {
if (fs.existsSync(tempMvnTree)) {
const mvnTreeString = fs.readFileSync(tempMvnTree, {
encoding: "utf-8"
});
const parsedList = utils.parseMavenTree(mvnTreeString);
const dlist = parsedList.pkgList;
parentComponent = dlist.splice(0, 1)[0];
parentComponent.type = "application";
if (dlist && dlist.length) {
pkgList = pkgList.concat(dlist);
}
if (parsedList.dependenciesList && parsedList.dependenciesList) {
dependencies = dependencies.concat(parsedList.dependenciesList);
}
fs.unlinkSync(tempMvnTree);
}
}
}
} // for
const bomFiles = utils.getAllFiles(path, "**/target/bom.xml");
const bomJsonFiles = utils.getAllFiles(path, "**/target/*.json");
for (const abjson of bomJsonFiles) {
let bomJsonObj = undefined;
try {
if (DEBUG_MODE) {
console.log(`Extracting data from generated bom file ${abjson}`);
}
bomJsonObj = JSON.parse(
fs.readFileSync(abjson, {
encoding: "utf-8"
})
);
if (bomJsonObj) {
if (
bomJsonObj.metadata &&
bomJsonObj.metadata.component &&
!Object.keys(parentComponent).length
) {
parentComponent = bomJsonObj.metadata.component;
pkgList = [];
}
if (bomJsonObj.components) {
pkgList = pkgList.concat(bomJsonObj.components);
}
if (bomJsonObj.dependencies && !options.requiredOnly) {
dependencies = mergeDependencies(
dependencies,
bomJsonObj.dependencies
);
}
}
} catch (err) {
if (options.failOnError || DEBUG_MODE) {
console.log(err);
options.failOnError && process.exit(1);
}
}
}
if (pkgList) {
pkgList = trimComponents(pkgList, "json");
pkgList = await utils.getMvnMetadata(pkgList);
return buildBomNSData(options, pkgList, "maven", {
src: path,
filename: pomFiles.join(", "),
nsMapping: jarNSMapping,
dependencies,
parentComponent
});
} else if (bomJsonFiles.length) {
const bomNSData = {};
bomNSData.bomXmlFiles = bomFiles;
bomNSData.bomJsonFiles = bomJsonFiles;
bomNSData.nsMapping = jarNSMapping;
bomNSData.dependencies = dependencies;
bomNSData.parentComponent = parentComponent;
return bomNSData;
}
}
// gradle
let gradleFiles = utils.getAllFiles(
path,
(options.multiProject ? "**/" : "") + "build.gradle*"
);
let allProjects = [];
const allProjectsAddedPurls = [];
const rootDependsOn = [];
// Execute gradle properties
if (gradleFiles && gradleFiles.length) {
let retMap = utils.executeGradleProperties(path, null, null);
const allProjectsStr = retMap.projects || [];
let rootProject = retMap.rootProject;
if (rootProject) {
parentComponent = {
name: rootProject,
type: "application",
...(retMap.metadata || {})
};
const parentPurl = decodeURIComponent(
new PackageURL(
"maven",
parentComponent.group || "",
parentComponent.name,
parentComponent.version,
{ type: "jar" },
null
).toString()
);
parentComponent["purl"] = parentPurl;
parentComponent["bom-ref"] = parentPurl;
}
// Get the sub-project properties and set the root dependencies
if (allProjectsStr && allProjectsStr.length) {
for (let spstr of allProjectsStr) {
retMap = utils.executeGradleProperties(path, null, spstr);
let rootSubProject = retMap.rootProject;
if (rootSubProject) {
let rspName = rootSubProject.replace(/^:/, "").replace(/:/, "/");
const rootSubProjectObj = {
name: rspName,
type: "application",
qualifiers: { type: "jar" },
...(retMap.metadata || {})
};
const rootSubProjectPurl = decodeURIComponent(
new PackageURL(
"maven",
rootSubProjectObj.group || parentComponent.group || "",
rootSubProjectObj.name,
rootSubProjectObj.version,
rootSubProjectObj.qualifiers,
null
).toString()
);
rootSubProjectObj["purl"] = rootSubProjectPurl;
rootSubProjectObj["bom-ref"] = rootSubProjectPurl;
if (!allProjectsAddedPurls.includes(rootSubProjectPurl)) {
allProjects.push(rootSubProjectObj);
rootDependsOn.push(rootSubProjectPurl);
allProjectsAddedPurls.push(rootSubProjectPurl);
}
}
}
// Bug #317 fix
parentComponent.components = allProjects.flatMap((s) => {
delete s.qualifiers;
return s;
});
dependencies.push({
ref: parentComponent["bom-ref"],
dependsOn: rootDependsOn
});
}
}
if (gradleFiles && gradleFiles.length && options.installDeps) {
let gradleCmd = utils.getGradleCommand(path, null);
const defaultDepTaskArgs = ["-q", "--console", "plain", "--build-cache"];
allProjects.push(parentComponent);
let depTaskWithArgs = ["dependencies"];
if (process.env.GRADLE_DEPENDENCY_TASK) {
depTaskWithArgs = process.env.GRADLE_DEPENDENCY_TASK.split(" ");
}
for (let sp of allProjects) {
let gradleDepArgs = [
sp.purl === parentComponent.purl
? depTaskWithArgs[0]
: `:${sp.name}:${depTaskWithArgs[0]}`
];
gradleDepArgs = gradleDepArgs
.concat(depTaskWithArgs.slice(1))
.concat(defaultDepTaskArgs);
// Support custom GRADLE_ARGS such as --configuration runtimeClassPath
if (process.env.GRADLE_ARGS) {
const addArgs = process.env.GRADLE_ARGS.split(" ");
gradleDepArgs = gradleDepArgs.concat(addArgs);
}
console.log(
"Executing",
gradleCmd,
gradleDepArgs.join(" "),
"in",
path
);
const sresult = spawnSync(gradleCmd, gradleDepArgs, {
cwd: path,
encoding: "utf-8",
timeout: TIMEOUT_MS
});
if (sresult.status !== 0 || sresult.error) {
if (options.failOnError || DEBUG_MODE) {
console.error(sresult.stdout, sresult.stderr);
}
options.failOnError && process.exit(1);
}
const sstdout = sresult.stdout;
if (sstdout) {
const cmdOutput = Buffer.from(sstdout).toString();
const parsedList = utils.parseGradleDep(
cmdOutput,
sp.group,
sp.name,
sp.version
);
const dlist = parsedList.pkgList;
if (parsedList.dependenciesList && parsedList.dependenciesList) {
dependencies = mergeDependencies(
dependencies,
parsedList.dependenciesList
);
}
if (dlist && dlist.length) {
if (DEBUG_MODE) {
console.log(
"Found",
dlist.length,
"packages in gradle project",
sp.name
);
}
pkgList = pkgList.concat(dlist);
}
}
} // for
if (pkgList.length) {
console.log(
"Obtained",
pkgList.length,
"from this gradle project. De-duping this list ..."
);
} else {
console.log(
"No packages found. Set the environment variable 'CDXGEN_DEBUG_MODE=debug' to troubleshoot any gradle related errors."
);
options.failOnError && process.exit(1);
}
pkgList = await utils.getMvnMetadata(pkgList);
// Should we attempt to resolve class names
if (options.resolveClass) {
console.log(
"Creating class names list based on available jars. This might take a few mins ..."
);
jarNSMapping = utils.collectJarNS(GRADLE_CACHE_DIR);
}
return buildBomNSData(options, pkgList, "maven", {
src: path,
filename: gradleFiles.join(", "),
nsMapping: jarNSMapping,
dependencies,
parentComponent
});
}
// Bazel
// Look for the BUILD file only in the root directory
let bazelFiles = utils.getAllFiles(path, "BUILD");
if (bazelFiles && bazelFiles.length) {
let BAZEL_CMD = "bazel";
if (process.env.BAZEL_HOME) {
BAZEL_CMD = pathLib.join(process.env.BAZEL_HOME, "bin", "bazel");
}
for (let f of bazelFiles) {
const basePath = pathLib.dirname(f);
// Invoke bazel build first
const bazelTarget = process.env.BAZEL_TARGET || ":all";
console.log(
"Executing",
BAZEL_CMD,
"build",
bazelTarget,
"in",
basePath
);
let result = spawnSync(BAZEL_CMD, ["build", bazelTarget], {
cwd: basePath,
shell: true,
encoding: "utf-8",
timeout: TIMEOUT_MS
});
if (result.status !== 0 || result.error) {
if (result.stderr) {
console.error(result.stdout, result.stderr);
}
console.log(
"1. Check if bazel is installed and available in PATH.\n2. Try building your app with bazel prior to invoking cdxgen"
);
options.failOnError && process.exit(1);
} else {
console.log(
"Executing",
BAZEL_CMD,
"aquery --output=textproto --skyframe_state in",
basePath
);
result = spawnSync(
BAZEL_CMD,
["aquery", "--output=textproto", "--skyframe_state"],
{ cwd: basePath, encoding: "utf-8", timeout: TIMEOUT_MS }
);
if (result.status !== 0 || result.error) {
console.error(result.stdout, result.stderr);
options.failOnError && process.exit(1);
}
let stdout = result.stdout;
if (stdout) {
const cmdOutput = Buffer.from(stdout).toString();
const dlist = utils.parseBazelSkyframe(cmdOutput);
if (dlist && dlist.length) {
pkgList = pkgList.concat(dlist);
} else {
console.log(
"No packages were detected.\n1. Build your project using bazel build command before running cdxgen\n2. Try running the bazel aquery command manually to see if skyframe state can be retrieved."
);
console.log(
"If your project requires a different query, please file a bug at cyclonedx/cdxgen repo!"
);
options.failOnError && process.exit(1);
}
} else {
console.log("Bazel unexpectedly didn't produce any output");
options.failOnError && process.exit(1);
}
pkgList = await utils.getMvnMetadata(pkgList);
return buildBomNSData(options, pkgList, "maven", {
src: path,
filename: "BUILD",
nsMapping: {},
dependencies,
parentComponent
});
}
}
}
// scala sbt
// Identify sbt projects via its `project` directory:
// - all SBT project _should_ define build.properties file with sbt version info
// - SBT projects _typically_ have some configs/plugins defined in .sbt files
// - SBT projects that are still on 0.13.x, can still use the old approach,
// where configs are defined via Scala files
// Detecting one of those should be enough to determine an SBT project.
let sbtProjectFiles = utils.getAllFiles(
path,
(options.multiProject ? "**/" : "") +
"project/{build.properties,*.sbt,*.scala}"
);
let sbtProjects = [];
for (let i in sbtProjectFiles) {
// parent dir of sbtProjectFile is the `project` directory
// parent dir of `project` is the sbt root project directory
const baseDir = pathLib.dirname(pathLib.dirname(sbtProjectFiles[i]));
sbtProjects = sbtProjects.concat(baseDir);
}
// Fallback in case sbt's project directory is non-existent
if (!sbtProjects.length) {
sbtProjectFiles = utils.getAllFiles(
path,
(options.multiProject ? "**/" : "") + "*.sbt"
);
for (let i in sbtProjectFiles) {
const baseDir = pathLib.dirname(sbtProjectFiles[i]);
sbtProjects = sbtProjects.concat(baseDir);
}
}
sbtProjects = [...new Set(sbtProjects)]; // eliminate duplicates
let sbtLockFiles = utils.getAllFiles(
path,
(options.multiProject ? "**/" : "") + "build.sbt.lock"
);
if (sbtProjects && sbtProjects.length) {
let pkgList = [];
// If the project use sbt lock files
if (sbtLockFiles && sbtLockFiles.length) {
for (let f of sbtLockFiles) {
const dlist = utils.parseSbtLock(f);
if (dlist && dlist.length) {
pkgList = pkgList.concat(dlist);
}
}
} else {
let SBT_CMD = process.env.SBT_CMD || "sbt";
let sbtVersion = utils.determineSbtVersion(path);
if (DEBUG_MODE) {
console.log("Detected sbt version: " + sbtVersion);
}
// Introduced in 1.2.0 https://www.scala-sbt.org/1.x/docs/sbt-1.2-Release-Notes.html#addPluginSbtFile+command,
// however working properly for real only since 1.3.4: https://github.com/sbt/sbt/releases/tag/v1.3.4
const standalonePluginFile =
sbtVersion != null &&
semver.gte(sbtVersion, "1.3.4") &&
semver.lte(sbtVersion, "1.4.0");
const useSlashSyntax = semver.gte(sbtVersion, "1.5.0");
const isDependencyTreeBuiltIn =
sbtVersion != null && semver.gte(sbtVersion, "1.4.0");
let tempDir = fs.mkdtempSync(pathLib.join(os.tmpdir(), "cdxsbt-"));
let tempSbtgDir = fs.mkdtempSync(pathLib.join(os.tmpdir(), "cdxsbtg-"));
fs.mkdirSync(tempSbtgDir, { recursive: true });
// Create temporary plugins file
let tempSbtPlugins = pathLib.join(tempSbtgDir, "dep-plugins.sbt");
// Requires a custom version of `sbt-dependency-graph` that
// supports `--append` for `toFile` subtask.
let sbtPluginDefinition = `\naddSbtPlugin("io.shiftleft" % "sbt-dependency-graph" % "0.10.0-append-to-file3")\n`;
if (isDependencyTreeBuiltIn) {
sbtPluginDefinition = `\naddDependencyTreePlugin\n`;
if (DEBUG_MODE) {
console.log("Using addDependencyTreePlugin as the custom plugin");
}
}
fs.writeFileSync(tempSbtPlugins, sbtPluginDefinition);
for (let i in sbtProjects) {
const basePath = sbtProjects[i];
let dlFile = pathLib.join(tempDir, "dl-" + i + ".tmp");
console.log(
"Executing",
SBT_CMD,
"dependencyList in",
basePath,
"using plugins",
tempSbtgDir
);
var sbtArgs = [];
var pluginFile = null;
if (standalonePluginFile) {
sbtArgs = [
`-addPluginSbtFile=${tempSbtPlugins}`,
`"dependencyList::toFile ${dlFile} --force"`
];
} else {
// write to the existing plugins file
if (useSlashSyntax) {
sbtArgs = [`"dependencyList / toFile ${dlFile} --force"`];
} else {
sbtArgs = [`"dependencyList::toFile ${dlFile} --force"`];
}
pluginFile = utils.addPlugin(basePath, sbtPluginDefinition);
}
// Note that the command has to be invoked with `shell: true` to properly execut sbt
const result = spawnSync(SBT_CMD, sbtArgs, {
cwd: basePath,
shell: true,
encoding: "utf-8",
timeout: TIMEOUT_MS
});
if (result.status !== 0 || result.error) {
console.error(result.stdout, result.stderr);
console.log(
`1. Check if scala and sbt is installed and available in PATH. Only scala 2.10 + sbt 0.13.6+ and 2.12 + sbt 1.0+ is supported for now.`
);
console.log(
`2. Check if the plugin net.virtual-void:sbt-dependency-graph 0.10.0-RC1 can be used in the environment`
);
console.log(
"3. Consider creating a lockfile using sbt-dependency-lock plugin. See https://github.com/stringbean/sbt-dependency-lock"
);
options.failOnError && process.exit(1);
}
if (!standalonePluginFile) {
utils.cleanupPlugin(basePath, pluginFile);
}
if (fs.existsSync(dlFile)) {
const cmdOutput = fs.readFileSync(dlFile, { encoding: "utf-8" });
const dlist = utils.parseKVDep(cmdOutput);
if (dlist && dlist.length) {
pkgList = pkgList.concat(dlist);
}
} else {
if (options.failOnError || DEBUG_MODE) {
console.log(`sbt dependencyList did not yield ${dlFile}`);
}
options.failOnError && process.exit(1);
}
}
// Cleanup
fs.unlinkSync(tempSbtPlugins);
} // else
if (DEBUG_MODE) {
console.log(`Found ${pkgList.length} packages`);
}
pkgList = await utils.getMvnMetadata(pkgList);
// Should we attempt to resolve class names
if (options.resolveClass) {
console.log(
"Creating class names list based on available jars. This might take a few mins ..."
);
jarNSMapping = utils.collectJarNS(SBT_CACHE_DIR);
}
return buildBomNSData(options, pkgList, "maven", {
src: path,
filen