@cyclonedx/cyclonedx-npm
Version:
Create CycloneDX Software Bill of Materials (SBOM) from NPM projects.
480 lines (475 loc) • 19.7 kB
JavaScript
"use strict";
/*!
This file is part of CycloneDX generator for NPM projects.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
SPDX-License-Identifier: Apache-2.0
Copyright (c) OWASP Foundation. All Rights Reserved.
*/
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.TreeBuilder = exports.BomBuilder = void 0;
const node_fs_1 = require("node:fs");
const node_path_1 = __importDefault(require("node:path"));
const Bom_1 = require("@cyclonedx/cyclonedx-library/Contrib/Bom");
const FromNodePackageJson_1 = require("@cyclonedx/cyclonedx-library/Contrib/FromNodePackageJson");
const Enums_1 = require("@cyclonedx/cyclonedx-library/Enums");
const Models_1 = require("@cyclonedx/cyclonedx-library/Models");
const _helpers_1 = require("./_helpers");
class BomBuilder {
npmRunner;
componentBuilder;
leGatherer;
treeBuilder;
purlFactory;
ignoreNpmErrors;
metaComponentType;
packageLockOnly;
omitDependencyTypes;
reproducible;
flattenComponents;
shortPURLs;
gatherLicenseTexts;
workspace;
includeWorkspaceRoot;
workspaces;
console;
constructor(npmRunner, componentBuilder, treeBuilder, purlFactory, leGatherer, options, console_) {
this.npmRunner = npmRunner;
this.componentBuilder = componentBuilder;
this.treeBuilder = treeBuilder;
this.purlFactory = purlFactory;
this.leGatherer = leGatherer;
this.ignoreNpmErrors = options.ignoreNpmErrors ?? false;
this.metaComponentType = options.metaComponentType ?? Enums_1.ComponentType.Library;
this.packageLockOnly = options.packageLockOnly ?? false;
this.omitDependencyTypes = new Set(options.omitDependencyTypes ?? []);
this.reproducible = options.reproducible ?? false;
this.flattenComponents = options.flattenComponents ?? false;
this.shortPURLs = options.shortPURLs ?? false;
this.gatherLicenseTexts = options.gatherLicenseTexts ?? false;
this.workspace = options.workspace ?? [];
this.includeWorkspaceRoot = options.includeWorkspaceRoot;
this.workspaces = options.workspaces;
this.console = console_;
}
buildFromProjectDir(projectDir, process_) {
return this.buildFromNpmLs(this.fetchNpmLs(projectDir, process_), this.npmRunner.getVersion({ env: process_.env }));
}
fetchNpmLs(projectDir, process_) {
const args = [
'ls',
'--json',
'--long',
'--all'
];
if (this.packageLockOnly) {
args.push('--package-lock-only');
}
for (const odt of this.omitDependencyTypes) {
args.push(`--omit=${odt}`);
}
for (const workspace of this.workspace) {
args.push(`--workspace=${workspace}`);
}
if (this.includeWorkspaceRoot !== undefined) {
args.push(`--include-workspace-root=${this.includeWorkspaceRoot}`);
}
if (this.workspaces !== undefined) {
args.push(`--workspaces=${this.workspaces}`);
}
this.console.info('INFO | gathering dependency tree ...');
this.console.debug('DEBUG | npm-ls: run npm with %j in %j', args, projectDir);
let npmLsReturns;
try {
npmLsReturns = this.npmRunner.run(args, {
cwd: projectDir,
env: process_.env,
stdio: ['ignore', 'pipe', 'pipe'],
encoding: 'buffer',
maxBuffer: Number.MAX_SAFE_INTEGER
});
}
catch (runError) {
this.console.group('WARN | npm-ls: MESSAGE');
this.console.warn('%s', runError.message);
this.console.groupEnd();
this.console.group('ERROR | npm-ls: STDERR');
this.console.error('%s', runError.stderr);
this.console.groupEnd();
if (!this.ignoreNpmErrors) {
throw new Error(`npm-ls exited with errors: ${runError.status ?? 'noStatus'} ${runError.signal ?? 'noSignal'}`, { cause: runError });
}
this.console.debug('DEBUG | npm-ls exited with errors that are to be ignored.');
npmLsReturns = runError.stdout ?? Buffer.alloc(0);
}
try {
return JSON.parse(npmLsReturns.toString());
}
catch (jsonParseError) {
throw new Error('failed to parse npm-ls response', { cause: jsonParseError });
}
}
buildFromNpmLs(data, npmVersion) {
this.console.info('INFO | building BOM ...');
const rootPath = data.path;
if (!(0, _helpers_1.isString)(rootPath) || rootPath.length === 0) {
throw new Error(`unexpected path ${JSON.stringify(rootPath)}`);
}
const allPackages = this.gatherPackages(data);
const allComponents = new Map((0, _helpers_1.iterableMap)(allPackages, ([p, packageData]) => [p, this.makeComponentWithPackageData(packageData, p)]));
let rootComponent;
try {
rootComponent = this.makeComponentFromPackagePath(rootPath, this.metaComponentType);
allComponents.set(rootPath, rootComponent);
}
catch (err) {
this.console.debug('DEBUG | failed make rootComponent, fallback to existing one.', err);
const _rootComponent = allComponents.get(rootPath);
if (_rootComponent === undefined) {
throw new TypeError('missing rootComponent', { cause: err });
}
rootComponent = _rootComponent;
rootComponent.type = this.metaComponentType;
}
const [relativePath, dirSep, dirSepRE] = rootPath.startsWith('/')
? [node_path_1.default.posix.relative, '/', /\//g]
: [node_path_1.default.win32.relative, '\\', /\\/g];
allComponents.forEach((c, p) => {
c.properties.add(new Models_1.Property("cdx:npm:package:path", relativePath(rootPath, p).replace(dirSepRE, '/')));
});
const pTree = this.treeBuilder.fromPaths(rootPath, allComponents.keys(), dirSep);
const bom = new Models_1.Bom();
bom.metadata.component = rootComponent;
bom.metadata.tools.components.add(new Models_1.Component(Enums_1.ComponentType.Application, 'npm', {
version: npmVersion
}));
for (const toolC of this.makeToolCs()) {
bom.metadata.tools.components.add(toolC);
}
if (this.reproducible) {
bom.metadata.properties.add(new Models_1.Property("cdx:reproducible", "true"));
}
else {
bom.serialNumber = Bom_1.Utils.randomSerialNumber();
bom.metadata.timestamp = new Date();
}
if (this.flattenComponents) {
for (const c of allComponents.values()) {
if (c === rootComponent) {
continue;
}
bom.components.add(c);
}
}
else {
bom.components = this.nestComponents(allComponents, pTree);
bom.components.delete(rootComponent);
rootComponent.components.forEach(c => bom.components.add(c));
rootComponent.components.clear();
}
this.setNestedBomRefs(allComponents, pTree);
this.makeDependencyGraph(allComponents, allPackages);
return bom;
}
setNestedBomRefs(allComponents, tree, pref = '') {
const bRefCs = {};
const treeI = this.reproducible
? Array.from(tree).sort(([k1,], [k2,]) => k1.localeCompare(k2))
: tree;
for (const [p, cTree] of treeI) {
const component = allComponents.get(p);
if (component === undefined) {
throw new TypeError(`missing component for ${p}`);
}
const parts = [];
if (component.group !== undefined && component.group.length > 0) {
parts.push(component.group, '/');
}
parts.push(component.name);
if (component.version !== undefined && component.version.length > 0) {
parts.push('@', component.version);
}
const bRefD = parts.join('');
const bRefC = bRefCs[bRefD] = (bRefCs[bRefD] ?? 0) + 1;
component.bomRef.value = `${pref}${bRefD}${bRefC > 1 ? '#' + bRefC : ''}`;
this.setNestedBomRefs(allComponents, cTree, `${component.bomRef.value}|`);
}
}
nestComponents(allComponents, tree) {
const children = new Models_1.ComponentRepository();
for (const [p, pTree] of tree) {
const component = allComponents.get(p);
if (component === undefined) {
throw new TypeError(`missing component for ${p}`);
}
component.components = this.nestComponents(allComponents, pTree);
children.add(component);
}
return children;
}
gatherPackages(data) {
const packages = new Map();
const todo = [data];
let w;
while ((w = todo.shift()) !== undefined) {
const wpath = w.path;
if (!(0, _helpers_1.isString)(wpath)) {
continue;
}
let d = packages.get(wpath);
if (d === undefined) {
packages.set(wpath, d = {
name: w.name,
dependencies: new Set()
});
}
d.private ??= w.private;
d.version ??= w.version;
d.license ??= w.license;
d.resolved ??= w.resolved;
d.integrity ??= w.integrity;
d.inBundle ??= w.inBundle;
d.extraneous ??= w.extraneous;
d.optional ??= w.optional;
d.devOptional ??= w.devOptional;
d.dev ??= w.dev;
const dependencies = Object.values(w.dependencies ?? {});
for (const { path: depPath } of dependencies) {
if (!(0, _helpers_1.isString)(depPath)) {
continue;
}
d.dependencies.add(depPath);
}
todo.push(...dependencies);
}
return packages;
}
*fetchLicenseEvidence(dirPath) {
const files = this.leGatherer.getFileAttachments(dirPath, error => {
this.console.info(`INFO | ${error.message}`);
this.console.debug(`DEBUG | ${error.message} -`, error);
});
try {
for (const { file, text } of files) {
yield new Models_1.NamedLicense(`file: ${file}`, { text });
}
}
catch (e) {
this.console.warn('WARN | collecting license evidence in', dirPath, 'failed:', e);
}
}
makeComponentFromPackagePath(ppath, type) {
const manifest = (0, _helpers_1.loadJsonFile)(node_path_1.default.join(ppath, 'package.json'));
(0, _helpers_1.normalizePackageManifest)(manifest);
const component = this.componentBuilder.makeComponent(manifest, type);
if (component === undefined) {
throw new TypeError('created no component');
}
component.licenses.forEach(l => {
l.acknowledgement = Enums_1.LicenseAcknowledgement.Declared;
});
if (this.gatherLicenseTexts) {
component.evidence = new Models_1.ComponentEvidence();
for (const le of this.fetchLicenseEvidence(ppath)) {
component.evidence.licenses.add(le);
}
}
component.purl = this.finalizePurl(this.purlFactory.makeFromPackageJson(manifest));
if (manifest.private === true) {
component.properties.add(new Models_1.Property("cdx:npm:package:private", "true"));
}
return component;
}
makeComponentWithPackageData(data, ppath, type = Enums_1.ComponentType.Library) {
const isOptional = data.optional === true || data.devOptional === false;
let isExcluded = false;
let component = undefined;
if (!this.packageLockOnly) {
try {
component = this.makeComponentFromPackagePath(ppath, type);
}
catch (err) {
if (err.code === 'ENOENT' && isOptional) {
isExcluded = true;
}
else {
this.console.debug('DEBUG | creating DummyComponent for ', ppath, err);
}
}
}
if (component === undefined) {
const manifest = {
private: data.private,
name: data.name,
version: data.version,
license: data.license
};
(0, _helpers_1.normalizePackageManifest)(manifest);
component = this.componentBuilder.makeComponent(manifest, type);
}
if (component === undefined) {
this.console.info('INFO | creating DummyComponent for ', ppath);
component = new DummyComponent(type, ppath);
}
else {
component.licenses.forEach(l => {
l.acknowledgement = Enums_1.LicenseAcknowledgement.Declared;
});
component.purl = this.finalizePurl(this.purlFactory.makeFromPackageData(data));
}
if (isExcluded) {
component.scope = Enums_1.ComponentScope.Excluded;
}
else if (isOptional) {
component.scope = Enums_1.ComponentScope.Optional;
}
if (data.dev === true || data.devOptional === true) {
component.properties.add(new Models_1.Property("cdx:npm:package:development", "true"));
}
if (data.extraneous === true) {
component.properties.add(new Models_1.Property("cdx:npm:package:extraneous", "true"));
}
if (data.inBundle === true) {
component.properties.add(new Models_1.Property("cdx:npm:package:bundled", "true"));
}
const rref = this.makeExtRefDistFromPackageData(data);
if (rref !== undefined) {
component.externalReferences.add(rref);
}
return component;
}
makeExtRefDistFromPackageData(data) {
const { resolved, integrity } = data;
if (!(0, _helpers_1.isString)(resolved) || _helpers_1.npmResolvedIgnoreMatcher.test(resolved)) {
return undefined;
}
const rref = new Models_1.ExternalReference((0, _helpers_1.tryRemoveSecretsFromUrl)(resolved), Enums_1.ExternalReferenceType.Distribution, { comment: 'as detected from npm-ls property "resolved"' });
if ((0, _helpers_1.isString)(integrity)) {
try {
rref.hashes.set(...FromNodePackageJson_1.Utils.parsePackageIntegrity(integrity));
rref.comment += ' and property "integrity"';
}
catch { }
}
return rref;
}
finalizePurl(purl) {
if (purl === undefined) {
return purl;
}
if (this.shortPURLs) {
purl.qualifiers = undefined;
purl.subpath = undefined;
}
return purl.toString();
}
*makeToolCs() {
const packageJsonPaths = [
[node_path_1.default.resolve(module.path, '..', 'package.json'), Enums_1.ComponentType.Application]
];
const libs = [
'@cyclonedx/cyclonedx-library'
].map(s => s.split('/', 2));
const nodeModulePaths = require.resolve.paths('__some_none-native_package__') ?? [];
libsLoop: for (const lib of libs) {
for (const nodeModulePath of nodeModulePaths) {
const packageJsonPath = node_path_1.default.resolve(nodeModulePath, ...lib, 'package.json');
if ((0, node_fs_1.existsSync)(packageJsonPath)) {
packageJsonPaths.push([packageJsonPath, Enums_1.ComponentType.Library]);
continue libsLoop;
}
}
}
for (const [packageJsonPath, cType] of packageJsonPaths) {
const packageData = (0, _helpers_1.loadJsonFile)(packageJsonPath);
(0, _helpers_1.normalizePackageManifest)(packageData);
const toolC = this.componentBuilder.makeComponent(packageData, cType);
if (toolC !== undefined) {
yield toolC;
}
}
}
makeDependencyGraph(allComponents, allPackages) {
for (const [p, comp] of allComponents) {
const pkg = allPackages.get(p);
if (pkg === undefined) {
throw new TypeError(`missing pkg for ${p}`);
}
for (const depPkg of pkg.dependencies) {
const depComp = allComponents.get(depPkg);
if (depComp === undefined) {
throw new TypeError(`missing depComp for ${depPkg}`);
}
comp.dependencies.add(depComp.bomRef);
}
}
}
}
exports.BomBuilder = BomBuilder;
class DummyComponent extends Models_1.Component {
constructor(type, name) {
super(type, `DummyComponent.${name}`, {
bomRef: `DummyComponent.${name}`,
description: `This is a dummy component "${name}" that fills the gap where the actual built failed.`
});
}
}
class TreeBuilder {
fromPaths(root, paths, dirSeparator) {
root += dirSeparator;
const upaths = new Set((0, _helpers_1.iterableMap)(paths, p => `${p}${dirSeparator}`));
const outs = new Set((0, _helpers_1.iterableFilter)(upaths, p => !p.startsWith(root)));
const inTree = new Map((0, _helpers_1.iterableMap)((0, _helpers_1.setDifference)(upaths, outs), p => [p, new Map()]));
this.nestPT(inTree);
const outTree = new Map((0, _helpers_1.iterableMap)(outs, p => [p, new Map()]));
this.nestPT(outTree);
const tree = new Map();
outTree.forEach((v, k) => { tree.set(k, v); });
inTree.forEach((v, k) => { tree.set(k, v); });
this.renderPR(tree, '');
return tree;
}
renderPR(tree, pref) {
for (const [p, pTree] of Array.from(tree)) {
tree.delete(p);
const pFull = pref + p;
this.renderPR(pTree, pFull);
tree.set(pFull.slice(undefined, -1), pTree);
}
}
nestPT(tree) {
if (tree.size < 2) {
return;
}
const treeI = Array.from(tree);
for (const [a, aTree] of treeI) {
for (const [b, bTree] of treeI) {
if (a === b) {
continue;
}
if (b.startsWith(a)) {
aTree.set(b.slice(a.length), bTree);
tree.delete(b);
}
else if (a.startsWith(b)) {
bTree.set(a.slice(b.length), aTree);
tree.delete(a);
}
}
}
for (const c of tree.values()) {
this.nestPT(c);
}
}
}
exports.TreeBuilder = TreeBuilder;