@nodesecure/scanner
Version:
A package API to run a static analysis of your module's dependencies.
319 lines • 14.3 kB
JavaScript
// Import Node.js Dependencies
import path from "node:path";
import { readFileSync } from "node:fs";
// Import Third-party Dependencies
import pacote from "pacote";
import * as npmRegistrySDK from "@nodesecure/npm-registry-sdk";
import { Mutex, MutexRelease } from "@openally/mutex";
import { extractAndResolve, scanDirOrArchive } from "@nodesecure/tarball";
import { DefaultCollectableSet } from "@nodesecure/js-x-ray";
import * as Vulnera from "@nodesecure/vulnera";
import { npm } from "@nodesecure/tree-walker";
import { parseAuthor } from "@nodesecure/utils";
import { ManifestManager, parseNpmSpec } from "@nodesecure/mama";
import { getNpmRegistryURL } from "@nodesecure/npm-registry-sdk";
import { fromData } from "ssri";
import semver from "semver";
// Import Internal Dependencies
import { getDependenciesWarnings, addMissingVersionFlags, getUsedDeps, getManifestLinks, NPM_TOKEN } from "./utils/index.js";
import { NpmRegistryProvider } from "./registry/NpmRegistryProvider.js";
import { StatsCollector } from "./class/StatsCollector.class.js";
import { RegistryTokenStore } from "./registry/RegistryTokenStore.js";
import { TempDirectory } from "./class/TempDirectory.class.js";
import { Logger, ScannerLoggerEvents } from "./class/logger.class.js";
import { parseSemverRange } from "./utils/parseSemverRange.js";
// CONSTANTS
const kDefaultDependencyVersionFields = {
description: "",
size: 0,
author: null,
engines: {},
scripts: {},
licenses: [],
uniqueLicenseIds: [],
composition: {
extensions: [],
files: [],
minified: [],
unused: [],
missing: [],
required_files: [],
required_nodejs: [],
required_thirdparty: [],
required_subpath: []
}
};
const kDefaultDependencyMetadata = {
publishedCount: 0,
lastUpdateAt: new Date(),
lastVersion: "N/A",
hasChangedAuthor: false,
hasManyPublishers: false,
hasReceivedUpdateInOneYear: true,
homepage: null,
author: null,
publishers: [],
maintainers: [],
integrity: {}
};
const kRootDependencyId = 0;
const kCollectableTypes = ["url", "hostname", "ip", "email"];
const { version: packageVersion } = JSON.parse(readFileSync(new URL(path.join("..", "package.json"), import.meta.url), "utf-8"));
export async function depWalker(manifest, options, logger = new Logger()) {
const { scanRootNode = false, includeDevDeps = false, isVerbose = false, packageLock, maxDepth, location, vulnerabilityStrategy = Vulnera.strategies.NONE, registry, npmRcConfig } = options;
const statsCollector = new StatsCollector({ logger }, { isVerbose });
const collectables = kCollectableTypes.map((type) => new DefaultCollectableSet(type));
const pacoteProvider = {
async extract(spec, dest, opts) {
await statsCollector.track(`pacote.extract ${spec}`, "tarball-scan", () => pacote.extract(spec, dest, opts));
}
};
const isRemoteScanning = typeof location === "undefined";
const tokenStore = new RegistryTokenStore(npmRcConfig, NPM_TOKEN.token);
await using tempDir = await TempDirectory.create();
const dependencyConfusionWarnings = [];
const payload = {
id: tempDir.id,
rootDependency: {
name: manifest.name ?? "workspace",
version: manifest.version ?? "0.0.0",
integrity: null
},
scannerVersion: packageVersion,
vulnerabilityStrategy,
warnings: []
};
const dependencies = new Map();
const highlightedPackages = new Set();
const identifiersToHighlight = new Set(options.highlight?.identifiers ?? []);
const npmTreeWalker = new npm.TreeWalker({
registry,
providers: {
pacote: {
manifest: (spec, opts) => statsCollector.track(`pacote.manifest ${spec}`, "tree-walk", () => pacote.manifest(spec, opts)),
packument: (spec, opts) => statsCollector.track(`pacote.packument ${spec}`, "tree-walk", () => pacote.packument(spec, opts))
}
}
});
const npmApiClient = {
packument: (name, opts) => statsCollector.track(`npmRegistrySDK.packument ${name}`, "metadata-fetch", () => npmRegistrySDK.packument(name, opts)),
packumentVersion: (name, version, opts) => statsCollector.track(`npmRegistrySDK.packumentVersion ${name}@${version}`, "metadata-fetch", () => npmRegistrySDK.packumentVersion(name, version, opts)),
org: (namespace) => statsCollector.track(`npmRegistrySDK.org ${namespace}`, "metadata-fetch", () => npmRegistrySDK.org(namespace))
};
{
logger
.start(ScannerLoggerEvents.analysis.tree)
.start(ScannerLoggerEvents.analysis.tarball)
.start(ScannerLoggerEvents.analysis.registry);
const fetchedMetadataPackages = new Set();
const operationsQueue = [];
const locker = new Mutex({ concurrency: 5 });
locker.on(MutexRelease, () => logger.tick(ScannerLoggerEvents.analysis.tarball));
const rootDepsOptions = {
maxDepth,
includeDevDeps,
packageLock
};
for await (const current of npmTreeWalker.walk(manifest, rootDepsOptions)) {
const { name, version, integrity, ...currentVersion } = current;
const dependency = {
versions: {
[version]: {
...currentVersion,
...structuredClone(kDefaultDependencyVersionFields)
}
},
vulnerabilities: [],
metadata: structuredClone(kDefaultDependencyMetadata)
};
let proceedDependencyScan = true;
const org = parseNpmSpec(name)?.org;
if (dependencies.has(name)) {
const dep = dependencies.get(name);
operationsQueue.push(new NpmRegistryProvider(name, version, {
registry,
tokenStore,
npmApiClient
}).enrichDependencyVersion(dep, dependencyConfusionWarnings, org));
if (version in dep.versions) {
// The dependency has already entered the analysis
// This happens if the package is used by multiple packages in the tree
proceedDependencyScan = false;
}
else {
dep.versions[version] = dependency.versions[version];
}
}
else {
dependencies.set(name, dependency);
}
const isRoot = current.id === kRootDependencyId;
if (isRoot && payload.rootDependency.integrity) {
payload.rootDependency.integrity = integrity;
}
else if (isRoot) {
const isWorkspace = options.location && "workspaces" in manifest;
payload.rootDependency.integrity = isWorkspace ?
null :
fromData(JSON.stringify(manifest), { algorithms: ["sha512"] }).toString();
}
// If the dependency is a DevDependencies we ignore it.
if (current.isDevDependency || !proceedDependencyScan) {
continue;
}
logger.tick(ScannerLoggerEvents.analysis.tree);
// There is no need to fetch 'N' times the npm metadata for the same package.
if (fetchedMetadataPackages.has(name) || !current.existOnRemoteRegistry) {
logger.tick(ScannerLoggerEvents.analysis.registry);
}
else {
fetchedMetadataPackages.add(name);
const provider = new NpmRegistryProvider(name, version, {
registry,
tokenStore
});
operationsQueue.push(provider.enrichDependency(logger, dependency));
if (registry !== getNpmRegistryURL() && org) {
operationsQueue.push(new NpmRegistryProvider(name, version, {
registry,
tokenStore
}).enrichScopedDependencyConfusionWarnings(dependencyConfusionWarnings, org));
}
}
const scanDirOptions = {
ref: dependency.versions[version],
location,
isRootNode: scanRootNode && name === manifest.name,
registry,
statsCollector,
pacoteProvider,
collectables
};
operationsQueue.push(scanDirOrArchiveEx(name, version, locker, tempDir, scanDirOptions));
}
logger.end(ScannerLoggerEvents.analysis.tree);
await Promise.allSettled(operationsQueue);
logger
.end(ScannerLoggerEvents.analysis.tarball)
.end(ScannerLoggerEvents.analysis.registry);
}
const { hydratePayloadDependencies, strategy } = Vulnera.setStrategy(vulnerabilityStrategy);
const isVulnHydratable = (strategy === "github-advisory" || strategy === "snyk")
&& isRemoteScanning;
if (!isVulnHydratable) {
await hydratePayloadDependencies(dependencies, {
useStandardFormat: true,
path: location
});
}
payload.vulnerabilityStrategy = strategy;
// We do this because it "seem" impossible to link all dependencies in the first walk.
// Because we are dealing with package only one time it may happen sometimes.
const globalWarnings = [];
for (const [packageName, dependency] of dependencies) {
const metadataIntegrities = dependency.metadata?.integrity ?? {};
for (const [version, integrity] of Object.entries(metadataIntegrities)) {
const dependencyVer = dependency.versions[version];
const isEmptyPackage = dependencyVer.warnings.some((warning) => warning.kind === "empty-package");
if (isEmptyPackage) {
globalWarnings.push({
type: "empty-package",
message: `${packageName}@${version} only contain a package.json file!`
});
}
if (!("integrity" in dependencyVer) || dependencyVer.flags.includes("isGit")) {
continue;
}
if (dependencyVer.integrity !== integrity) {
globalWarnings.push({
type: "integrity-mismatch",
message: `${packageName}@${version} manifest & tarball integrity doesn't match!`
});
}
}
const semverRanges = parseSemverRange(options.highlight?.packages ?? {});
for (const version of Object.entries(dependency.versions)) {
const [verStr, verDescriptor] = version;
const range = semverRanges?.[packageName];
if (range && semver.satisfies(verStr, range)) {
highlightedPackages.add(`${packageName}@${verStr}`);
}
verDescriptor.flags.push(...addMissingVersionFlags(new Set(verDescriptor.flags), dependency));
if (isLocalManifest(verDescriptor, manifest, packageName)) {
Object.assign(dependency.metadata, {
author: parseAuthor(manifest.author),
homepage: manifest.homepage
});
Object.assign(verDescriptor, {
author: parseAuthor(manifest.author),
links: getManifestLinks(manifest),
repository: manifest.repository
});
}
const usedDeps = npmTreeWalker.relationsMap.get(`${packageName}@${verStr}`) || new Set();
if (usedDeps.size === 0) {
continue;
}
const usedBy = Object.create(null);
for (const [name, version] of getUsedDeps(usedDeps)) {
usedBy[name] = version;
}
Object.assign(verDescriptor.usedBy, usedBy);
}
}
try {
const { warnings, illuminated } = await getDependenciesWarnings(dependencies, options.highlight?.contacts, isRemoteScanning);
payload.warnings = globalWarnings.concat(dependencyConfusionWarnings).concat(warnings);
payload.highlighted = {
contacts: illuminated,
packages: [...highlightedPackages],
identifiers: extractHighlightedIdentifiers(collectables, identifiersToHighlight)
};
payload.dependencies = Object.fromEntries(dependencies);
payload.metadata = statsCollector.getStats();
return payload;
}
finally {
logger.emit(ScannerLoggerEvents.done);
}
}
function extractHighlightedIdentifiers(collectables, identifiersToHighlight) {
if (identifiersToHighlight.size === 0) {
return [];
}
return collectables.flatMap((collectableSet) => Array.from(collectableSet)
.flatMap(({ value, locations }) => (identifiersToHighlight.has(value) ?
locations.map(({ file, metadata, location }) => {
return {
value,
spec: metadata?.spec,
location: {
file,
lines: location
}
};
}) : [])));
}
// eslint-disable-next-line max-params
async function scanDirOrArchiveEx(name, version, locker, tempDir, options) {
using _ = await locker.acquire();
const spec = `${name}@${version}`;
const { registry, location = process.cwd(), isRootNode, ref, statsCollector, pacoteProvider, collectables } = options;
const mama = await (isRootNode ?
ManifestManager.fromPackageJSON(location) :
extractAndResolve(tempDir.location, {
spec,
registry,
pacoteProvider
}));
await statsCollector.track(`tarball.scanDirOrArchive ${spec}`, "tarball-scan", () => scanDirOrArchive(mama, ref, {
astAnalyserOptions: {
optionalWarnings: typeof location !== "undefined",
collectables
}
}));
}
function isLocalManifest(verDescriptor, manifest, packageName) {
return verDescriptor.existOnRemoteRegistry === false && (packageName === manifest.name || manifest.name === undefined);
}
//# sourceMappingURL=depWalker.js.map