@eagleoutice/flowr-dev
Version:
Static Dataflow Analyzer and Program Slicer for the R Programming Language
567 lines • 27.9 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.FlowrAnalyzerPackageVersionsSigDbPlugin = exports.ExportIndex = exports.sigDbLog = exports.SigDbPluginName = void 0;
exports.reconstructS3Generics = reconstructS3Generics;
const flowr_analyzer_package_versions_plugin_1 = require("./flowr-analyzer-package-versions-plugin");
const semver_1 = require("semver");
const path_1 = __importDefault(require("path"));
const package_1 = require("./package");
const reader_1 = require("../../sigdb/reader");
const schema_1 = require("../../sigdb/schema");
const manifest_1 = require("../../sigdb/manifest");
const decompress_1 = require("../../sigdb/decompress");
const codec_1 = require("../../sigdb/codec");
const log_1 = require("../../../util/log");
const flowr_file_1 = require("../../context/flowr-file");
const config_1 = require("../../../config");
const r_version_1 = require("../../../util/r-version");
const r_base_packages_1 = require("../../../util/r-base-packages");
/** the plugin's instance name (pass to `unregisterPlugins` to disable the default sigdb resolver) */
exports.SigDbPluginName = 'flowr-analyzer-package-versions-sigdb-plugin';
/** map a resolved source's compressed extension to a codec name for display (`undefined` extension is a plain file) */
function codecNameOf(ext) {
switch (ext) {
case '.zst': return 'zstd';
case '.br': return 'brotli';
case '.gz': return 'gzip';
default: return 'plain';
}
}
/** the codec name of a manifest's resolved shard + dict sources, or `mixed` when they do not all agree */
function manifestFormat(set) {
const paths = [
...set.manifest.shards.map(s => s.path),
...(set.manifest.dicts?.map(d => d.path) ?? [])
];
const codecs = new Set(paths.map(p => codecNameOf((0, codec_1.compressedExtOf)((0, decompress_1.resolveSource)(set.baseDir, p)))));
return codecs.size === 1 ? [...codecs][0] : 'mixed';
}
/** describe one loaded source for `:version` / diagnostics: a single bundle, a sharded set, or an in-memory source */
function describeLoadedDatabase(src) {
if (src instanceof reader_1.SigDatabase) {
return { scope: 'signatures', version: src.content?.version ?? 0, date: src.content?.date ?? '', hash: src.content?.hash ?? '' };
}
if (src instanceof reader_1.SigDatabaseSet) {
// the scope (base/current/history) is the leading segment of the manifest's own shard/dict filenames
const file = src.manifest.dicts?.[0]?.path ?? src.manifest.shards[0]?.path ?? '';
const scope = file.split('.')[0] || 'signatures';
return { scope, version: src.manifest.schema, date: src.manifest.date, hash: src.manifest.shards.map(s => s.hash).join(','), format: manifestFormat(src) };
}
// an in-memory source carries no bundle version/date
return { scope: 'signatures', version: 0, date: '', hash: src.packageNames().join(',') };
}
exports.sigDbLog = log_1.log.getSubLogger({ name: exports.SigDbPluginName });
/** additional sources from `$FLOWR_SIGDB` (path-delimiter separated), consulted after the explicit ones */
function envSources() {
const env = typeof process !== 'undefined' ? process.env?.FLOWR_SIGDB : undefined;
return env ? env.split(path_1.default.delimiter).filter(s => s.length > 0) : [];
}
/**
* The database flattens S3 methods into the export list, so we recover the `generic -> classes` map for
* dispatch: an export `generic.class` is a method when the `generic` is itself exported (e.g. `print.foo`).
*/
function reconstructS3Generics(exported) {
const names = new Set(exported);
const generics = new Map();
for (const name of exported) {
const dot = name.indexOf('.');
const generic = dot > 0 ? name.slice(0, dot) : undefined;
if (generic !== undefined && names.has(generic)) {
const classes = generics.get(generic) ?? [];
classes.push(name.slice(dot + 1));
generics.set(generic, classes);
}
}
return generics;
}
/** the built {@link ExportIndex} of each source, see {@link ExportIndex.of} for why it is keyed by the source */
const exportIndices = new WeakMap();
/**
* The reverse `export name -> packages exporting it` view of a signature source, each entry ordered by download
* count (descending, ties by name) so whoever has to pick one exporter starts with the package a script most
* likely means.
*
* Building the view reads every package blob of a bundle, which is why {@link of} memoizes it on the source
* object rather than on the caller: signature sources are opened once per process (see {@link getSharedSigSource})
* and are immutable, so every analyzer mounting the same bundle shares one index instead of re-scanning the
* database per analysis. It is deliberately analyzer-independent -- self-package exclusion belongs to the caller
* (see {@link FlowrAnalyzerPackageVersionsSigDbPlugin.packagesExporting}), not to the index.
*/
exports.ExportIndex = {
name: 'ExportIndex',
/** The index of `src`, built on first use and shared by every later caller; see {@link ExportIndex}. */
of(src) {
const cached = exportIndices.get(src);
if (cached !== undefined) {
return cached;
}
const index = new Map();
for (const pkg of src.packageNames()) {
for (const exp of src.lookup(pkg)?.exported ?? []) {
const owners = index.get(exp);
if (owners === undefined) {
index.set(exp, pkg);
}
else if (typeof owners === 'string') {
if (owners !== pkg) {
index.set(exp, [owners, pkg]);
}
}
else if (owners[owners.length - 1] !== pkg) {
owners.push(pkg);
}
}
}
// sorting once per name here beats sorting at every lookup
for (const owners of index.values()) {
if (typeof owners !== 'string') {
owners.sort((a, b) => src.downloads(b) - src.downloads(a) || a.localeCompare(b));
}
}
exportIndices.set(src, index);
return index;
},
/** The packages an {@link ExportIndexEntry} names, as a list; empty when no package exports the name. */
owners(entry) {
return entry === undefined ? [] : typeof entry === 'string' ? [entry] : entry;
}
};
/**
* Resolves `library(pkg)` / `use(pkg, fn)` from precomputed `flowr-sigdb` databases via the
* {@link PackageSignatureSource} contract. For an R-core package it picks the version shipped with the assumed
* R release (`solver.sigdb.assumedRVersion`), so `library(stats)` attaches that release's exports. Plain-file
* sources load lazily; a `.br` or manifest source is mounted by {@link preload}. On by default.
*/
class FlowrAnalyzerPackageVersionsSigDbPlugin extends flowr_analyzer_package_versions_plugin_1.FlowrAnalyzerPackageVersionsPlugin {
name = exports.SigDbPluginName;
description = 'Resolves library exports (and versioned base R) from precomputed flowr-sigdb databases.';
version = new semver_1.SemVer('0.1.0');
extraSources;
sources;
/** the `additionalPaths` the current {@link sources} were assembled with, so a later config resolves a rebuild */
sourcesKey;
analyzerCtx;
/** `packagesExporting` answers, merged across the source set and self-filtered; the scan itself is {@link ExportIndex} */
exportsByName = new Map();
/** `pkg@assumedR` keys already reported via {@link baseVersionFor}'s fallback, so the info is logged once */
baseFallbacksLogged = new Set();
/** installed package versions for `versionSelection: 'system'`, read once from R (see {@link warmInstalledVersions}) */
installedVersions;
/** guards the one-time async warm-up of {@link installedVersions} */
installedVersionsPromise;
/** invalidate the assembled source list and derived caches (after a source or config change) */
resetAssembled() {
this.sources = undefined;
this.sourcesKey = undefined;
this.exportsByName = new Map();
}
constructor(...sources) {
super();
this.extraSources = sources;
}
/**
* Dynamically add signature sources after construction (opened instances, plain `.sigs.ndjson`, or `.br`/
* manifest paths). Added sources take precedence over the bundled default, so they can override or extend it.
*/
addSource(...sources) {
this.extraSources.push(...sources);
this.resetAssembled(); // invalidate the caches so the next resolve picks the new sources up
}
process(ctx) {
this.resetAssembled(); // reload on (re)registration so config/source changes take effect (shared bundles are reused)
this.analyzerCtx = ctx;
if ((0, config_1.isSigDbEnabled)(ctx.config)) {
ctx.deps.addLazyResolver((name, existing) => this.resolve(name, existing));
if (ctx.config.solver.sigdb.warmInBackground) {
this.startBackgroundWarm();
}
if (ctx.config.solver.sigdb.autoSync) {
this.startBackgroundSync(ctx);
}
if (ctx.config.solver.sigdb.versionSelection === config_1.VersionSelection.System) {
this.warmInstalledVersions();
}
}
}
syncPromise;
/**
* Opt-in (`solver.sigdb.autoSync`) startup re-sync.
* If the committed `sigdb.remote.json` link file lists shards whose cached copies are missing or hash-mismatched,
* this will sync them.
*/
startBackgroundSync(ctx) {
if (this.syncPromise !== undefined) {
return;
}
this.syncPromise = (async () => {
// dynamic import: sigdb-download pulls in node http/https, keep it off the hot load path
const dl = await Promise.resolve().then(() => __importStar(require('../../sigdb/sigdb-download')));
if (!dl.sigDbNeedsSync()) {
return;
}
exports.sigDbLog.info('sigdb: committed link file changed, re-syncing shards in the background');
const { files } = await dl.downloadFullSigDb({
repo: ctx.config.solver.sigdb.downloadRepo,
onProgress: msg => exports.sigDbLog.info(`sigdb sync: ${msg}`)
});
for (const manifest of files.filter(f => /\.manifest\.json(\.br)?$/.test(f))) {
await ctx.deps.addDatabaseSource(manifest);
}
})().catch((e) => {
exports.sigDbLog.warn(`background sigdb sync failed (keeping the cached shards): ${e.message}`);
});
}
warmPromise;
/**
* Warm the hot shards (base + most-downloaded packages) of any sharded source in a background task, so the
* first `library()` lookup no longer blocks on decompression. Idempotent.
*/
startBackgroundWarm() {
if (this.warmPromise !== undefined) {
return;
}
this.warmPromise = (async () => {
for (const src of this.loadSources()) {
if (src instanceof reader_1.SigDatabaseSet) {
await src.preloadShards(s => s.tier === 'current' && s.shard !== 'rest');
}
}
})().catch((e) => {
exports.sigDbLog.warn(`background sigdb warm failed: ${e.message}`);
});
}
/**
* Read the system's installed package versions once (for `versionSelection: 'system'`), off the hot path. Only
* an R-backed parser exposes `installedPackageVersions`; a tree-sitter (no-R) parser skips this, so `system`
* gracefully falls back to `newest` in {@link resolve}. Idempotent; failures leave the map empty (same fallback).
*/
warmInstalledVersions() {
if (this.installedVersionsPromise !== undefined || this.installedVersions !== undefined) {
return;
}
const info = this.analyzerCtx?.analyzer?.parserInformation();
if (!info || !('installedPackageVersions' in info) || typeof info.installedPackageVersions !== 'function') {
return; // no R available: system selection falls back to newest
}
this.installedVersionsPromise = info.installedPackageVersions()
.then(versions => {
this.installedVersions = versions;
})
.catch((e) => {
this.installedVersions = new Map();
exports.sigDbLog.warn(`sigdb: could not read installed package versions for system version selection, falling back to newest: ${e.message}`);
});
}
/** Mount the databases up front instead of on the first library load (see `solver.sigdb.eagerlyLoad`). */
preloadDatabasesSync() {
this.loadSources();
}
/** whether any loaded source carries a versioned base-R package (so base namespaces can be attached eagerly) */
providesBaseRPackages() {
const base = (0, r_base_packages_1.baseRPackages)();
return this.loadSources().some(src => base.some(p => src.has(p) && src.isBaseR(p)));
}
signatureSources(config) {
return this.loadSources(config);
}
loadedDatabases() {
return this.loadSources().map(describeLoadedDatabase);
}
/**
* Packages in the loaded sources (respecting `self`-package exclusion) that export `name`, **most downloaded
* first**, so whoever has to pick one (or show only a few) starts with the package a script most likely means.
* Backed by {@link ExportIndex}, a reverse index built once per *source* (and hence shared by every analyzer
* mounting the same bundle), so repeated hint lookups (e.g. from the `undefined-symbol` linter) do not re-scan
* every package. Self-package exclusion is applied here rather than baked into the index, which keeps the
* index analyzer-independent.
*/
packagesExporting(name) {
if (!(0, config_1.isSigDbEnabled)(this.analyzerCtx?.config)) {
return [];
}
const cached = this.exportsByName.get(name);
if (cached !== undefined) {
return cached;
}
const sources = this.loadSources();
const seen = new Set();
const owners = [];
for (const src of sources) {
for (const pkg of src.packagesExporting(name)) {
if (!seen.has(pkg) && !this.isSelfPackage(pkg)) {
seen.add(pkg);
owners.push(pkg);
}
}
}
// each source's list is already sorted; a merge of several needs one more pass to be globally ordered
if (owners.length > 1 && sources.length > 1) {
/* the comparator runs O(n log n) times, so each package is counted once up front */
const downloads = new Map(owners.map(pkg => [pkg, sources.reduce((m, s) => Math.max(m, s.has(pkg) ? s.downloads(pkg) : 0), 0)]));
owners.sort((a, b) => downloads.get(b) - downloads.get(a) || a.localeCompare(b));
}
this.exportsByName.set(name, owners);
return owners;
}
/**
* The raw sources in priority order: explicit constructor sources, `$FLOWR_SIGDB`, then **every** bundled
* database discovered in the data dirs (see {@link defaultSigDbPaths}), so an extra bundle dropped next to
* the default (e.g. a downloaded full-history one) is mounted automatically. All bundled defaults are skipped
* when `$FLOWR_DISABLE_DEFAULT_SIGDB` is set; explicit sources are always honored.
*/
rawSources(config) {
const sources = [...this.extraSources, ...envSources()];
const disableBundled = typeof process !== 'undefined' && process.env?.FLOWR_DISABLE_DEFAULT_SIGDB;
if (!disableBundled) {
// `additionalPaths` from the config are prioritized
const extra = (config ?? this.analyzerCtx?.config)?.solver.sigdb.additionalPaths ?? [];
sources.push(...(0, manifest_1.defaultSigDbPaths)(extra));
sources.push(...extra.filter(p => /\.(manifest\.json|sigs\.ndjson)(\.br|\.gz)?$/.test(p)));
}
return sources;
}
/** synchronously openable sources (instances + plain `.sigs.ndjson`); `.br`/manifests are added by {@link preload}. */
loadSources(config) {
const cfg = config ?? this.analyzerCtx?.config;
// `solver.sigdb.enabled: false` disables the database for this analyzer only: drop any loaded sources so it
// frees memory and every consumer (resolve, base-R link, queries) sees nothing; other analyzers are untouched.
// An absent config is the pre-analysis query path (default enabled), so only an explicit `false` disables.
if (cfg !== undefined && !(0, config_1.isSigDbEnabled)(cfg)) {
if (this.sources !== undefined) {
this.resetAssembled();
}
return [];
}
// a query may run before an analysis has set `analyzerCtx`, memoizing sources without the config's
// `additionalPaths`; re-key on them so the config-aware call rebuilds rather than reusing the stale set
const key = (cfg?.solver.sigdb.additionalPaths ?? []).join('\0');
if (this.sources === undefined || this.sourcesKey !== key) {
this.sourcesKey = key;
this.sources = this.rawSources(config)
.map(s => this.loadSync(s))
.filter((s) => s !== undefined);
}
return this.sources;
}
loadSync(source) {
if (typeof source !== 'string') {
return source;
}
try {
const opened = (0, reader_1.getSharedSigSourceSync)(source);
if (opened === undefined) {
exports.sigDbLog.warn(`sigdb source ${source} needs preload() (only plain ${schema_1.SigDbExt} files and manifests open synchronously)`);
}
return opened;
}
catch (e) {
exports.sigDbLog.warn(`Could not load sigdb source: ${e.message}`);
}
return undefined;
}
/**
* Open every source, including compressed bundles (`.br`/`.gz`) and manifests (`*.manifest.json`).
* Call once at the analyzer boundary so resolution can fall through to them.
*/
async preload() {
for (const source of this.rawSources()) {
if (typeof source !== 'string') {
continue;
}
try {
await (0, reader_1.getSharedSigSource)(source);
}
catch (e) {
exports.sigDbLog.warn(`Could not load sigdb source: ${e.message}`);
}
}
this.resetAssembled();
}
async addDatabaseSource(source) {
this.addSource(source);
await this.preload();
}
/** Whether the given name is the analyzed project itself (so we must not shadow its own definitions). */
isSelfPackage(name) {
const desc = this.analyzerCtx?.files.getFilesByRole(flowr_file_1.FileRole.Description) ?? [];
return desc.some(d => d.packageName() === name);
}
/**
* For a base package, the newest core version `<=` the assumed R version (see
* {@link FlowrAnalyzerContext.resolvedRVersion}). If the assumed version predates every recorded core
* release, the closest supported one (the earliest) is used and the substitution is logged once as info.
*/
baseVersionFor(src, name) {
const versions = src.coreVersions(name);
if (!versions || versions.length === 0) {
return undefined;
}
const target = this.analyzerCtx?.resolvedRVersion ?? (0, config_1.resolveAssumedRVersion)(undefined);
let chosen; // versions are ascending; keep the newest core release <= target
for (const v of versions) {
if (r_version_1.RVersion.compare(v.str, target) <= 0) {
chosen = v;
}
}
if (chosen === undefined) {
// the assumed R version is older than the earliest recorded core release: fall back to the closest one
chosen = versions[0];
const key = `${name}@${target}`;
if (!this.baseFallbacksLogged.has(key)) {
this.baseFallbacksLogged.add(key);
exports.sigDbLog.info(`Assumed R version ${target} predates the earliest recorded core release of ${name}; falling back to the closest supported version ${chosen.str}.`);
}
}
return chosen.str;
}
resolve(name, existing) {
if (this.isSelfPackage(name)) {
return undefined;
}
const sigdb = this.analyzerCtx?.config.solver.sigdb;
const override = sigdb?.versionOverrides?.[name];
const selection = sigdb?.versionSelection ?? config_1.VersionSelection.Newest;
const range = existing?.derivedRange;
let fallback;
for (const src of this.loadSources()) {
if (!src.has(name)) {
continue;
}
// base R always resolves against the assumed R version, independent of override/selection
if (src.isBaseR(name)) {
const info = src.lookup(name, this.baseVersionFor(src, name)) ?? src.lookup(name);
if (info && (range === undefined || r_version_1.RRange.satisfies(info.version, range))) {
return this.toResolvedPackage(name, info);
}
fallback ??= info;
continue;
}
// a per-package override wins over both the constraint and the newest/oldest/system policy
if (override !== undefined) {
const info = src.lookup(name, override);
if (info) {
return this.toResolvedPackage(name, info);
}
fallback ??= src.lookup(name);
continue;
}
const info = this.selectVersion(src, name, range, selection);
if (info) {
return this.toResolvedPackage(name, info);
}
fallback ??= src.lookup(name); // only a version outside the constraint is stored; keep it as a last resort
}
if (fallback !== undefined) {
const constraint = override !== undefined ? `override ${override}` : (range?.raw ?? 'a version');
exports.sigDbLog.warn(`project constrains ${name} to ${constraint} but the signature database only has ${fallback.version}; analyzing with ${fallback.version}`);
return this.toResolvedPackage(name, fallback);
}
return undefined;
}
/** the concrete export view for a non-base package under the active {@link VersionSelection} policy, or `undefined` if none satisfies */
selectVersion(src, name, range, selection) {
if (selection === config_1.VersionSelection.System) {
const installed = this.installedVersions?.get(name);
const info = installed !== undefined ? src.lookup(name, installed) : undefined;
if (info) {
return info; // the version that actually runs on this system (even if outside the declared constraint)
}
// not installed, not in the db, or no R available: fall through to newest-satisfying
}
if (selection === config_1.VersionSelection.Oldest) {
return this.oldestSatisfying(src, name, range);
}
return this.newestSatisfying(src, name, range);
}
/**
* Newest version satisfying the constraint. Fast path: prefer the source's latest (no history decompression for
* the common `>=` case) and accept it when it satisfies; only otherwise enumerate the stored versions and pick
* the highest satisfying one (e.g. an upper-bound or exact-old pin).
*/
newestSatisfying(src, name, range) {
const pinned = range ? (0, semver_1.minVersion)(range)?.version : undefined;
const info = src.lookup(name, pinned) ?? src.lookup(name);
if (info && (range === undefined || r_version_1.RRange.satisfies(info.version, range))) {
return info;
}
if (range !== undefined) {
const versions = this.availableVersions(src, name);
for (let i = versions.length - 1; i >= 0; i--) {
if (r_version_1.RRange.satisfies(versions[i], range)) {
return src.lookup(name, versions[i]);
}
}
}
return undefined;
}
/** Lowest stored version satisfying the constraint (using the store's actual R-form version strings). */
oldestSatisfying(src, name, range) {
for (const version of this.availableVersions(src, name)) { // ascending
if (range === undefined || r_version_1.RRange.satisfies(version, range)) {
return src.lookup(name, version);
}
}
return undefined;
}
/**
* The versions the source can answer for a package (dated releases, base-R core releases, and the latest),
* ascending. Versions that differ in writing but not in order (`1.2` and `1.2.0`) are settled by release date.
*/
availableVersions(src, pkg) {
return (0, reader_1.availableVersionEntries)(src, pkg).map(e => e.version);
}
/** build the resolved {@link Package} (namespace + version) from a source's export view */
toResolvedPackage(name, info) {
const exported = info.exported.slice();
const namespaceInfo = {
exportedSymbols: exported,
exportedFunctions: [],
exportS3Generics: reconstructS3Generics(exported),
exportedPatterns: [],
importedPackages: new Map(),
loadsWithSideEffects: false,
callable: exported
};
/* a source may know the exports without knowing which release they are from, and then the package
carries no version rather than a made-up one */
return new package_1.Package({ name, namespaceInfo, resolvedVersion: info.version || undefined });
}
}
exports.FlowrAnalyzerPackageVersionsSigDbPlugin = FlowrAnalyzerPackageVersionsSigDbPlugin;
//# sourceMappingURL=flowr-analyzer-package-versions-sigdb-plugin.js.map