@eagleoutice/flowr-dev
Version:
Static Dataflow Analyzer and Program Slicer for the R Programming Language
1,060 lines • 47.3 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.SigDatabaseSet = exports.SigDatabase = exports.MergedSignatureSource = void 0;
exports.readSignatureDb = readSignatureDb;
exports.availableVersionEntries = availableVersionEntries;
exports.classOwnerIndexFor = classOwnerIndexFor;
exports.sourceForPackage = sourceForPackage;
exports.getSharedSigSourceSync = getSharedSigSourceSync;
exports.getSharedSigSource = getSharedSigSource;
exports.verifyShardedDatabase = verifyShardedDatabase;
/**
* The read path for the `flowr-sigdb` package database: fast partial readers for a single bundle
* ({@link SigDatabase}) and a sharded set ({@link SigDatabaseSet}), the process-wide shared-source cache,
* whole-bundle reading, and the post-write verification gate. This is the surface the package-version plugin
* uses; the format/codec/writer building blocks live in the sibling `sigdb/*` modules (imported directly).
*/
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const readline_1 = __importDefault(require("readline"));
const r_version_1 = require("../../util/r-version");
const schema_1 = require("./schema");
const sigdb_version_1 = require("./sigdb-version");
const index_format_1 = require("./index-format");
const decode_1 = require("./decode");
const decompress_1 = require("./decompress");
const codec_1 = require("./codec");
const hash_1 = require("./hash");
const manifest_1 = require("./manifest");
/** apply one `d` line (`["d", start, payload]`, new newline-blob or legacy `string[]` form) to the dictionary in place */
function applyDictLine(json, strings) {
const [, start, payload] = JSON.parse(json);
const batch = typeof payload === 'string' ? payload.split('\n') : payload;
for (let k = 0; k < batch.length; k++) {
strings[start + k] = batch[k];
}
}
function readDictSection(buf, strings) {
let off = 0;
while (off < buf.length) {
let nl = buf.indexOf(0x0a, off);
if (nl < 0) {
nl = buf.length;
}
if (nl > off) {
applyDictLine(buf.toString('utf8', off, nl), strings);
}
off = nl + 1;
}
}
/** stream-read a whole bundle into a {@link SigDb} (any size; never one string). Prefer {@link SigDatabase} for partial access. */
async function readSignatureDb(file) {
const rl = readline_1.default.createInterface({ input: (0, decompress_1.sigDbStream)(file), crlfDelay: Infinity });
let header;
const strings = [];
const blobs = [];
const pkgs = {};
const meta = {};
for await (const line of rl) {
if (line.length === 0) {
continue;
}
if (header === undefined) {
header = JSON.parse(line);
continue;
}
const tag = line.charCodeAt(2); // '["X",...' -> the tag char
if (tag === 100 /* d */) {
applyDictLine(line, strings);
}
else if (tag === 98 /* b */) {
const [, i, tuple] = JSON.parse(line);
blobs[i] = (0, decode_1.tupleToBlob)(tuple);
}
else if (tag === 109 /* m */) {
const [, , batch] = JSON.parse(line);
for (const [name, m] of batch) {
meta[name] = m;
}
}
else if (tag === 112 /* p */) {
const [, , batch] = JSON.parse(line);
for (const [name, i] of batch) {
pkgs[name] = i;
}
}
}
return { ...header, strings, blobs, pkgs, meta };
}
/**
* The versions a source can answer for a package (dated releases, base-R core releases, and the recorded latest),
* deduplicated and ascending by R-version order. This is the single enumeration both the signature query and the
* version-guessing query build on.
*/
function availableVersionEntries(src, pkg) {
const map = new Map();
for (const r of src.releaseDates(pkg)) {
if (!map.has(r.version.str)) {
map.set(r.version.str, r.date);
}
}
for (const v of src.coreVersions(pkg) ?? []) {
if (!map.has(v.str)) {
map.set(v.str, src.releaseDate(pkg, v.str));
}
}
const latest = src.latestVersion(pkg);
if (latest && !map.has(latest.str)) {
map.set(latest.str, src.releaseDate(pkg, latest.str));
}
return [...map.entries()]
.map(([version, date]) => ({ version, ...(date ? { date } : {}) }))
// versions that differ in writing but not in R-version order (`1.2` vs `1.2.0`) are settled by release date
.sort((a, b) => r_version_1.RVersion.compare(a.version, b.version) || (a.date && b.date ? a.date.getTime() - b.date.getTime() : 0));
}
/**
* The reverse index `class -> owning package` over `candidates`, at each one's latest version. S3 ownership (a
* same-named constructor plus a registered method) is a stronger signal than an S4 `exportClasses`, so S3-owned
* classes are indexed first and an S4 class only claims a name no S3 owner already took.
*
* Restricting the candidates is the targeted counterpart of {@link PackageSignatureSource.classOwner}: class names
* collide heavily across CRAN, so a whole-database answer is decided by database order, and deriving one means
* reading every package in the database.
*/
function classOwnerIndexFor(src, candidates) {
const index = new Map();
const libs = [];
for (const pkg of candidates) {
if (src.has(pkg)) {
libs.push({ pkg, lib: src.lookup(pkg) });
}
}
for (const pick of [(l) => l?.s3Classes, (l) => l?.s4Classes]) {
for (const { pkg, lib } of libs) {
for (const cls of pick(lib) ?? []) {
if (!index.has(cls)) {
index.set(cls, pkg);
}
}
}
}
return index;
}
/** the package owning `className` at a specific version (linear scan; S3 and S4 both count) */
function classOwnerAtVersion(src, className, version) {
return src.packageNames().find(pkg => {
const lib = src.lookup(pkg, version);
return (lib?.s3Classes.includes(className) ?? false) || (lib?.s4Classes.includes(className) ?? false);
});
}
/** union view over multiple sources for the same package; routes queries to the appropriate source */
class MergedSignatureSource {
sources;
constructor(sources) {
this.sources = sources;
}
/** source carrying a version, or the one with the newest release when unpinned */
pick(pkg, version) {
if (version !== undefined) {
return this.sources.find(s => s.hasVersion(pkg, version));
}
let best, bestSource;
for (const s of this.sources) {
const latest = s.has(pkg) ? s.latestVersion(pkg) : undefined;
if (latest && (best === undefined || r_version_1.RVersion.compare(latest.str, best.str) > 0)) {
best = latest;
bestSource = s;
}
}
return bestSource ?? this.sources.find(s => s.has(pkg));
}
has(pkg) {
return this.sources.some(s => s.has(pkg));
}
hasVersion(pkg, version) {
return this.sources.some(s => s.hasVersion(pkg, version));
}
isCranVersion(pkg, version) {
return this.pick(pkg, version)?.isCranVersion(pkg, version) ?? false;
}
lookup(pkg, version) {
return this.pick(pkg, version)?.lookup(pkg, version);
}
packagesExporting(name) {
const found = new Set();
for (const s of this.sources) {
for (const pkg of s.packagesExporting(name)) {
found.add(pkg);
}
}
return [...found].sort((a, b) => this.downloads(b) - this.downloads(a) || a.localeCompare(b));
}
classOwner(className, version) {
for (const s of this.sources) {
const owner = s.classOwner(className, version);
if (owner !== undefined) {
return owner;
}
}
return undefined;
}
functions(pkg, version) {
return this.pick(pkg, version)?.functions(pkg, version);
}
functionByName(pkg, name, version) {
return this.pick(pkg, version)?.functionByName(pkg, name, version);
}
transitiveCallees(pkg, name, version) {
return this.pick(pkg, version)?.transitiveCallees(pkg, name, version);
}
dependencies(pkg, version) {
return this.pick(pkg, version)?.dependencies(pkg, version);
}
packageNames() {
return [...new Set(this.sources.flatMap(s => s.packageNames()))];
}
isBaseR(pkg) {
return this.sources.some(s => s.has(pkg) && s.isBaseR(pkg));
}
downloads(pkg) {
return Math.max(0, ...this.sources.map(s => s.downloads(pkg)));
}
coreVersions(pkg) {
return this.sources.find(s => s.has(pkg))?.coreVersions(pkg);
}
releaseDate(pkg, version) {
return this.pick(pkg, version)?.releaseDate(pkg, version);
}
releaseDates(pkg) {
const map = new Map();
for (const s of this.sources) {
for (const r of s.releaseDates(pkg)) {
if (!map.has(r.version.str)) {
map.set(r.version.str, r);
}
}
}
return [...map.values()].sort((a, b) => r_version_1.RVersion.compare(a.version.str, b.version.str));
}
latestVersion(pkg) {
let best;
for (const s of this.sources) {
const latest = s.latestVersion(pkg);
if (latest && (best === undefined || r_version_1.RVersion.compare(latest.str, best.str) > 0)) {
best = latest;
}
}
return best;
}
// merged view owns no handles; underlying sources keep theirs
close() { }
}
exports.MergedSignatureSource = MergedSignatureSource;
/** source that answers for pkg, merging all sources that carry it; undefined if none do */
function sourceForPackage(sources, pkg) {
const having = sources.filter(s => s.has(pkg));
return having.length === 0 ? undefined : having.length === 1 ? having[0] : new MergedSignatureSource(having);
}
/**
* Fast, partial reader for a single bundle. `open()`/`openSync()` load the string dictionary + `.idx`
* once (a single ranged read of the dictionary section, no full parse), then every query seeks straight
* to one package blob on demand. `open()` additionally decompresses a `.br`/`.gz` source into a
* hash-keyed cache and reuses it on later startups. Implements {@link PackageSignatureSource}.
*/
/** the {@link SigDatabase.fd} of a database that has no file behind it (see {@link SigDatabase.fromMemory}) */
const NoFile = -1;
class SigDatabase {
closed = false;
/** parsed blobs by blob index so repeated lookups skip the re-read + JSON.parse; FIFO-bounded to cap memory */
blobCache = new Map();
static BlobCacheCap = 2048;
/** a version's function indices plus its `name -> index` view; FIFO-bounded like {@link blobCache} */
versionFnCache = new Map();
/**
* Which names a package can offer at all, as sorted dictionary ids -- integers, so the whole database of
* them costs a few MB where the parsed blobs cost hundreds. Filled for a package the first time its blob is
* read, so a later lookup for a name it does not have answers without touching the file again.
*/
nameIdsOfBlob = new Map();
/** dictionary id per name asked for, so only the names someone actually queried are ever resolved */
nameIds = new Map();
/** reverse index `S3 class -> owning package`, over every package's latest version; built once (see {@link classOwner}) */
classIndex;
fd;
strings;
index;
content;
cranBase;
constructor(fd, strings, index, content, cranBase) {
this.fd = fd;
this.strings = strings;
this.index = index;
this.content = content;
this.cranBase = cranBase;
}
/**
* Use an already-built {@link SigDb} directly, without writing or reading a file: its blobs simply start out in
* the cache. It holds exactly what a bundle carries, so every query answers as it would from disk; the
* {@link SigDbBuilder} plus this is all a test needs.
*/
static fromMemory(db) {
const index = { byteCount: 0, dict: [0, 0], blobs: [], pkgs: db.pkgs, meta: db.meta };
const source = new SigDatabase(NoFile, db.strings, index, db.content, db.cranBase ?? schema_1.DefaultCranBase);
db.blobs.forEach((blob, i) => source.blobCache.set(i, blob));
return source;
}
/**
* Open a plain, seekable `.sigs.ndjson` synchronously. Pass `index` to skip reading the `.idx`,
* and `strings` to use an already-loaded shared dictionary instead of the file's own `d` section (for a
* blob-only shard). One ranged read loads the dictionary, no readline overhead.
*/
static openSync(plainFile, opts = {}) {
if ((0, decompress_1.isCompressed)(plainFile)) {
throw new Error('SigDatabase.openSync needs the plain .sigs.ndjson; use open() for .br/.zst/.gz');
}
const index = opts.index ?? (0, index_format_1.readSigDbIndex)(plainFile);
const fd = fs_1.default.openSync(plainFile, 'r');
const head = Buffer.allocUnsafe(Math.min(65536, index.byteCount));
fs_1.default.readSync(fd, head, 0, head.length, 0);
const header = (0, decompress_1.parseHeader)(head.toString('utf8'));
let strings = opts.strings;
if (!strings) {
strings = [];
const [dictStart, dictBytes] = index.dict;
if (dictBytes > 0) {
const buf = Buffer.allocUnsafe(dictBytes);
fs_1.default.readSync(fd, buf, 0, dictBytes, dictStart);
readDictSection(buf, strings);
}
}
const cranBase = header?.cranBase ?? schema_1.DefaultCranBase;
return new SigDatabase(fd, strings, index, header?.content, cranBase);
}
/** open a `.sigs.ndjson`, `.br` or `.gz`; compressed sources are decompressed into a hash-keyed cache once */
static async open(source, opts = {}) {
return SigDatabase.openSync(await (0, decompress_1.ensurePlain)(source, opts));
}
/**
* Like {@link open} but fully synchronous (blocking decompression); a `hash` keys the cache when `source`
* is compressed. Pass `strings` for a blob-only shard that shares an already-loaded dictionary.
*/
static openSyncFrom(source, opts) {
const plain = (0, decompress_1.isCompressed)(source)
? (opts.hash !== undefined ? (0, decompress_1.ensurePlainSync)(source, { cacheDir: opts.cacheDir, hash: opts.hash, index: opts.index })
: (() => {
throw new Error('openSyncFrom needs a hash to key the cache for a compressed source');
})())
: source;
return SigDatabase.openSync(plain, { index: opts.index, strings: opts.strings });
}
has(pkg) {
return this.index.pkgs[pkg] !== undefined;
}
packageNames() {
return Object.keys(this.index.pkgs);
}
/** load a single package's blob by seeking to its line (undefined if absent); cached by blob index */
blob(pkg) {
const blobIdx = this.index.pkgs[pkg];
if (blobIdx === undefined) {
return undefined;
}
const cached = this.blobCache.get(blobIdx);
if (cached !== undefined) {
return cached;
}
if (this.fd === NoFile) {
return undefined; // an in-memory database starts out with every blob cached
}
const blob = this.readBlobAt(this.index.blobs[blobIdx]);
if (!this.nameIdsOfBlob.has(blobIdx)) {
this.nameIdsOfBlob.set(blobIdx, Int32Array.from(new Set(blob.fns.map(fn => fn[0]))).sort());
}
return SigDatabase.cache(this.blobCache, blobIdx, blob);
}
/** the dictionary id of `name`, or `-1` if the dictionary does not hold it, so no package can offer it */
nameId(name) {
let id = this.nameIds.get(name);
if (id === undefined) {
this.nameIds.set(name, id = this.strings.indexOf(name));
}
return id;
}
/**
* Whether `pkg` can offer `name` in any of its versions, answered from {@link nameIdsOfBlob} alone.
* `true` whenever nothing is known about the package yet, so this only ever skips work that would have
* found nothing: the ids cover every function record, of which a version selects a subset.
*/
mayOffer(pkg, name) {
const blobIdx = this.index.pkgs[pkg];
const ids = blobIdx === undefined ? undefined : this.nameIdsOfBlob.get(blobIdx);
if (ids === undefined) {
return true;
}
const id = this.nameId(name);
if (id < 0) {
return false;
}
let lo = 0, hi = ids.length - 1;
while (lo <= hi) {
const mid = (lo + hi) >> 1;
if (ids[mid] === id) {
return true;
}
else if (ids[mid] < id) {
lo = mid + 1;
}
else {
hi = mid - 1;
}
}
return false;
}
/** store `value` under `key`, dropping the oldest entry first once the cache sits at {@link BlobCacheCap} */
static cache(cache, key, value) {
if (cache.size >= SigDatabase.BlobCacheCap) {
const oldest = cache.keys().next().value;
if (oldest !== undefined) {
cache.delete(oldest);
}
}
cache.set(key, value);
return value;
}
/** seek to a byte range, read + decode the package blob there (no caching) */
readBlobAt([start, bytes]) {
const buf = Buffer.allocUnsafe(bytes);
fs_1.default.readSync(this.fd, buf, 0, bytes, start);
const [, , tuple] = JSON.parse(buf.toString('utf8'));
return (0, decode_1.tupleToBlob)(tuple);
}
/** read every unique package blob in index order (used to re-hash a whole shard during verification) */
allBlobs() {
// an in-memory database has no byte ranges to re-read: its blobs are the cached ones, in blob-index order
return this.fd === NoFile
? [...this.blobCache.entries()].sort(([a], [b]) => a - b).map(([, blob]) => blob)
: this.index.blobs.map(range => this.readBlobAt(range));
}
/** recompute this bundle's self-contained content hash from its re-read data (matches {@link writeSignatureDb}) */
contentHash(blobs = this.allBlobs()) {
// use only this bundle's own package metadata, in package-index order, since a shared manifest may hoist a
// superset of metadata that the self-contained bundle was NOT hashed over
const meta = {};
for (const pkg of Object.keys(this.index.pkgs)) {
meta[pkg] = this.index.meta[pkg];
}
return (0, hash_1.contentHash)({ strings: this.strings, blobs, pkgs: this.index.pkgs, meta });
}
/** whether this bundle actually carries the given version of a package (not just the package) */
hasVersion(pkg, version) {
return this.blob(pkg)?.versions[version] !== undefined;
}
/** whether a version is a current CRAN release (not in the package's `noncran`/removed set) */
isCranVersion(pkg, version) {
return !this.blob(pkg)?.noncran?.includes(version);
}
lookup(pkg, version) {
const blob = this.blob(pkg);
const meta = this.index.meta[pkg];
if (!blob || !meta) {
return undefined;
}
return (0, decode_1.deriveLibraryExports)(this.strings, blob, meta, pkg, version, this.cranBase);
}
packagesExporting(name) {
if (this.nameId(name) < 0) {
return [];
}
const found = [];
for (const pkg of this.packageNames()) {
if (this.mayOffer(pkg, name) && this.lookup(pkg)?.exported.includes(name)) {
found.push(pkg);
}
}
return found.sort((a, b) => this.downloads(b) - this.downloads(a) || a.localeCompare(b));
}
classOwner(className, version) {
if (version !== undefined) {
return classOwnerAtVersion(this, className, version);
}
this.classIndex ??= classOwnerIndexFor(this, this.packageNames());
return this.classIndex.get(className);
}
/** resolve a package version to its blob and the function-record indices of that version (the shared prologue of {@link functions}/{@link functionByName}) */
versionFns(pkg, version) {
const blob = this.blob(pkg);
const meta = this.index.meta[pkg];
if (!blob || !meta) {
return undefined;
}
// keyed on the resolved version, so `undefined` and the version it stands for share one entry
const ver = (0, sigdb_version_1.resolveVersion)(blob, meta[0], version);
if (ver === undefined) {
return undefined;
}
const key = `${pkg}\0${ver}`;
const cached = this.versionFnCache.get(key);
if (cached !== undefined) {
return cached;
}
const idxs = (0, decode_1.versionFnIndices)(blob, ver);
if (idxs === undefined) {
return undefined;
}
return SigDatabase.cache(this.versionFnCache, key, { blob, idxs });
}
functions(pkg, version) {
const r = this.versionFns(pkg, version);
return r?.idxs.map(i => (0, decode_1.decodeFunction)(this.strings, r.blob, i));
}
functionByName(pkg, name, version) {
if (!this.mayOffer(pkg, name)) {
return undefined;
}
const r = this.versionFns(pkg, version);
if (r === undefined) {
return undefined;
}
if (r.byName === undefined) {
const byName = new Map();
for (const i of r.idxs) {
const fn = this.strings[r.blob.fns[i][0]];
// first record wins, as the linear scan did
if (!byName.has(fn)) {
byName.set(fn, i);
}
}
r.byName = byName;
}
const hit = r.byName.get(name);
return hit !== undefined ? (0, decode_1.decodeFunction)(this.strings, r.blob, hit) : undefined;
}
transitiveCallees(pkg, name, version) {
const fns = this.functions(pkg, version);
return fns?.some(f => f.name === name) ? (0, decode_1.transitiveCallees)(fns, name) : undefined;
}
dependencies(pkg, version) {
const blob = this.blob(pkg);
const meta = this.index.meta[pkg];
if (!blob || !meta) {
return undefined;
}
const ver = (0, sigdb_version_1.resolveVersion)(blob, meta[0], version);
return ver !== undefined ? (0, decode_1.decodeDependencies)(this.strings, blob, ver) : undefined;
}
/** whether this is an R-core / base package (its versions are the R releases it shipped with; see {@link SigDbPkgMeta}) */
isBaseR(pkg) {
return this.index.meta[pkg]?.[3] === 1;
}
/** the download count recorded for the package, `0` when this source does not carry it */
downloads(pkg) {
return this.index.meta[pkg]?.[2] ?? 0;
}
/**
* The R versions a base package was part of core, in ascending R-version order (exactly its stored
* versions). `undefined` for a non-base package. E.g. `mva` returns `…1.9.1`, `parallel` `2.14.0…`.
*/
coreVersions(pkg) {
if (!this.isBaseR(pkg)) {
return undefined;
}
return Object.keys(this.blob(pkg)?.versions ?? {}).map(r_version_1.RVersion.parseOrZero).sort((a, b) => r_version_1.RVersion.compare(a.str, b.str));
}
/** the release date of a package version (defaulting to the newest release), or `undefined` if unknown */
releaseDate(pkg, version) {
const blob = this.blob(pkg);
const meta = this.index.meta[pkg];
if (!blob || !meta) {
return undefined;
}
const ver = version ?? (0, sigdb_version_1.newestVersion)(blob, meta[0]);
const day = ver !== undefined ? blob.dates[ver] : undefined;
return day !== undefined ? new Date((0, sigdb_version_1.dayToMillis)(day)) : undefined;
}
/** every known release date of a package, in ascending R-version order (empty when no dates were stored) */
releaseDates(pkg) {
return (0, sigdb_version_1.releasesOf)(this.blob(pkg));
}
/** the newest version of a package by release date (falling back to the recorded latest, then SemVer order) */
latestVersion(pkg) {
const blob = this.blob(pkg);
const meta = this.index.meta[pkg];
const ver = blob && meta ? (0, sigdb_version_1.newestVersion)(blob, meta[0]) : undefined;
return ver !== undefined ? r_version_1.RVersion.parseOrZero(ver) : undefined;
}
/** close the underlying file descriptor (idempotent; safe to call more than once) */
close() {
if (!this.closed) {
this.closed = true;
this.blobCache.clear();
this.classIndex = undefined;
if (this.fd !== NoFile) {
fs_1.default.closeSync(this.fd);
}
}
}
}
exports.SigDatabase = SigDatabase;
/** current-tier shards are preferred over full-tier ones (smaller/faster) when both can serve a request */
function tierRank(ref) {
return ref.tier === 'current' ? 0 : 1;
}
/** apply the include/exclude shard filters (include first, then exclude), preserving manifest order */
function selectShards(shards, include, exclude) {
let out = include ? shards.filter(s => include.includes(s.id)) : [...shards];
if (exclude) {
out = out.filter(s => !exclude.includes(s.id));
}
return out;
}
/**
* A transparent, read-only view over several {@link SigDatabase} shards described by a {@link SigDbManifest}.
* When the manifest embeds each shard's index (the default), `openManifest()` reads only that small file to
* build the package to shard routing table.
*/
class SigDatabaseSet {
opened;
manifest;
/** the directory the manifest's shard/dict paths are relative to */
baseDir;
indices;
/** package name to shard indices, ordered by preference (current before full) */
routes;
cacheDir;
/** reverse index `S3 class -> owning package`, over every package's latest version; built once (see {@link classOwner}) */
classIndex;
constructor(manifest, baseDir, indices, routes, cacheDir) {
this.manifest = manifest;
this.baseDir = baseDir;
this.indices = indices;
this.routes = routes;
this.cacheDir = cacheDir;
this.opened = new Array(manifest.shards.length).fill(undefined);
}
/** read + validate a manifest and apply the include/exclude shard filter */
static prepManifest(manifestFile, opts) {
const baseDir = path_1.default.dirname(manifestFile);
const full = (0, manifest_1.readManifestFile)(manifestFile);
if (full.format !== manifest_1.SigDbManifestMagic) {
throw new Error(`not a ${manifest_1.SigDbManifestMagic} (got ${String(full.format)})`);
}
const active = selectShards(full.shards, opts.includeShards, opts.excludeShards);
if (active.length === 0) {
throw new Error('openManifest: no shards left after include/exclude filtering');
}
return { baseDir, manifest: { ...full, shards: active } };
}
/** build the package to shard routing (current before full) and construct the set from resolved indices */
static assemble(manifest, baseDir, indices, cacheDir) {
const order = manifest.shards.map((_, i) => i).sort((a, b) => tierRank(manifest.shards[a]) - tierRank(manifest.shards[b]));
const routes = new Map();
for (const i of order) {
for (const pkg of Object.keys(indices[i].pkgs)) {
const list = routes.get(pkg);
if (list) {
list.push(i);
}
else {
routes.set(pkg, [i]);
}
}
}
return new SigDatabaseSet(manifest, baseDir, indices, routes, cacheDir);
}
static async openManifest(manifestFile, opts = {}) {
const { baseDir, manifest } = SigDatabaseSet.prepManifest(manifestFile, opts);
// prefer the embedded (compact) index with hoisted meta
const indices = await Promise.all(manifest.shards.map(async (s) => s.idx ? (0, index_format_1.decodeIndex)(s.idx, manifest.meta) : (0, index_format_1.readSigDbIndex)(await (0, decompress_1.ensurePlain)((0, decompress_1.resolveSource)(baseDir, s.path), { cacheDir: opts.cacheDir, hash: s.hash }))));
return SigDatabaseSet.assemble(manifest, baseDir, indices, opts.cacheDir);
}
/**
* Synchronous {@link openManifest}, needing every shard to embed its index (the default for the bundles
* flowR ships). Shards and dictionaries still decompress lazily.
*/
static openManifestSync(manifestFile, opts = {}) {
const { baseDir, manifest } = SigDatabaseSet.prepManifest(manifestFile, opts);
const indices = manifest.shards.map(s => {
if (!s.idx) {
throw new Error(`openManifestSync needs every shard to embed its index; shard '${s.id}' does not, use openManifest`);
}
return (0, index_format_1.decodeIndex)(s.idx, manifest.meta);
});
return SigDatabaseSet.assemble(manifest, baseDir, indices, opts.cacheDir);
}
/** shared dictionaries, loaded (decompressed + parsed) once and cached by id */
dictCache = new Map();
/** load (and cache) a shared dictionary's strings, decompressing its `.br` into the cache once */
dictionaryStrings(dictId) {
const cached = this.dictCache.get(dictId);
if (cached) {
return cached;
}
const ref = this.manifest.dicts?.find(d => d.id === dictId);
if (!ref) {
throw new Error(`manifest references unknown dictionary '${dictId}'`);
}
const plain = (0, decompress_1.ensurePlainSync)((0, decompress_1.resolveSource)(this.baseDir, ref.path), { cacheDir: this.cacheDir, hash: ref.hash, indexless: true });
const strings = [];
const fd = fs_1.default.openSync(plain, 'r');
try {
const [start, bytes] = ref.range;
const buf = Buffer.allocUnsafe(bytes);
fs_1.default.readSync(fd, buf, 0, bytes, start);
readDictSection(buf, strings);
}
finally {
fs_1.default.closeSync(fd);
}
this.dictCache.set(dictId, strings);
return strings;
}
/** lazily open a shard, decompressing its `.br` (and its shared dictionary) into the cache on first access */
shard(i) {
const existing = this.opened[i];
if (existing) {
return existing;
}
const ref = this.manifest.shards[i];
const strings = ref.dict ? this.dictionaryStrings(ref.dict) : undefined;
const db = SigDatabase.openSyncFrom((0, decompress_1.resolveSource)(this.baseDir, ref.path), { cacheDir: this.cacheDir, hash: ref.hash, index: this.indices[i], strings });
this.opened[i] = db;
return db;
}
/**
* Warm the shards (and their shared dictionaries) needed for `pkgs`, or **everything** when omitted.
* Afterward, the synchronous query methods, for the latest *and* historical versions, do no I/O or decompression.
*/
async preload(pkgs) {
const need = new Set();
if (pkgs) {
for (const p of pkgs) {
for (const i of this.routes.get(p) ?? []) {
need.add(i);
}
}
}
else {
this.manifest.shards.forEach((_, i) => need.add(i));
}
await this.warmShards(need);
}
/**
* Warm just the shards matching `include`, e.g., only the current-tier top shards (the base + most-downloaded
* packages) to speed up common lookups without paying for the long tail or the history shards. See {@link preload}.
*/
async preloadShards(include) {
const need = new Set();
this.manifest.shards.forEach((s, i) => {
if (include(s)) {
need.add(i);
}
});
await this.warmShards(need);
}
/** decompress the given shards + their shared dictionaries concurrently, then open them (see {@link preload}) */
async warmShards(need) {
const dicts = new Set();
for (const i of need) {
const d = this.manifest.shards[i].dict;
if (d) {
dicts.add(d);
}
}
const shardJobs = [...need].map(i => (0, decompress_1.ensurePlain)((0, decompress_1.resolveSource)(this.baseDir, this.manifest.shards[i].path), { cacheDir: this.cacheDir, hash: this.manifest.shards[i].hash, index: this.indices[i] }));
const dictJobs = [...dicts].map(id => {
const ref = this.manifest.dicts?.find(d => d.id === id);
return ref ? (0, decompress_1.ensurePlain)((0, decompress_1.resolveSource)(this.baseDir, ref.path), { cacheDir: this.cacheDir, hash: ref.hash, indexless: true }) : Promise.resolve('');
});
await Promise.all([...shardJobs, ...dictJobs]);
// open each shard from the now-decompressed cache (cheap; parses each shared dictionary once) so later
// synchronous queries, including historical pinned-version lookups, never block
for (const i of need) {
this.shard(i);
}
}
/** the shard indices that can serve this package, preferred order; optionally requiring a specific version */
route(pkg, version) {
const candidates = this.routes.get(pkg) ?? [];
if (version === undefined) {
return candidates;
}
// keep only shards that actually carry the requested version (current shards may hold only the latest)
return candidates.filter(i => this.shard(i).hasVersion(pkg, version));
}
/** read (once) the blob from the shard with the most complete history: a `full` or `history` tier if present */
historyBlob(pkg) {
const candidates = this.routes.get(pkg);
if (!candidates || candidates.length === 0) {
return undefined;
}
const full = candidates.find(i => this.manifest.shards[i].tier === 'full' || this.manifest.shards[i].tier === 'history');
return this.shard(full ?? candidates[0]).blob(pkg);
}
has(pkg) {
return this.routes.has(pkg);
}
/** whether any active shard actually carries the given version of a package (not just the package) */
hasVersion(pkg, version) {
return this.route(pkg, version).length > 0;
}
/** whether a version is a current CRAN release (not in the package's `noncran`/removed set) */
isCranVersion(pkg, version) {
return !this.historyBlob(pkg)?.noncran?.includes(version);
}
packageNames() {
return [...this.routes.keys()];
}
/** the on-demand load state (opened + unpacked) of every shard in this set */
shardStatus() {
return this.manifest.shards.map((ref, i) => {
const compressed = (0, decompress_1.isCompressed)((0, decompress_1.resolveSource)(this.baseDir, ref.path));
return {
id: ref.id,
compressed,
accessed: this.opened[i] !== undefined,
unpacked: !compressed || (0, decompress_1.isUnpacked)(ref.hash, this.cacheDir)
};
});
}
/** open (blocking) and return every shard database with its manifest ref, for whole-set verification */
allShards() {
return this.manifest.shards.map((ref, i) => ({ ref, db: this.shard(i) }));
}
/** load (and cache) a shared dictionary's strings by id, for verification/inspection */
sharedDictionary(id) {
return this.dictionaryStrings(id);
}
/** the first non-empty result from the shards that can serve `pkg` (in preferred order: current before full) */
firstOf(pkg, version, read) {
for (const i of this.route(pkg, version)) {
const r = read(this.shard(i));
if (r) {
return r;
}
}
return undefined;
}
lookup(pkg, version) {
return this.firstOf(pkg, version, db => db.lookup(pkg, version));
}
packagesExporting(name) {
const found = new Set();
for (const idx of this.indices.keys()) {
for (const pkg of this.shard(idx)?.packagesExporting(name) ?? []) {
found.add(pkg);
}
}
return [...found].sort((a, b) => this.downloads(b) - this.downloads(a) || a.localeCompare(b));
}
classOwner(className, version) {
if (version !== undefined) {
return classOwnerAtVersion(this, className, version);
}
this.classIndex ??= classOwnerIndexFor(this, this.packageNames());
return this.classIndex.get(className);
}
functions(pkg, version) {
return this.firstOf(pkg, version, db => db.functions(pkg, version));
}
functionByName(pkg, name, version) {
return this.firstOf(pkg, version, db => db.functionByName(pkg, name, version));
}
transitiveCallees(pkg, name, version) {
const fns = this.functions(pkg, version);
return fns?.some(f => f.name === name) ? (0, decode_1.transitiveCallees)(fns, name) : undefined;
}
dependencies(pkg, version) {
return this.firstOf(pkg, version, db => db.dependencies(pkg, version));
}
/** whether this is an R-core / base package (see {@link SigDatabase.isBaseR}); O(1) via the hoisted metadata */
isBaseR(pkg) {
const meta = this.manifest.meta?.[pkg];
if (meta) {
return meta[3] === 1;
}
return this.route(pkg).some(i => this.shard(i).isBaseR(pkg));
}
/** the download count of the package; O(1) via the hoisted metadata, so no shard has to be unpacked for it */
downloads(pkg) {
const meta = this.manifest.meta?.[pkg];
return meta ? meta[2] : Math.max(0, ...this.route(pkg).map(i => this.shard(i).downloads(pkg)));
}
/** the R versions a base package was part of core (ascending); `undefined` if not a base package */
coreVersions(pkg) {
if (!this.isBaseR(pkg)) {
return undefined;
}
return Object.keys(this.historyBlob(pkg)?.versions ?? {}).map(r_version_1.RVersion.parseOrZero).sort((a, b) => r_version_1.RVersion.compare(a.str, b.str));
}
/** every known release of a package (version + date), ascending, read once from the most complete shard */
releaseDates(pkg) {
return (0, sigdb_version_1.releasesOf)(this.historyBlob(pkg));
}
/** the release date of a package version (defaulting to the newest release), or `undefined` if unknown */
releaseDate(pkg, version) {
const blob = this.historyBlob(pkg);
if (!blob) {
return undefined;
}
const ver = version ?? (0, sigdb_version_1.newestVersion)(blob, this.manifest.meta?.[pkg]?.[0] ?? '');
const day = ver !== undefined ? blob.dates[ver] : undefined;
return day !== undefined ? new Date((0, sigdb_version_1.dayToMillis)(day)) : undefined;
}
/** the newest version of a package by release date (falling back to the recorded latest, then SemVer order) */
latestVersion(pkg) {
const blob = this.historyBlob(pkg);
const ver = blob ? (0, sigdb_version_1.newestVersion)(blob, this.manifest.meta?.[pkg]?.[0] ?? '') : undefined;
return ver !== undefined ? r_version_1.RVersion.parseOrZero(ver) : undefined;
}
/**
* Close every opened shard's file descriptor and drop the in-memory caches (opened shards + shared
* dictionaries), so the (potentially large) dictionary strings can be reclaimed by the GC. Idempotent.
*/
close() {
for (const db of this.opened) {
db?.close();
}
this.opened.fill(undefined);
this.dictCache.clear();
this.classIndex = undefined;
}
}
exports.SigDatabaseSet = SigDatabaseSet;
const sharedSources = new Map();
/** whether a bundle path can be opened synchronously (a plain `.sigs.ndjson` or an index-embedding manifest) */
function isSyncOpenable(source) {
return (0, codec_1.stripCompressedExt)(source).endsWith('.manifest.json') || source.endsWith(schema_1.SigDbExt);
}
/**
* Open a path-based source once, process-wide, synchronously. Returns the shared instance (opening it on the
* first call), or `undefined` if the path needs async opening (a `.br`/`.gz` bundle, use {@link getSharedSigSource}).
* Throws only if a sync-openable source fails to open.
*/
function getSharedSigSourceSync(source) {
const cached = sharedSources.get(source);
if (cached) {
return cached;
}
if (!isSyncOpenable(source)) {
return undefined;
}
const opened = source.endsWith(schema_1.SigDbExt) ? SigDatabase.openSync(source) : SigDatabaseSet.openManifestSync(source);
sharedSources.set(source, opened);
return opened;
}
/** Open a path-based source once, process-wide (async: also handles `.br`/`.zst`/`.gz` bundles and non-embedded manifests). */
async function getSharedSigSource(source) {
const cached = sharedSources.get(source);
if (cached) {
return cached;
}
const opened = (0, codec_1.stripCompressedExt)(source).endsWith('.manifest.json')
? await SigDatabaseSet.openManifest(source)
: await SigDatabase.open(source);
const raced = sharedSources.get(source); // a concurrent opener may have won the race
if (raced) {
opened.close();
return raced;
}
sharedSources.set(source, opened);
return opened;
}
/**
* Re-read a written sharded database from its (compressed) files and check it is internally consistent:
* every shard's content hash recomputed from its re-read blobs matches both the manifest and the file
* header; every shared dictionary's hash matches; the manifest routes every package a shard holds; a
* sample of packages decodes (functions + dependencies) with all string indices in range; and any
* `requirePackages` (e.g. base R) are present. This is a strong correctness gate: correctness over speed.
*/
async function verifyShardedDatabase(manifestFile, opts = {}) {
const errors = [];
const set = await SigDatabaseSet.openManifest(manifestFile, opts);
const manifest = set.manifest;
// 1. every shared dictionary hash matches its re-read strings
let dictsOk = true;
for (const dref of manifest.dicts ?? []) {
const strings = set.sharedDictionary(dref.id);
const actual = (0, hash_1.dictionaryHash)(strings);
if (strings.length !== dref.strings) {
dictsOk = false;
errors.push(`dictionary '${dref.id}': expected ${dref.strings} strings, re-read ${strings.length}`);
}
if (actual !== dref.hash) {
dictsOk = false;
errors.push(`dictionary '${dref.id}': hash ${actual} != manifest ${dref.hash}`);
}
}
// 2. every shard's content hash, recomputed from its re-read blobs, matches the manifest and the file header
const shardResults = [];
for (const { ref, db } of set.allShards()) {
const blobs = db.allBlobs();
// shared-dictionary shards are hashed over blobs+pkgs only; self-contained shards over their whole content
const actual = ref.dict ? (0, hash_1.shardHash)(blobs, db.index.pkgs) : db.contentHash(blobs);
const headerHash = db.content?.hash;
const hashOk = actual === ref.hash && (headerHash === undefined || headerHash === ref.hash);
if (!hashOk) {
errors.push(`shard '${ref.id}': recomputed hash ${actual} vs manifest ${ref.hash}${headerHash && headerHash !== ref.hash ? ` (header ${headerHash})` : ''}`);
}
const packages = Object.keys(db.index.pkgs).length;
if (packages !== ref.packages) {
errors.push(`shard '${ref.id}': manifest says ${ref.packages} packages, index has ${packages}`);
}
shardResults.push({ id: ref.id, packages, hashOk, expectedHash: ref.hash, actualHash: actual });
}
// 3. routing covers every package that any shard holds
const routed = new Set(set.packageNames());
for (const { ref, db } of set.allShards()) {
for (const pkg of Object.keys(db.index.pkgs)) {
if (!routed.has(pkg)) {
errors.push(`package '${pkg}' in shard '${ref.id}' is not routed by the manifest`);
}
}
}
// 4. spot-check: a spread of packages decodes with all string indices resolving to real strings
const names = [...routed].sort();
const sample = opts.sample ?? 200;
const step = Math.max(1, Math.floor(names.length / sample));
let spotChecked = 0;
for (let i = 0; i < names.length; i += step) {
const pkg = names[i];
const exp = set.lookup(pkg);
if (!exp) {
errors.push(`spot-check: lookup('${pkg}') returned nothing though it is routed`);
continue;
}
const fns = set.functions(pkg);
for (const f of fns ?? []) {
if (typeof f.name !== 'string' || (f.file !== undefined && typeof f.file !== 'string')) {
errors.push(`spot-check: '${pkg}' function decoded with a non-string name/file (dictionary index out of range?)`);
break;
}
for (const p of f.signature) {
if (typeof p.name !== 'string' || (p.default !== undefined && typeof p.default !== 'string')) {
errors.push(`spot-check: '${pkg}' param decoded with a non-string name/default (dictionary index out of range?)`);
break;
}
}
}
for (const d of set.dependencies(pkg) ?? []) {
if (typeof d.name !== 'string' || (d.constraint !== undefined && typeof d.constraint !== 'string')) {
errors.push(`spot-check: '${pkg}' dependency decoded with a non-string name/constraint (dictionary index out of range?)`);
break;
}
}
spotChecked++;
}
// 5. required packages (e.g. base R) are present
const missingRequired = (opts.requirePackages ?? []).filter(p => !routed.has(p));
for (const p of missingRequired) {
errors.push(`required package '${p}' is missing`);
}
set.close();
return {
ok: errors.length === 0, dictsOk, shards: shardResults,
routedPackages: routed.size, spotChecked, missingRequired, errors
};
}
//# sourceMappingURL=reader.js.map