@eagleoutice/flowr-dev/project/sigdb/reader.d.ts

Static Dataflow Analyzer and Program Slicer for the R Programming Language

import { RVersion, type VersionString } from '../../util/r-version';
import { type LibraryExports, type PkgBlob, type SigDb, type SigDbContent } from './schema';
import { type VersionRelease } from './sigdb-version';
import { type SigDbIndex } from './index-format';
import { type DecodedFunction, type ResolvedDependency } from './decode';
import { type SigDbManifest, type SigDbShardRef } from './manifest';
/** stream-read a whole bundle into a {@link SigDb} (any size; never one string). Prefer {@link SigDatabase} for partial access. */
export declare function readSignatureDb(file: string): Promise<SigDb>;
/**
 * The read interface every package-signature source implements, so a single {@link SigDatabase} and a
 * sharded {@link SigDatabaseSet} are interchangeable. Queries are synchronous; any decompression/caching
 * happens once during `open`.
 */
export interface PackageSignatureSource {
    /** whether the source can resolve the package at all */
    has(pkg: string): boolean;
    /** whether the source actually carries the given version of a package (not just the package itself) */
    hasVersion(pkg: string, version: string): boolean;
    /** whether a version is a current CRAN release (i.e. not in the package's `noncran`/removed set) */
    isCranVersion(pkg: string, version: string): boolean;
    /** the export view of a package version (defaults to its latest) */
    lookup(pkg: string, version?: string): LibraryExports | undefined;
    /**
     * The package that OWNS the class `className`: an S3 class (a same-named constructor plus a registered method,
     * see {@link LibraryExports.s3Classes}) or an S4 class (exported via `exportClasses`, see
     * {@link LibraryExports.s4Classes}); S3 ownership wins a tie. `undefined` if none does. Without `version`, backed
     * by a reverse index over every package's latest version, built once and cached.
     */
    /**
     * The packages exporting `name`, ordered by downloads (descending, ties by name). Answered without building
     * a reverse index: a name the database never stores is rejected outright, and only the blobs that may hold
     * it are decoded.
     */
    packagesExporting(name: string): readonly string[];
    classOwner(className: string, version?: string): string | undefined;
    /** rich per-function view (signatures + call graphs) of a package version */
    functions(pkg: string, version?: string): DecodedFunction[] | undefined;
    /** the rich view of a single function by name, decoding only it (unlike {@link functions}, which decodes the whole package) */
    functionByName(pkg: string, name: string, version?: string): DecodedFunction | undefined;
    /** the transitive callees of a function within one package version, expanding the stored local call graphs */
    transitiveCallees(pkg: string, name: string, version?: string): string[] | undefined;
    /** declared dependencies (Depends/Imports/…) of a package version, with version qualifiers */
    dependencies(pkg: string, version?: string): ResolvedDependency[] | undefined;
    /** every package name this source can resolve */
    packageNames(): string[];
    /** whether the package is an R-core / base package (its versions are the R releases it shipped with) */
    isBaseR(pkg: string): boolean;
    /** how often the package was downloaded when the database was built, the popularity {@link SigDbPkgMeta} records */
    downloads(pkg: string): number;
    /** for a base package, the R versions it was part of core (ascending); `undefined` otherwise */
    coreVersions(pkg: string): RVersion[] | undefined;
    /** the release date of a package version (defaulting to the newest release), or `undefined` if unknown */
    releaseDate(pkg: string, version?: string): Date | undefined;
    /** every known release of a package (version + date), ascending by R-version order */
    releaseDates(pkg: string): VersionRelease[];
    /** the newest version of a package by release date (falling back to the recorded latest) */
    latestVersion(pkg: string): RVersion | undefined;
    /** release any held file handles */
    close(): void;
}
/** one version a source can answer for a package, with its release date when known */
export interface AvailableVersion {
    readonly version: VersionString;
    readonly date?: Date;
}
/** the on-demand load state of one shard of a {@link SigDatabaseSet} */
export interface ShardStatus {
    /** the shard id, e.g. `current-top` (its `base`/`current`/`history` prefix is the scope it belongs to) */
    readonly id: string;
    /** whether the shard ships only as a compressed `.br`/`.zst` bundle (so it must be unpacked to be read) */
    readonly compressed: boolean;
    /** whether this session has opened (mounted) the shard */
    readonly accessed: boolean;
    /** whether the shard's decompressed cache exists on disk (unpacked, this session or an earlier one) */
    readonly unpacked: boolean;
}
/**
 * The versions a source can answer for a package (dated releases, base-R core releases, and the recorded latest),
 * deduplicated and ascending by R-version order. This is the single enumeration both the signature query and the
 * version-guessing query build on.
 */
export declare function availableVersionEntries(src: PackageSignatureSource, pkg: string): AvailableVersion[];
/**
 * The reverse index `class -> owning package` over `candidates`, at each one's latest version. S3 ownership (a
 * same-named constructor plus a registered method) is a stronger signal than an S4 `exportClasses`, so S3-owned
 * classes are indexed first and an S4 class only claims a name no S3 owner already took.
 *
 * Restricting the candidates is the targeted counterpart of {@link PackageSignatureSource.classOwner}: class names
 * collide heavily across CRAN, so a whole-database answer is decided by database order, and deriving one means
 * reading every package in the database.
 */
export declare function classOwnerIndexFor(src: PackageSignatureSource, candidates: Iterable<string>): Map<string, string>;
/** union view over multiple sources for the same package; routes queries to the appropriate source */
export declare class MergedSignatureSource implements PackageSignatureSource {
    private readonly sources;
    constructor(sources: readonly PackageSignatureSource[]);
    /** source carrying a version, or the one with the newest release when unpinned */
    private pick;
    has(pkg: string): boolean;
    hasVersion(pkg: string, version: string): boolean;
    isCranVersion(pkg: string, version: string): boolean;
    lookup(pkg: string, version?: string): LibraryExports | undefined;
    packagesExporting(name: string): readonly string[];
    classOwner(className: string, version?: string): string | undefined;
    functions(pkg: string, version?: string): DecodedFunction[] | undefined;
    functionByName(pkg: string, name: string, version?: string): DecodedFunction | undefined;
    transitiveCallees(pkg: string, name: string, version?: string): string[] | undefined;
    dependencies(pkg: string, version?: string): ResolvedDependency[] | undefined;
    packageNames(): string[];
    isBaseR(pkg: string): boolean;
    downloads(pkg: string): number;
    coreVersions(pkg: string): RVersion[] | undefined;
    releaseDate(pkg: string, version?: string): Date | undefined;
    releaseDates(pkg: string): VersionRelease[];
    latestVersion(pkg: string): RVersion | undefined;
    close(): void;
}
/** source that answers for pkg, merging all sources that carry it; undefined if none do */
export declare function sourceForPackage(sources: readonly PackageSignatureSource[], pkg: string): PackageSignatureSource | undefined;
/** options controlling where {@link SigDatabase}/{@link SigDatabaseSet} materialize decompressed caches */
export interface SigDbOpenOptions {
    /** directory for the decompressed, hash-keyed cache (default: see {@link sigDbCacheDir}) */
    cacheDir?: string;
    /** content hash to key the cache (avoids reading the source header; supplied from a manifest) */
    hash?: string;
    /** index to use instead of a sibling `.idx` (supplied from a manifest so no `.idx` file need ship) */
    index?: SigDbIndex;
}
/** a caller-supplied index/dictionary for {@link SigDatabase.openSync} (both derived from the source otherwise) */
export interface OpenSyncOptions {
    index?: SigDbIndex;
    strings?: string[];
}
/** {@link SigDatabase.openSyncFrom} options: cache settings plus an optional precomputed hash/index/dictionary */
export interface OpenSyncFromOptions extends SigDbOpenOptions, OpenSyncOptions {
    hash?: string;
}
export declare class SigDatabase implements PackageSignatureSource {
    private closed;
    /** parsed blobs by blob index so repeated lookups skip the re-read + JSON.parse; FIFO-bounded to cap memory */
    private readonly blobCache;
    private static readonly BlobCacheCap;
    /** a version's function indices plus its `name -> index` view; FIFO-bounded like {@link blobCache} */
    private readonly versionFnCache;
    /**
     * Which names a package can offer at all, as sorted dictionary ids -- integers, so the whole database of
     * them costs a few MB where the parsed blobs cost hundreds. Filled for a package the first time its blob is
     * read, so a later lookup for a name it does not have answers without touching the file again.
     */
    private readonly nameIdsOfBlob;
    /** dictionary id per name asked for, so only the names someone actually queried are ever resolved */
    private readonly nameIds;
    /** reverse index `S3 class -> owning package`, over every package's latest version; built once (see {@link classOwner}) */
    private classIndex;
    private readonly fd;
    readonly strings: string[];
    readonly index: SigDbIndex;
    readonly content: SigDbContent | undefined;
    private readonly cranBase;
    private constructor();
    /**
     * Use an already-built {@link SigDb} directly, without writing or reading a file: its blobs simply start out in
     * the cache. It holds exactly what a bundle carries, so every query answers as it would from disk; the
     * {@link SigDbBuilder} plus this is all a test needs.
     */
    static fromMemory(db: SigDb): SigDatabase;
    /**
     * Open a plain, seekable `.sigs.ndjson` synchronously. Pass `index` to skip reading the `.idx`,
     * and `strings` to use an already-loaded shared dictionary instead of the file's own `d` section (for a
     * blob-only shard). One ranged read loads the dictionary, no readline overhead.
     */
    static openSync(plainFile: string, opts?: OpenSyncOptions): SigDatabase;
    /** open a `.sigs.ndjson`, `.br` or `.gz`; compressed sources are decompressed into a hash-keyed cache once */
    static open(source: string, opts?: SigDbOpenOptions): Promise<SigDatabase>;
    /**
     * Like {@link open} but fully synchronous (blocking decompression); a `hash` keys the cache when `source`
     * is compressed. Pass `strings` for a blob-only shard that shares an already-loaded dictionary.
     */
    static openSyncFrom(source: string, opts: OpenSyncFromOptions): SigDatabase;
    has(pkg: string): boolean;
    packageNames(): string[];
    /** load a single package's blob by seeking to its line (undefined if absent); cached by blob index */
    blob(pkg: string): PkgBlob | undefined;
    /** the dictionary id of `name`, or `-1` if the dictionary does not hold it, so no package can offer it */
    private nameId;
    /**
     * Whether `pkg` can offer `name` in any of its versions, answered from {@link nameIdsOfBlob} alone.
     * `true` whenever nothing is known about the package yet, so this only ever skips work that would have
     * found nothing: the ids cover every function record, of which a version selects a subset.
     */
    private mayOffer;
    /** store `value` under `key`, dropping the oldest entry first once the cache sits at {@link BlobCacheCap} */
    private static cache;
    /** seek to a byte range, read + decode the package blob there (no caching) */
    private readBlobAt;
    /** read every unique package blob in index order (used to re-hash a whole shard during verification) */
    allBlobs(): PkgBlob[];
    /** recompute this bundle's self-contained content hash from its re-read data (matches {@link writeSignatureDb}) */
    contentHash(blobs?: PkgBlob[]): string;
    /** whether this bundle actually carries the given version of a package (not just the package) */
    hasVersion(pkg: string, version: string): boolean;
    /** whether a version is a current CRAN release (not in the package's `noncran`/removed set) */
    isCranVersion(pkg: string, version: string): boolean;
    lookup(pkg: string, version?: string): LibraryExports | undefined;
    packagesExporting(name: string): readonly string[];
    classOwner(className: string, version?: string): string | undefined;
    /** resolve a package version to its blob and the function-record indices of that version (the shared prologue of {@link functions}/{@link functionByName}) */
    private versionFns;
    functions(pkg: string, version?: string): DecodedFunction[] | undefined;
    functionByName(pkg: string, name: string, version?: string): DecodedFunction | undefined;
    transitiveCallees(pkg: string, name: string, version?: string): string[] | undefined;
    dependencies(pkg: string, version?: string): ResolvedDependency[] | undefined;
    /** whether this is an R-core / base package (its versions are the R releases it shipped with; see {@link SigDbPkgMeta}) */
    isBaseR(pkg: string): boolean;
    /** the download count recorded for the package, `0` when this source does not carry it */
    downloads(pkg: string): number;
    /**
     * The R versions a base package was part of core, in ascending R-version order (exactly its stored
     * versions). `undefined` for a non-base package. E.g. `mva` returns `…1.9.1`, `parallel` `2.14.0…`.
     */
    coreVersions(pkg: string): RVersion[] | undefined;
    /** the release date of a package version (defaulting to the newest release), or `undefined` if unknown */
    releaseDate(pkg: string, version?: string): Date | undefined;
    /** every known release date of a package, in ascending R-version order (empty when no dates were stored) */
    releaseDates(pkg: string): VersionRelease[];
    /** the newest version of a package by release date (falling back to the recorded latest, then SemVer order) */
    latestVersion(pkg: string): RVersion | undefined;
    /** close the underlying file descriptor (idempotent; safe to call more than once) */
    close(): void;
}
/** options for {@link SigDatabaseSet.openManifest}: the base cache options plus per-shard enable/disable */
export interface SigDbSetOpenOptions extends SigDbOpenOptions {
    /** only load these shard ids (e.g. `['base-current','current-top']`); omit to load all */
    includeShards?: readonly string[];
    /** load every shard except these ids (e.g. `['full-top','full-rest']` for a current-only view) */
    excludeShards?: readonly string[];
}
/** one mounted shard: its manifest entry paired with the opened {@link SigDatabase} */
interface MountedShard {
    ref: SigDbShardRef;
    db: SigDatabase;
}
/**
 * A transparent, read-only view over several {@link SigDatabase} shards described by a {@link SigDbManifest}.
 * When the manifest embeds each shard's index (the default), `openManifest()` reads only that small file to
 * build the package to shard routing table.
 */
export declare class SigDatabaseSet implements PackageSignatureSource {
    private readonly opened;
    readonly manifest: SigDbManifest;
    /** the directory the manifest's shard/dict paths are relative to */
    readonly baseDir: string;
    private readonly indices;
    /** package name to shard indices, ordered by preference (current before full) */
    private readonly routes;
    private readonly cacheDir?;
    /** reverse index `S3 class -> owning package`, over every package's latest version; built once (see {@link classOwner}) */
    private classIndex;
    private constructor();
    /** read + validate a manifest and apply the include/exclude shard filter */
    private static prepManifest;
    /** build the package to shard routing (current before full) and construct the set from resolved indices */
    private static assemble;
    static openManifest(manifestFile: string, opts?: SigDbSetOpenOptions): Promise<SigDatabaseSet>;
    /**
     * Synchronous {@link openManifest}, needing every shard to embed its index (the default for the bundles
     * flowR ships). Shards and dictionaries still decompress lazily.
     */
    static openManifestSync(manifestFile: string, opts?: SigDbSetOpenOptions): SigDatabaseSet;
    /** shared dictionaries, loaded (decompressed + parsed) once and cached by id */
    private readonly dictCache;
    /** load (and cache) a shared dictionary's strings, decompressing its `.br` into the cache once */
    private dictionaryStrings;
    /** lazily open a shard, decompressing its `.br` (and its shared dictionary) into the cache on first access */
    private shard;
    /**
     * Warm the shards (and their shared dictionaries) needed for `pkgs`, or **everything** when omitted.
     * Afterward, the synchronous query methods, for the latest *and* historical versions, do no I/O or decompression.
     */
    preload(pkgs?: readonly string[]): Promise<void>;
    /**
     * Warm just the shards matching `include`, e.g., only the current-tier top shards (the base + most-downloaded
     * packages) to speed up common lookups without paying for the long tail or the history shards. See {@link preload}.
     */
    preloadShards(include: (shard: SigDbShardRef) => boolean): Promise<void>;
    /** decompress the given shards + their shared dictionaries concurrently, then open them (see {@link preload}) */
    private warmShards;
    /** the shard indices that can serve this package, preferred order; optionally requiring a specific version */
    private route;
    /** read (once) the blob from the shard with the most complete history: a `full` or `history` tier if present */
    private historyBlob;
    has(pkg: string): boolean;
    /** whether any active shard actually carries the given version of a package (not just the package) */
    hasVersion(pkg: string, version: string): boolean;
    /** whether a version is a current CRAN release (not in the package's `noncran`/removed set) */
    isCranVersion(pkg: string, version: string): boolean;
    packageNames(): string[];
    /** the on-demand load state (opened + unpacked) of every shard in this set */
    shardStatus(): ShardStatus[];
    /** open (blocking) and return every shard database with its manifest ref, for whole-set verification */
    allShards(): MountedShard[];
    /** load (and cache) a shared dictionary's strings by id, for verification/inspection */
    sharedDictionary(id: string): string[];
    /** the first non-empty result from the shards that can serve `pkg` (in preferred order: current before full) */
    private firstOf;
    lookup(pkg: string, version?: string): LibraryExports | undefined;
    packagesExporting(name: string): readonly string[];
    classOwner(className: string, version?: string): string | undefined;
    functions(pkg: string, version?: string): DecodedFunction[] | undefined;
    functionByName(pkg: string, name: string, version?: string): DecodedFunction | undefined;
    transitiveCallees(pkg: string, name: string, version?: string): string[] | undefined;
    dependencies(pkg: string, version?: string): ResolvedDependency[] | undefined;
    /** whether this is an R-core / base package (see {@link SigDatabase.isBaseR}); O(1) via the hoisted metadata */
    isBaseR(pkg: string): boolean;
    /** the download count of the package; O(1) via the hoisted metadata, so no shard has to be unpacked for it */
    downloads(pkg: string): number;
    /** the R versions a base package was part of core (ascending); `undefined` if not a base package */
    coreVersions(pkg: string): RVersion[] | undefined;
    /** every known release of a package (version + date), ascending, read once from the most complete shard */
    releaseDates(pkg: string): VersionRelease[];
    /** the release date of a package version (defaulting to the newest release), or `undefined` if unknown */
    releaseDate(pkg: string, version?: string): Date | undefined;
    /** the newest version of a package by release date (falling back to the recorded latest, then SemVer order) */
    latestVersion(pkg: string): RVersion | undefined;
    /**
     * Close every opened shard's file descriptor and drop the in-memory caches (opened shards + shared
     * dictionaries), so the (potentially large) dictionary strings can be reclaimed by the GC. Idempotent.
     */
    close(): void;
}
/**
 * Open a path-based source once, process-wide, synchronously. Returns the shared instance (opening it on the
 * first call), or `undefined` if the path needs async opening (a `.br`/`.gz` bundle, use {@link getSharedSigSource}).
 * Throws only if a sync-openable source fails to open.
 */
export declare function getSharedSigSourceSync(source: string): PackageSignatureSource | undefined;
/** Open a path-based source once, process-wide (async: also handles `.br`/`.zst`/`.gz` bundles and non-embedded manifests). */
export declare function getSharedSigSource(source: string): Promise<PackageSignatureSource | undefined>;
/** per-shard result of {@link verifyShardedDatabase} */
export interface ShardVerifyResult {
    id: string;
    packages: number;
    /** the shard's content hash recomputed from its re-read blobs matches the manifest + file header */
    hashOk: boolean;
    expectedHash: string;
    actualHash: string;
}
/** the outcome of {@link verifyShardedDatabase} */
export interface SigDbVerifyReport {
    ok: boolean;
    /** the shared dictionaries' recomputed hashes all match the manifest */
    dictsOk: boolean;
    shards: ShardVerifyResult[];
    /** number of distinct packages routed by the manifest */
    routedPackages: number;
    /** functions/dependencies decoded during the spot-check without an out-of-range string */
    spotChecked: number;
    /** required packages (e.g. base R) that were requested but not found */
    missingRequired: string[];
    /** every problem found (empty when `ok`) */
    errors: string[];
}
/** {@link verifyShardedDatabase} options: open settings plus which packages must exist and how many to spot-check */
export interface VerifyOptions extends SigDbOpenOptions {
    requirePackages?: readonly string[];
    sample?: number;
}
/**
 * Re-read a written sharded database from its (compressed) files and check it is internally consistent:
 * every shard's content hash recomputed from its re-read blobs matches both the manifest and the file
 * header; every shared dictionary's hash matches; the manifest routes every package a shard holds; a
 * sample of packages decodes (functions + dependencies) with all string indices in range; and any
 * `requirePackages` (e.g. base R) are present. This is a strong correctness gate: correctness over speed.
 */
export declare function verifyShardedDatabase(manifestFile: string, opts?: VerifyOptions): Promise<SigDbVerifyReport>;
export {};