@eagleoutice/flowr-dev/project/sigdb/build.d.ts

Static Dataflow Analyzer and Program Slicer for the R Programming Language

import { SigDbMagic, SigDbSchema, type PkgBlob, type SigDb, type SigDbFeatures, type SigDbPkgMeta, type SigDbShard, type SigDbTier, type SigVersionInfo } from './schema';
import { type SigDbIndex, type SigShardIndexWire } from './index-format';
import { type SigDbManifest, type SigDbShardRef, type SigDbDictRef } from './manifest';
/** per-package metadata handed to {@link SigDbBuilder.addPackage} (everything but `latest` defaults) */
export interface AddPackageOptions {
    readonly latest: string;
    readonly archived?: boolean;
    readonly downloads?: number;
    readonly core?: boolean;
}
/**
 * Accumulates analyzed functions and serializes a {@link SigDb}. Feed it with {@link addPackage} and
 * {@link addVersion}, then {@link build}. Pooling (dictionary, per-package blobs, whole-package dedup,
 * frequency reordering) happens in {@link build} so the result is deterministic for identical inputs.
 */
export declare class SigDbBuilder {
    private readonly raw;
    addPackage(name: string, opts: AddPackageOptions): void;
    addVersion(name: string, version: string, info: SigVersionInfo): void;
    /** the package names once, alphabetically -- the stable build order, computed a single time (see {@link selectPackages}) */
    private sortedNames;
    private namesCache;
    /** the package names to include, in build (sorted) order, honoring the R-core policy and popularity shard */
    private selectPackages;
    /**
     * Build one {@link SigDb} bundle. `tier: 'current'` keeps only each package's latest version (small,
     * fast to load); `tier: 'full'` keeps every version. `topN` + `shard` further restrict to the most-
     * downloaded packages (`'top'`) or the remainder (`'rest'`), so a database can be split into several
     * small shards routed by a {@link SigDbManifest}.
     */
    build(opts: SigDbBuildOptions): SigDb;
    /**
     * Build several shards that all reindex into a **single shared string dictionary** (stored once, not
     * per shard). All shards' blobs are pooled into one dictionary and frequency-sorted together, so the
     * dictionary loads once and no strings are duplicated across shards. Package metadata is likewise
     * collected once. This is the compact, fast-loading counterpart of calling {@link build} per shard.
     */
    buildSharded(opts: Omit<SigDbBuildOptions, 'tier' | 'shard' | 'topN'>, specs: readonly ShardSpec[]): ShardedSigDb;
}
/** a shard to build: a temporal tier, optionally restricted to a popularity shard and/or the R-core packages */
export interface ShardSpec {
    tier?: SigDbTier;
    shard?: SigDbShard;
    topN?: number;
    core?: CorePolicy;
}
/** the id of a shard, e.g. `base-current`, `current-top` or `full` */
export declare function shardId(spec: ShardSpec): string;
/** one shard produced by {@link SigDbBuilder.buildSharded}: blobs referencing the shared dictionary */
export interface SigShard {
    id: string;
    tier: SigDbTier;
    shard?: SigDbShard;
    topN?: number;
    core?: CorePolicy;
    blobs: PkgBlob[];
    pkgs: Record<string, number>;
    versions: number;
    functions: number;
    /** hash over this shard's blobs + pkgs (the dictionary is hashed separately) */
    hash: string;
}
/** several shards sharing one dictionary + one package-metadata map */
export interface ShardedSigDb {
    format: typeof SigDbMagic;
    schema: typeof SigDbSchema;
    scope: 'signatures';
    date: string;
    generated: number;
    cranBase?: string;
    features: Required<SigDbFeatures>;
    /** the single shared string dictionary */
    strings: string[];
    dictHash: string;
    /** package name to metadata, shared by every shard */
    meta: Record<string, SigDbPkgMeta>;
    shards: SigShard[];
}
/** options for {@link SigDbBuilder.build} */
export interface SigDbBuildOptions {
    /** dataset date `YYYY-MM-DD` */
    date: string;
    /** build timestamp (ms since epoch) */
    generated: number;
    /** CRAN base url (only stored when non-default) */
    cranBase?: string;
    /** renumber the string dictionary by frequency for better compression (default: true) */
    optimizeStrings?: boolean;
    /** temporal tier (default `full`) */
    tier?: SigDbTier;
    /** with {@link SigDbBuildOptions.topN}: include only the top-N most-downloaded packages (`top`) or the rest (`rest`) */
    shard?: SigDbShard;
    /** download-rank cutoff for {@link SigDbBuildOptions.shard} */
    topN?: number;
    /** restrict to R-core / base packages (`only`) or exclude them (`exclude`); default: no restriction */
    core?: CorePolicy;
    /** which information to store (default: everything) */
    features?: SigDbFeatures;
}
/** how a shard treats R-core / base packages: keep only them, exclude them, or don't care */
export type CorePolicy = 'only' | 'exclude';
export interface CompressOptions {
    /** zstd compression level (1..22) for the `.zst` output (default 19) */
    level?: number;
    /** brotli quality for the `.br` output; 11 is smallest but slow, lower is much faster for a small size cost */
    brotliQuality?: number;
    /** brotli window bits (10..30); above 24 enables the non-standard large-window mode (reader must opt in) */
    brotliLgwin?: number;
}
/** Write `<outBase>.sigs.ndjson` (+ `.br`, `.zst` when supported, `.idx`) -- a single self-contained bundle (its own dictionary). */
export declare function writeSignatureDb(outBase: string, db: SigDb, compress?: CompressOptions): Promise<SigDbIndex>;
/** the extension of a standalone shared-dictionary file */
export declare const SigDbDictExt: ".dict.sigs.ndjson";
/** Write a shared string dictionary to `<outBase>.dict.sigs.ndjson` (+ `.br`/`.zst`). Returns where its lines sit. */
export declare function writeDictionary(outBase: string, id: string, strings: readonly string[], compress?: CompressOptions): Promise<SigDbDictRef>;
/** Write one blob-only shard (references a shared dictionary) to `<outBase>.<id>.sigs.ndjson` (+ `.br`/`.zst`). */
export declare function writeShardBundle(outBase: string, shard: SigShard, cranBase: string | undefined, compress?: CompressOptions): Promise<SigShardIndexWire>;
/** options for {@link writeShardedDatabase} */
export interface ShardedWriteOptions extends CompressOptions {
    /** assemble a clean copy-into-flowR folder holding just the compressed shards, the dictionary and the manifest */
    pack?: string;
    /** invoked after each shard file is written (for progress logging) */
    onShard?: (shard: SigShard, ref: SigDbShardRef) => void;
}
/**
 * Write a {@link ShardedSigDb}: one shared dictionary file, one blob-only file per shard, and a
 * {@link SigDbManifest} that embeds each shard's index and references the shared dictionary by id. Every
 * shard reindexes into that single dictionary (stored once, not per shard). A reader needs only the compressed
 * files plus the manifest -- no `.idx` sidecars. With `pack`, also assembles a clean copy-into-flowR folder.
 */
export declare function writeShardedDatabase(outBase: string, db: ShardedSigDb, manifestFile: string, opts?: ShardedWriteOptions): Promise<SigDbManifest>;