@eagleoutice/flowr-dev
Version:
Static Dataflow Analyzer and Program Slicer for the R Programming Language
638 lines • 28.5 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.SigDbDictExt = exports.SigDbBuilder = void 0;
exports.shardId = shardId;
exports.writeSignatureDb = writeSignatureDb;
exports.writeDictionary = writeDictionary;
exports.writeShardBundle = writeShardBundle;
exports.writeShardedDatabase = writeShardedDatabase;
/**
* The build/write half of the sigdb format: the {@link SigDbBuilder} (accumulate analyzed packages, pool +
* frequency-reorder the dictionary, emit a {@link SigDb}) and the NDJSON writers (single bundle, shared
* dictionary, blob-only shards, and the sharded {@link SigDbManifest}). Split out of `../sigdb` so the reader
* there is not weighed down by the (build-time only) encoder; imports only sibling format/codec modules.
*/
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const events_1 = require("events");
const schema_1 = require("./schema");
const r_version_1 = require("../../util/r-version");
const decode_1 = require("./decode");
const hash_1 = require("./hash");
const index_format_1 = require("./index-format");
const manifest_1 = require("./manifest");
const codec_1 = require("./codec");
const log_1 = require("../../util/log");
// warn once (build-time) when this Node cannot produce `.zst`, so only the brotli `.br` fallback is written
let zstdSkipLogged = false;
function noteZstdSupport() {
if (!(0, codec_1.zstdSupported)() && !zstdSkipLogged) {
zstdSkipLogged = true;
log_1.log.warn('sigdb: this Node lacks zstd (node:zlib); writing only the brotli .br fallback (upgrade to Node >= 22.15 for smaller .zst bundles)');
}
}
/** resolve the effective features (everything defaults to on) */
function resolveFeatures(f) {
if (f === undefined) {
return { signatures: true, callGraphs: true, locations: true, dependencies: true };
}
return {
signatures: f.signatures ?? true,
callGraphs: f.callGraphs ?? true,
locations: f.locations ?? true,
dependencies: f.dependencies ?? true
};
}
/** first-order delta encoding of an ascending integer list (smaller, more repetitive, compresses better) */
function deltaEncode(sorted) {
const out = new Array(sorted.length);
let prev = 0;
for (let i = 0; i < sorted.length; i++) {
out[i] = sorted[i] - prev;
prev = sorted[i];
}
return out;
}
/**
* Index based string internalization
*/
function internalize(arr, map, key, value) {
let i = map.get(key);
if (i === undefined) {
i = arr.length;
arr.push(value);
map.set(key, i);
}
return i;
}
/** pack a raw package's metadata, appending the R-core marker only for base packages */
function packMeta(p) {
return p.core ? [p.latest, p.archived ? 1 : 0, p.downloads, 1] : [p.latest, p.archived ? 1 : 0, p.downloads];
}
/** an append-once dedup pool: {@link internalize}s a value under a string key, returning its stable index */
class Pool {
items = [];
index = new Map();
intern(key, value) {
return internalize(this.items, this.index, key, value);
}
}
/** the versions of `p` to keep for `tier`: `current` only the latest, `history` all but it, `full` all (see {@link SigDbTier}) */
function keptVersions(p, tier) {
const all = [...p.versions.keys()].sort();
if (tier === 'full') {
return all;
}
// fall back to the highest by R-version order (not lexical) when the recorded latest is absent
const latest = p.versions.has(p.latest) ? p.latest : r_version_1.RVersion.highest(all);
return tier === 'current'
? (latest !== undefined ? [latest] : [])
: all.filter(v => v !== latest);
}
/** build one self-contained blob for the given tier + features (functions sorted for determinism) */
function buildBlob(p, strings, tier, feats) {
const sigs = new Pool();
const cgs = new Pool();
const fns = new Pool();
const deps = new Pool();
const sig = (params) => {
if (!feats.signatures || params.length === 0) {
return -1;
}
const value = params.map(par => {
const nameI = strings.str(par.name);
const flags = (par.forced ? 1 /* ParamFlag.Forced */ : 0) | (par.missing ? 2 /* ParamFlag.Missing */ : 0);
if (par.default !== undefined) {
// cap very long default expressions (e.g. a 7 KB `c(...)` column list): they are unique, so they never
// dedupe and bloat the dictionary. A truncation marker keeps "has a default" plus a preview. Newlines
// are flattened so the string is safe as a delimiter in the newline-blob dictionary.
const d0 = par.default.replace(/[\r\n]+/g, ' ');
const def = d0.length > schema_1.MaxDefaultLength ? d0.slice(0, schema_1.MaxDefaultLength) + '…' : d0;
return [nameI, flags, strings.str(def)];
}
return flags === 0 ? nameI : [nameI, flags];
});
return sigs.intern(JSON.stringify(value), value);
};
const cg = (callees) => {
if (!feats.callGraphs || callees.length === 0) {
return -1;
}
const idxs = Array.from(new Set(callees), c => strings.str(c)).sort((a, b) => a - b);
return cgs.intern(idxs.join(','), deltaEncode(idxs));
};
const fn = (f) => {
const base = [strings.str(f.name), sig(f.params), cg(f.callees), f.props,
feats.locations && f.file ? strings.str(f.file) : -1, feats.locations ? f.line ?? -1 : -1];
const rec = f.topic && f.topic !== f.name ? [...base, strings.str(f.topic)] : base;
return fns.intern(rec.join(','), rec);
};
const depList = (list) => {
// sorted by (type, name) for a canonical, poolable form
const value = list
.toSorted((a, b) => a.type - b.type || a.name.localeCompare(b.name))
.map(d => d.constraint !== undefined
? [strings.str(d.name), d.type, strings.str(d.constraint)]
: [strings.str(d.name), d.type]);
return deps.intern(JSON.stringify(value), value);
};
const versions = {};
const depsByVersion = {};
const dates = {};
const noncran = [];
let versionCount = 0;
let functionCount = 0;
for (const version of keptVersions(p, tier)) {
const info = p.versions.get(version);
const idxs = info.functions
.toSorted((a, b) => a.name.localeCompare(b.name) || (a.file ?? '').localeCompare(b.file ?? '') || (a.line ?? -1) - (b.line ?? -1))
.map(fn).sort((a, b) => a - b);
versions[version] = deltaEncode(idxs);
if (feats.dependencies && info.dependencies && info.dependencies.length > 0) {
depsByVersion[version] = depList(info.dependencies);
}
if (info.date !== undefined && Number.isFinite(info.date)) {
dates[version] = Math.round(info.date / 86_400_000); // days since the Unix epoch (compact; day precision)
}
versionCount++;
functionCount += info.functions.length;
if (!info.cran) {
noncran.push(version);
}
}
const blob = { sigs: sigs.items, cgs: cgs.items, fns: fns.items, versions, noncran: noncran.length ? noncran : undefined, deps: deps.items, depsByVersion, dates };
return { blob, versionCount, functionCount };
}
/**
* Accumulates analyzed functions and serializes a {@link SigDb}. Feed it with {@link addPackage} and
* {@link addVersion}, then {@link build}. Pooling (dictionary, per-package blobs, whole-package dedup,
* frequency reordering) happens in {@link build} so the result is deterministic for identical inputs.
*/
class SigDbBuilder {
raw = new Map();
addPackage(name, opts) {
const p = this.raw.get(name);
if (p) {
p.latest = opts.latest;
p.archived = opts.archived ?? p.archived;
p.downloads = opts.downloads ?? p.downloads;
p.core = opts.core ?? p.core;
}
else {
this.raw.set(name, { latest: opts.latest, archived: opts.archived ?? false, downloads: opts.downloads ?? 0, core: opts.core ?? false, versions: new Map() });
this.namesCache = undefined; // a new package name invalidates the memoized sorted order
}
}
addVersion(name, version, info) {
let p = this.raw.get(name);
if (!p) {
p = { latest: version, archived: false, downloads: 0, core: false, versions: new Map() };
this.raw.set(name, p);
this.namesCache = undefined; // a new package name invalidates the memoized sorted order
}
p.versions.set(version, info);
}
/** the package names once, alphabetically -- the stable build order, computed a single time (see {@link selectPackages}) */
sortedNames() {
return (this.namesCache ??= [...this.raw.keys()].sort());
}
namesCache;
/** the package names to include, in build (sorted) order, honoring the R-core policy and popularity shard */
selectPackages(opts) {
let names = this.sortedNames(); // filters below reassign to fresh arrays, so no copy needed
if (opts.core === 'only') {
names = names.filter(n => this.raw.get(n).core);
}
else if (opts.core === 'exclude') {
names = names.filter(n => !this.raw.get(n).core);
}
if (opts.topN === undefined || opts.shard === undefined) {
return names;
}
// rank by downloads within the (already core-filtered) selection, then split into top-N / rest
const ranked = names
.map(n => [n, this.raw.get(n).downloads])
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
.map(([n]) => n);
const top = new Set(ranked.slice(0, opts.topN));
return names.filter(n => opts.shard === 'top' ? top.has(n) : !top.has(n));
}
/**
* Build one {@link SigDb} bundle. `tier: 'current'` keeps only each package's latest version (small,
* fast to load); `tier: 'full'` keeps every version. `topN` + `shard` further restrict to the most-
* downloaded packages (`'top'`) or the remainder (`'rest'`), so a database can be split into several
* small shards routed by a {@link SigDbManifest}.
*/
build(opts) {
const tier = opts.tier ?? 'full';
const feats = resolveFeatures(opts.features);
const strings = new StringPool();
const blobs = [];
const blobIdx = new Map();
const pkgs = {};
const pkgMeta = {};
let versionCount = 0;
let functionCount = 0;
for (const name of this.selectPackages(opts)) {
const p = this.raw.get(name);
const built = buildBlob(p, strings, tier, feats);
if (built.versionCount === 0) {
continue; // a package with no versions in this tier (e.g. a single-version package under `history`) is not routed
}
versionCount += built.versionCount;
functionCount += built.functionCount;
const key = JSON.stringify((0, decode_1.blobTuple)(built.blob));
pkgs[name] = internalize(blobs, blobIdx, key, built.blob);
pkgMeta[name] = packMeta(p);
}
const dict = (opts.optimizeStrings ?? true) ? optimizeStringOrder(strings.strings, blobs) : strings.strings;
const cranBase = opts.cranBase && opts.cranBase !== schema_1.DefaultCranBase ? opts.cranBase : undefined;
const partial = {
format: schema_1.SigDbMagic, schema: schema_1.SigDbSchema, scope: 'signatures',
...(cranBase ? { cranBase } : {}), strings: dict, blobs, pkgs, meta: pkgMeta
};
return {
...partial,
content: {
version: Number(opts.date.replace(/-/g, '')),
date: opts.date,
generated: opts.generated,
tier,
...(opts.shard ? { shard: opts.shard, topN: opts.topN } : {}),
features: feats,
packages: Object.keys(pkgs).length,
versions: versionCount,
functions: functionCount,
uniquePackages: blobs.length,
strings: dict.length,
hash: (0, hash_1.contentHash)(partial)
}
};
}
/**
* Build several shards that all reindex into a **single shared string dictionary** (stored once, not
* per shard). All shards' blobs are pooled into one dictionary and frequency-sorted together, so the
* dictionary loads once and no strings are duplicated across shards. Package metadata is likewise
* collected once. This is the compact, fast-loading counterpart of calling {@link build} per shard.
*/
buildSharded(opts, specs) {
const feats = resolveFeatures(opts.features);
const strings = new StringPool(); // SHARED across every shard
const meta = {};
const shards = specs.map(spec => {
const tier = spec.tier ?? 'full';
const blobs = [];
const blobIdx = new Map();
const pkgs = {};
let versions = 0;
let functions = 0;
for (const name of this.selectPackages(spec)) {
const p = this.raw.get(name);
const built = buildBlob(p, strings, tier, feats); // interns into the SHARED dictionary
if (built.versionCount === 0) {
continue; // no versions in this tier (e.g. a single-version package under `history`) -> not routed
}
versions += built.versionCount;
functions += built.functionCount;
pkgs[name] = internalize(blobs, blobIdx, JSON.stringify((0, decode_1.blobTuple)(built.blob)), built.blob);
meta[name] = packMeta(p);
}
return { id: shardId(spec), tier, shard: spec.shard, topN: spec.topN, core: spec.core, blobs, pkgs, versions, functions, hash: '' };
});
// ONE frequency reorder over every shard's blobs, remapping the shared dictionary in place
const dict = (opts.optimizeStrings ?? true) ? optimizeStringOrder(strings.strings, shards.flatMap(s => s.blobs)) : strings.strings;
for (const s of shards) {
s.hash = (0, hash_1.shardHash)(s.blobs, s.pkgs);
}
const cranBase = opts.cranBase && opts.cranBase !== schema_1.DefaultCranBase ? opts.cranBase : undefined;
return {
format: schema_1.SigDbMagic, schema: schema_1.SigDbSchema, scope: 'signatures',
date: opts.date, generated: opts.generated, ...(cranBase ? { cranBase } : {}),
features: feats, strings: dict, dictHash: (0, hash_1.dictionaryHash)(dict), meta, shards
};
}
}
exports.SigDbBuilder = SigDbBuilder;
/** the id of a shard, e.g. `base-current`, `current-top` or `full` */
function shardId(spec) {
const tier = spec.tier ?? 'full';
if (spec.core === 'only') {
return `base-${tier}`;
}
return spec.shard ? `${tier}-${spec.shard}` : tier;
}
/** the global, frequency-orderable string dictionary shared by every package blob */
class StringPool {
strings = [];
idx = new Map();
str(s) {
// the dictionary is newline-delimited on disk, so a stored string must never contain one -- otherwise it splits
// on read and shifts every later index. Flatten defensively here so no field (name, file, topic, ...) can corrupt it.
const safe = s.includes('\n') || s.includes('\r') ? s.replace(/[\r\n]+/g, ' ') : s;
return internalize(this.strings, this.idx, safe, safe);
}
}
/** renumber the dictionary so the most-referenced strings get the smallest indices (a pure bijection) */
function optimizeStringOrder(strings, blobs) {
const n = strings.length;
const counts = new Float64Array(n);
const bump = (i) => {
if (i >= 0) {
counts[i]++;
}
};
for (const blob of blobs) {
for (const s of blob.sigs) {
for (const p of s) {
if (typeof p === 'number') {
bump(p);
}
else {
bump(p[0]);
if (p.length === 3) {
bump(p[2]);
}
}
}
}
for (const c of blob.cgs) {
let prev = 0;
for (const d of c) {
prev += d;
bump(prev);
}
}
for (const f of blob.fns) {
bump(f[0]);
bump(f[4]);
if (f[6] !== undefined) {
bump(f[6]);
}
}
for (const list of blob.deps) {
for (const d of list) {
bump(d[0]);
if (d.length === 3) {
bump(d[2]);
}
}
}
}
const order = Array.from({ length: n }, (_, i) => i).sort((a, b) => counts[b] - counts[a] || a - b);
const remap = new Int32Array(n);
for (let newI = 0; newI < n; newI++) {
remap[order[newI]] = newI;
}
for (const blob of blobs) {
for (const s of blob.sigs) {
for (let k = 0; k < s.length; k++) {
const p = s[k];
if (typeof p === 'number') {
s[k] = remap[p];
}
else {
p[0] = remap[p[0]];
if (p.length === 3) {
p[2] = remap[p[2]];
}
}
}
}
for (let c = 0; c < blob.cgs.length; c++) {
const idxs = [];
let prev = 0;
for (const d of blob.cgs[c]) {
prev += d;
idxs.push(remap[prev]);
}
idxs.sort((a, b) => a - b);
blob.cgs[c] = deltaEncode(idxs);
}
for (const f of blob.fns) {
f[0] = remap[f[0]];
if (f[4] >= 0) {
f[4] = remap[f[4]];
}
if (f[6] !== undefined && f[6] >= 0) {
f[6] = remap[f[6]];
}
}
for (const list of blob.deps) {
for (const d of list) {
d[0] = remap[d[0]];
if (d.length === 3) {
d[2] = remap[d[2]];
}
}
}
}
return order.map(oldI => strings[oldI]);
}
/** flush a batch line once its serialized size reaches roughly this many bytes (keeps lines under the string cap) */
const NdjsonBatchBytes = 8_000_000;
/**
* Batch the string dictionary into `["d", startIdx, "s1\ns2\n…"]` lines: the strings are joined into ONE
* newline-delimited blob per line rather than a JSON array of quoted strings. On load this parses as a single
* string and `split('\n')` -- far faster than `JSON.parse`-ing an array of ~1.4M elements, and a touch smaller
* Safe because no dictionary string contains a newline (defaults are normalized).
*/
function* dictLines(strings) {
let start = 0;
let parts = [];
let bytes = 0;
const line = (from) => `["d",${from},${JSON.stringify(parts.join('\n'))}]`;
for (let i = 0; i < strings.length; i++) {
const s = strings[i];
if (parts.length > 0 && bytes + s.length + 1 > NdjsonBatchBytes) {
yield line(start);
start = i;
parts = [];
bytes = 0;
}
parts.push(s);
bytes += s.length + 1;
}
yield line(start);
}
/** batch an array into `[tag, startIdx, [...]]` lines that never approach V8's string cap */
function* batchLines(tag, arr) {
let start = 0;
let parts = [];
let bytes = 0;
const line = (from) => `["${tag}",${from},[${parts.join(',')}]]`;
for (let i = 0; i < arr.length; i++) {
const s = JSON.stringify(arr[i]);
if (parts.length > 0 && bytes + s.length + 1 > NdjsonBatchBytes) {
yield line(start);
start = i;
parts = [];
bytes = 0;
}
parts.push(s);
bytes += s.length + 1;
}
yield line(start);
}
// large window (30) shaves ~15% off the `.br`; every {@link SigDatabase} read path decodes with
// `BROTLI_DECODER_PARAM_LARGE_WINDOW` enabled, so this is always safe for bundles flowR reads back.
const DefaultBrotliLgwin = 30;
/** codec-appropriate compression options: brotli takes the quality/window, zstd its own level */
function codecOptions(spec, compress) {
return spec.codec === 'zstd'
? { level: compress.level }
: { level: compress.brotliQuality, lgwin: compress.brotliLgwin ?? DefaultBrotliLgwin };
}
/**
* Streams NDJSON lines to `<plain>` plus every codec {@link writeCodecs} yields (`.br` always, `.zst` when this
* Node supports it) at once, tracking the plain byte offset for seek indexes. A `.br` fallback is thus always
* produced beside any `.zst`.
*/
class LineWriter {
plainOut;
sinks = [];
byteOff = 0;
constructor(plain, compress) {
noteZstdSupport();
fs_1.default.mkdirSync(path_1.default.dirname(path_1.default.resolve(plain)), { recursive: true });
this.plainOut = fs_1.default.createWriteStream(plain);
for (const spec of (0, codec_1.writeCodecs)()) {
const file = fs_1.default.createWriteStream(`${plain}${spec.ext}`);
const stream = spec.createCompress(codecOptions(spec, compress));
stream.pipe(file);
this.sinks.push({ stream, file });
}
}
async write(text) {
const chunk = text + '\n';
const back = [];
if (!this.plainOut.write(chunk)) {
back.push((0, events_1.once)(this.plainOut, 'drain'));
}
for (const { stream } of this.sinks) {
if (!stream.write(chunk)) {
back.push((0, events_1.once)(stream, 'drain'));
}
}
if (back.length > 0) {
await Promise.all(back);
}
this.byteOff += Buffer.byteLength(chunk);
}
async close() {
this.plainOut.end();
for (const { stream } of this.sinks) {
stream.end();
}
await Promise.all([(0, events_1.once)(this.plainOut, 'close'), ...this.sinks.map(s => (0, events_1.once)(s.file, 'close'))]);
}
}
/** Write `<outBase>.sigs.ndjson` (+ `.br`, `.zst` when supported, `.idx`) -- a single self-contained bundle (its own dictionary). */
async function writeSignatureDb(outBase, db, compress = {}) {
const plain = `${outBase}${schema_1.SigDbExt}`;
const w = new LineWriter(plain, compress);
await w.write(JSON.stringify({
format: db.format, schema: db.schema, scope: db.scope,
...(db.cranBase ? { cranBase: db.cranBase } : {}), content: db.content
}));
const dictStartByte = w.byteOff;
for (const line of dictLines(db.strings)) {
await w.write(line);
}
const dict = [dictStartByte, w.byteOff - dictStartByte];
const blobs = [];
for (let i = 0; i < db.blobs.length; i++) {
const startByte = w.byteOff;
await w.write(`["b",${i},${JSON.stringify((0, decode_1.blobTuple)(db.blobs[i]))}]`);
blobs.push([startByte, w.byteOff - startByte]);
}
for (const line of batchLines('m', Object.entries(db.meta))) {
await w.write(line);
}
for (const line of batchLines('p', Object.entries(db.pkgs))) {
await w.write(line);
}
await w.close();
const index = { byteCount: w.byteOff, dict, blobs, pkgs: db.pkgs, meta: db.meta };
fs_1.default.writeFileSync(`${plain}.idx`, JSON.stringify((0, index_format_1.encodeIndex)(index)));
return index;
}
/** the extension of a standalone shared-dictionary file */
exports.SigDbDictExt = '.dict.sigs.ndjson';
/** Write a shared string dictionary to `<outBase>.dict.sigs.ndjson` (+ `.br`/`.zst`). Returns where its lines sit. */
async function writeDictionary(outBase, id, strings, compress = {}) {
const plain = `${outBase}${exports.SigDbDictExt}`;
const w = new LineWriter(plain, compress);
await w.write(JSON.stringify({ format: 'flowr-sigdb-dict', schema: schema_1.SigDbSchema, strings: strings.length }));
const start = w.byteOff;
for (const line of dictLines(strings)) {
await w.write(line);
}
const range = [start, w.byteOff - start];
await w.close();
return { id, path: `${path_1.default.basename(outBase)}${exports.SigDbDictExt}`, hash: (0, hash_1.dictionaryHash)(strings), range, byteCount: w.byteOff, strings: strings.length };
}
/** Write one blob-only shard (references a shared dictionary) to `<outBase>.<id>.sigs.ndjson` (+ `.br`/`.zst`). */
async function writeShardBundle(outBase, shard, cranBase, compress = {}) {
const plain = `${outBase}.${shard.id}${schema_1.SigDbExt}`;
const w = new LineWriter(plain, compress);
await w.write(JSON.stringify({
format: schema_1.SigDbMagic, schema: schema_1.SigDbSchema, scope: 'signatures', shared: true,
...(cranBase ? { cranBase } : {}),
content: { tier: shard.tier, ...(shard.shard ? { shard: shard.shard, topN: shard.topN } : {}), packages: Object.keys(shard.pkgs).length, versions: shard.versions, functions: shard.functions, hash: shard.hash }
}));
const blobs = [];
for (let i = 0; i < shard.blobs.length; i++) {
const startByte = w.byteOff;
await w.write(`["b",${i},${JSON.stringify((0, decode_1.blobTuple)(shard.blobs[i]))}]`);
blobs.push([startByte, w.byteOff - startByte]);
}
for (const line of batchLines('p', Object.entries(shard.pkgs))) {
await w.write(line);
}
await w.close();
return { n: w.byteOff, b: blobs, p: shard.pkgs };
}
/**
* Write a {@link ShardedSigDb}: one shared dictionary file, one blob-only file per shard, and a
* {@link SigDbManifest} that embeds each shard's index and references the shared dictionary by id. Every
* shard reindexes into that single dictionary (stored once, not per shard). A reader needs only the compressed
* files plus the manifest -- no `.idx` sidecars. With `pack`, also assembles a clean copy-into-flowR folder.
*/
async function writeShardedDatabase(outBase, db, manifestFile, opts = {}) {
const compress = { level: opts.level, brotliQuality: opts.brotliQuality, brotliLgwin: opts.brotliLgwin };
const exts = (0, codec_1.writeCodecs)().map(c => c.ext); // every compressed variant produced (`.br` always, `.zst` when supported)
const base = path_1.default.basename(outBase);
const dictRef = await writeDictionary(outBase, 'shared', db.strings, compress);
const shardRefs = [];
for (const s of db.shards) {
const idx = await writeShardBundle(outBase, s, db.cranBase, compress);
const ref = {
id: s.id, tier: s.tier, ...(s.shard ? { shard: s.shard, topN: s.topN } : {}),
path: `${base}.${s.id}${schema_1.SigDbExt}`, hash: s.hash,
packages: Object.keys(s.pkgs).length, versions: s.versions, dict: dictRef.id, idx
};
shardRefs.push(ref);
opts.onShard?.(s, ref);
}
const manifest = {
format: manifest_1.SigDbManifestMagic, schema: manifest_1.SigDbManifestSchema, date: db.date, generated: db.generated,
...(db.cranBase ? { cranBase: db.cranBase } : {}), meta: db.meta, dicts: [dictRef], shards: shardRefs
};
(0, manifest_1.writeManifest)(manifestFile, manifest);
if (opts.pack) {
// a self-contained folder to copy into flowR: every compressed variant of the dictionary + shards + the (index-embedding) manifest
fs_1.default.mkdirSync(opts.pack, { recursive: true });
for (const ext of exts) {
fs_1.default.copyFileSync(`${outBase}${exports.SigDbDictExt}${ext}`, path_1.default.join(opts.pack, `${dictRef.path}${ext}`));
for (const ref of shardRefs) {
fs_1.default.copyFileSync(`${outBase}.${ref.id}${schema_1.SigDbExt}${ext}`, path_1.default.join(opts.pack, `${ref.path}${ext}`));
}
}
(0, manifest_1.writeManifest)(path_1.default.join(opts.pack, `${base}.manifest.json`), manifest);
}
return manifest;
}
//# sourceMappingURL=build.js.map