@eagleoutice/flowr-dev/project/sigdb/manifest.js

Static Dataflow Analyzer and Program Slicer for the R Programming Language

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.SigDbManifestSchema = exports.SigDbManifestMagic = void 0;
exports.writeManifest = writeManifest;
exports.readManifestFile = readManifestFile;
exports.readManifestDate = readManifestDate;
exports.defaultSigDbPath = defaultSigDbPath;
exports.defaultSigDbPaths = defaultSigDbPaths;
/**
 * The sharded-database manifest format (shard/dictionary descriptors that route a set of shard files as one
 * database), reading/writing it, and discovering the bundled manifests/bundles on flowR's search path. Split
 * out of `../sigdb` as pure format + filesystem discovery, with no dependency on the reader/writer classes.
 */
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const schema_1 = require("./schema");
const codec_1 = require("./codec");
const decompress_1 = require("./decompress");
exports.SigDbManifestMagic = 'flowr-sigdb-manifest';
exports.SigDbManifestSchema = 2;
/** write a {@link SigDbManifest} (compact JSON) plus a compressed copy per available codec (`.br` always, `.zst` when supported) beside it */
function writeManifest(file, manifest) {
    fs_1.default.mkdirSync(path_1.default.dirname(path_1.default.resolve(file)), { recursive: true });
    const json = JSON.stringify(manifest);
    fs_1.default.writeFileSync(file, json);
    for (const spec of (0, codec_1.writeCodecs)()) {
        fs_1.default.writeFileSync(`${file}${spec.ext}`, spec.compressSync(json, { level: 11, sizeHint: json.length }));
    }
}
/** how cheap a file is to read (lower is better); one we cannot decompress here sorts last */
function codecRank(file) {
    const ext = (0, codec_1.compressedExtOf)(file);
    if (ext === undefined) {
        return -1;
    }
    const preferred = (0, codec_1.readableExtsPreferred)().indexOf(ext);
    return preferred < 0 ? Number.MAX_SAFE_INTEGER : preferred;
}
/** read a manifest file (transparently decompressing a `.br`/`.zst`/`.gz`) */
function readManifestFile(manifestFile) {
    return JSON.parse(readManifestText(manifestFile));
}
function readManifestText(manifestFile) {
    const raw = fs_1.default.readFileSync(manifestFile);
    return (0, codec_1.compressedExtOf)(manifestFile) ? (0, codec_1.decompressSyncFor)(manifestFile, raw).toString('utf8') : raw.toString('utf8');
}
/**
 * The `date` of a manifest without parsing the rest of it, whose `meta` is megabytes of packages.
 * Falls back to a full parse if the date is not where we expect it.
 */
function readManifestDate(manifestFile) {
    const text = readManifestText(manifestFile);
    return /"date"\s*:\s*"([^"]*)"/.exec(text.slice(0, 4096))?.[1]
        ?? JSON.parse(text).date;
}
/** richest first: a container shipping the full set uses it, else the slim `current`, else the `base` floor */
const SigDbScopeOrder = ['full', 'current', 'base'];
/** layouts a bundled sigdb may sit in, relative to a search root -- the root itself (e.g. a `$FLOWR_SIGDB_DIR` data mount), then the dev `src`, build `dist` and data-dir layouts */
const SigDbSubDirs = ['', 'data/sigdb', 'src/data/sigdb', 'dist/src/data/sigdb'];
/** a `<scope>.manifest.json`, in any of the codecs we can read */
const ManifestFilePattern = new RegExp(`\\.manifest\\.json${codec_1.CompressedExtPattern}$`);
/** a `<name>.sigs.ndjson` bundle, in any of the codecs we can read */
const BundleFilePattern = new RegExp(`${schema_1.SigDbExt.replace(/\./g, '\\.')}${codec_1.CompressedExtPattern}$`);
function sigDbBundleDirs() {
    if (typeof fs_1.default?.readdirSync !== 'function') {
        return [];
    }
    try {
        const bundles = path_1.default.join((0, decompress_1.sigDbCacheDir)(undefined, false), 'bundles');
        return fs_1.default.readdirSync(bundles, { withFileTypes: true })
            .filter(e => e.isDirectory())
            .map(e => path_1.default.join(bundles, e.name));
    }
    catch {
        return [];
    }
}
/** roots to search for a bundled sigdb; extendable via `$FLOWR_SIGDB_DIR` (path-delimiter separated) */
function sigDbSearchRoots(extra) {
    const roots = [...(extra ?? [])];
    const env = typeof process !== 'undefined' ? process.env?.FLOWR_SIGDB_DIR : undefined;
    if (env) {
        roots.push(...env.split(path_1.default.delimiter).filter(Boolean));
    }
    if (typeof __dirname !== 'undefined') {
        roots.push(__dirname);
    }
    if (typeof process !== 'undefined' && typeof process.cwd === 'function') {
        roots.push(process.cwd());
    }
    return roots;
}
/**
 * Location of a bundled sigdb **manifest**, found by walking up from several roots (this module,
 * `$FLOWR_SIGDB_DIR`, the working directory) across the dev (`src`), build (`dist`) and data-mount
 * layouts. With no `scope` it returns the richest available (`full` &gt; `current` &gt; `base`), so a
 * container that ships the full set uses it automatically while a plain npm install falls back to the
 * bundled `base`. Node only (needs `fs`); pass `searchRoots` to override where it looks.
 */
function defaultSigDbPath(scope, searchRoots) {
    if (typeof fs_1.default?.existsSync !== 'function') {
        return undefined;
    }
    const scopes = scope ? [scope] : SigDbScopeOrder;
    const probe = (dir) => {
        for (const sub of SigDbSubDirs) {
            for (const s of scopes) {
                for (const suffix of ['', ...(0, codec_1.readableExtsPreferred)()]) {
                    const candidate = path_1.default.join(dir, sub, `${s}.manifest.json${suffix}`);
                    if (fs_1.default.existsSync(candidate)) {
                        return candidate;
                    }
                }
            }
        }
        return undefined;
    };
    for (const root of sigDbSearchRoots(searchRoots)) {
        for (let dir = root, i = 0; i < 10; i++) {
            const hit = probe(dir);
            if (hit !== undefined) {
                return hit;
            }
            const parent = path_1.default.dirname(dir);
            if (parent === dir) {
                break;
            }
            dir = parent;
        }
    }
    for (const bundle of sigDbBundleDirs()) {
        const hit = probe(bundle);
        if (hit !== undefined) {
            return hit;
        }
    }
    return undefined;
}
/**
 * Every distinct sigdb bundle discoverable in the search dirs (see {@link defaultSigDbPath}) -- not just the
 * richest scope. So dropping an extra bundle next to the shipped default (a downloaded full-history
 * `full.manifest.json.br`, a custom `*.manifest.json`, or a standalone `*.sigs.ndjson`) makes flowR mount it
 * automatically. Manifests come first (scope-named leading, richest scope first), then standalone bundles; the
 * shard and dictionary files a manifest already owns are skipped. Deduped by filename, first search location wins.
 */
function defaultSigDbPaths(searchRoots) {
    if (typeof fs_1.default?.readdirSync !== 'function') {
        return [];
    }
    const manifests = new Map(); // `<name>.manifest.json` (ignoring compression ext) -> first-found path
    const standalones = new Map(); // `<name>.sigs.ndjson` (ignoring compression ext) -> first-found path
    const foundIn = new Map(); // where a key was first found, to only compare codecs within one directory
    /** keeps the first location, but picks the cheapest readable codec among the copies that location offers */
    const keep = (into, key, file, dir) => {
        const previous = into.get(key);
        if (previous === undefined) {
            into.set(key, file);
            foundIn.set(key, dir);
        }
        else if (foundIn.get(key) === dir && codecRank(file) < codecRank(previous)) {
            into.set(key, file);
        }
    };
    const scanDir = (base) => {
        for (const sub of SigDbSubDirs) {
            let entries;
            try {
                entries = fs_1.default.readdirSync(path_1.default.join(base, sub));
            }
            catch {
                continue; // directory does not exist on this root
            }
            const dir = path_1.default.join(base, sub);
            for (const file of entries) {
                if (ManifestFilePattern.test(file)) {
                    keep(manifests, (0, codec_1.stripCompressedExt)(file), path_1.default.join(dir, file), dir);
                }
                else if (BundleFilePattern.test(file) && !file.includes('.dict' + schema_1.SigDbExt)) {
                    keep(standalones, (0, codec_1.stripCompressedExt)(file), path_1.default.join(dir, file), dir);
                }
            }
        }
    };
    for (const root of sigDbSearchRoots(searchRoots)) {
        for (let dir = root, i = 0; i < 10; i++) {
            scanDir(dir);
            const parent = path_1.default.dirname(dir);
            if (parent === dir) {
                break;
            }
            dir = parent;
        }
    }
    for (const bundle of sigDbBundleDirs()) {
        scanDir(bundle);
    }
    // a standalone bundle is a `.sigs.ndjson` that is not a shard of a discovered manifest (`<manifest>.<shard>...`)
    const prefixes = [...manifests.keys()].map(k => k.replace(/\.manifest\.json$/, ''));
    const isShard = (name) => {
        const base = name.replace(new RegExp(`${schema_1.SigDbExt.replace('.', '\\.')}$`), '');
        return prefixes.some(p => base === p || base.startsWith(p + '.'));
    };
    const scopeRank = (name) => {
        const scope = SigDbScopeOrder.indexOf(name.replace(/\.manifest\.json$/, ''));
        return scope === -1 ? SigDbScopeOrder.length : scope; // custom bundles sort after the known scopes
    };
    const orderedManifests = [...manifests.entries()]
        .sort((a, b) => scopeRank(a[0]) - scopeRank(b[0]) || a[0].localeCompare(b[0])).map(([, p]) => p);
    const bundles = standalones.entries().filter(([name]) => !isShard(name)).map(([, p]) => p);
    return [...orderedManifests, ...bundles];
}
//# sourceMappingURL=manifest.js.map