@eagleoutice/flowr-dev
Version:
Static Dataflow Analyzer and Program Slicer for the R Programming Language
184 lines • 8.15 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.isCompressed = void 0;
exports.parseHeader = parseHeader;
exports.sigDbStream = sigDbStream;
exports.resolveSource = resolveSource;
exports.sigDbCacheDir = sigDbCacheDir;
exports.readHeaderOf = readHeaderOf;
exports.isUnpacked = isUnpacked;
exports.ensurePlain = ensurePlain;
exports.ensurePlainSync = ensurePlainSync;
/**
* On-disk (de)compression + the hash-keyed decompress cache: turn a `.br`/`.zst`/`.gz` bundle into a seekable
* plain `.sigs.ndjson` (materialized once, reused on later startups), read a bundle's header cheaply, and resolve
* the cache directory. Split out of `../sigdb` so the reader there only consumes plain, seekable files. The codec
* is detected by extension (see `./codec`), so existing `.br` and new `.zst` bundles read transparently.
*/
const fs_1 = __importDefault(require("fs"));
const os_1 = __importDefault(require("os"));
const path_1 = __importDefault(require("path"));
const readline_1 = __importDefault(require("readline"));
const promises_1 = require("stream/promises");
const schema_1 = require("./schema");
const index_format_1 = require("./index-format");
const hash_1 = require("../../util/hash");
const codec_1 = require("./codec");
/** whether a bundle path is a compressed (`.br`/`.zst`/`.gz`) source that must be decompressed to be seekable */
const isCompressed = (f) => (0, codec_1.isCompressedExt)(f);
exports.isCompressed = isCompressed;
/** read and parse the header (line 1) from a buffer/string of NDJSON */
function parseHeader(text) {
const nl = text.indexOf('\n');
try {
return JSON.parse(nl >= 0 ? text.slice(0, nl) : text);
}
catch {
return undefined;
}
}
function decompressStream(file) {
return fs_1.default.createReadStream(file).pipe((0, codec_1.createDecompressFor)(file));
}
/** a readable stream of the bundle's plain NDJSON, transparently decompressing `.gz`/`.br`/`.zst` inputs */
function sigDbStream(file) {
return (0, exports.isCompressed)(file) ? decompressStream(file) : fs_1.default.createReadStream(file);
}
/**
* Resolve a manifest-relative file to the best source this runtime can read: a plain (already seekable) file
* if present, else the most-preferred compressed variant that exists AND is decompressible here -- `.zst` first
* when this Node supports zstd, otherwise `.br` (then `.gz`). A `.zst` is never returned on a Node without zstd
* (it could not be decompressed), so `.br`-only bundles read on any Node. Falls back to the plain path.
*/
function resolveSource(baseDir, relPath) {
const plain = path_1.default.resolve(baseDir, relPath);
if (fs_1.default.existsSync(plain)) {
return plain;
}
for (const ext of (0, codec_1.readableExtsPreferred)()) {
const compressed = `${plain}${ext}`;
if (fs_1.default.existsSync(compressed)) {
return compressed;
}
}
return plain;
}
/**
* Directory for decompressed, hash-keyed caches. Honours `$FLOWR_SIGDB_CACHE` / `$FLOWR_CACHE_DIR`, then the
* platform cache home (`$XDG_CACHE_HOME` on Linux, `%LOCALAPPDATA%` on Windows), then `~/.cache/flowr`, falling
* back to the OS temp dir (so it works in a read-only Docker image where only `/tmp` is writable -- mount a
* volume at the cache dir to persist it).
*/
function sigDbCacheDir(override, create = true) {
const base = override
?? process.env.FLOWR_SIGDB_CACHE ?? process.env.FLOWR_CACHE_DIR
?? (process.env.XDG_CACHE_HOME ? path_1.default.join(process.env.XDG_CACHE_HOME, 'flowr') : undefined)
?? (process.env.LOCALAPPDATA ? path_1.default.join(process.env.LOCALAPPDATA, 'flowr', 'cache') : undefined)
?? path_1.default.join(os_1.default.homedir?.() || os_1.default.tmpdir(), '.cache', 'flowr');
const dir = path_1.default.join(base, 'sigdb');
if (!create) {
return dir;
}
try {
fs_1.default.mkdirSync(dir, { recursive: true });
return dir;
}
catch {
const tmp = path_1.default.join(os_1.default.tmpdir(), 'flowr-sigdb-cache');
fs_1.default.mkdirSync(tmp, { recursive: true });
return tmp;
}
}
/** read the (small) header line of a possibly-compressed bundle without decompressing the whole thing */
async function readHeaderOf(source) {
if (!(0, exports.isCompressed)(source)) {
const fd = fs_1.default.openSync(source, 'r');
try {
const buf = Buffer.allocUnsafe(65536);
const n = fs_1.default.readSync(fd, buf, 0, buf.length, 0);
return parseHeader(buf.toString('utf8', 0, n));
}
finally {
fs_1.default.closeSync(fd);
}
}
const input = decompressStream(source);
const rl = readline_1.default.createInterface({ input, crlfDelay: Infinity });
try {
const line = (await rl[Symbol.asyncIterator]().next()).value; // just the first line (the header)
return line !== undefined ? JSON.parse(line) : undefined;
}
finally {
rl.close();
input.destroy();
}
}
function cachePaths(hash, cacheDir) {
const plain = path_1.default.join(sigDbCacheDir(cacheDir), `sigdb-${hash}${schema_1.SigDbExt}`);
return { plain, idx: `${plain}.idx` };
}
/** whether a compressed source with the given content hash has already been unpacked into its decompressed cache */
function isUnpacked(hash, cacheDir) {
return fs_1.default.existsSync(cachePaths(hash, cacheDir).plain);
}
/** materialize the `.idx` for a freshly decompressed cache file -- from the supplied index or the source's sibling */
function writeCacheIndex(source, idx, index) {
if (index) {
fs_1.default.writeFileSync(idx, JSON.stringify((0, index_format_1.encodeIndex)(index)));
return;
}
const srcIdx = source.replace(/\.(br|zst|gz)$/, '') + '.idx';
if (!fs_1.default.existsSync(srcIdx)) {
throw new Error(`missing sidecar index next to ${source} (expected ${srcIdx}), and none was supplied`);
}
fs_1.default.copyFileSync(srcIdx, idx);
}
/**
* Ensure a seekable plain `.sigs.ndjson` (+ its `.idx`) exists for `source`, decompressing a `.br`/`.zst`/`.gz`
* once into a hash-keyed cache the first time and reusing it on every later startup. The index may be
* supplied by the caller (e.g. embedded in a manifest) so no separate `.idx` file needs to ship.
*/
async function ensurePlain(source, opts = {}) {
if (!(0, exports.isCompressed)(source)) {
return source;
}
let hash = opts.hash;
if (hash === undefined) {
const content = (await readHeaderOf(source))?.content;
hash = content?.hash ?? new hash_1.Hash53().update(source).digest();
}
const { plain, idx } = cachePaths(hash, opts.cacheDir);
if (fs_1.default.existsSync(plain) && (opts.indexless || fs_1.default.existsSync(idx))) {
return plain;
}
const tmp = `${plain}.${process.pid}.tmp`;
await (0, promises_1.pipeline)(decompressStream(source), fs_1.default.createWriteStream(tmp));
fs_1.default.renameSync(tmp, plain);
if (!opts.indexless) {
writeCacheIndex(source, idx, opts.index);
}
return plain;
}
/** synchronous {@link ensurePlain} (blocking decompression); the supplied `hash` keys the decompress cache */
function ensurePlainSync(source, opts) {
if (!(0, exports.isCompressed)(source)) {
return source;
}
const { plain, idx } = cachePaths(opts.hash, opts.cacheDir);
if (fs_1.default.existsSync(plain) && (opts.indexless || fs_1.default.existsSync(idx))) {
return plain;
}
const raw = fs_1.default.readFileSync(source);
const out = (0, codec_1.decompressSyncFor)(source, raw);
const tmp = `${plain}.${process.pid}.tmp`;
fs_1.default.writeFileSync(tmp, out);
fs_1.default.renameSync(tmp, plain);
if (!opts.indexless) {
writeCacheIndex(source, idx, opts.index);
}
return plain;
}
//# sourceMappingURL=decompress.js.map