@eagleoutice/flowr-dev
Version:
Static Dataflow Analyzer and Program Slicer for the R Programming Language
258 lines • 12.5 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.SigDbRemoteFileName = exports.sha256File = void 0;
exports.selectDownloadVariants = selectDownloadVariants;
exports.downloadFullSigDb = downloadFullSigDb;
exports.syncedSigDbDir = syncedSigDbDir;
exports.sigDbRemoteRelease = sigDbRemoteRelease;
exports.sigDbCacheComplete = sigDbCacheComplete;
exports.sigDbNeedsSync = sigDbNeedsSync;
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const https_1 = __importDefault(require("https"));
const crypto_1 = __importDefault(require("crypto"));
const decompress_1 = require("./decompress");
const codec_1 = require("./codec");
const version_1 = require("../../util/version");
/** the GitHub `owner/repo` the full-history bundle is published to (see `scripts/publish-sigdb.mjs`) */
const DefaultRepo = 'flowr-analysis/flowr';
/** GitHub request headers; the token (used only for the API's rate limit and private repos) is attached only when `withAuth` */
function ghHeaders(accept, withAuth) {
const token = withAuth ? (process.env.GH_TOKEN ?? process.env.GITHUB_TOKEN) : undefined;
return { 'User-Agent': 'flowr', Accept: accept, ...(token ? { Authorization: `Bearer ${token}` } : {}) };
}
/** GET following redirects (GitHub asset URLs redirect to a storage host); resolves the final response for streaming */
function httpGet(url, accept, redirects = 5, withAuth = true) {
return new Promise((resolve, reject) => {
https_1.default.get(url, { headers: ghHeaders(accept, withAuth) }, res => {
const status = res.statusCode ?? 0;
if (status >= 300 && status < 400 && res.headers.location && redirects > 0) {
res.resume();
const next = new URL(res.headers.location, url);
// GitHub redirects a release-asset download to a pre-signed storage host: never forward the token
// across hosts (it is unnecessary there, leaks the token, and the signed URL rejects a second auth header)
const keepAuth = withAuth && next.host === new URL(url).host;
httpGet(next.toString(), accept, redirects - 1, keepAuth).then(resolve, reject);
}
else if (status >= 200 && status < 300) {
resolve(res);
}
else {
res.resume();
reject(new Error(`GET ${url} -> HTTP ${status}`));
}
}).on('error', reject);
});
}
async function readJson(url) {
const res = await httpGet(url, 'application/vnd.github+json');
const chunks = [];
for await (const c of res) {
chunks.push(c);
}
return JSON.parse(Buffer.concat(chunks).toString('utf8'));
}
async function downloadTo(url, dest) {
const res = await httpGet(url, 'application/octet-stream');
await new Promise((resolve, reject) => {
const out = fs_1.default.createWriteStream(dest);
res.on('error', reject);
out.on('error', reject).on('finish', () => out.close(err => err ? reject(err) : resolve()));
res.pipe(out);
});
}
/** hex sha256 of a file's contents (shared by the runtime verify and the `sigdb-remote` pointer generator) */
const sha256File = (p) => crypto_1.default.createHash('sha256').update(fs_1.default.readFileSync(p)).digest('hex');
exports.sha256File = sha256File;
/** the committed link file naming every downloadable shard (base floor + CRAN sets); only this pointer is committed */
exports.SigDbRemoteFileName = 'sigdb.remote.json';
/** locate the committed `sigdb.remote.json` link file in the sigdb data dir (dev `src`, built `dist`) */
function findRemotePointer() {
const candidates = [
path_1.default.join(__dirname, '../../data/sigdb', exports.SigDbRemoteFileName), // src/ and dist/src/ layouts (same depth)
path_1.default.join(process.cwd(), 'src/data/sigdb', exports.SigDbRemoteFileName),
path_1.default.join(process.cwd(), 'dist/src/data/sigdb', exports.SigDbRemoteFileName),
];
return candidates.find(p => {
try {
return fs_1.default.existsSync(p);
}
catch {
return false;
}
});
}
const readPointer = (p) => JSON.parse(fs_1.default.readFileSync(p, 'utf8'));
/** the direct release-asset CDN URL -- bypasses the REST API (and its rate limit) entirely */
const assetUrl = (repo, tag, name) => `https://github.com/${repo}/releases/download/${tag}/${encodeURIComponent(name)}`;
/** runtime-decodable variant extensions, most-preferred first, ending in plain (`''`) as a last resort */
function downloadVariantOrder() {
return [...(0, codec_1.readableExtsPreferred)(), ''];
}
/**
* Group physical asset names by their logical (compression-ext-stripped) name and pick, per logical shard, the
* single best variant this runtime can use: `.zst` when zstd is supported, otherwise `.br` (then `.gz`/plain).
* On a Node without zstd, a `.zst`-only logical shard is skipped entirely (it could not be decompressed). So the
* downloader fetches exactly one variant per shard/dictionary -- never both -- matching what the reader resolves.
*/
function selectDownloadVariants(names) {
const groups = new Map(); // logical name -> (ext -> physical name)
for (const name of names) {
const logical = (0, codec_1.stripCompressedExt)(name);
const ext = (0, codec_1.compressedExtOf)(name) ?? '';
let byExt = groups.get(logical);
if (!byExt) {
byExt = new Map();
groups.set(logical, byExt);
}
byExt.set(ext, name);
}
const order = downloadVariantOrder();
const picked = [];
for (const byExt of groups.values()) {
const ext = order.find(e => byExt.has(e));
if (ext !== undefined) {
picked.push(byExt.get(ext));
}
}
return picked;
}
/** pick the richest manifest among downloaded files (a `full`/scope manifest first, else any) */
const pickManifest = (files) => files.find(f => new RegExp(`full\\.manifest\\.json${codec_1.CompressedExtPattern}$`).test(f))
?? files.find(f => new RegExp(`\\.manifest\\.json${codec_1.CompressedExtPattern}$`).test(f));
/**
* Download the signature-database shards into the cache directory and return where they landed. Prefers the
* committed `sigdb.remote.json`, we verify each download by content hash, and **skip shards already
* present with the right hash**.
* Use the returned {@link SigDbDownloadResult.manifest}, or point `solver.sigdb.additionalPaths` at the dir.
*/
async function downloadFullSigDb(opts = {}) {
const progress = opts.onProgress ?? (() => { });
const pointerPath = findRemotePointer();
if (pointerPath) {
const remote = readPointer(pointerPath);
const repo = opts.repo ?? remote.repo ?? DefaultRepo;
const tag = remote.tag;
const dir = path_1.default.join((0, decompress_1.sigDbCacheDir)(), 'bundles', tag);
fs_1.default.mkdirSync(dir, { recursive: true });
// one variant per logical shard/dictionary -- the best this runtime can decompress (never both codecs)
const picked = selectDownloadVariants(Object.keys(remote.shards));
progress(`syncing ${picked.length} shards for ${tag} from ${repo}`);
const files = [];
for (const name of picked) {
const meta = remote.shards[name];
const dest = path_1.default.join(dir, name);
if (!opts.force && fs_1.default.existsSync(dest) && (0, exports.sha256File)(dest) === meta.sha256) {
progress(`have ${name}`);
}
else {
progress(`downloading ${name} (${(meta.bytes / 1e6).toFixed(1)} MB)`);
await downloadTo(assetUrl(repo, tag, name), dest);
if ((0, exports.sha256File)(dest) !== meta.sha256) {
throw new Error(`sha256 mismatch for ${name} (${tag}); the download is corrupt -- retry, or regenerate sigdb.remote.json`);
}
}
files.push(dest);
}
return { dir, manifest: pickManifest(files), files };
}
// fallback: no committed link file -> list the release via the API and download by size
const repo = opts.repo ?? DefaultRepo;
const version = opts.version ?? (0, version_1.flowrVersion)().toString();
const tag = `sigdb-v${version}`;
progress(`fetching release ${tag} from ${repo}`);
const release = await readJson(`https://api.github.com/repos/${repo}/releases/tags/${tag}`);
const assets = (release.assets ?? [])
.filter(a => a.name.endsWith('.br') || a.name.endsWith('.zst') || a.name.endsWith('.manifest.json'));
if (assets.length === 0) {
throw new Error(`release ${tag} in ${repo} has no signature-database assets${typeof release.message === 'string' ? ` (${release.message})` : ''}`);
}
const dir = path_1.default.join((0, decompress_1.sigDbCacheDir)(), 'bundles', tag);
fs_1.default.mkdirSync(dir, { recursive: true });
const byName = new Map(assets.map(a => [a.name, a]));
const files = [];
for (const name of selectDownloadVariants(byName.keys())) {
const a = byName.get(name);
const dest = path_1.default.join(dir, name);
if (!opts.force && fs_1.default.existsSync(dest) && fs_1.default.statSync(dest).size === a.size) {
progress(`have ${name}`);
}
else {
progress(`downloading ${name} (${(a.size / 1e6).toFixed(1)} MB)`);
await downloadTo(a.browser_download_url, dest);
}
files.push(dest);
}
return { dir, manifest: pickManifest(files), files };
}
/**
* The cache dir a {@link downloadFullSigDb} for the committed link file would populate, or `undefined` if no
* pointer is committed. Lets a caller mount an already-synced bundle via `solver.sigdb.additionalPaths` without
* hitting the network.
*/
function syncedSigDbDir() {
const pointerPath = findRemotePointer();
if (!pointerPath) {
return undefined;
}
return path_1.default.join((0, decompress_1.sigDbCacheDir)(), 'bundles', readPointer(pointerPath).tag);
}
/** the GitHub release (`{repo, tag, url}`) the committed pointer downloads from, or `undefined` when no pointer is committed */
function sigDbRemoteRelease() {
const pointerPath = findRemotePointer();
if (!pointerPath) {
return undefined;
}
try {
const remote = readPointer(pointerPath);
const repo = remote.repo ?? DefaultRepo;
return { repo, tag: remote.tag, url: `https://github.com/${repo}/releases/tag/${remote.tag}` };
}
catch {
return undefined;
}
}
/** Fast presence check (existence + byte size only, no hashing) for the shards the committed pointer selects for this runtime; `false` when no pointer is committed or any selected shard is missing/wrong-size. */
function sigDbCacheComplete() {
const pointerPath = findRemotePointer();
if (!pointerPath) {
return false;
}
try {
const remote = readPointer(pointerPath);
const dir = path_1.default.join((0, decompress_1.sigDbCacheDir)(undefined, false), 'bundles', remote.tag);
return selectDownloadVariants(Object.keys(remote.shards)).every(name => {
try {
return fs_1.default.statSync(path_1.default.join(dir, name)).size === remote.shards[name].bytes;
}
catch {
return false;
}
});
}
catch {
return false;
}
}
/** Startup check: `true` when a committed link file lists shards whose cached copies are missing or hash-mismatched */
function sigDbNeedsSync() {
const pointerPath = findRemotePointer();
if (!pointerPath) {
return false;
}
try {
const remote = readPointer(pointerPath);
const dir = path_1.default.join((0, decompress_1.sigDbCacheDir)(), 'bundles', remote.tag);
return Object.entries(remote.shards).some(([name, meta]) => {
const dest = path_1.default.join(dir, name);
return !fs_1.default.existsSync(dest) || (0, exports.sha256File)(dest) !== meta.sha256;
});
}
catch {
return false;
}
}
//# sourceMappingURL=sigdb-download.js.map