@struktoai/mirage-core/dist/workspace/snapshot/tar_io.js

A Unified Virtual Filesystem For AI Agents

// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { drain } from "../../io/stream.js";
import { closeStream, openStream, serveRequests } from "../../rs/stream.js";
import { byteView, textView } from "../../shell/bytes.js";
import { VFSName } from "../../types.js";
import { concat } from "../../utils/bytes.js";
import { fileChunks, fileSize, regularChunks, stagedPath, writeStreamToFile } from "./fs.js";
import { MountKey, StateKey, VFSStateKey } from "./keys.js";
import { resolveManifest } from "./manifest.js";
import { BLOB_REF_KEY, isSafeBlobPath } from "./utils.js";
const ENC = new TextEncoder();
const DEC = new TextDecoder('utf-8', { fatal: false });
const MANIFEST_NAME = 'manifest.json';
// The name the archive goes by inside the tar engine.
const ARCHIVE = 'snapshot.tar';
/**
 * The tar as one buffer, for a caller that needs bytes (an agents SDK that
 * stores snapshots as buffers). A blob that is a host path is read through
 * `regularChunks`, refused if a link has replaced it.
 */
export async function writeSnapshotTar(manifest, blobs, compress = null) {
    const parts = [];
    await pack(manifest, blobs, compress, (data) => {
        parts.push(data);
        return Promise.resolve();
    });
    return concat(parts);
}
export async function readSnapshotTar(data, compress = null, maxBytes) {
    return unpack(once(data), null, compress, maxBytes);
}
/**
 * Write manifest + blobs as a tar at `target`, streaming: a blob that is a
 * host path (a disk mount's file) is read a chunk at a time, opened only
 * while it is read. Returns the tar's size.
 */
export async function writeTar(target, manifest, blobs) {
    const [writer, written] = fileWriter(target);
    try {
        await pack(manifest, blobs, null, (data) => writer.write(data));
        await writer.close();
    }
    catch (error) {
        await writer.abort(error);
        await written.catch(() => undefined);
        throw error;
    }
    return written;
}
/**
 * Read a tar written by `writeTar` from the file at `source`, streaming.
 * Each disk mount's files are extracted under `staging` and come back as
 * host paths, so a restore copies them into place instead of holding them
 * in memory; the caller removes the directory once the state is loaded.
 */
export async function readTar(source, staging, maxBytes) {
    return unpack(fileChunks(source), staging, null, maxBytes);
}
/**
 * Run mirage-rs's tar to its end, `serve` answering each request; a refusal
 * throws in tar's own words.
 */
async function run(config, serve) {
    const stream = await openStream('tar', config);
    const errors = [];
    const host = {
        errors: {
            append: (data) => {
                errors.push(data);
                return Promise.resolve();
            },
        },
        serve,
    };
    try {
        await drain(serveRequests(stream, host));
        if (stream.status() !== 0) {
            const message = DEC.decode(concat(errors)).trim();
            throw new Error(message === 'tar: decoded input exceeds size limit'
                ? 'snapshot exceeds expanded size limit'
                : message);
        }
    }
    finally {
        await closeStream(stream);
    }
}
/** The manifest, then each blob, as tar's -c writes them to `write`. */
async function pack(manifest, blobs, compress, write) {
    const entries = [
        [MANIFEST_NAME, ENC.encode(JSON.stringify(manifest, null, 2))],
        ...Object.entries(blobs),
    ];
    const taken = counts(entries.map(([name]) => memberName(name)));
    for (const [name] of entries) {
        if ((taken.get(memberName(name)) ?? 0) > 1)
            throw new Error(`Blob paths name one member: ${name}`);
    }
    const members = [];
    for (const [index, [name, data]] of entries.entries()) {
        const size = typeof data === 'string' ? await fileSize(data) : data.byteLength;
        members.push({ name: byteView(name), path: byteView(name), operand: index + 2, size });
    }
    const inputs = new Map();
    try {
        await run({
            mode: 'create',
            archive: ARCHIVE,
            output: ARCHIVE,
            codec: compress === 'gz' ? 'gzip' : null,
            members,
        }, async (request, data) => {
            if (request.op === 'write')
                await write(data);
            if (request.op !== 'read')
                return { reply: {} };
            const fd = request.fd;
            if (typeof request.operand === 'number') {
                const blob = entries[request.operand - 2]?.[1] ?? new Uint8Array();
                inputs.set(fd, typeof blob === 'string' ? regularChunks(blob) : once(blob));
            }
            const next = await inputs.get(fd)?.next();
            if (next === undefined || next.done === true) {
                inputs.delete(fd);
                return { reply: { end: true } };
            }
            return { reply: {}, data: next.value };
        });
    }
    finally {
        for (const input of inputs.values())
            await input.return(undefined);
    }
}
/**
 * Read a snapshot tar from `source` in one pass: the manifest first, then
 * each blob it names, a disk mount's file written under `staging` when there
 * is one (and held as well when the manifest names it elsewhere too), every
 * other blob held in memory; a member it names no blob for is read past, and
 * one stored twice is refused.
 */
async function unpack(source, staging, compress, maxBytes) {
    if (maxBytes !== undefined && maxBytes < 0)
        throw new Error('maxBytes must be non-negative');
    let total = 0;
    const held = new Map();
    const staged = new Map();
    const files = new Map();
    let manifest = null;
    let named = new Map();
    let disk = new Map();
    const begin = async (name) => {
        if (held.has(name) || staged.has(name))
            throw new Error(`Blob stored twice: ${name}`);
        if (name === MANIFEST_NAME) {
            held.set(name, []);
            return;
        }
        if (manifest === null) {
            manifest = manifestOf(held);
            named = counts(blobPaths(manifest).map(memberName));
            disk = counts(diskFileRefs(manifest).map(([, , blobPath]) => memberName(blobPath)));
        }
        if (!named.has(name))
            return;
        if (staging !== null && disk.has(name)) {
            const target = await stagedPath(staging, name);
            files.set(name, fileWriter(target));
            staged.set(name, target);
        }
        if (!staged.has(name) || (named.get(name) ?? 0) > (disk.get(name) ?? 0))
            held.set(name, []);
    };
    const serve = async (request, data) => {
        if (request.op === 'read') {
            const next = await source.next();
            return next.done === true ? { reply: { end: true } } : { reply: {}, data: next.value };
        }
        if (request.op === 'stat')
            return { reply: { kind: 'dir' } };
        if ((request.op !== 'write' && request.op !== 'flush') || typeof request.path !== 'string') {
            return { reply: {} };
        }
        const name = textView(request.path).replace(/^\/+/, '');
        if (request.create === true || request.whole === true)
            await begin(name);
        if (maxBytes !== undefined) {
            const copies = name === MANIFEST_NAME ? 1 : Math.max(1, named.get(name) ?? 0);
            const amount = data.byteLength * copies;
            if (amount > maxBytes - total)
                throw new Error('snapshot exceeds expanded size limit');
            total += amount;
        }
        held.get(name)?.push(data);
        const file = files.get(name);
        if (file === undefined)
            return { reply: {} };
        await file[0].write(data);
        if (request.op === 'flush' || request.whole === true) {
            files.delete(name);
            await file[0].close();
            await file[1];
        }
        return { reply: {} };
    };
    try {
        await run({
            mode: 'extract',
            archive: ARCHIVE,
            base: '',
            codec: compress === 'gz' ? 'gzip' : null,
            max_decoded_bytes: maxBytes ?? null,
        }, serve);
    }
    finally {
        for (const [writer, written] of files.values()) {
            await writer.abort();
            await written.catch(() => undefined);
        }
        await source.return(undefined);
    }
    manifest ??= manifestOf(held);
    for (const [map, rel, blobPath] of diskFileRefs(manifest)) {
        const path = staged.get(memberName(blobPath));
        if (path !== undefined)
            map[rel] = path;
    }
    return resolveManifest(manifest, (blobPath) => {
        if (!isSafeBlobPath(blobPath))
            throw new Error(`Unsafe blob path: ${String(blobPath)}`);
        if (memberName(blobPath) === MANIFEST_NAME)
            throw new Error('Manifest cannot reference itself as a blob');
        const blob = held.get(memberName(blobPath));
        if (blob === undefined)
            throw new Error(`Manifest references missing blob: ${blobPath}`);
        return concat(blob);
    });
}
/** A writer into a new file at `path`, whose writes fail once the copy into the file does. */
function fileWriter(path) {
    const { readable, writable } = new TransformStream();
    const writer = writable.getWriter();
    const written = writeStreamToFile(path, readable);
    void written.catch((error) => writer.abort(error));
    return [writer, written];
}
/** The name tar's -x gives the member a blob path was written under: the path without its empty and `.` components. */
function memberName(path) {
    return path
        .split('/')
        .filter((part) => part !== '' && part !== '.')
        .join('/');
}
/** The blob path of every reference in the manifest, each checked safe. */
function blobPaths(manifest) {
    const paths = [];
    resolveManifest(manifest, (path) => {
        paths.push(path);
        return new Uint8Array();
    });
    return paths;
}
/** How many times each name comes. */
function counts(names) {
    const out = new Map();
    for (const name of names)
        out.set(name, (out.get(name) ?? 0) + 1);
    return out;
}
async function* once(data) {
    await Promise.resolve();
    yield data;
}
/** The manifest, the archive's first member, parsed. */
function manifestOf(held) {
    const raw = held.get(MANIFEST_NAME);
    if (raw === undefined)
        throw new Error(`${MANIFEST_NAME} missing or unreadable`);
    return JSON.parse(DEC.decode(concat(raw)));
}
/**
 * Every disk mount file's blob reference in the manifest, as the files map
 * holding it, its key and its tar path: what a restore stages on disk.
 */
function diskFileRefs(manifest) {
    const refs = [];
    for (const mount of manifest[StateKey.MOUNTS] ?? []) {
        const vfsState = mount[MountKey.VFS_STATE];
        if (vfsState?.[VFSStateKey.TYPE] !== VFSName.DISK)
            continue;
        const files = vfsState[VFSStateKey.FILES] ?? {};
        for (const [rel, ref] of Object.entries(files)) {
            const blobPath = ref[BLOB_REF_KEY];
            if (blobPath !== undefined)
                refs.push([files, rel, blobPath]);
        }
    }
    return refs;
}
//# sourceMappingURL=tar_io.js.map