@struktoai/mirage-core/dist/runtime/python/pyodide/fs/fs.js

A Unified Virtual Filesystem For AI Agents

// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { epochToIso } from "../../../../utils/dates.js";
import { FileStat } from "../../../../types.js";
import { BLKSIZE, DIR_MODE, FILE_MODE, FLAGS, LINK_MODE, SEEK_CUR, SEEK_END, UMASK, } from "./constants.js";
import { errnoError } from "./errors.js";
import { classify } from "../../../../errors/index.js";
import { isMissingPath } from "../../../../errors/fs.js";
import { entryName } from "../../../../utils/remnants.js";
import { S_IFCHR, S_IFMT } from "../../../../utils/stat_view.js";
import { isUnclassified } from "../../../files.js";
import { openMode } from "../../../handles/mode.js";
import { posixStat } from "../../../stat.js";
import { NodeTable } from "./nodes.js";
// Emscripten's MEMFS makes these three at startup and its own stderr capture
// reopens /dev/stderr when a run ends (`API.restore_stderr`). Mounting mirage's
// /dev on top hid them, so that reopen raised ENOENT out of a filesystem
// callback nobody catches, and the unhandled rejection took the process down
// with it. Recreated here so the interpreter still finds them. rdev matches
// MEMFS's own numbering.
const STD_STREAM_RDEV = new Map([
    ['stdin', 6],
    ['stdout', 7],
    ['stderr', 8],
]);
// /dev/null's device number. It and the three streams above answer a read with
// EOF; only /dev/zero hands back an endless run of zeroes, and a stream that
// did the same would hang `open('/dev/stdin').read()` forever.
const NULL_RDEV = 0x103;
const EOF_ON_READ_RDEV = new Set([NULL_RDEV, ...STD_STREAM_RDEV.values()]);
function isCharDevice(mode) {
    return (mode & S_IFMT) === S_IFCHR;
}
/**
 * The metadata a `setattr` actually changes, or null when it changes
 * none of it.
 *
 * Emscripten hands this callback more than a guest's chmod and utime:
 * a stamp arrives beside a resize and beside a mode, and a write that
 * moved no field would still journal an op. Sending one costs a
 * dispatch and, worse, splits two writes the journal would otherwise
 * coalesce. Comparing against the node costs one read.
 *
 * The one case comparing cannot catch is a create, whose finalizing
 * chmod does lower a real mode; `PyodideFs.fresh` is what suppresses
 * that one.
 *
 * `ctime` is never included: no POSIX call sets it directly, so a mount
 * has nothing to write it to. `size` is not metadata here either, it is
 * the resize branch's bytes.
 *
 * Args:
 *   node: the node as it stands before the write.
 *   attr: the fields Emscripten passed, times as epoch ms.
 */
export function changedAttrs(node, attr) {
    const fields = {};
    if (attr.mode !== undefined && (attr.mode & 0o7777) !== (node.mode & 0o7777)) {
        fields.mode = attr.mode & 0o7777;
    }
    if (attr.atime !== undefined && attr.atime !== node.atime) {
        fields.atime = epochToIso(attr.atime / 1000);
    }
    if (attr.mtime !== undefined && attr.mtime !== node.mtime) {
        fields.mtime = epochToIso(attr.mtime / 1000);
    }
    return Object.keys(fields).length === 0 ? null : fields;
}
/**
 * A name a repeat listing found, classified by its own stat as the
 * first listing's file adapter classifies it, or left as the listing's row
 * when that stat fails, the way the file adapter degrades one.
 */
function ownRow(sync, path, entry) {
    if (entry.isDir)
        return entry;
    try {
        return sync.stat(path);
    }
    catch (error) {
        if (!isMissingPath(error))
            console.warn(`mirage: cannot stat ${path}: ${String(error)}`);
        return entry;
    }
}
/** The owner a mount's row reports, kept on the node `getattr` reads; the
 * owner and the group each on its own, since a chown may set either. */
function takeOwner(node, stat) {
    if (typeof stat.uid === 'number')
        node.uid = stat.uid;
    if (typeof stat.gid === 'number')
        node.gid = stat.gid;
}
/**
 * An Emscripten filesystem backed by a mirage mount.
 *
 * This is the pyodide runtime's interception layer. Mounting it at a mount
 * prefix puts mirage below the guest's syscall boundary, where every
 * spelling of an operation (`open`, `os.open`, `pathlib`, `shutil`'s
 * fd-relative walk) arrives as the same callback. Nothing is patched
 * inside the interpreter.
 *
 * Callbacks are synchronous: lookups, reads and writes wait for the host
 * through SyncVFS, which needs the worker. A file's stream reads and
 * writes through a handle on the run's open-file core, which holds its
 * bytes as every other runtime's open files are held; names and metadata
 * enter the journal and land at once, in guest order.
 *
 * The node table lives in `NodeTable`; what is left here is one method
 * per Emscripten callback, plus the errno translation only this layer
 * performs.
 */
export class PyodideFs {
    sync;
    deferred;
    streams;
    type;
    host;
    errno;
    journal;
    nodes;
    mountOf;
    // The file node FS.open just created, whose finalizing chmod is the
    // filesystem's own and not a guest metadata write. Cleared by the
    // first setattr, so it can never outlive the create it belongs to.
    fresh = null;
    /**
     * Args:
     *   host: the Emscripten FS namespace (`pyodide.FS`).
     *   errno: Emscripten's errno table (`pyodide.ERRNO_CODES`).
     *   journal: the write-ahead log guest mutations are recorded on.
     *   prefix: the mount prefix this filesystem serves.
     *   mountOf: the mirage mount owning a path (`RuntimeFiles.mountOf`).
     *     One mountpoint serves every mirage mount nested under its
     *     prefix, so this is the only boundary fact left to check ops
     *     against; Emscripten's own cross-mount checks cannot see it.
     *   sync: the worker's blocking way to the mounts.
     *   deferred: the refusals the mount gave a path whose call has not
     *     come yet, shared by every mountpoint as the journal is.
     *   streams: the file streams that hold a core handle, shared the same
     *     way, which the run's end closes.
     */
    constructor(host, errno, journal, prefix, mountOf, sync, deferred = new Map(), streams = new Set()) {
        this.sync = sync;
        this.deferred = deferred;
        this.streams = streams;
        this.host = host;
        this.errno = errno;
        this.journal = journal;
        this.mountOf = mountOf;
        const nodeOps = {
            getattr: this.getattr.bind(this),
            setattr: this.setattr.bind(this),
            lookup: this.lookup.bind(this),
            mknod: this.mknod.bind(this),
            rename: this.rename.bind(this),
            unlink: this.unlink.bind(this),
            rmdir: this.rmdir.bind(this),
            readdir: this.readdir.bind(this),
            symlink: this.symlink.bind(this),
            readlink: this.readlink.bind(this),
        };
        const streamOps = {
            open: this.streamOpen.bind(this),
            close: this.streamClose.bind(this),
            read: this.streamRead.bind(this),
            write: this.streamWrite.bind(this),
            llseek: this.llseek.bind(this),
            getattr: (stream) => this.attrOf(stream.node, stream.opened ?? this.nodes.pathOf(stream.node), this.host.isFile(stream.node.mode) ? this.handleOf(stream) : undefined),
        };
        this.nodes = new NodeTable(host, prefix, nodeOps, streamOps);
        this.type = { mount: this.nodes.mount.bind(this.nodes) };
    }
    getattr(node) {
        return this.attrOf(node, this.nodes.pathOf(node));
    }
    /**
     * One node's stat, its inode named by `path`.
     *
     * Args:
     *   node: the node to describe.
     *   path: the name its inode is reported under.
     *   fh: the core handle a stream stats through, which counts what it
     *     holds; absent stats the node's path.
     */
    attrOf(node, path, fh) {
        // The core's row, so one path reports one inode whoever asks, what an
        // open handle wrote and has not landed counted and the policies
        // judging it as a stat of its path. A device is the tree's own.
        let row = {
            size: 0,
            isDir: false,
            mode: node.mode,
            uid: node.uid ?? 0,
            gid: node.gid ?? 0,
            rdev: node.rdev,
            atimeMs: node.atime,
            mtimeMs: node.mtime,
        };
        if (!isCharDevice(node.mode)) {
            const sync = this.sync;
            row = this.readThrough([], () => fh === undefined ? sync.stat(this.nodes.pathOf(node)) : sync.fstat(fh));
            if (node.unclassified === true)
                this.classifyNode(node, row);
            if (row.isDir)
                row = { ...row, size: BLKSIZE };
        }
        const posix = posixStat(row, path, '/');
        return {
            dev: posix.dev,
            ino: posix.ino,
            mode: posix.mode,
            nlink: node.unlinked === true ? 0 : posix.nlink,
            uid: posix.uid,
            gid: posix.gid,
            rdev: posix.rdev,
            size: posix.size,
            atime: new Date(posix.atimeMs),
            mtime: new Date(posix.mtimeMs),
            ctime: new Date(node.ctime),
            blksize: posix.blksize,
            blocks: posix.blocks,
        };
    }
    setattr(node, attr) {
        // Read the changes before applying them, and only outside a create:
        // the same callback carries a guest's chmod and the chmod Emscripten
        // itself performs to finalize a new file, and the two are the same
        // shape. `fresh` is what tells them apart; the comparison then drops
        // a stamp write that changes nothing.
        const finalizing = this.fresh === node;
        this.fresh = null;
        // Read before the mode below is applied. A character device is not mount
        // content, so none of its metadata is a guest write: Emscripten chmods one
        // right after creating it, and journaling that queued a setattr against
        // the real mount for a node the mount does not have.
        const isDevice = isCharDevice(node.mode);
        const fields = finalizing || isDevice ? null : changedAttrs(node, attr);
        const size = isDevice ? undefined : attr.size;
        if (fields !== null && node.unlinked !== true) {
            // A link's own attrs go to the link, since the tree resolved to
            // the link node itself and the target's row is not what changed.
            if (this.host.isLink(node.mode))
                fields.nofollow = true;
            this.journal.markSetattr(this.nodes.pathOf(node), fields);
        }
        // A resize goes as a truncate to the new length. Recording here rather
        // than at close is what makes a bare `os.truncate(path, n)`, which
        // opens no handle at all, reach the mount.
        if (size !== undefined && node.unlinked !== true)
            this.journal.markTruncate(this.nodes.pathOf(node), size);
        if (node.unlinked !== true)
            this.settle(this.nodes.pathOf(node));
        else if (size !== undefined)
            this.resizeHeld(node, size);
        // Once the mount took the guest's own edit, the backend row the policies
        // are shown carries it; the times Emscripten filled in stay out.
        if (fields !== null && node.unlinked !== true && node.source !== undefined) {
            node.source = new FileStat(node.source).with({
                ...(fields.mode !== undefined ? { mode: fields.mode } : {}),
                ...(fields.mtime !== undefined ? { modified: fields.mtime } : {}),
                ...(fields.atime !== undefined ? { atime: fields.atime } : {}),
            });
        }
        // The chmod that finalizes a create carries the mode the create asked
        // for, so the umask comes off it as it did off the node.
        if (attr.mode !== undefined)
            node.mode = finalizing ? attr.mode & ~UMASK : attr.mode;
        if (attr.atime !== undefined)
            node.atime = attr.atime;
        if (attr.mtime !== undefined)
            node.mtime = attr.mtime;
        if (attr.ctime !== undefined)
            node.ctime = attr.ctime;
    }
    /**
     * Resize a file whose name is gone through a stream that writes it: its
     * bytes are only what the core's handles on it hold.
     *
     * Args:
     *   node: the file with no name left.
     *   size: the length it is left at.
     */
    resizeHeld(node, size) {
        const writes = FLAGS.O_WRONLY | FLAGS.O_RDWR;
        for (const stream of this.streams) {
            const fh = stream.fh;
            if (stream.node !== node || fh === undefined || (stream.flags & writes) === 0)
                continue;
            this.readThrough([], () => {
                this.sync.ftruncate(fh, size);
            });
            return;
        }
    }
    lookup(parent, name) {
        const sync = this.sync;
        let found = this.nodes.childOf(parent, name);
        if (found === undefined) {
            const path = this.nodes.pathOf(parent) + '/' + name;
            found = this.readThrough([path], () => {
                let stat;
                try {
                    stat = sync.stat(path);
                }
                catch (error) {
                    const rdev = STD_STREAM_RDEV.get(name);
                    if (this.nodes.pathOf(parent) === '/dev' &&
                        rdev !== undefined &&
                        classify(error) === 'ENOENT') {
                        return this.nodes.makeNode(parent, name, S_IFCHR | 0o666, rdev);
                    }
                    throw error;
                }
                return this.placeEntry(parent, name, stat);
            });
        }
        return found;
    }
    mknod(parent, name, mode, rdev) {
        // A character device is not mount content. pyodide makes one of its own
        // at runtime -- `API.capture_stderr` calls FS.createDevice, which lands
        // here as mknod -- and journaling it queued a write of /dev/capture_stderr
        // against the real mount, which then failed on replay. Devices live in
        // this tree only.
        if (isCharDevice(mode)) {
            const device = this.nodes.makeNode(parent, name, mode);
            device.rdev = rdev;
            return device;
        }
        const path = this.nodes.pathOf(parent) + '/' + name;
        if (this.host.isDir(mode)) {
            this.journal.markMkdir(path);
            this.settle(path);
        }
        // The create is what carries a file that is made and never written
        // (`Path.touch()`, `open(p,'w').close()`) through to the mount. The
        // open FS.open makes next takes it back and creates the file itself,
        // one open with O_CREAT as on every other entry point; a mknod no open
        // follows lands it with the next call, or at the run's end.
        else
            this.journal.markCreate(path);
        const node = this.nodes.makeNode(parent, name, mode & ~UMASK);
        node.rdev = rdev;
        // FS.open finalizes a new file with a chmod of its own, right here
        // and on this node. Only a file is marked: a directory gets no such
        // call, so a marker left on one would swallow the guest's next chmod
        // instead.
        if (!this.host.isDir(mode))
            this.fresh = node;
        return node;
    }
    rename(node, newDir, newName) {
        const from = this.nodes.pathOf(node);
        const to = this.nodes.pathOf(newDir) + '/' + newName;
        // A nested mirage mount is served through this same mountpoint, so
        // Emscripten's kernel cannot see the boundary; refuse the crossing
        // here, before the tree moves and the journal records a rename the
        // post-run replay would reject anyway.
        if (this.mountOf(from) !== this.mountOf(to)) {
            throw errnoError(this.host, this.errno, 'EXDEV');
        }
        const found = this.nodes.childOf(newDir, newName);
        const replaced = found !== node ? found : undefined;
        this.journal.markRename(from, to);
        this.settle(from, to);
        if (replaced !== undefined)
            this.orphan(replaced);
        this.nodes.move(node, newDir, newName);
    }
    unlink(parent, name) {
        const path = this.nodes.pathOf(parent) + '/' + name;
        const node = this.nodes.childOf(parent, name);
        this.journal.markUnlink(path);
        this.settle(path);
        if (node !== undefined)
            this.orphan(node);
        this.nodes.detach(parent, name);
    }
    /**
     * A file whose name the mount removed or replaced. One a stream has open
     * reads and writes on through the core, which keeps its bytes; marked
     * only once the mount took the change, so a refused removal leaves it
     * named.
     *
     * Args:
     *   node: the file that lost its name.
     */
    orphan(node) {
        if ((node.opens ?? 0) > 0)
            node.unlinked = true;
    }
    rmdir(parent, name) {
        if (this.readdir(this.lookup(parent, name)).length > 2) {
            throw errnoError(this.host, this.errno, 'ENOTEMPTY');
        }
        const path = this.nodes.pathOf(parent) + '/' + name;
        this.journal.markRmdir(path);
        this.settle(path);
        this.nodes.detach(parent, name);
    }
    /**
     * The directory's names, asking the mount for any it has gained.
     *
     * The first listing is classified, since Emscripten's getdents reads
     * every name's kind off its node. A node is never re-asked about once
     * placed, so a later listing asks for names alone and stats only the
     * names the tree lacks, rather than classifying every entry again.
     */
    readdir(node) {
        const sync = this.sync;
        this.readThrough([], () => {
            const dir = this.nodes.pathOf(node) + '/';
            const again = node.listed === true;
            for (const entry of sync.readdir(dir, !again)) {
                const name = entryName(entry.path);
                if (this.nodes.childOf(node, name) !== undefined)
                    continue;
                this.placeEntry(node, name, again ? ownRow(sync, dir + name, entry) : entry);
            }
            node.listed = true;
        });
        return ['.', '..', ...this.nodes.childNames(node)];
    }
    invalidate() {
        this.nodes.invalidate();
    }
    /**
     * Send the journal to the mount now, in guest order. An entry the mount
     * refuses fails the call that touched its path with the mount's errno,
     * this call or that file's next one (its close), as a deferred write
     * error does; until then nothing more for that path reaches the mount,
     * and every other file's entries still land. Without a worker there is
     * no mount to send to yet, and the journal waits for the run's end.
     *
     * Args:
     *   paths: the paths this call touched.
     */
    settle(...paths) {
        const sync = this.sync;
        const sendable = (m) => !this.deferred.has(m.path) && !(m.kind === 'rename' && this.deferred.has(m.dst));
        let pending = this.journal.takeMutations().filter(sendable);
        try {
            while (pending.length > 0) {
                const failure = sync.flush(pending);
                if (failure === undefined)
                    break;
                const rest = pending.length - failure.skipped;
                const failed = pending[rest - 1];
                if (failed === undefined)
                    break;
                this.deferred.set(failed.path, failure);
                pending = pending.slice(rest).filter(sendable);
            }
        }
        catch (error) {
            throw errnoError(this.host, this.errno, classify(error) ?? 'EIO');
        }
        for (const path of paths) {
            const failure = this.deferred.get(path);
            if (failure === undefined)
                continue;
            this.deferred.delete(path);
            throw errnoError(this.host, this.errno, failure.code ?? 'EIO');
        }
    }
    readThrough(paths, read) {
        this.settle(...paths);
        try {
            return read();
        }
        catch (error) {
            throw errnoError(this.host, this.errno, classify(error) ?? 'EIO');
        }
    }
    placeEntry(parent, name, stat) {
        const target = stat.isLink
            ? this.sync.readlink(this.nodes.pathOf(parent) + '/' + name)
            : undefined;
        const mode = stat.isLink ? LINK_MODE : (stat.mode ?? (stat.isDir ? DIR_MODE : FILE_MODE));
        const node = this.nodes.makeNode(parent, name, mode, stat.rdev ?? 0);
        if ('source' in stat)
            node.source = stat.source;
        if (stat.mtimeMs !== undefined)
            node.atime = node.mtime = node.ctime = stat.mtimeMs;
        if (stat.atimeMs !== undefined)
            node.atime = stat.atimeMs;
        takeOwner(node, stat);
        if (target !== undefined)
            node.link = target;
        // Emscripten's getdents looks up every name it lists, so a row the
        // file adapter could not classify still needs a node; it goes in as a
        // regular file and is asked about before its first stat.
        if (isUnclassified(stat))
            node.unclassified = true;
        return node;
    }
    /**
     * Restamp a node placed from an unclassified row with what the mount
     * said it is.
     *
     * The listing that placed it swallowed a failed stat so the rest of
     * the directory could list; the guest's own stat is where that
     * failure belongs, so that stat asks the mount rather than answering
     * from the guess. Its answer turns the node into a directory if that
     * is what it is.
     *
     * Args:
     *   node: the node to classify.
     *   stat: the row the guest's stat got.
     */
    classifyNode(node, stat) {
        node.unclassified = false;
        node.source = stat.source;
        this.nodes.retype(node, stat.mode);
        node.rdev = stat.rdev ?? 0;
        node.atime = node.mtime = node.ctime = stat.mtimeMs ?? 0;
        if (stat.atimeMs !== undefined)
            node.atime = stat.atimeMs;
        takeOwner(node, stat);
    }
    /**
     * Create a symlink node and record it for the mounts.
     *
     * A link is namespace state, which is exactly why this can be served:
     * the op reaches the node table rather than a backend, so a link lands
     * on an s3 or notion mount whose store has no such thing. The target
     * is stored as typed and never resolved here.
     *
     * Args:
     *   parent: directory the link is created in.
     *   name: the link's own name.
     *   target: what it points at, verbatim.
     */
    symlink(parent, name, target) {
        const path = this.nodes.pathOf(parent) + '/' + name;
        this.journal.markSymlink(path, target);
        this.settle(path);
        const node = this.nodes.makeNode(parent, name, LINK_MODE);
        node.link = target;
        return node;
    }
    /**
     * The target of a symlink node.
     *
     * Args:
     *   node: the node to read.
     */
    readlink(node) {
        if (!this.host.isLink(node.mode))
            throw errnoError(this.host, this.errno, 'EINVAL');
        return node.link ?? '';
    }
    /**
     * Open a file's stream on the run's core, which judges, holds and lands
     * it as it does every runtime's open files. Emscripten made the open's
     * create and truncate before it got here; what is left is how the
     * stream reads and writes.
     */
    streamOpen(stream) {
        const node = stream.node;
        const path = this.nodes.pathOf(node);
        stream.opened = path;
        if (this.host.isFile(node.mode)) {
            const creating = this.journal.takeCreate(path);
            try {
                this.handleOf(stream, creating);
            }
            catch (error) {
                // A create the mount refused leaves no file behind.
                if (creating)
                    this.nodes.detach(node.parent, node.name);
                throw error;
            }
        }
        node.opens = (node.opens ?? 0) + 1;
    }
    /**
     * The core handle a file's stream reads and writes through. One a run's
     * end closed opens again, as long as its file still has a name: the
     * core held the file for that run only. A guest that exited
     * (`os._exit`, caught) has no files left. `create` makes the file the
     * open's own mknod named.
     */
    handleOf(stream, create = false) {
        if (this.journal.sealed)
            throw errnoError(this.host, this.errno, 'EBADF');
        if (stream.fh !== undefined)
            return stream.fh;
        if (stream.node.unlinked === true)
            throw errnoError(this.host, this.errno, 'EBADF');
        const path = this.nodes.pathOf(stream.node);
        const mode = { ...openMode(stream.flags, FLAGS), create };
        const fh = this.readThrough([path], () => this.sync.open(path, mode));
        stream.fh = fh;
        this.streams.add(stream);
        return fh;
    }
    /** Land what the stream wrote and let its handle go. */
    streamClose(stream) {
        const node = stream.node;
        node.opens = Math.max(0, (node.opens ?? 1) - 1);
        const fh = stream.fh;
        this.streams.delete(stream);
        delete stream.fh;
        try {
            if (node.unlinked === true)
                this.settle();
            else
                this.settle(this.nodes.pathOf(node));
        }
        finally {
            if (fh !== undefined)
                this.readThrough([], () => {
                    this.sync.close(fh);
                });
        }
    }
    streamRead(stream, buffer, offset, length, position) {
        if (length === 0)
            return 0;
        const node = stream.node;
        if (isCharDevice(node.mode)) {
            if (EOF_ON_READ_RDEV.has(node.rdev))
                return 0;
            buffer.fill(0, offset, offset + length);
            return length;
        }
        const fh = this.handleOf(stream);
        const out = this.readThrough([], () => this.sync.pread(fh, position, length)).subarray(0, length);
        buffer.set(out, offset);
        return out.length;
    }
    streamWrite(stream, buffer, offset, length, position) {
        if (length === 0)
            return 0;
        if (isCharDevice(stream.node.mode))
            return length;
        const fh = this.handleOf(stream);
        // A copy: the guest's buffer is a view into WASM memory, and a message
        // carries a view's whole buffer.
        const bytes = buffer.slice(offset, offset + length);
        this.readThrough([], () => {
            this.sync.pwrite(fh, position, bytes);
        });
        return length;
    }
    llseek(stream, offset, whence) {
        let position = offset;
        if (whence === SEEK_CUR)
            position += stream.position;
        else if (whence === SEEK_END && this.host.isFile(stream.node.mode)) {
            const fh = this.handleOf(stream);
            position += this.readThrough([], () => this.sync.fstat(fh)).size;
        }
        if (position < 0)
            throw errnoError(this.host, this.errno, 'EINVAL');
        return position;
    }
}
//# sourceMappingURL=fs.js.map