@struktoai/mirage-core/dist/core/mongodb/stream.js

A Unified Virtual Filesystem For AI Agents

// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { stripSlash } from "../../utils/slash.js";
import { EJSON } from 'bson';
import { PathSpec } from "../../types.js";
import { countDocuments, findDocuments, findSnapshot, iterDocuments, iterInserts, } from "./client.js";
import { perKind } from "../hierarchy/read.js";
import { entityGuard } from "./readdir.js";
import { detectScope, entityKind } from "./scope.js";
import { EntityKind, PRIMARY_KEY } from "./types.js";
import { efbig, enoent } from "../../errors/fs.js";
export function applyElision(value, paths) {
    const grouped = new Map();
    const leaves = new Set();
    for (const p of paths) {
        const dot = p.indexOf('.');
        if (dot === -1) {
            leaves.add(p);
        }
        else {
            const head = p.slice(0, dot);
            const tail = p.slice(dot + 1);
            const tails = grouped.get(head) ?? new Set();
            tails.add(tail);
            grouped.set(head, tails);
        }
    }
    const out = {};
    for (const [k, v] of Object.entries(value)) {
        if (leaves.has(k))
            continue;
        const tails = grouped.get(k);
        if (tails !== undefined &&
            typeof v === 'object' &&
            v !== null &&
            !Array.isArray(v) &&
            Object.getPrototypeOf(v) === Object.prototype) {
            out[k] = applyElision(v, tails);
        }
        else {
            out[k] = v;
        }
    }
    return out;
}
export function elisionPaths(accessor, database, name) {
    const key = `${database}.${name}`;
    const fields = accessor.config.elideFields[key] ?? [];
    return new Set(fields);
}
// An Int64 the driver leaves as a Long is past 2^53, where a JS number
// would round it; it is written with its exact digits.
function isLong(value) {
    return (typeof value === 'object' &&
        value !== null &&
        Object.getPrototypeOf(value) !== Object.prototype &&
        value._bsontype === 'Long');
}
// Relaxed Extended JSON writes a $date as ISO-8601 for years 1970 through
// 9999 only; bson's own bound runs five hours into year 10000.
const ISO_DATE_END = 253402300800000;
function relaxedEjson(value) {
    if (value instanceof Date) {
        const ms = value.getTime();
        if (ms >= 0 && ms < ISO_DATE_END)
            return EJSON.serialize(value, { relaxed: true });
        return { $date: { $numberLong: String(ms) } };
    }
    if (isLong(value))
        return value;
    if (Array.isArray(value))
        return value.map(relaxedEjson);
    if (typeof value !== 'object' || value === null)
        return EJSON.serialize(value, { relaxed: true });
    if (Object.getPrototypeOf(value) === Object.prototype) {
        const out = {};
        for (const [k, v] of Object.entries(value))
            out[k] = relaxedEjson(v);
        return out;
    }
    if (value._bsontype === 'DBRef') {
        const ref = value;
        return relaxedEjson({
            $ref: ref.collection,
            $id: ref.oid,
            ...(ref.db ? { $db: ref.db } : {}),
            ...ref.fields,
        });
    }
    return EJSON.serialize(value, { relaxed: true });
}
function pyJsonDumps(value) {
    if (value === null || value === undefined)
        return 'null';
    if (typeof value === 'string')
        return JSON.stringify(value);
    if (typeof value === 'number')
        return Number.isFinite(value) ? String(value) : 'null';
    if (typeof value === 'boolean')
        return value ? 'true' : 'false';
    if (Array.isArray(value))
        return `[${value.map(pyJsonDumps).join(', ')}]`;
    if (typeof value === 'object') {
        if (isLong(value))
            return value.toString();
        const parts = [];
        for (const [k, v] of Object.entries(value)) {
            parts.push(`${JSON.stringify(k)}: ${pyJsonDumps(v)}`);
        }
        return `{${parts.join(', ')}}`;
    }
    return 'null';
}
export function renderDoc(doc) {
    return pyJsonDumps(relaxedEjson(doc));
}
export async function* readStream(accessor, path, options = {}) {
    const ps = typeof path === 'string'
        ? new PathSpec({ vfsPath: stripSlash(path), virtual: path, directory: path })
        : path;
    const scope = detectScope(ps);
    if (scope.kind !== 'documents') {
        throw enoent(ps.virtual);
    }
    // The entity guard is what applies the mount's `databases` filter; this
    // stream is the read_stream op, which a caller reaches without a stat first
    // (a redirect, a runtime's open), so it proves the collection itself
    // rather than trusting the names in the path.
    await entityGuard(accessor, scope, ps.virtual);
    const elide = elisionPaths(accessor, scope.slots.database ?? '', scope.slots.name ?? '');
    const batchSize = options.batchSize ?? 100;
    for await (const doc of iterDocuments(accessor, scope.slots.database ?? '', scope.slots.name ?? '', {
        sort: { [PRIMARY_KEY]: 1 },
        batchSize,
    })) {
        const final = elide.size > 0 ? applyElision(doc, elide) : doc;
        yield new TextEncoder().encode(`${renderDoc(final)}\n`);
    }
}
/**
 * The last `n` documents of a collection or view, as its documents.jsonl
 * renders them. Sorts by primary key descending with a limit instead of
 * streaming the whole collection. `maxDocLimit` is the most documents one such
 * read returns: a count past it that the collection could fill throws EFBIG.
 * Mirrors `_last_documents` in `mirage/core/mongodb/stream.py`.
 */
async function lastDocuments(accessor, scope, path, n) {
    const database = scope.slots.database ?? '';
    const name = scope.slots.name ?? '';
    const docs = await findDocuments(accessor, database, name, {}, { limit: await docLimit(accessor, path, database, name, n), sort: { [PRIMARY_KEY]: -1 } });
    return renderLines(accessor, database, name, docs.reverse());
}
// How many of the last `n` documents one read returns: `n` up to
// `maxDocLimit`, and EFBIG for a count past it that the collection fills.
// Mirrors Python's `_doc_limit`.
async function docLimit(accessor, path, database, name, n) {
    const cap = accessor.config.maxDocLimit;
    if (n > cap && (await countDocuments(accessor, database, name)) > cap)
        throw efbig(path);
    return Math.min(n, cap);
}
// Documents as their documents.jsonl lines, with the mount's hidden fields
// left out. Mirrors Python's `_render_lines`.
function renderLines(accessor, database, name, docs) {
    const elide = elisionPaths(accessor, database, name);
    const lines = docs.map((doc) => `${renderDoc(elide.size > 0 ? applyElision(doc, elide) : doc)}\n`);
    return new TextEncoder().encode(lines.join(''));
}
/**
 * How many documents a collection or view holds, one line each in its
 * documents.jsonl, by `countDocuments`. Mirrors `_document_count` in
 * `mirage/core/mongodb/stream.py`.
 */
function documentCount(accessor, scope, _path) {
    return countDocuments(accessor, scope.slots.database ?? '', scope.slots.name ?? '');
}
/**
 * The last `n` documents of a collection, then each one inserted after them,
 * one documents.jsonl line each. Null for a view, which has no change stream,
 * and for a server outside a replica set, which has none either. The last
 * documents are read at one snapshot and the change stream holds each insert
 * after it, so the two meet at one point: no insert is lost or printed twice.
 * Mirrors `_follow_documents` in `mirage/core/mongodb/stream.py`.
 */
async function followDocuments(accessor, scope, path, n) {
    if (entityKind(scope) !== EntityKind.COLLECTION)
        return null;
    await entityGuard(accessor, scope, path.virtual);
    const database = scope.slots.database ?? '';
    const name = scope.slots.name ?? '';
    const limit = n > 0 ? await docLimit(accessor, path, database, name, n) : 1;
    const snapshot = await findSnapshot(accessor, database, name, {
        limit,
        sort: { [PRIMARY_KEY]: -1 },
    });
    if (snapshot === null)
        return null;
    const [docs, after] = snapshot;
    return followed(accessor, database, name, after, docs.slice(0, n).reverse());
}
// `docs`, then each document inserted after `after`, rendered as lines. An
// iterator rather than a generator: a generator queues `return` behind a pull
// waiting for the next insert, so an idle `tail -f` could never close its
// change stream. Mirrors `_followed` in `mirage/core/mongodb/stream.py`.
function followed(accessor, database, name, after, docs) {
    const first = renderLines(accessor, database, name, docs);
    let pending = first.byteLength > 0 ? first : null;
    let inserts = null;
    let closed = false;
    const done = { value: undefined, done: true };
    return {
        async next() {
            if (pending !== null) {
                const value = pending;
                pending = null;
                return { value, done: false };
            }
            if (closed)
                return done;
            inserts ??= iterInserts(accessor, database, name, after);
            const step = await inserts.next();
            if (step.done === true)
                return done;
            return { value: renderLines(accessor, database, name, [step.value]), done: false };
        },
        async return() {
            closed = true;
            pending = null;
            await inserts?.return?.();
            return done;
        },
        [Symbol.asyncIterator]() {
            return this;
        },
    };
}
export const lastLines = perKind(detectScope, { documents: lastDocuments }, { guard: entityGuard });
export const lineCount = perKind(detectScope, { documents: documentCount }, { guard: entityGuard });
export const follow = perKind(detectScope, { documents: followDocuments });
//# sourceMappingURL=stream.js.map