UNPKG

@struktoai/mirage-core

Version:

A Unified Virtual Filesystem For AI Agents

748 lines • 32.2 kB
import { YieldBudget } from "../../../io/yield_budget.js"; // ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // ========= Copyright 2026 @ Strukto.AI All Rights Reserved. ========= import { isStdin, operandHeader, showsHeaders } from "../utils/stream.js"; import { stdinStream, stdinStat } from "../utils/stream.js"; import { specOf } from "../../spec/builtins.js"; import { FlagView } from "../../spec/flag_view.js"; import { closeQuietly } from "../../../io/stream.js"; import { abortable, hasAborted } from "../../../utils/abort.js"; import { IOResult, materialize } from "../../../io/types.js"; import { FileType, } from "../../../types.js"; import { enoent, fsStrerror, isEisdir, isFsError } from "../../../errors/fs.js"; import { READ_FAILURES } from "../../../errors/constants.js"; import { shellQuote } from "../../../utils/quote.js"; import { reportFsError, splitOpened } from "../utils/operands.js"; import { CHUNK_SIZE, chunks } from "../../../io/cooperative.js"; import { checkMemory } from "../../../io/memory.js"; import { concat } from "../../../utils/bytes.js"; import { encodeText } from "../../../shell/bytes.js"; import { posixPhrase } from "../../../errors/posix.js"; import { revisionFor } from "../../../observe/context.js"; import { dispatchOf } from "../utils/links.js"; import { StreamRefusal } from "../../../rs/errors.js"; import { refused, scan } from "../../../rs/scan.js"; /** * Fill in whatever a caller left off a `TailCounts`. * * The struct is a plain object, so a caller that omits a field — or a * JavaScript one that passes something else entirely — supplies undefined, * and a bare `!== null` test waves that straight into the first branch, * where `slice(NaN)` quietly returns the whole input. Python's twin is a * dataclass whose four fields all default to None and has no such hole; * reading every field through `?? null` gives this side the same floor. */ export function normalizeCounts(counts) { return { lines: counts.lines ?? null, fromLine: counts.fromLine ?? null, byteCount: counts.byteCount ?? null, fromByte: counts.fromByte ?? null, }; } // -f, --follow[=HOW] and -F as (follow, by name, retry), `how` the --follow // word mirage-rs read, if typed. -F is --follow=name --retry. The mode is // whichever of -f/--follow and -F came last, GNU's own order // (`-F --follow=descriptor` follows the descriptor), while -F's --retry half // stays on either way. function followFlags(fl, how) { const raw = fl.raw('follow'); const typed = fl .typedOrder('follow', 'F') .filter((k) => k === 'F' ? fl.asBool('F') : raw !== undefined && raw !== null && raw !== false); const follow = typed.length > 0; const byName = follow && (typed[typed.length - 1] === 'F' || how === 'name'); return [follow, byName, fl.asBool('retry') || typed.includes('F')]; } // A count mirage-rs read, `{count, from_start}` or null, as (from the end, // from the start). function countsOf(value) { if (value === null) return [null, null]; if (typeof value !== 'object' || Array.isArray(value) || typeof value.count !== 'number') { throw new Error(`tail: mirage-rs read a count as ${JSON.stringify(value)}`); } return value.from_start === true ? [null, value.count] : [value.count, null]; } // Append a follow-time diagnostic to the result's stderr. A following // tail's stdout is a stream the caller drains as the file grows, and its // stderr is the bytes on the result, which the caller reads once the // stream ends; a notice raised mid-follow lands there. function note(io, message) { const prior = io.stderr instanceof Uint8Array ? io.stderr : new Uint8Array(); io.stderr = concat([prior, encodeText(message)]); } // Sleep for the poll interval, or until the caller's signal fires. async function pause(seconds, signal) { if (hasAborted(signal)) return; await new Promise((resolve) => { // An infinite interval waits on the abort signal alone, never on a // timer. `setTimeout` holds a 32-bit signed delay, so Node warns // (`TimeoutOverflowWarning`) and clamps `Infinity` to 1ms, which // would poll the backend continuously where python's // `asyncio.sleep(inf)` waits. GNU accepts `-s inf` (measured, // coreutils 9.4), so the acceptance is right and only the wait was // wrong. With no signal there is nothing to wake on, which is the // indefinite wait python performs. if (!Number.isFinite(seconds)) { signal?.addEventListener('abort', () => { resolve(); }, { once: true }); return; } const onAbort = () => { clearTimeout(timer); resolve(); }; const timer = setTimeout(() => { signal?.removeEventListener('abort', onAbort); resolve(); }, seconds * 1000); signal?.addEventListener('abort', onAbort, { once: true }); }); } // What a followed file gained past `pos`, and where the next poll // starts: a size-unknown file is read whole and measured, and a file // shorter than `pos` was truncated, which is noted and read from the // start. async function catchUp(stream, readRange, io, p, size, pos) { let whole = null; let length = size; if (length === null) { whole = await materialize(stream(p)); length = whole.byteLength; } let start = pos; if (length < start) { note(io, `tail: ${p.rawPath}: file truncated\n`); start = 0; } if (length <= start) return [new Uint8Array(), start]; const data = whole !== null ? whole.slice(start) : await window(stream, readRange, p, start, length - start); return [data, start + data.byteLength]; } /** * A follow poll's stat: what the backend holds now, at the dispatcher, past the * mount's index and past the stat the command's freshness probe kept: the poll * looks for exactly the change neither has yet. Mirrors Python's `_polled_stat`. */ async function polledStat(dispatch, p) { const [found] = await dispatch('stat', p, [], { fresh: true }); if (found === null || found === undefined) throw enoent(p); return found; } /** * A follow poll's read, whole or a window: what the backend holds now, at the * dispatcher. A direct read, never served from the file cache nor kept in it: * the poll looks for exactly the bytes a cached copy does not have yet. Mirrors * Python's `_polled_window`. */ async function* polledWindow(dispatch, p, offset = 0, size = null) { const [data] = await dispatch('read', p, [], { filetype: null, offset, size, direct: true }); yield await materialize(data); } async function* tailRange(read, path, offset, end) { // A range-capable backend may return bytes, so bound the request too. while (offset < end) { const size = Math.min(1 << 20, end - offset); yield* chunks(read(path, offset, size)); offset += size; } } async function* tailFile(path, read, stat, readRange, lastLines, counts) { const n = counts.lines ?? 10; const fromEnd = counts.fromLine === null && counts.fromByte === null; if (!isStdin(path) && fromEnd) { if (counts.byteCount === null && lastLines !== undefined) { const answer = await lastLines(path, Math.abs(n)); if (answer !== null) { yield* chunks(answer); return; } } if (readRange !== null && revisionFor(path.virtual) === null) { const info = await stat(path); if (info.type === FileType.FILE && info.size !== null && info.size > 0) { const size = info.size; if (counts.byteCount !== null) { const offset = Math.max(0, size - Math.abs(counts.byteCount)); yield* tailRange(readRange, path, offset, size); return; } let end = size; let wanted = Math.abs(n); let window = CHUNK_SIZE; const budget = new YieldBudget(); while (end > 0) { await budget.run(); const start = Math.max(0, end - window); const data = await materialize(readRange(path, start, end - start)); if (data.byteLength !== end - start) break; if (end === size && data.at(-1) === 10) wanted++; let found = 0; for (let at = data.indexOf(10); at >= 0; at = data.indexOf(10, at + 1)) found++; if (found < wanted && start > 0) { wanted -= found; end = start; window = Math.min(window * 2, 1 << 20); continue; } let at = data.length; while (wanted > 0) { at = at === 0 ? -1 : data.lastIndexOf(10, at - 1); if (at < 0) break; wanted--; } if (wanted === 0 || start === 0) { const piece = wanted === 0 ? data.subarray(at + 1) : data; yield* chunks(piece); yield* tailRange(readRange, path, end, size); return; } end = start; window = Math.min(window * 2, 1 << 20); } } } } yield* tail(read(path), counts); } // The tail of one source, read as it streams. Mirrors Python's `tail`. async function* tail(source, rawCounts) { const counts = normalizeCounts(rawCounts); if (counts.fromByte !== null || counts.fromLine !== null) { let skipBytes = counts.fromByte === null ? 0 : Math.max(0, counts.fromByte - 1); let skipLines = counts.fromLine === null ? 0 : Math.max(0, counts.fromLine - 1); for await (const part of chunks(source)) { let start = Math.min(skipBytes, part.length); skipBytes -= start; while (skipLines > 0 && start < part.length) { const next = part.indexOf(10, start); if (next === -1) { start = part.length; break; } start = next + 1; skipLines--; } if (start < part.length && skipLines === 0) yield part.subarray(start); } return; } if (counts.byteCount !== null) { const count = Math.abs(counts.byteCount); if (count === 0) return; const kept = []; let held = 0; for await (const part of chunks(source)) { kept.push(part.slice(-count)); held += Math.min(part.length, count); while (kept[0] !== undefined && held - kept[0].length >= count) { held -= kept[0].length; kept.shift(); } } if (held > 0) yield concat(kept).slice(-count); return; } const count = Math.abs(counts.lines ?? 10); if (count === 0) return; // The bounded chunks that may hold the last lines, each with its // newlines: the first goes once the others hold a newline more than the // count, which covers a last line that lacks its own. const ends = []; let next = 0; let held = 0; let size = 0; for await (const part of chunks(source)) { if (part.length === 0) continue; let newlines = 0; for (let at = part.indexOf(10); at >= 0; at = part.indexOf(10, at + 1)) newlines++; ends.push([part, newlines]); held += newlines; size += part.length; for (let first = ends[next]; first !== undefined && held - first[1] > count; first = ends[next]) { held -= first[1]; size -= first[0].length; next++; } checkMemory(size); if (next * 2 > ends.length) { ends.splice(0, next); next = 0; } } const last = ends.at(-1); if (last === undefined || next >= ends.length) return; let wanted = count + (last[0].at(-1) === 10 ? 1 : 0); const pieces = []; for (const [part, newlines] of ends.slice(next).reverse()) { if (newlines < wanted) { wanted -= newlines; pieces.push(part); continue; } let at = part.length; for (; wanted > 0; wanted--) at = part.lastIndexOf(10, at - 1); pieces.push(part.subarray(at + 1)); break; } yield concat(pieces.reverse()); } // Each opened operand's tail under its header. An unread operand (a // directory) prints only its header; a read that fails once the stat passed // is reported and the next operand still prints. Mirrors Python's // `tail_multi`. async function* tailMulti(paths, read, counts, showHeaders, unread, io, selected = false) { let printed = 0; for (const p of paths) { let pending = showHeaders ? operandHeader(p, printed > 0) : null; if (unread.has(p.virtual)) { if (pending !== null) { yield pending; printed++; } continue; } const source = read(p); try { for await (const part of selected ? chunks(source) : tail(source, counts)) { if (pending !== null) { yield pending; pending = null; } yield part; } if (pending !== null) yield pending; printed++; } catch (err) { if (!isFsError(err)) throw err; const code = err.code; await reportFsError(io, 'tail', p, err); if (pending !== null && code !== undefined && READ_FAILURES.has(code)) { yield pending; printed++; } } finally { await closeQuietly(source); } } } async function window(stream, readRange, p, offset, size) { if (readRange !== null) return await materialize(readRange(p, offset, size)); const whole = await materialize(stream(p)); return whole.slice(offset, offset + size); } // Print each operand's tail, then keep printing what it gains. GNU tail // -f is a poll: every -s seconds each followed file is stat'ed, bytes // past the last position are printed under that file's header when the // previous output was another file's, and a size that shrank is `file // truncated` and a restart from the top. An operand whose first read // fails after its stat passed is a failed open, reported as one and, // under --retry, waited for like a file that was never there. A file that // goes away later is dropped with `has become inaccessible` under // --follow=name; --retry // keeps polling for it (and for one that was never there) and announces // `has appeared` when it turns up, reading it from the start as GNU does // after a rotation. The loop ends only when nothing is left to follow // (`no files remaining`, exit 1) or the caller's signal fires, which is // how `timeout` and a killed job end it. A file whose stat carries no // size (a backend that cannot know one without reading reports null, // never a guess) is polled by reading it whole every interval and // measuring that: one read per poll, rather than a follow that never // prints. State is per operand, not per path: `tail -f f f` prints what // f gains twice, under a header each time, as GNU does. Standard input is // printed once and never followed: POSIX has tail ignore -f on a pipe, and // GNU extends that to every `-` operand; a line that named nothing else has // nothing left to wait for, so it ends there, without `no files remaining`. // An unread operand (a directory) prints its header and is not followed. // One file followed by descriptor, under a count of lines from the end, is // asked of the mount's `follow` first: a mount that is told about new lines // (a change stream), or can be asked for only the new ones every -s seconds, // hands over the last lines and then each new one as it lands, so the file is // not read again on each poll and the two meet at one point. async function* follow(paths, pending, stream, pollStream, stat, readRange, counts, showHeaders, flags, io, signal, unread, onlyStdin, followLines = null, followOp) { const positions = new Map(); let added = null; const active = paths.map((p, i) => [i, p]); const waiting = pending.map(([p, how], i) => [ paths.length + i, p, how, ]); let last = null; try { for (const entry of [...active]) { const [slot, p] = entry; if (unread.has(p.virtual)) { if (showHeaders) yield operandHeader(p, last !== null); last = slot; active.splice(active.indexOf(entry), 1); continue; } let raw = null; try { if (followOp !== undefined && followLines !== null && paths.length === 1 && pending.length === 0 && !flags.byName && !isStdin(p)) { added = await followOp(p, followLines, flags.interval); } // A follow's stream holds the first print too. if (added === null) raw = await materialize(stream(p)); } catch (err) { await closeQuietly(added); added = null; if (!isFsError(err)) throw err; // There is no handle to hold, so this first read is the open: one // that fails after the operand's stat passed is GNU's failed open, // reported the way the stat's failure would have been, and under // --retry waited for like a file that was never there (this is the // initial open that a descriptor follow's --retry covers). await reportFsError(io, 'tail', p, err); active.splice(active.indexOf(entry), 1); if (flags.retry) waiting.push([slot, p, APPEARED]); continue; } if (showHeaders) yield operandHeader(p, last !== null); last = slot; if (raw !== null) { yield* tail(raw, counts); positions.set(slot, raw.byteLength); } } if (added !== null) { // A stream waits for the next line, so an abort ends the wait rather // than the next line. const lines = added[Symbol.asyncIterator](); try { for (;;) { const next = await abortable(lines.next(), signal); if (next.done === true) break; yield next.value; } } catch (err) { if (hasAborted(signal)) return; throw err; } note(io, 'tail: no files remaining\n'); io.exitCode = 1; return; } } finally { // The stream follow opened holds the mount busy, so it closes however // the print ends, a reader that stops early included; it may be // waiting for the next line, so the close is not waited on. if (added !== null) void closeQuietly(added); } for (const entry of active.filter(([, p]) => isStdin(p))) active.splice(active.indexOf(entry), 1); if (onlyStdin) return; while ((active.length > 0 || waiting.length > 0) && !hasAborted(signal)) { await pause(flags.interval, signal); if (hasAborted(signal)) return; for (const entry of [...waiting]) { const [slot, p, how] = entry; let found; try { found = await stat(p); } catch (err) { if (!isFsError(err)) throw err; // The name is gone now, so whatever stands there next has // appeared, whatever stood there before. if (how !== APPEARED) waiting.splice(waiting.indexOf(entry), 1, [slot, p, APPEARED]); continue; } if (found.type === FileType.DIRECTORY) continue; note(io, `tail: '${p.rawPath}' ${how}\n`); waiting.splice(waiting.indexOf(entry), 1); active.push([slot, p]); positions.set(slot, 0); } for (const entry of [...active]) { const [slot, p] = entry; let grown; try { const current = await stat(p); grown = current.type === FileType.DIRECTORY ? null : await catchUp(pollStream, readRange, io, p, current.size, positions.get(slot) ?? 0); } catch (err) { if (!isFsError(err)) throw err; if (!isEisdir(err)) { // A path that went away, whether its stat failed or the read // right after it did (a rotation between the two). Only // name-following notices; under a descriptor --retry covers // the initial open alone, as in GNU, which words the loss as an // inaccessible name only when it means to wait for it. if (flags.byName) { const strerror = fsStrerror(err) ?? posixPhrase('ENOENT'); note(io, flags.retry ? `tail: '${p.rawPath}' has become inaccessible: ${strerror}\n` : `tail: ${shellQuote(p.rawPath)}: ${strerror}\n`); active.splice(active.indexOf(entry), 1); if (flags.retry) waiting.push([slot, p, APPEARED]); } continue; } grown = null; } if (grown === null) { // A directory replaced the file: name-following gives the name // up, or under --retry waits for a file to stand there again; a // descriptor follow prints nothing, as GNU's does while it holds // the old descriptor. if (!flags.byName) continue; const line = `tail: '${p.rawPath}' has been replaced with an untailable file`; note(io, line + (flags.retry ? '\n' : '; giving up on this name\n')); active.splice(active.indexOf(entry), 1); if (flags.retry) waiting.push([slot, p, ACCESSIBLE]); continue; } const [data, pos] = grown; if (data.byteLength > 0) { if (showHeaders && last !== slot) yield operandHeader(p, true); last = slot; yield data; } positions.set(slot, pos); } } if (hasAborted(signal)) return; note(io, 'tail: no files remaining\n'); io.exitCode = 1; } // Sort the operands that did not open into the ones --retry waits for and // the ones tail gives up on, wording the latter. A directory cannot be // followed. Without --retry that is `giving up on this name`; with it // GNU drops the suffix, and under --follow=name keeps polling the name // until something tailable replaces it, announced as `has become // accessible` rather than the `has appeared` a missing file gets. async function unfollowable(paths, opened, stat, flags, io) { const pending = []; for (const p of paths) { if (opened.has(p.virtual)) continue; let isDir; try { isDir = (await stat(p)).type === FileType.DIRECTORY; } catch (err) { if (!isFsError(err)) throw err; if (!isEisdir(err)) { if (flags.retry) pending.push([p, APPEARED]); continue; } isDir = true; } if (!isDir) continue; const line = `tail: ${p.rawPath}: cannot follow end of this type of file`; if (!flags.retry) { note(io, `${line}; giving up on this name\n`); continue; } note(io, `${line}\n`); if (flags.byName) pending.push([p, ACCESSIBLE]); } return pending; } const RETRY_IGNORED = 'tail: warning: --retry ignored; --retry is useful only when following\n'; const RETRY_INITIAL = 'tail: warning: --retry only effective for the initial open\n'; // A name is what -F follows, and standard input has none. const STDIN_BY_NAME = "tail: cannot follow '-' by name\n"; // What a waited-for name is announced as when it turns up: a missing file // has appeared, an untailable one (a directory) has become accessible. const APPEARED = 'has appeared; following new file'; const ACCESSIBLE = 'has become accessible'; /** * tail's options, the values read by mirage-rs before any operand is resolved: * the flags, or tail's refusal of a value. Mirrors Python's `parse_flags`. */ export function parseFlags(bag) { const fl = new FlagView(bag, specOf('tail')); const raw = fl.raw('follow'); const typed = fl.texts('n', 'c', 'sleep_interval'); if (typeof raw === 'string') typed.push(['follow', raw]); let values; try { values = scan('tail', typed); } catch (err) { if (!(err instanceof StreamRefusal)) throw err; return refused(err); } const how = values.follow; if (how !== null && typeof how !== 'string') { throw new Error(`tail: mirage-rs read --follow as ${JSON.stringify(how)}`); } const [follow, byName, retry] = followFlags(fl, how); // The last of -q and -v decides, as in GNU tail, and so does the last of // -n and -c. const headers = fl.typedOrder('q', 'v').at(-1); const unit = fl.typedOrder('n', 'c').at(-1); const [lines, fromLine] = countsOf(unit === 'c' ? null : values.lines); const [byteCount, fromByte] = countsOf(unit === 'n' ? null : values.bytes); // mirage-rs answers -s, typed or GNU's default of 1, as null when it is // infinite. const interval = values.sleep_interval; if (interval !== null && typeof interval !== 'number') { throw new Error(`tail: mirage-rs read -s as ${JSON.stringify(interval)}`); } return { counts: { lines, fromLine, byteCount, fromByte }, quiet: headers === 'q', verbose: headers === 'v', following: { follow, byName, retry, interval: interval ?? Number.POSITIVE_INFINITY, }, }; } /** * Run tail over resolved operands, GNU semantics. A line count from the end * (`-n N`, or the default 10) is asked of `lastLines` first, so a mount that * can fetch the end of a file on its own does not send the rest; `-f` on one * file asks `followOp(p, n, interval)` (see `follow`). Null, or no such op, reads * or polls the file. Mirrors Python's `tail_generic`; `parsed` holds the * options, read before the operands were. */ export async function tailGeneric(parsed, paths, texts, ctx, stream, stat, readRange = null, lastLines, followOp) { stat = stdinStat(stat); const backend = stream; stream = stdinStream(stream, ctx.stdin); const { counts, quiet: qFlag, verbose: vFlag, following } = parsed; const n = counts.lines ?? 10; const fromEnd = counts.byteCount === null && counts.fromLine === null && counts.fromByte === null; if (following.follow && following.byName && (paths.length === 0 || paths.some((p) => isStdin(p)))) { return [null, new IOResult({ exitCode: 1, stderr: encodeText(STDIN_BY_NAME) })]; } // GNU warns first, then tails as if --retry were not there; a descriptor // it follows only after the initial open says so too. const retryWarning = following.retry && !following.follow ? RETRY_IGNORED : following.retry && !following.byName ? RETRY_INITIAL : ''; // GNU opens nothing when a tail from the end keeps no line or byte. if (!following.follow && counts.fromLine === null && counts.fromByte === null && (counts.byteCount ?? n) === 0) { return [null, new IOResult({ stderr: retryWarning === '' ? null : encodeText(retryWarning) })]; } if (paths.length > 0) { const showHeaders = showsHeaders(vFlag, qFlag, paths.length); const [opened, unread, err] = await splitOpened(paths, stat, 'tail'); const io = new IOResult({ exitCode: err === '' ? 0 : 1, stderr: retryWarning + err === '' ? null : encodeText(retryWarning + err), }); if (!following.follow) { if (opened.length === 0) return [null, io]; return [ tailMulti(opened, (p) => tailFile(p, stream, stat, readRange, lastLines, counts), counts, showHeaders, unread, io, true), io, ]; } const readable = opened.filter((p) => !unread.has(p.virtual)); const pending = await unfollowable(paths, new Set(readable.map((p) => p.virtual)), stat, following, io); if (readable.length === 0 && pending.length === 0 && !(showHeaders && opened.length > 0)) { note(io, 'tail: no files remaining\n'); io.exitCode = 1; return [null, io]; } // The first print reads like any other tail; the polls ask the dispatcher // for what the backend holds now, past the file cache and past the // stat the command's freshness probe kept: they look for exactly the // change neither has yet. const first = stdinStream(backend, ctx.stdin); const dispatch = dispatchOf(ctx); const polled = dispatch === undefined ? { stream: first, stat, readRange } : { stream: (p) => polledWindow(dispatch, p), stat: (p) => polledStat(dispatch, p), readRange: (p, offset, size) => polledWindow(dispatch, p, offset, size), }; return [ follow(opened, pending, first, polled.stream, polled.stat, polled.readRange, counts, showHeaders, following, io, ctx.signal, unread, paths.every((p) => isStdin(p)), fromEnd ? n : null, followOp), io, ]; } async function* output() { if (vFlag && !qFlag) yield operandHeader(null); yield* tail(ctx.stdin ?? new Uint8Array(), counts); } return [output(), new IOResult({ stderr: retryWarning === '' ? null : encodeText(retryWarning) })]; } //# sourceMappingURL=tail.js.map