@struktoai/mirage-core
Version:
A Unified Virtual Filesystem For AI Agents
507 lines • 18.9 kB
JavaScript
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
import { concat, findBytes } from "../utils/bytes.js";
import { abortable } from "../utils/abort.js";
import { YieldBudget } from "./yield_budget.js";
import { chunks } from "./cooperative.js";
import { MemoryExhausted } from "./errors.js";
import { checkMemory, memoryRoom } from "./memory.js";
import { DeviceInput, PipeStream } from "./types.js";
/**
* The read end an `exec` keeps on a live pipe (`exec 3<"$1"`). It holds the
* pipe, not the name it was opened by. Each read takes a stream of its own
* under the reading command's signal and closes it after one chunk, so a read
* that gives up (`timeout`) leaves the rest to the next read. Mirrors
* Python's PipeInput.
*/
export class PipeInput {
path;
end;
signal;
/** `path` is the pipe's own (`/dev/fd/N`); `signal` gives each read its reader's. */
constructor(path, end, signal) {
this.path = path;
this.end = end;
this.signal = signal;
}
[Symbol.asyncIterator]() {
return this;
}
async next() {
const iterator = this.end.stream(this.path, this.signal())[Symbol.asyncIterator]();
try {
return await iterator.next();
}
finally {
await iterator.return?.();
}
}
}
export class AsyncLineIterator {
input;
source;
buf = new Uint8Array(0);
pulled = 0;
exhausted = false;
budget = new YieldBudget();
linesSinceCheck = 0;
pulling = false;
started = false;
holders = 0;
released = false;
constructor(input) {
this.input = input;
const s = this.input;
if (this.input instanceof Uint8Array) {
this.source = chunks(this.input);
}
else if (this.input instanceof PipeInput) {
this.source = this.input;
}
else if (typeof s[Symbol.asyncIterator] === 'function') {
this.source = chunks(s);
}
else {
this.source = chunks({
[Symbol.asyncIterator]: () => this.input,
});
}
}
[Symbol.asyncIterator]() {
return this;
}
get position() {
return this.pulled - this.buf.byteLength;
}
unread(data) {
this.buf = concat([data, this.buf]);
}
get loaded() {
return this.pulled;
}
async next() {
const line = await this.readline();
if (line === null)
return { done: true, value: undefined };
return { done: false, value: line };
}
async readline(signal, eol = 0x0a) {
try {
// A buffered line is handed out without a pull, so the signal is
// checked here as well, on both sides of the yield.
signal?.throwIfAborted();
// Amortize clock reads on short-line workloads; chunk pulls also check.
if (++this.linesSinceCheck >= 64) {
this.linesSinceCheck = 0;
const pending = this.budget.run();
if (pending !== undefined) {
await pending;
signal?.throwIfAborted();
}
}
const idx = this.buf.indexOf(eol);
if (idx >= 0) {
const line = this.buf.subarray(0, idx);
this.buf = this.buf.subarray(idx + 1);
return line;
}
}
catch (error) {
await this.failed(error);
throw error;
}
const [line, found] = await this.readDelimited(eol, signal);
return found || line.byteLength > 0 ? line : null;
}
// The stdin buffer survives individual builtins; cancellation belongs to each read.
/** Close the source and drop what it buffered, for input a failed line abandoned. */
discard() {
return this.close();
}
/** Close once no descriptor names this reader, or when the last job still holding it ends. */
async release() {
this.released = true;
if (this.holders === 0)
await this.close();
}
/** End a job's hold, closing a reader its shell already closed. */
async unhold() {
this.holders -= 1;
if (this.holders === 0 && this.released)
await this.close();
}
/**
* Discard this reader for one over `source` that keeps its holds and its
* release, for a descriptor a write moved. Mirrors Python's reopen.
*/
async reopen(source) {
await this.discard();
const lines = new AsyncLineIterator(source);
lines.holders = this.holders;
lines.released = this.released;
return lines;
}
/**
* Read what the source has left into the buffer, for a descriptor that
* outlives its pipe: reads go on from memory. Mirrors Python's load_rest.
*/
async loadRest() {
try {
while (!this.exhausted) {
const pending = this.check();
if (pending !== undefined)
await pending;
const result = await this.pull();
if (result.done === true)
this.exhausted = true;
else {
checkMemory(this.buf.byteLength + result.value.byteLength);
this.buf = concat([this.buf, result.value]);
}
}
}
catch (error) {
await this.failed(error);
throw error;
}
}
/**
* Close the source after a failed read, unless it was a reader's own abort
* on a pipe an `exec` keeps, which stays open with `taken`, what the read
* had taken, put back. Mirrors Python's _failed.
*/
async failed(error, taken = []) {
const aborted = error instanceof DOMException && error.name === 'AbortError';
if (this.input instanceof PipeInput && aborted) {
if (taken.length > 0)
this.buf = concat([...taken, this.buf]);
return;
}
await this.close();
}
check(signal) {
signal?.throwIfAborted();
const pending = this.budget.run();
if (pending !== undefined)
return pending.then(() => signal?.throwIfAborted());
}
async close() {
this.exhausted = true;
this.buf = new Uint8Array(0);
// A return queued behind a pull that never settles would hang the
// abort itself; that one is not awaited.
const closing = this.source.return?.();
if (closing !== undefined) {
if (this.pulling)
void closing.catch(() => undefined);
else
await closing;
}
if (!this.started) {
const original = this.input;
await original.return?.();
}
}
async pull(signal) {
// Left set when the pull fails: close() reads it to know the source
// is still busy with the pull the abort abandoned.
this.pulling = true;
this.started = true;
const result = await abortable(this.source.next(), signal);
this.pulling = false;
if (result.done !== true)
this.pulled += result.value.byteLength;
return result;
}
async readDelimited(delim, signal) {
const parts = [];
let held = 0;
const width = typeof delim === 'number' ? 1 : delim.length;
if (width === 0)
throw new Error('empty separator');
try {
for (;;) {
const pending = this.check(signal);
if (pending !== undefined)
await pending;
const idx = typeof delim === 'number' ? this.buf.indexOf(delim) : findBytes(this.buf, delim);
if (idx >= 0) {
checkMemory(held + idx);
const tail = this.buf.subarray(0, idx);
this.buf = this.buf.subarray(idx + width);
return [parts.length === 0 ? tail : concat([...parts, tail]), true];
}
if (this.exhausted) {
checkMemory(held + this.buf.byteLength);
parts.push(this.buf);
this.buf = new Uint8Array();
return [concat(parts), false];
}
const keep = Math.min(width - 1, this.buf.length);
const cut = this.buf.length - keep;
if (cut > 0)
parts.push(this.buf.subarray(0, cut));
held += cut;
checkMemory(held);
this.buf = this.buf.subarray(cut);
// Raced, not just checked between pulls: a stalled stdin must lose
// to the read's own signal, or the reader outlives the caller.
const result = await this.pull(signal);
if (result.done === true)
this.exhausted = true;
else
this.buf = concat([this.buf, result.value]);
}
}
catch (error) {
await this.failed(error, parts);
throw error;
}
}
/**
* Read up to (not including) `delim`, or to EOF. Returns the bytes and
* whether the delimiter was found (false means EOF, which `read`/
* `mapfile` report as status 1).
*/
async readUntil(delim, signal) {
const [data, found] = await this.readDelimited(delim, signal);
return [new Uint8Array(data), found];
}
/**
* Whether another byte follows, consuming nothing. Pulls the source's next
* chunk only when nothing is buffered, so a line still arriving never holds
* the answer up. Mirrors Python's peek.
*/
async peek(signal) {
try {
while (this.buf.byteLength === 0) {
if (this.exhausted)
return false;
const pending = this.check(signal);
if (pending !== undefined)
await pending;
const result = await this.pull(signal);
if (result.done === true)
this.exhausted = true;
else
this.buf = new Uint8Array(result.value);
}
return true;
}
catch (error) {
await this.failed(error);
throw error;
}
}
/** Hand over what is buffered, else the source's next chunk; null at end of input. */
async readChunk() {
if (this.buf.byteLength > 0) {
const data = this.buf;
this.buf = new Uint8Array(0);
return data;
}
if (this.exhausted)
return null;
try {
const pending = this.check();
if (pending !== undefined)
await pending;
const result = await this.pull();
if (result.done === true) {
this.exhausted = true;
return null;
}
return result.value;
}
catch (error) {
await this.failed(error);
throw error;
}
}
/**
* At most `size` bytes with at most one pull of the source, as a
* read(2) of a pipe answers (Python's `read1`): what has arrived, the
* rest kept for the next read; empty at the end of input.
*/
async read1(size) {
if (size <= 0)
return new Uint8Array(0);
const chunk = await this.readChunk();
if (chunk === null)
return new Uint8Array(0);
if (chunk.byteLength > size)
this.unread(chunk.subarray(size));
return chunk.subarray(0, size);
}
/**
* Read at most `count` characters, stopping early at `delim` (null
* reads through delimiters). `read -n` is the delimited form, `read
* -N` the null one. The delimiter is consumed and not returned.
* Returns the bytes and whether the read ended on its own terms
* rather than EOF.
*
* Characters, not bytes: bash counts them in the shell's locale, so
* `read -n 1` on `éx` assigns `é` and leaves `x`. Counting bytes
* would hand back half a character and leave the other half to
* corrupt the next read.
*/
async readChars(count, delim, signal) {
let out = new Uint8Array(0);
const room = memoryRoom();
try {
let taken = 0;
while (taken < count) {
const pending = this.check(signal);
if (pending !== undefined)
await pending;
// One pull can split a character across chunks, so top the buffer
// up to the widest one before reading its first byte as a whole.
if (this.buf.byteLength < 4 && !this.exhausted) {
const result = await this.pull(signal);
if (result.done === true)
this.exhausted = true;
else
this.buf = concat([this.buf, result.value]);
continue;
}
if (this.buf.byteLength === 0)
return [new Uint8Array(out), false];
if (delim !== null && this.buf[0] === delim) {
this.buf = this.buf.subarray(1);
return [new Uint8Array(out), true];
}
const width = charWidth(this.buf);
if (room !== null && out.byteLength + width > room)
throw new MemoryExhausted();
out = concat([out, this.buf.subarray(0, width)]);
this.buf = this.buf.subarray(width);
taken++;
}
return [new Uint8Array(out), true];
}
catch (error) {
await this.failed(error, [out]);
throw error;
}
}
}
/**
* Standard input that the commands of one group, loop or shell read in
* turn, as bash's all read one open descriptor.
*
* What one command reads the next does not see again: `read` takes its
* line off `lines` and leaves the rest buffered there, and any other
* command iterates this object for that rest, then for what the source
* still holds. It has no `return`, so a command that stops early never
* closes the source a later command may still read; whoever opened the
* source closes it, and a failed line discards it.
*/
export class SharedInput {
seekable;
lines;
/**
* The live pipe this reads while the substitution's command runs; when that
* ends, its scope loads the rest for a descriptor the shell still holds
* (`f() { exec 3<"$1"; }; f <(cmd)`) and clears the name.
*/
pipe;
/** `source` is what the descriptor reads, or the line buffer of the
* descriptor it duplicates, whose live pipe `pipe` names; one built over
* a `PipeStream` takes that stream's. */
constructor(source, pipe = null, seekable = false) {
this.seekable = seekable;
this.lines = source instanceof AsyncLineIterator ? source : new AsyncLineIterator(source);
this.pipe = source instanceof PipeStream || source instanceof PipeInput ? source.path : pipe;
}
[Symbol.asyncIterator]() {
return this;
}
/** Another descriptor on the same open file, as `dup` makes: a read
* through either moves the one offset. */
dup() {
return new SharedInput(this.lines, this.pipe, this.seekable);
}
async next() {
const chunk = await this.lines.readChunk();
if (chunk === null)
return { done: true, value: undefined };
return { done: false, value: chunk };
}
/** Close the source for good, for a line that failed reading it. */
discard() {
return this.lines.discard();
}
}
/**
* The one descriptor a construct hands every command it runs. `<
* /dev/null` stays as it is: it reads nothing, so there is no position
* to share, and its type tells a command no file is attached.
*/
export function share(stdin) {
if (stdin === null || stdin instanceof SharedInput || stdin instanceof DeviceInput)
return stdin;
return new SharedInput(stdin);
}
/**
* The line reader `read`, `mapfile` and `select` take input from: a
* shared descriptor's own, so what they leave the next command reads,
* else one over `stdin` alone.
*/
export function lineBuffer(stdin) {
return stdin instanceof SharedInput ? stdin.lines : new AsyncLineIterator(stdin);
}
/**
* How many bytes `data`'s first character spans, decoded as UTF-8.
*
* Always at least one and never more than what is there, so a caller
* stepping by this never splits a character and never stalls. Bytes that
* decode to one replacement character answer 1, which is what a
* fatal:false TextDecoder makes of them: a stray continuation byte, a
* lead the encoding never uses, and a sequence cut short by a byte that
* cannot continue it.
*/
export function charWidth(data) {
const lead = data[0] ?? 0;
if (lead < 0xc2 || lead >= 0xf5)
return 1;
const width = lead < 0xe0 ? 2 : lead < 0xf0 ? 3 : 4;
const limit = Math.min(width, data.byteLength);
for (let i = 1; i < limit; i++) {
const byte = data[i] ?? 0;
if (byte < 0x80 || byte >= 0xc0)
return i;
}
return limit;
}
/** Close unreferenced cursors once, preserving duplicate descriptors. */
export async function releaseInputs(inputs, keep = []) {
const retained = new Set(keep.map((source) => source.lines));
for (const source of inputs) {
if (!retained.has(source.lines)) {
retained.add(source.lines);
await source.lines.release();
}
}
}
/**
* Keep each reader a job inherits open until the job ends, as the
* descriptor's: a write that replaces it hands the hold on.
*/
export function holdInputs(inputs) {
const held = [...new Map(inputs.map((source) => [source.lines, source])).values()];
for (const source of held)
source.lines.holders += 1;
return held;
}
//# sourceMappingURL=async_line_iterator.js.map