agentfootprint
Version:
The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.
284 lines • 14.1 kB
JavaScript
/**
* fileObservability — the typed event stream, one JSON line per event, in a
* local file.
*
* The sink for the shop that has no collector. Every other adapter in this
* folder ships somewhere: CloudWatch, X-Ray, an OTLP endpoint. A great many
* on-premises deployments have none of those — they have a directory, a log
* shipper (Filebeat, Fluent Bit, Vector, `promtail`, `journald`, or a person
* with `grep`), and a rule that nothing leaves the network. NDJSON on disk is
* the format all of those already read, so the sink is the file.
*
* Zero dependencies: `node:fs`, lazily required at construction so merely
* importing `agentfootprint/observe` stays browser-safe (this factory is
* Node-only; calling it in a browser throws by name).
*
* ## The line
*
* One `JSON.stringify(event)` per line, newline-terminated, appended in
* dispatch order — the SAME envelope `cloudwatchObservability` puts in a log
* event, so a query written against one reads the other:
*
* ```jsonl
* {"type":"agentfootprint.agent.turn_start","payload":{…},"meta":{"runId":"…","sessionId":"…"}}
* {"type":"agentfootprint.stream.tool_end","payload":{…},"meta":{…}}
* ```
*
* Nothing is summarized, bounded or redacted on the way out. **A payload that
* must not be on that disk must not reach this strategy** — narrow it with
* `eventTypes` / `tier` / `sampleRate`, or apply a footprintjs
* `RedactionPolicy` upstream, exactly as with every other sink. (For a bounded
* record by construction, `auditExport({ payloadMode: 'bounded' })` is the
* adapter that does that job.)
*
* ## Buffered, not synchronous
*
* `exportEvent` is sync and never touches the disk: it serializes, buffers, and
* returns. Batches are appended asynchronously on a size trigger
* (`maxBufferEvents` / `maxBufferBytes`), on a timer (`flushIntervalMs`), and on
* `flush()` — all through ONE queue (`outbox.ts`), so one append is in flight
* at a time: this sink's appends never interleave, and lines land in dispatch
* order (#223). That is not atomicity — a disk that fails partway through an
* append (ENOSPC) can still leave part of a line behind, and that batch is
* reported dropped. A hard kill loses the buffer and can interrupt the append
* in flight — the price of not making telemetry a term in agent-loop latency.
* Call `flush()` (or `agent.shutdown()`, which does) at process end; see the
* 8.12.0 lifecycle laws on {@link BaseStrategy.flush}.
*
* ## Rotation is ONE generation, and that is deliberate
*
* With `maxBytes` set, a batch that would push the file past the ceiling first
* renames it to `<path>.1` — **replacing any previous `.1`** — and starts a
* fresh file. That is the whole policy. There is no `.2`, no compression, no
* time-based schedule, no cross-process coordination (two processes writing one
* file each keep their own byte count and will both rotate it). It exists so an
* unattended agent cannot fill a disk, and for nothing else: **retention is a
* log-management daemon's job**, and `logrotate` with `copytruncate`, Fluent Bit,
* or a systemd timer will do it properly. Omit `maxBytes` — the default — and
* this adapter never renames anything, which is the right choice when a real
* rotator already owns the file.
*
* @example
* ```ts
* import { fileObservability } from 'agentfootprint/observe';
*
* const telemetry = agent.enable.observability({
* strategy: fileObservability({
* path: '/var/log/agentfootprint/events.ndjson',
* maxBytes: 64 * 1024 * 1024, // safety ceiling; logrotate owns retention
* }),
* });
*
* // … run …
* await agent.shutdown(); // flushes + stops everything enabled
* ```
*/
import { lazyRequire } from '../../lib/lazyRequire.js';
import { toWireJson } from '../../lib/wireJson.js';
import { rateLimitedConsoleSink } from './deliveryErrors.js';
import { createOutbox } from './outbox.js';
// ─── Public factory ──────────────────────────────────────────────────
/**
* NDJSON-to-a-local-file observability strategy. See
* {@link FileObservabilityOptions} for the per-option contract, and this
* module's docstring for the rotation policy and what is NOT bounded.
*/
export function fileObservability(opts) {
const strategyName = 'file';
if (!opts.path || typeof opts.path !== 'string' || opts.path.trim() === '') {
throw new TypeError(`[${strategyName}Observability] \`path\` is required — the file this sink writes. ` +
`Pass one, e.g. '/var/log/agentfootprint/events.ndjson'. There is no default: ` +
`where a run's telemetry lands on your disk is not a decision this library makes.`);
}
if (opts.maxBytes !== undefined && !(opts.maxBytes > 0)) {
throw new TypeError(`[${strategyName}Observability] \`maxBytes\` must be a positive number of bytes ` +
`(got ${String(opts.maxBytes)}). Omit it for no rotation at all, which is the ` +
`right choice when logrotate or a log shipper already owns '${opts.path}'.`);
}
const path = opts.path;
const rotatedPath = `${path}.1`;
const maxBufferEvents = opts.maxBufferEvents ?? 100;
const maxBufferBytes = opts.maxBufferBytes ?? 65_536;
const flushIntervalMs = opts.flushIntervalMs ?? 1000;
const fs = opts._fs ?? createNodeFileSink(strategyName);
// Claim the file NOW. Two things this buys, both worth the syscall:
// 1. an unwritable path (no directory, no permission, a directory where a
// file should be, a read-only mount) is refused at construction, where
// the caller can still read the message;
// 2. `statSync` below gives the rotation counter a true starting size, so a
// restart appends to an existing file rather than believing it is empty.
let bytesOnDisk = 0;
try {
const dir = parentDir(path);
if (dir)
fs.mkdirSync(dir, { recursive: true });
fs.appendFileSync(path, '');
bytesOnDisk = fs.statSync(path).size;
}
catch (err) {
throw new Error(`[${strategyName}Observability] cannot write '${path}': ${errorMessage(err)}. ` +
`The parent directory is created for you; a failure here means the path is a ` +
`directory, the mount is read-only, or the process user lacks permission. ` +
`Fix the path or the permission — this sink refuses to start rather than ` +
`silently drop every event.`);
}
// Set by `stop()`: this sink accepts no more events. It is NOT a guard on
// writing — `stop()` does not authorise throwing away what was already
// accepted, and `flush()` after it still writes (the 8.11.1 stance).
let stopped = false;
// The fallback when the consumer wires nothing. Rate-limited; a
// consumer-supplied sink is not.
const consoleSink = rateLimitedConsoleSink(strategyName);
/** Route a delivery failure through whatever `_onError` IS RIGHT NOW —
* reading it at call time, so a consumer who assigns `_onError` after
* construction still receives them (the 8.11.0 fix). */
function reportFailure(err) {
strategy._onError?.(err);
}
/** ONE generation. A file already at or over the ceiling is renamed to
* `<path>.1` (replacing any previous `.1`) and the counter resets. A batch
* bigger than the whole ceiling is still written — truncating a run's
* telemetry to fit a dial nobody set for that purpose would be worse. */
async function rotateIfNeeded(incomingBytes) {
const ceiling = opts.maxBytes;
if (ceiling === undefined)
return;
if (bytesOnDisk === 0)
return;
if (bytesOnDisk + incomingBytes <= ceiling)
return;
await fs.rename(path, rotatedPath);
bytesOnDisk = 0;
}
/** One batch: rotate if it would cross the ceiling, then append. Runs
* inside the outbox's queue, so no other batch — and no other rotation —
* can start until this one settles. */
async function writeBatch(lines) {
const data = `${lines.join('\n')}\n`;
const bytes = byteLength(data);
await rotateIfNeeded(bytes);
await fs.appendFile(path, data);
bytesOnDisk += bytes;
}
// Buffered lines, and the one queue every write goes through — size
// trigger, timer, `flush()`, shutdown (outbox.ts; #223).
const outbox = createOutbox({
name: strategyName,
maxItems: maxBufferEvents,
maxBytes: maxBufferBytes,
flushIntervalMs,
// Never hold the process open for a local write.
timerHoldsProcess: false,
deliver: writeBatch,
dropped: (lines, cause) => new Error(`${lines.length} event(s) dropped writing to '${path}': ${errorMessage(cause)}`),
report: reportFailure,
});
function enqueue(event) {
if (stopped)
return;
// The hot path must not throw (port law). `JSON.stringify` can — a cycle, a
// BigInt, a throwing getter — and an event that cannot be serialized is a
// reportable fact, not a reason to break the agent loop.
let line;
try {
// The wire rule (`lib/wireJson.ts`): an Error writes its name, message,
// code and cause — never the custom properties a client library hung on
// it (an axios error's `config.headers.authorization`).
line = toWireJson(event);
}
catch (err) {
reportFailure(new Error(`event '${event?.type ?? 'unknown'}' could not be serialized: ` + errorMessage(err)));
return;
}
// `JSON.stringify` returns undefined for an undefined input — nothing to
// write, and a bare "undefined" line would corrupt the NDJSON stream.
if (line === undefined)
return;
// + the newline this line will carry. The outbox decides when it is
// written, and writes it after every line accepted before it — the file's
// line order IS the dispatch order, the one thing an offline reader can
// rely on.
outbox.add(line, byteLength(line) + 1);
}
const strategy = {
name: strategyName,
capabilities: { events: true, logs: true },
...(opts.eventTypes && { relevantEventTypes: opts.eventTypes }),
exportEvent: enqueue,
/**
* Write what is buffered. Called for you on shutdown (8.12.0) — by the
* handle `enable.observability()` returns, by `agent.shutdown()`, and by a
* `standingAgent` closing. Safe at any time, including after `stop()`.
*
* Resolves once every event accepted before the call is written (or
* dropped and reported) — one a timed write already had in flight
* included — after one more sweep of what arrived while it waited. It
* does not chase a producer that keeps emitting. A write that never
* settles holds it: nothing can cancel an in-flight write.
*/
flush() {
return outbox.drain();
},
/** Clear the timer and stop accepting events. Terminal: there is no
* restart. What it does NOT do is discard the buffer — `flush()` after
* `stop()` still writes it. */
stop() {
stopped = true;
outbox.stop();
},
/** Where failures go. Two callers: the dispatch layer (when `exportEvent`
* itself throws) and this adapter's own write path. */
_onError(err, event) {
(opts.onError ?? consoleSink)(err, event);
},
};
return strategy;
}
// ─── node:fs binding (lazy) ──────────────────────────────────────────
/**
* Bind {@link FileSinkFs} to `node:fs`. Lazily required so that importing
* `agentfootprint/observe` in a browser bundle never resolves `node:fs` — only
* CALLING this factory does, and a runtime without it is refused by name.
*/
function createNodeFileSink(strategyName) {
let mod;
try {
mod = lazyRequire('node:fs');
}
catch {
throw new Error(`[${strategyName}Observability] needs \`node:fs\`, and this runtime has none. ` +
`It is a Node-only sink by definition — a browser has no local file to append to. ` +
`In a browser, ship events over your own transport, or pass \`_fs\` to write ` +
`somewhere you control.`);
}
return {
mkdirSync: (dir, options) => void mod.mkdirSync(dir, options),
appendFileSync: (file, data) => mod.appendFileSync(file, data),
statSync: (file) => mod.statSync(file),
appendFile: (file, data) => mod.promises.appendFile(file, data),
rename: (from, to) => mod.promises.rename(from, to),
};
}
// ─── Small helpers ───────────────────────────────────────────────────
function errorMessage(err) {
return err instanceof Error ? err.message : String(err);
}
/** UTF-8 byte length. `Buffer` where there is one, `TextEncoder` otherwise —
* the byte count decides rotation, so a multi-byte prompt must not be counted
* as its character length. */
function byteLength(s) {
const B = globalThis.Buffer;
if (B)
return B.byteLength(s, 'utf8');
return new TextEncoder().encode(s).length;
}
/** The directory part of a path, POSIX or Windows separators, or `''` for a
* bare filename (nothing to create). Done by hand rather than through
* `node:path` so the module has exactly one Node import, behind one seam. */
function parentDir(file) {
const idx = Math.max(file.lastIndexOf('/'), file.lastIndexOf('\\'));
if (idx <= 0)
return '';
return file.slice(0, idx);
}
//# sourceMappingURL=file.js.map