UNPKG

@sema-agent/core

Version:

Stateless, task-oriented AI agent core

781 lines (780 loc) • 27.7 kB
import { appendFileSync, closeSync, constants as FS, existsSync, fstatSync, ftruncateSync, linkSync, mkdirSync, openSync, readFileSync, readSync, renameSync, truncateSync, unlinkSync, writeFileSync, writeSync, fsyncSync, } from "node:fs"; import { constants as bufferConstants } from "node:buffer"; import { homedir, hostname, uptime } from "node:os"; import { hrtime } from "node:process"; import { basename, dirname, join, resolve as resolvePath } from "node:path"; import { createHash, randomBytes } from "node:crypto"; import { execFileSync } from "node:child_process"; import { scrubSecretEnv } from "../../core/secret-env.js"; import { physicalPathOf, physicalPathOfExisting } from "../../core/physical-path.js"; import { thrownText } from "../../core/thrown-value.js"; const SAFE_COMPONENT = /^[A-Za-z0-9_.-]+$/; export function sanitizePathComponent(raw) { if (raw === "" || raw === "." || raw === ".." || !SAFE_COMPONENT.test(raw)) { throw new Error(`file store: unsafe path component ${JSON.stringify(raw)}`); } return raw; } export function sanitizeScope(scope) { const base = scope.replace(/[^A-Za-z0-9]/g, "-"); return `${(base || "_").slice(0, 120)}-${createHash("sha256").update(scope, "utf8").digest("hex")}`; } export function resolveDataRoot(explicit) { const raw = (explicit ?? process.env.AGENT_DATA_DIR ?? join(homedir(), ".ai-agent")).normalize("NFC"); mkdirSync(raw, { recursive: true, mode: 0o700 }); return physicalPathOf(raw); } export function writeThenLink(target, content) { ensureDir(dirnameOf(target)); const tmp = `${target}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`; let linked = false; try { const fd = openSync(tmp, FS.O_CREAT | FS.O_EXCL | FS.O_WRONLY, 0o600); try { writeFileSync(fd, content); fsyncSync(fd); } finally { closeSync(fd); } linkSync(tmp, target); linked = true; } finally { try { unlinkSync(tmp); } catch { } } if (linked) fsyncDir(dirnameOf(target)); } export function ensureDir(dir) { mkdirSync(dir, { recursive: true, mode: 0o700 }); } export function atomicWriteFile(tmpDir, target, bytes) { ensureDir(tmpDir); const tmp = join(tmpDir, `${basenameOf(target)}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`); let fd; try { fd = openSync(tmp, "w", 0o600); writeFileSync(fd, bytes); fsyncSync(fd); closeSync(fd); fd = undefined; renameSync(tmp, target); fsyncDir(dirnameOf(target)); } catch (err) { if (fd !== undefined) { try { closeSync(fd); } catch { } } try { if (existsSync(tmp)) unlinkSync(tmp); } catch { } throw err; } } function fsyncDir(dir) { let dfd; try { dfd = openSync(dir, "r"); fsyncSync(dfd); } catch { } finally { if (dfd !== undefined) { try { closeSync(dfd); } catch { } } } } function basenameOf(p) { return basename(p); } function dirnameOf(p) { return dirname(p); } function jsonlTextFormOf(text) { let value; try { value = JSON.parse(text); } catch { return { form: "unparseable" }; } if (value === null || typeof value !== "object" || Array.isArray(value)) return { form: "not-an-object", value }; return { form: "record", record: value }; } export function readJsonlRecords(path, onCorrupt, judge) { let raw; try { raw = readFileSync(path, "utf8"); } catch (err) { if (err.code === "ENOENT") return []; throw err; } if (raw === "") return []; const parts = raw.split("\n"); const out = []; let dropped = 0; const discloseSkip = (form, line, what) => { dropped++; onCorrupt?.({ kind: "record", path, reason: `${form}: ${what} at line ${line} (skipped)`, line, form }); }; for (let i = 0; i < parts.length; i++) { const line = parts[i]; if (line === "") continue; const read = jsonlTextFormOf(line); if (read.form === "unparseable") { if (i === parts.length - 1) break; discloseSkip("torn-json", i + 1, "unparseable jsonl record"); continue; } if (read.form === "not-an-object") { discloseSkip("not-an-object", i + 1, `jsonl record is ${jsonTypeName(read.value)}, not an object`); continue; } if (judge !== undefined && !judge.accepts(read.record)) { discloseSkip("missing-required", i + 1, judge.what); continue; } out.push(read.record); } if (dropped > 0) { const total = out.length + dropped; const reason = out.length === 0 ? `replay: none of ${total} records readable (all skipped)` : `replay: ${dropped} of ${total} records skipped, ${out.length} kept`; onCorrupt?.({ kind: "replay", path, reason, kept: out.length, dropped, total }); } return out; } function jsonTypeName(value) { if (value === null) return "null"; if (Array.isArray(value)) return "an array"; return `a ${typeof value}`; } export function canonicalStoreKey(p) { return physicalPathOfExisting(p).toLowerCase(); } export class AppendLog { fd; disposed = false; path; boundary = -1; constructor(path) { this.path = path; ensureDir(dirnameOf(path)); repairJsonlTail(path); this.fd = openSync(path, "a+", 0o600); this.boundary = this.sizeOrUnknown(); } sizeOrUnknown() { try { return fstatSync(this.fd).size; } catch { return -1; } } repairTail() { const size = this.sizeOrUnknown(); if (size < 0) { this.boundary = -1; return; } if (size === this.boundary) return; if (size === 0) { this.boundary = 0; return; } const tail = Buffer.allocUnsafe(1); try { if (readSync(this.fd, tail, 0, 1, size - 1) === 1 && tail[0] === 0x0a) { this.boundary = size; return; } } catch { this.boundary = -1; return; } repairJsonlTail(this.path); this.boundary = this.sizeOrUnknown(); } append(record, fsync) { if (this.fd < 0 && !this.disposed) { repairJsonlTail(this.path); this.fd = openSync(this.path, "a+", 0o600); this.boundary = this.sizeOrUnknown(); } if (this.fd < 0) { const e = new Error("append: this log was closed (its session/box was deleted or replaced) — refusing to write through a released descriptor"); e.code = "log_closed"; throw e; } this.repairTail(); const buf = Buffer.from(`${JSON.stringify(record)}\n`, "utf8"); const startedAt = this.boundary; let written = 0; try { while (written < buf.length) { const n = writeSync(this.fd, buf, written, buf.length - written); if (n <= 0) throw new Error(`append: write made no progress (${written}/${buf.length} bytes) — refusing to leave a torn record`); written += n; } } catch (err) { this.boundary = this.rollBackFailedWrite(startedAt, written) ? startedAt : -1; throw err; } if (this.boundary >= 0) this.boundary += buf.length; if (fsync) fsyncSync(this.fd); } rollBackFailedWrite(startedAt, written) { if (startedAt < 0) return false; try { if (fstatSync(this.fd).size !== startedAt + written) return false; ftruncateSync(this.fd, startedAt); return true; } catch { return false; } } close() { this.disposed = true; this.releaseFd(); } closeForSwap() { this.releaseFd(); } releaseFd() { if (this.fd < 0) return; const fd = this.fd; this.fd = -1; try { closeSync(fd); } catch { } } } export const JSONL_TAIL_READ_BLOCK = 64 * 1024; export const JSONL_TAIL_SEGMENT_LIMIT = bufferConstants.MAX_STRING_LENGTH; function readExactly(fd, into, length, position) { let got = 0; while (got < length) { const n = readSync(fd, into, got, length - got, position + got); if (n <= 0) throw new Error(`jsonl tail: the file ended ${length - got} bytes early while its last line was read — it shrank under the reader`); got += n; } } export function readTailSegment(fd, size, limit = JSONL_TAIL_SEGMENT_LIMIT) { const block = Buffer.allocUnsafe(Math.min(JSONL_TAIL_READ_BLOCK, size)); let start = 0; for (let end = size; end > 0;) { const from = Math.max(0, end - JSONL_TAIL_READ_BLOCK); readExactly(fd, block, end - from, from); const newline = block.subarray(0, end - from).lastIndexOf(0x0a); if (newline >= 0) { start = from + newline + 1; break; } end = from; if (size - end > limit) return { unjudgeable: true }; } if (size - start > limit) return { unjudgeable: true }; const bytes = Buffer.allocUnsafe(size - start); readExactly(fd, bytes, bytes.length, start); return { unjudgeable: false, start, bytes }; } function repairJsonlTail(path) { let fd; try { fd = openSync(path, "r"); } catch (err) { if (err.code === "ENOENT") return; throw err; } let segment; try { segment = readTailSegment(fd, fstatSync(fd).size); } finally { closeSync(fd); } if (segment.unjudgeable) { appendFileSync(path, "\n"); try { console.warn(`[sema] file store: a tail segment larger than the judgeable bound (${JSONL_TAIL_SEGMENT_LIMIT} bytes) was ended as a line, not judged — ` + `${path}. Its bytes are untouched; a reader meets it as an ordinary line.`); } catch { } return; } if (segment.bytes.length === 0) return; if (jsonlTextFormOf(segment.bytes.toString("utf8")).form === "record") appendFileSync(path, "\n"); else truncateSync(path, segment.start); } export class FileStoreLockError extends Error { code; lockPath; ownerPid; name = "FileStoreLockError"; constructor(code, message, lockPath, ownerPid) { super(message); this.code = code; this.lockPath = lockPath; this.ownerPid = ownerPid; } } const MAX_RECLAIM_DEPTH = 8; const MAX_ACQUIRE_ATTEMPTS = 4; const abandonedLocks = new Set(); const LOCK_RECORD_VERSION = 2; function serializeLockRecord() { const fp = processFingerprint(); return JSON.stringify({ v: LOCK_RECORD_VERSION, pid: fp.pid, startedAtMs: fp.startedAtMs, startedSinceBootMs: fp.startedSinceBootMs, host: fp.host, }); } function parseLockRecord(raw) { if (/^[1-9][0-9]*$/.test(raw)) { const barePid = Number.parseInt(raw, 10); return Number.isSafeInteger(barePid) && barePid > 0 ? { pid: barePid, startedAtMs: null, startedSinceBootMs: null, host: null } : null; } if (!raw.startsWith("{")) return null; let parsed; try { parsed = JSON.parse(raw); } catch { return null; } if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null; const rec = parsed; if (typeof rec.pid !== "number" || !Number.isSafeInteger(rec.pid) || rec.pid <= 0) return null; const known = rec.v === LOCK_RECORD_VERSION; const measure = (raw) => (known && typeof raw === "number" && Number.isFinite(raw) ? raw : null); return { pid: rec.pid, startedAtMs: measure(rec.startedAtMs), startedSinceBootMs: measure(rec.startedSinceBootMs), host: known && typeof rec.host === "string" && rec.host !== "" ? rec.host : null, }; } function readLockRecord(lockPath) { let raw; try { raw = readFileSync(lockPath, "utf8").trim(); } catch (err) { return err.code === "ENOENT" ? { state: "absent" } : { state: "unreadable" }; } const owner = parseLockRecord(raw); return owner === null ? { state: "unreadable" } : { state: "owner", ...owner }; } function sameOwner(a, b) { return (a.pid === b.pid && a.startedAtMs === b.startedAtMs && a.startedSinceBootMs === b.startedSinceBootMs && a.host === b.host); } export class BootLock { lockPath; held = false; spentVerdict; constructor(lockPath) { this.lockPath = lockPath; } acquire() { this.acquireWithin(0); } acquireWithin(depth) { for (let attempt = 0; attempt < MAX_ACQUIRE_ATTEMPTS; attempt++) { try { this.writeLock(); this.held = true; abandonedLocks.delete(this.abandonKey); return; } catch (err) { if (err.code !== "EEXIST") throw err; } const owner = this.inspectLock(); if (owner.state === "absent") continue; if (owner.state === "unreadable") { throw new FileStoreLockError("store.lock_unreadable", `file store: lock file at ${this.lockPath} is unreadable/corrupt; remove it manually if it is stale`, this.lockPath); } if (owner.pid === process.pid && abandonedLocks.has(this.abandonKey)) { this.pruneOwnAbandoned(); return; } if (isProcessLive(owner)) { throw new FileStoreLockError("store.dir_in_use", `file store: another instance (pid ${owner.pid}) owns this data dir (${dirnameOf(this.lockPath)})`, this.lockPath, owner.pid); } if (depth >= MAX_RECLAIM_DEPTH) { throw new FileStoreLockError("store.lock_unreadable", `file store: lock file at ${this.lockPath} is buried under ${depth} nested stale reclaim gates ` + `(repeated crashes mid-reclaim); remove the ${basenameOf(this.lockPath)}.claim.* files manually`, this.lockPath); } const gate = new BootLock(`${this.lockPath}.claim.${owner.pid}`); try { gate.acquireWithin(depth + 1); } catch (err) { if (err instanceof FileStoreLockError && err.code === "store.dir_in_use" && err.ownerPid !== undefined) { throw new FileStoreLockError("store.dir_claiming", `file store: another instance${err.ownerPid !== undefined ? ` (pid ${err.ownerPid})` : ""} is claiming ` + `this data dir (${dirnameOf(this.lockPath)}) from a stale lock`, this.lockPath, err.ownerPid); } throw err; } try { const under = this.inspectLock(); if (under.state === "owner" && sameOwner(under, owner)) { try { unlinkSync(this.lockPath); } catch { } } else if (under.state !== "absent") { continue; } this.publishOrLoseRace(); return; } finally { gate.release(); } } throw new FileStoreLockError("store.dir_in_use", `file store: ownership of this data dir (${dirnameOf(this.lockPath)}) kept changing under us ` + `(${MAX_ACQUIRE_ATTEMPTS} attempts); another instance is starting and stopping on the same directory`, this.lockPath); } publishOrLoseRace() { try { this.writeLock(); this.held = true; abandonedLocks.delete(this.abandonKey); } catch (err) { if (err.code === "EEXIST") { throw new FileStoreLockError("store.dir_in_use", `file store: another instance won the lock race for this data dir`, this.lockPath); } throw err; } } pruneOwnAbandoned() { try { unlinkSync(this.lockPath); } catch (err) { throw new FileStoreLockError("store.lock_unreadable", `file store: this process still owns the lock file at ${this.lockPath} from an earlier release that ` + `could not remove it, and removing it now failed too (${thrownText(err)}); ` + `remove it manually`, this.lockPath, process.pid); } abandonedLocks.delete(this.abandonKey); this.publishOrLoseRace(); } writeLock() { writeThenLink(this.lockPath, serializeLockRecord()); } get abandonKey() { return resolvePath(this.lockPath); } inspectLock() { return readLockRecord(this.lockPath); } release() { if (!this.held) { return this.spentVerdict ?? true; } this.held = false; this.spentVerdict = this.removeOwnLock(); return this.spentVerdict; } removeOwnLock() { const before = this.inspectLock(); if (before.state === "absent" || (before.state === "owner" && before.pid !== process.pid)) { abandonedLocks.delete(this.abandonKey); return true; } if (before.state === "unreadable") { abandonedLocks.add(this.abandonKey); return false; } try { unlinkSync(this.lockPath); } catch { } const after = this.inspectLock(); const gone = after.state === "absent" || (after.state === "owner" && after.pid !== process.pid); if (gone) abandonedLocks.delete(this.abandonKey); else abandonedLocks.add(this.abandonKey); return gone; } resumeAbandoned() { if (this.held) return true; if (!abandonedLocks.has(this.abandonKey)) return false; const owner = this.inspectLock(); if (owner.state !== "owner" || owner.pid !== process.pid) return false; this.held = true; this.spentVerdict = undefined; abandonedLocks.delete(this.abandonKey); return true; } } const heldStoreDirLocks = new Map(); export function acquireStoreDirLock(dir, opts) { const fileName = opts.fileName ?? "LOCK"; const lockPath = join(dir, fileName); const key = `${fileName}@${canonicalStoreKey(dir)}`; const lease = (entry) => { let released = false; return { lockPath, release: () => { if (released) return; released = true; if (heldStoreDirLocks.get(key) !== entry) return; entry.refs -= 1; if (entry.refs > 0) return; if (entry.lock.release()) { heldStoreDirLocks.delete(key); return; } console.warn(`[sema] file store: the ${opts.label} directory fence at ${lockPath} could not be handed back — the ` + `removal could not be proven, so a lock file naming this live process (pid ${process.pid}) may still ` + `be there. While it is, another process opening that directory is refused by name; the next open of ` + `it in THIS process finishes the cleanup or refuses by name. After this process exits the file is ` + `reclaimed as a stale lock ONLY if it is readable and its directory writable — otherwise it has to ` + `be removed by hand.`); }, }; }; const existing = heldStoreDirLocks.get(key); if (existing !== undefined) { if (existing.refs > 0 || existing.lock.resumeAbandoned()) { existing.refs += 1; return lease(existing); } heldStoreDirLocks.delete(key); } const lock = new BootLock(lockPath); try { lock.acquire(); } catch (err) { if (err instanceof FileStoreLockError) { const who = err.ownerPid !== undefined ? ` (pid ${err.ownerPid})` : ""; const single = `This backend is SINGLE-WRITER per directory: a second process replays a private view (both sides can ` + `win a once-only decision) and its next compaction erases the records it never loaded.`; throw new FileStoreLockError(err.code, err.code === "store.lock_unreadable" ? `file store: the ${opts.label} directory (${dir}) cannot be opened — ${err.message}` : err.code === "store.dir_claiming" ? `file store: another OS process${who} is claiming this data dir — it is reclaiming the ${opts.label} ` + `directory (${dir}) from a lock a crashed owner left behind. ${single} Retry in a moment: either that ` + `process finishes and owns the directory, or it dies and the next start reclaims it.` : `file store: another OS process${who} is using the ${opts.label} directory (${dir}). ${single} Stop the ` + `other process, point this one at a different data root, or use the pg/tidb backend for real ` + `multi-process concurrency.`, err.code === "store.lock_unreadable" ? err.lockPath : lockPath, err.ownerPid); } throw err; } const entry = { lock, refs: 1 }; heldStoreDirLocks.set(key, entry); return lease(entry); } export function createFileConsolidationLock(lockDir) { return (scope) => { ensureDir(lockDir); const lock = new BootLock(join(lockDir, `${sanitizeScope(scope)}.consolidate.lock`)); try { lock.acquire(); } catch (err) { if (err instanceof FileStoreLockError) return undefined; throw err; } return () => { lock.release(); }; }; } function isProcessRunning(pid) { try { process.kill(pid, 0); return true; } catch (err) { return err.code === "EPERM"; } } const START_TIME_TOLERANCE_MS = 2_000; const START_TIME_PROBE_TIMEOUT_MS = 2_000; const START_TIME_PROBE_PRECISION_MS = 250; const ETIME_PATTERN = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)$/; let cachedHostname; let selfFingerprint; let startTimeProbeUnavailable = false; function thisHost() { return (cachedHostname ??= hostname()); } function announceStartTimeDegrade(reason) { if (startTimeProbeUnavailable) return; startTimeProbeUnavailable = true; console.warn(`[sema] file store: this host cannot report process start times (${reason}), so a directory lock's owner ` + `is judged by PID ALONE. A lock left behind by a crashed owner whose pid the OS has since handed to an ` + `unrelated process then reads as a LIVE owner, and that directory stays refused until the lock file is ` + `removed by hand.`); } const START_TIME_PROBE_ATTEMPTS = 3; function probeStartTimes(pid) { for (let attempt = 0; attempt < START_TIME_PROBE_ATTEMPTS; attempt++) { const reading = probeStartTimesOnce(pid); if (reading !== "imprecise") return reading; } return null; } function probeStartTimesOnce(pid) { if (startTimeProbeUnavailable) return null; const knownToExist = pid === process.pid; if (process.platform === "win32") { announceStartTimeDegrade("no ps(1) on win32"); return null; } const monotonicBefore = hrtime.bigint(); const wallBefore = Date.now(); const uptimeBefore = Math.round(uptime() * 1000); let raw; try { raw = execFileSync("ps", ["-p", String(pid), "-o", "etime="], { encoding: "utf8", timeout: START_TIME_PROBE_TIMEOUT_MS, stdio: ["ignore", "pipe", "ignore"], env: { ...scrubSecretEnv(process.env), LC_ALL: "C", LANG: "C" }, }); } catch (err) { if (typeof err.status === "number") { if (knownToExist) announceStartTimeDegrade("ps refused a query about this very process"); return null; } announceStartTimeDegrade(`ps failed (${err.code ?? "unknown"})`); return null; } const latency = Number(hrtime.bigint() - monotonicBefore) / 1e6; const wallDelta = Date.now() - wallBefore; const trimmed = raw.trim(); const parts = ETIME_PATTERN.exec(trimmed); if (parts === null) { if (trimmed !== "" || knownToExist) announceStartTimeDegrade("ps produced no elapsed time"); return null; } if (latency > START_TIME_PROBE_PRECISION_MS) return "imprecise"; if (Math.abs(wallDelta - latency) > START_TIME_PROBE_PRECISION_MS) return "imprecise"; const elapsedMs = (Number(parts[1] ?? "0") * 86_400 + Number(parts[2] ?? "0") * 3_600 + Number(parts[3] ?? "0") * 60 + Number(parts[4] ?? "0")) * 1000; return { startedAtMs: Math.round((wallBefore - elapsedMs) / 1000) * 1000, startedSinceBootMs: Math.round((uptimeBefore - elapsedMs) / 1000) * 1000, }; } export function processFingerprint(pid) { const host = thisHost(); const of = (id, probed) => ({ pid: id, startedAtMs: probed?.startedAtMs ?? null, startedSinceBootMs: probed?.startedSinceBootMs ?? null, host, }); if (pid === undefined || pid === process.pid) { if (selfFingerprint === undefined || selfFingerprint.startedAtMs === null) { const answer = of(process.pid, probeStartTimes(process.pid)); if (answer.startedAtMs === null) return answer; selfFingerprint = answer; } return selfFingerprint; } if (processFingerprint().startedAtMs === null) return of(pid, null); return of(pid, probeStartTimes(pid)); } export function isProcessLive(record) { if (!isProcessRunning(record.pid)) return false; if (record.host !== undefined && record.host !== null && record.host !== thisHost()) return true; const observed = processFingerprint(record.pid); const falsified = (was, now) => was !== undefined && was !== null && now !== null && now - was > START_TIME_TOLERANCE_MS; const bootIsOver = (was) => was !== undefined && was !== null && was - Math.round(uptime() * 1000) > START_TIME_TOLERANCE_MS; const wallSaysGone = falsified(record.startedAtMs, observed.startedAtMs); const uptimeSaysGone = falsified(record.startedSinceBootMs, observed.startedSinceBootMs) || bootIsOver(record.startedSinceBootMs); return !(wallSaysGone && uptimeSaysGone); }