@sema-agent/core
Version:
Stateless, task-oriented AI agent core
781 lines (780 loc) • 27.7 kB
JavaScript
import { appendFileSync, closeSync, constants as FS, existsSync, fstatSync, ftruncateSync, linkSync, mkdirSync, openSync, readFileSync, readSync, renameSync, truncateSync, unlinkSync, writeFileSync, writeSync, fsyncSync, } from "node:fs";
import { constants as bufferConstants } from "node:buffer";
import { homedir, hostname, uptime } from "node:os";
import { hrtime } from "node:process";
import { basename, dirname, join, resolve as resolvePath } from "node:path";
import { createHash, randomBytes } from "node:crypto";
import { execFileSync } from "node:child_process";
import { scrubSecretEnv } from "../../core/secret-env.js";
import { physicalPathOf, physicalPathOfExisting } from "../../core/physical-path.js";
import { thrownText } from "../../core/thrown-value.js";
const SAFE_COMPONENT = /^[A-Za-z0-9_.-]+$/;
export function sanitizePathComponent(raw) {
if (raw === "" || raw === "." || raw === ".." || !SAFE_COMPONENT.test(raw)) {
throw new Error(`file store: unsafe path component ${JSON.stringify(raw)}`);
}
return raw;
}
export function sanitizeScope(scope) {
const base = scope.replace(/[^A-Za-z0-9]/g, "-");
return `${(base || "_").slice(0, 120)}-${createHash("sha256").update(scope, "utf8").digest("hex")}`;
}
export function resolveDataRoot(explicit) {
const raw = (explicit ?? process.env.AGENT_DATA_DIR ?? join(homedir(), ".ai-agent")).normalize("NFC");
mkdirSync(raw, { recursive: true, mode: 0o700 });
return physicalPathOf(raw);
}
export function writeThenLink(target, content) {
ensureDir(dirnameOf(target));
const tmp = `${target}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`;
let linked = false;
try {
const fd = openSync(tmp, FS.O_CREAT | FS.O_EXCL | FS.O_WRONLY, 0o600);
try {
writeFileSync(fd, content);
fsyncSync(fd);
}
finally {
closeSync(fd);
}
linkSync(tmp, target);
linked = true;
}
finally {
try {
unlinkSync(tmp);
}
catch {
}
}
if (linked)
fsyncDir(dirnameOf(target));
}
export function ensureDir(dir) {
mkdirSync(dir, { recursive: true, mode: 0o700 });
}
export function atomicWriteFile(tmpDir, target, bytes) {
ensureDir(tmpDir);
const tmp = join(tmpDir, `${basenameOf(target)}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`);
let fd;
try {
fd = openSync(tmp, "w", 0o600);
writeFileSync(fd, bytes);
fsyncSync(fd);
closeSync(fd);
fd = undefined;
renameSync(tmp, target);
fsyncDir(dirnameOf(target));
}
catch (err) {
if (fd !== undefined) {
try {
closeSync(fd);
}
catch {
}
}
try {
if (existsSync(tmp))
unlinkSync(tmp);
}
catch {
}
throw err;
}
}
function fsyncDir(dir) {
let dfd;
try {
dfd = openSync(dir, "r");
fsyncSync(dfd);
}
catch {
}
finally {
if (dfd !== undefined) {
try {
closeSync(dfd);
}
catch {
}
}
}
}
function basenameOf(p) {
return basename(p);
}
function dirnameOf(p) {
return dirname(p);
}
function jsonlTextFormOf(text) {
let value;
try {
value = JSON.parse(text);
}
catch {
return { form: "unparseable" };
}
if (value === null || typeof value !== "object" || Array.isArray(value))
return { form: "not-an-object", value };
return { form: "record", record: value };
}
export function readJsonlRecords(path, onCorrupt, judge) {
let raw;
try {
raw = readFileSync(path, "utf8");
}
catch (err) {
if (err.code === "ENOENT")
return [];
throw err;
}
if (raw === "")
return [];
const parts = raw.split("\n");
const out = [];
let dropped = 0;
const discloseSkip = (form, line, what) => {
dropped++;
onCorrupt?.({ kind: "record", path, reason: `${form}: ${what} at line ${line} (skipped)`, line, form });
};
for (let i = 0; i < parts.length; i++) {
const line = parts[i];
if (line === "")
continue;
const read = jsonlTextFormOf(line);
if (read.form === "unparseable") {
if (i === parts.length - 1)
break;
discloseSkip("torn-json", i + 1, "unparseable jsonl record");
continue;
}
if (read.form === "not-an-object") {
discloseSkip("not-an-object", i + 1, `jsonl record is ${jsonTypeName(read.value)}, not an object`);
continue;
}
if (judge !== undefined && !judge.accepts(read.record)) {
discloseSkip("missing-required", i + 1, judge.what);
continue;
}
out.push(read.record);
}
if (dropped > 0) {
const total = out.length + dropped;
const reason = out.length === 0 ? `replay: none of ${total} records readable (all skipped)` : `replay: ${dropped} of ${total} records skipped, ${out.length} kept`;
onCorrupt?.({ kind: "replay", path, reason, kept: out.length, dropped, total });
}
return out;
}
function jsonTypeName(value) {
if (value === null)
return "null";
if (Array.isArray(value))
return "an array";
return `a ${typeof value}`;
}
export function canonicalStoreKey(p) {
return physicalPathOfExisting(p).toLowerCase();
}
export class AppendLog {
fd;
disposed = false;
path;
boundary = -1;
constructor(path) {
this.path = path;
ensureDir(dirnameOf(path));
repairJsonlTail(path);
this.fd = openSync(path, "a+", 0o600);
this.boundary = this.sizeOrUnknown();
}
sizeOrUnknown() {
try {
return fstatSync(this.fd).size;
}
catch {
return -1;
}
}
repairTail() {
const size = this.sizeOrUnknown();
if (size < 0) {
this.boundary = -1;
return;
}
if (size === this.boundary)
return;
if (size === 0) {
this.boundary = 0;
return;
}
const tail = Buffer.allocUnsafe(1);
try {
if (readSync(this.fd, tail, 0, 1, size - 1) === 1 && tail[0] === 0x0a) {
this.boundary = size;
return;
}
}
catch {
this.boundary = -1;
return;
}
repairJsonlTail(this.path);
this.boundary = this.sizeOrUnknown();
}
append(record, fsync) {
if (this.fd < 0 && !this.disposed) {
repairJsonlTail(this.path);
this.fd = openSync(this.path, "a+", 0o600);
this.boundary = this.sizeOrUnknown();
}
if (this.fd < 0) {
const e = new Error("append: this log was closed (its session/box was deleted or replaced) — refusing to write through a released descriptor");
e.code = "log_closed";
throw e;
}
this.repairTail();
const buf = Buffer.from(`${JSON.stringify(record)}\n`, "utf8");
const startedAt = this.boundary;
let written = 0;
try {
while (written < buf.length) {
const n = writeSync(this.fd, buf, written, buf.length - written);
if (n <= 0)
throw new Error(`append: write made no progress (${written}/${buf.length} bytes) — refusing to leave a torn record`);
written += n;
}
}
catch (err) {
this.boundary = this.rollBackFailedWrite(startedAt, written) ? startedAt : -1;
throw err;
}
if (this.boundary >= 0)
this.boundary += buf.length;
if (fsync)
fsyncSync(this.fd);
}
rollBackFailedWrite(startedAt, written) {
if (startedAt < 0)
return false;
try {
if (fstatSync(this.fd).size !== startedAt + written)
return false;
ftruncateSync(this.fd, startedAt);
return true;
}
catch {
return false;
}
}
close() {
this.disposed = true;
this.releaseFd();
}
closeForSwap() {
this.releaseFd();
}
releaseFd() {
if (this.fd < 0)
return;
const fd = this.fd;
this.fd = -1;
try {
closeSync(fd);
}
catch {
}
}
}
export const JSONL_TAIL_READ_BLOCK = 64 * 1024;
export const JSONL_TAIL_SEGMENT_LIMIT = bufferConstants.MAX_STRING_LENGTH;
function readExactly(fd, into, length, position) {
let got = 0;
while (got < length) {
const n = readSync(fd, into, got, length - got, position + got);
if (n <= 0)
throw new Error(`jsonl tail: the file ended ${length - got} bytes early while its last line was read — it shrank under the reader`);
got += n;
}
}
export function readTailSegment(fd, size, limit = JSONL_TAIL_SEGMENT_LIMIT) {
const block = Buffer.allocUnsafe(Math.min(JSONL_TAIL_READ_BLOCK, size));
let start = 0;
for (let end = size; end > 0;) {
const from = Math.max(0, end - JSONL_TAIL_READ_BLOCK);
readExactly(fd, block, end - from, from);
const newline = block.subarray(0, end - from).lastIndexOf(0x0a);
if (newline >= 0) {
start = from + newline + 1;
break;
}
end = from;
if (size - end > limit)
return { unjudgeable: true };
}
if (size - start > limit)
return { unjudgeable: true };
const bytes = Buffer.allocUnsafe(size - start);
readExactly(fd, bytes, bytes.length, start);
return { unjudgeable: false, start, bytes };
}
function repairJsonlTail(path) {
let fd;
try {
fd = openSync(path, "r");
}
catch (err) {
if (err.code === "ENOENT")
return;
throw err;
}
let segment;
try {
segment = readTailSegment(fd, fstatSync(fd).size);
}
finally {
closeSync(fd);
}
if (segment.unjudgeable) {
appendFileSync(path, "\n");
try {
console.warn(`[sema] file store: a tail segment larger than the judgeable bound (${JSONL_TAIL_SEGMENT_LIMIT} bytes) was ended as a line, not judged — ` +
`${path}. Its bytes are untouched; a reader meets it as an ordinary line.`);
}
catch {
}
return;
}
if (segment.bytes.length === 0)
return;
if (jsonlTextFormOf(segment.bytes.toString("utf8")).form === "record")
appendFileSync(path, "\n");
else
truncateSync(path, segment.start);
}
export class FileStoreLockError extends Error {
code;
lockPath;
ownerPid;
name = "FileStoreLockError";
constructor(code, message, lockPath, ownerPid) {
super(message);
this.code = code;
this.lockPath = lockPath;
this.ownerPid = ownerPid;
}
}
const MAX_RECLAIM_DEPTH = 8;
const MAX_ACQUIRE_ATTEMPTS = 4;
const abandonedLocks = new Set();
const LOCK_RECORD_VERSION = 2;
function serializeLockRecord() {
const fp = processFingerprint();
return JSON.stringify({
v: LOCK_RECORD_VERSION,
pid: fp.pid,
startedAtMs: fp.startedAtMs,
startedSinceBootMs: fp.startedSinceBootMs,
host: fp.host,
});
}
function parseLockRecord(raw) {
if (/^[1-9][0-9]*$/.test(raw)) {
const barePid = Number.parseInt(raw, 10);
return Number.isSafeInteger(barePid) && barePid > 0
? { pid: barePid, startedAtMs: null, startedSinceBootMs: null, host: null }
: null;
}
if (!raw.startsWith("{"))
return null;
let parsed;
try {
parsed = JSON.parse(raw);
}
catch {
return null;
}
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed))
return null;
const rec = parsed;
if (typeof rec.pid !== "number" || !Number.isSafeInteger(rec.pid) || rec.pid <= 0)
return null;
const known = rec.v === LOCK_RECORD_VERSION;
const measure = (raw) => (known && typeof raw === "number" && Number.isFinite(raw) ? raw : null);
return {
pid: rec.pid,
startedAtMs: measure(rec.startedAtMs),
startedSinceBootMs: measure(rec.startedSinceBootMs),
host: known && typeof rec.host === "string" && rec.host !== "" ? rec.host : null,
};
}
function readLockRecord(lockPath) {
let raw;
try {
raw = readFileSync(lockPath, "utf8").trim();
}
catch (err) {
return err.code === "ENOENT" ? { state: "absent" } : { state: "unreadable" };
}
const owner = parseLockRecord(raw);
return owner === null ? { state: "unreadable" } : { state: "owner", ...owner };
}
function sameOwner(a, b) {
return (a.pid === b.pid &&
a.startedAtMs === b.startedAtMs &&
a.startedSinceBootMs === b.startedSinceBootMs &&
a.host === b.host);
}
export class BootLock {
lockPath;
held = false;
spentVerdict;
constructor(lockPath) {
this.lockPath = lockPath;
}
acquire() {
this.acquireWithin(0);
}
acquireWithin(depth) {
for (let attempt = 0; attempt < MAX_ACQUIRE_ATTEMPTS; attempt++) {
try {
this.writeLock();
this.held = true;
abandonedLocks.delete(this.abandonKey);
return;
}
catch (err) {
if (err.code !== "EEXIST")
throw err;
}
const owner = this.inspectLock();
if (owner.state === "absent")
continue;
if (owner.state === "unreadable") {
throw new FileStoreLockError("store.lock_unreadable", `file store: lock file at ${this.lockPath} is unreadable/corrupt; remove it manually if it is stale`, this.lockPath);
}
if (owner.pid === process.pid && abandonedLocks.has(this.abandonKey)) {
this.pruneOwnAbandoned();
return;
}
if (isProcessLive(owner)) {
throw new FileStoreLockError("store.dir_in_use", `file store: another instance (pid ${owner.pid}) owns this data dir (${dirnameOf(this.lockPath)})`, this.lockPath, owner.pid);
}
if (depth >= MAX_RECLAIM_DEPTH) {
throw new FileStoreLockError("store.lock_unreadable", `file store: lock file at ${this.lockPath} is buried under ${depth} nested stale reclaim gates ` +
`(repeated crashes mid-reclaim); remove the ${basenameOf(this.lockPath)}.claim.* files manually`, this.lockPath);
}
const gate = new BootLock(`${this.lockPath}.claim.${owner.pid}`);
try {
gate.acquireWithin(depth + 1);
}
catch (err) {
if (err instanceof FileStoreLockError && err.code === "store.dir_in_use" && err.ownerPid !== undefined) {
throw new FileStoreLockError("store.dir_claiming", `file store: another instance${err.ownerPid !== undefined ? ` (pid ${err.ownerPid})` : ""} is claiming ` +
`this data dir (${dirnameOf(this.lockPath)}) from a stale lock`, this.lockPath, err.ownerPid);
}
throw err;
}
try {
const under = this.inspectLock();
if (under.state === "owner" && sameOwner(under, owner)) {
try {
unlinkSync(this.lockPath);
}
catch {
}
}
else if (under.state !== "absent") {
continue;
}
this.publishOrLoseRace();
return;
}
finally {
gate.release();
}
}
throw new FileStoreLockError("store.dir_in_use", `file store: ownership of this data dir (${dirnameOf(this.lockPath)}) kept changing under us ` +
`(${MAX_ACQUIRE_ATTEMPTS} attempts); another instance is starting and stopping on the same directory`, this.lockPath);
}
publishOrLoseRace() {
try {
this.writeLock();
this.held = true;
abandonedLocks.delete(this.abandonKey);
}
catch (err) {
if (err.code === "EEXIST") {
throw new FileStoreLockError("store.dir_in_use", `file store: another instance won the lock race for this data dir`, this.lockPath);
}
throw err;
}
}
pruneOwnAbandoned() {
try {
unlinkSync(this.lockPath);
}
catch (err) {
throw new FileStoreLockError("store.lock_unreadable", `file store: this process still owns the lock file at ${this.lockPath} from an earlier release that ` +
`could not remove it, and removing it now failed too (${thrownText(err)}); ` +
`remove it manually`, this.lockPath, process.pid);
}
abandonedLocks.delete(this.abandonKey);
this.publishOrLoseRace();
}
writeLock() {
writeThenLink(this.lockPath, serializeLockRecord());
}
get abandonKey() {
return resolvePath(this.lockPath);
}
inspectLock() {
return readLockRecord(this.lockPath);
}
release() {
if (!this.held) {
return this.spentVerdict ?? true;
}
this.held = false;
this.spentVerdict = this.removeOwnLock();
return this.spentVerdict;
}
removeOwnLock() {
const before = this.inspectLock();
if (before.state === "absent" || (before.state === "owner" && before.pid !== process.pid)) {
abandonedLocks.delete(this.abandonKey);
return true;
}
if (before.state === "unreadable") {
abandonedLocks.add(this.abandonKey);
return false;
}
try {
unlinkSync(this.lockPath);
}
catch {
}
const after = this.inspectLock();
const gone = after.state === "absent" || (after.state === "owner" && after.pid !== process.pid);
if (gone)
abandonedLocks.delete(this.abandonKey);
else
abandonedLocks.add(this.abandonKey);
return gone;
}
resumeAbandoned() {
if (this.held)
return true;
if (!abandonedLocks.has(this.abandonKey))
return false;
const owner = this.inspectLock();
if (owner.state !== "owner" || owner.pid !== process.pid)
return false;
this.held = true;
this.spentVerdict = undefined;
abandonedLocks.delete(this.abandonKey);
return true;
}
}
const heldStoreDirLocks = new Map();
export function acquireStoreDirLock(dir, opts) {
const fileName = opts.fileName ?? "LOCK";
const lockPath = join(dir, fileName);
const key = `${fileName}@${canonicalStoreKey(dir)}`;
const lease = (entry) => {
let released = false;
return {
lockPath,
release: () => {
if (released)
return;
released = true;
if (heldStoreDirLocks.get(key) !== entry)
return;
entry.refs -= 1;
if (entry.refs > 0)
return;
if (entry.lock.release()) {
heldStoreDirLocks.delete(key);
return;
}
console.warn(`[sema] file store: the ${opts.label} directory fence at ${lockPath} could not be handed back — the ` +
`removal could not be proven, so a lock file naming this live process (pid ${process.pid}) may still ` +
`be there. While it is, another process opening that directory is refused by name; the next open of ` +
`it in THIS process finishes the cleanup or refuses by name. After this process exits the file is ` +
`reclaimed as a stale lock ONLY if it is readable and its directory writable — otherwise it has to ` +
`be removed by hand.`);
},
};
};
const existing = heldStoreDirLocks.get(key);
if (existing !== undefined) {
if (existing.refs > 0 || existing.lock.resumeAbandoned()) {
existing.refs += 1;
return lease(existing);
}
heldStoreDirLocks.delete(key);
}
const lock = new BootLock(lockPath);
try {
lock.acquire();
}
catch (err) {
if (err instanceof FileStoreLockError) {
const who = err.ownerPid !== undefined ? ` (pid ${err.ownerPid})` : "";
const single = `This backend is SINGLE-WRITER per directory: a second process replays a private view (both sides can ` +
`win a once-only decision) and its next compaction erases the records it never loaded.`;
throw new FileStoreLockError(err.code, err.code === "store.lock_unreadable"
?
`file store: the ${opts.label} directory (${dir}) cannot be opened — ${err.message}`
: err.code === "store.dir_claiming"
?
`file store: another OS process${who} is claiming this data dir — it is reclaiming the ${opts.label} ` +
`directory (${dir}) from a lock a crashed owner left behind. ${single} Retry in a moment: either that ` +
`process finishes and owns the directory, or it dies and the next start reclaims it.`
: `file store: another OS process${who} is using the ${opts.label} directory (${dir}). ${single} Stop the ` +
`other process, point this one at a different data root, or use the pg/tidb backend for real ` +
`multi-process concurrency.`, err.code === "store.lock_unreadable" ? err.lockPath : lockPath, err.ownerPid);
}
throw err;
}
const entry = { lock, refs: 1 };
heldStoreDirLocks.set(key, entry);
return lease(entry);
}
export function createFileConsolidationLock(lockDir) {
return (scope) => {
ensureDir(lockDir);
const lock = new BootLock(join(lockDir, `${sanitizeScope(scope)}.consolidate.lock`));
try {
lock.acquire();
}
catch (err) {
if (err instanceof FileStoreLockError)
return undefined;
throw err;
}
return () => {
lock.release();
};
};
}
function isProcessRunning(pid) {
try {
process.kill(pid, 0);
return true;
}
catch (err) {
return err.code === "EPERM";
}
}
const START_TIME_TOLERANCE_MS = 2_000;
const START_TIME_PROBE_TIMEOUT_MS = 2_000;
const START_TIME_PROBE_PRECISION_MS = 250;
const ETIME_PATTERN = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)$/;
let cachedHostname;
let selfFingerprint;
let startTimeProbeUnavailable = false;
function thisHost() {
return (cachedHostname ??= hostname());
}
function announceStartTimeDegrade(reason) {
if (startTimeProbeUnavailable)
return;
startTimeProbeUnavailable = true;
console.warn(`[sema] file store: this host cannot report process start times (${reason}), so a directory lock's owner ` +
`is judged by PID ALONE. A lock left behind by a crashed owner whose pid the OS has since handed to an ` +
`unrelated process then reads as a LIVE owner, and that directory stays refused until the lock file is ` +
`removed by hand.`);
}
const START_TIME_PROBE_ATTEMPTS = 3;
function probeStartTimes(pid) {
for (let attempt = 0; attempt < START_TIME_PROBE_ATTEMPTS; attempt++) {
const reading = probeStartTimesOnce(pid);
if (reading !== "imprecise")
return reading;
}
return null;
}
function probeStartTimesOnce(pid) {
if (startTimeProbeUnavailable)
return null;
const knownToExist = pid === process.pid;
if (process.platform === "win32") {
announceStartTimeDegrade("no ps(1) on win32");
return null;
}
const monotonicBefore = hrtime.bigint();
const wallBefore = Date.now();
const uptimeBefore = Math.round(uptime() * 1000);
let raw;
try {
raw = execFileSync("ps", ["-p", String(pid), "-o", "etime="], {
encoding: "utf8",
timeout: START_TIME_PROBE_TIMEOUT_MS,
stdio: ["ignore", "pipe", "ignore"],
env: { ...scrubSecretEnv(process.env), LC_ALL: "C", LANG: "C" },
});
}
catch (err) {
if (typeof err.status === "number") {
if (knownToExist)
announceStartTimeDegrade("ps refused a query about this very process");
return null;
}
announceStartTimeDegrade(`ps failed (${err.code ?? "unknown"})`);
return null;
}
const latency = Number(hrtime.bigint() - monotonicBefore) / 1e6;
const wallDelta = Date.now() - wallBefore;
const trimmed = raw.trim();
const parts = ETIME_PATTERN.exec(trimmed);
if (parts === null) {
if (trimmed !== "" || knownToExist)
announceStartTimeDegrade("ps produced no elapsed time");
return null;
}
if (latency > START_TIME_PROBE_PRECISION_MS)
return "imprecise";
if (Math.abs(wallDelta - latency) > START_TIME_PROBE_PRECISION_MS)
return "imprecise";
const elapsedMs = (Number(parts[1] ?? "0") * 86_400 + Number(parts[2] ?? "0") * 3_600 + Number(parts[3] ?? "0") * 60 + Number(parts[4] ?? "0")) *
1000;
return {
startedAtMs: Math.round((wallBefore - elapsedMs) / 1000) * 1000,
startedSinceBootMs: Math.round((uptimeBefore - elapsedMs) / 1000) * 1000,
};
}
export function processFingerprint(pid) {
const host = thisHost();
const of = (id, probed) => ({
pid: id,
startedAtMs: probed?.startedAtMs ?? null,
startedSinceBootMs: probed?.startedSinceBootMs ?? null,
host,
});
if (pid === undefined || pid === process.pid) {
if (selfFingerprint === undefined || selfFingerprint.startedAtMs === null) {
const answer = of(process.pid, probeStartTimes(process.pid));
if (answer.startedAtMs === null)
return answer;
selfFingerprint = answer;
}
return selfFingerprint;
}
if (processFingerprint().startedAtMs === null)
return of(pid, null);
return of(pid, probeStartTimes(pid));
}
export function isProcessLive(record) {
if (!isProcessRunning(record.pid))
return false;
if (record.host !== undefined && record.host !== null && record.host !== thisHost())
return true;
const observed = processFingerprint(record.pid);
const falsified = (was, now) => was !== undefined && was !== null && now !== null && now - was > START_TIME_TOLERANCE_MS;
const bootIsOver = (was) => was !== undefined && was !== null && was - Math.round(uptime() * 1000) > START_TIME_TOLERANCE_MS;
const wallSaysGone = falsified(record.startedAtMs, observed.startedAtMs);
const uptimeSaysGone = falsified(record.startedSinceBootMs, observed.startedSinceBootMs) || bootIsOver(record.startedSinceBootMs);
return !(wallSaysGone && uptimeSaysGone);
}