@sema-agent/core/dist/stores/file/memory-store.js

Stateless, task-oriented AI agent core

import { join } from "node:path";
import { uuidv7 } from "../../internal/harness.js";
import { firstSentence, lexicalSearchMatch, parseNoteTimestamp, MemoryProjectionError, } from "../../core/memory.js";
import { cosineDistance, jaccardDistance, termSet } from "../../core/memory-vector.js";
import { canonicalStoreKey, AppendLog, atomicWriteFile, ensureDir, readJsonlRecords, sanitizeScope, } from "./fs-atomic.js";
import { assertAdoptionBootGate } from "./adoption/marker.js";
const MAX_OPEN_MEMORY_SCOPES = 64;
const sharedMemoryDirs = new Map();
export class FileMemoryStore {
    root;
    embedder;
    embedTimeoutMs;
    pending = new Set();
    get scopes() {
        return this.sharedScopes.scopes;
    }
    sharedScopes;
    scopeKey;
    closed = false;
    constructor(root, opts = {}) {
        assertAdoptionBootGate(root, "FileMemoryStore");
        this.root = join(root, "memory");
        this.scopeKey = canonicalStoreKey(this.root);
        const live = sharedMemoryDirs.get(this.scopeKey);
        if (live !== undefined) {
            live.refs += 1;
            this.sharedScopes = live;
        }
        else {
            this.sharedScopes = { scopes: new Map(), refs: 1, openLogs: new Set() };
            sharedMemoryDirs.set(this.scopeKey, this.sharedScopes);
        }
        this.embedder = opts.embedder;
        this.embedTimeoutMs = Math.max(0, opts.embedTimeoutMs ?? 10_000);
        ensureDir(this.root);
    }
    get vectorMode() {
        return this.embedder ? "portable" : "lexical";
    }
    get hasEmbedder() {
        return this.embedder !== undefined;
    }
    async flushEmbeds() {
        await Promise.all([...this.pending]);
    }
    scopeState(scope) {
        const existing = this.scopes.get(scope);
        if (existing)
            return existing;
        const dir = join(this.root, sanitizeScope(scope));
        ensureDir(dir);
        const notesPath = join(dir, "notes.jsonl");
        const entries = [];
        let cursor;
        for (const ev of readJsonlRecords(notesPath)) {
            switch (ev.op) {
                case "append":
                    entries.push(ev.entry);
                    break;
                case "clear":
                    entries.length = 0;
                    cursor = undefined;
                    break;
                case "cursor":
                    cursor = ev.cursor;
                    break;
                case "update": {
                    const target = entries.find((e) => e.id === ev.id);
                    if (target) {
                        target.text = ev.text;
                        target.ts = ev.ts;
                        if (ev.description !== undefined)
                            target.description = ev.description;
                    }
                    break;
                }
                case "delete": {
                    const i = entries.findIndex((e) => e.id === ev.id);
                    if (i !== -1)
                        entries.splice(i, 1);
                    break;
                }
                default:
                    break;
            }
        }
        const vectors = new Map();
        const vpath = join(dir, "vectors.jsonl");
        for (const ve of readJsonlRecords(vpath)) {
            if (ve && typeof ve.id === "string" && Array.isArray(ve.embedding)) {
                vectors.set(ve.id, { embedding: ve.embedding, h: typeof ve.h === "string" ? ve.h : "" });
            }
        }
        const state = { entries, log: new AppendLog(notesPath), dir, cursor, vectors, vpath };
        this.scopes.set(scope, state);
        this.touchOpenLogs(scope);
        return state;
    }
    touchOpenLogs(scope) {
        const open = this.sharedScopes.openLogs;
        open.delete(scope);
        open.add(scope);
        while (open.size > MAX_OPEN_MEMORY_SCOPES) {
            const coldest = open.values().next().value;
            if (coldest === undefined)
                break;
            open.delete(coldest);
            const st = this.scopes.get(coldest);
            st?.log.closeForSwap();
            st?.vlog?.closeForSwap();
        }
    }
    writeLog(scope, state) {
        this.touchOpenLogs(scope);
        return state.log;
    }
    vectorLog(scope, state) {
        if (state.vlog === undefined)
            state.vlog = new AppendLog(state.vpath);
        this.touchOpenLogs(scope);
        return state.vlog;
    }
    queueEmbed(scope, state, id, body) {
        const emb = this.embedder;
        if (!emb)
            return;
        const h = hashBody(body);
        const job = Promise.race([
            Promise.resolve().then(() => emb.embed(body)),
            new Promise((r) => setTimeout(() => r(null), this.embedTimeoutMs).unref?.()),
        ])
            .then((vec) => {
            if (this.closed)
                return;
            if (!Array.isArray(vec) || vec.length !== emb.dimensions)
                return;
            const cur = state.entries.find((e) => e.id === id);
            if (!cur || hashBody(cur.text) !== h)
                return;
            this.vectorLog(scope, state).append({ id, embedding: vec, h }, false);
            state.vectors.set(id, { embedding: vec, h });
        })
            .catch(() => { });
        this.pending.add(job);
        void job.finally(() => this.pending.delete(job));
    }
    writeProjection(state) {
        const body = state.entries.map(renderBullet).join("\n");
        const target = join(state.dir, "MEMORY.md");
        try {
            atomicWriteFile(state.dir, target, body ? `${body}\n` : "");
        }
        catch (err) {
            throw new MemoryProjectionError(target, err);
        }
    }
    read(scope) {
        const { entries } = this.scopeState(scope);
        if (entries.length === 0)
            return null;
        return entries.map(renderBullet).join("\n");
    }
    append(scope, note) {
        const text = note.trim().replace(/^#+\s*/, "");
        if (!text)
            return;
        const entry = { id: uuidv7(), text, ts: nowTs() };
        this.commit(scope, entry);
        return entry.id;
    }
    appendStructured(scope, note) {
        const body = note.body.trim().replace(/^#+\s*/, "");
        if (!body)
            return "";
        const entry = {
            id: uuidv7(),
            text: body,
            ts: nowTs(),
            name: note.name,
            type: note.type ?? "project",
            description: note.description?.trim() || firstSentence(body),
            links: note.links,
            ...(note.consolidationGenerated ? { consolidationGenerated: true } : {}),
        };
        this.commit(scope, entry);
        return entry.id;
    }
    commit(scope, entry) {
        const state = this.scopeState(scope);
        this.writeLog(scope, state).append({ op: "append", entry }, true);
        state.entries.push(entry);
        this.writeProjection(state);
        this.queueEmbed(scope, state, entry.id, entry.text);
    }
    clear(scope) {
        const state = this.scopeState(scope);
        this.writeLog(scope, state).append({ op: "clear" }, true);
        state.entries.length = 0;
        state.cursor = undefined;
        state.vectors.clear();
        this.writeProjection(state);
    }
    getConsolidationCursor(scope) {
        return this.scopeState(scope).cursor;
    }
    setConsolidationCursor(scope, cursor) {
        const state = this.scopeState(scope);
        this.writeLog(scope, state).append({ op: "cursor", cursor }, true);
        state.cursor = cursor;
    }
    search(scope, query, limit = 10) {
        const { entries } = this.scopeState(scope);
        if (entries.length === 0)
            return [];
        if (query.trim() === "")
            return [];
        return entries
            .map(renderBullet)
            .filter((b) => lexicalSearchMatch(query, b))
            .slice(-limit);
    }
    listStructuredNotes(scope) {
        const { entries } = this.scopeState(scope);
        return entries.map((e) => ({
            id: e.id,
            description: e.description ?? firstSentence(e.text),
            ...parseNoteTimestamp(e.ts),
            ...(e.name ? { name: e.name } : {}),
            ...(e.type ? { type: e.type } : {}),
            ...(e.consolidationGenerated ? { consolidationGenerated: true } : {}),
        }));
    }
    getByIds(scope, ids) {
        const { entries } = this.scopeState(scope);
        const want = new Set(ids);
        return entries
            .filter((e) => want.has(e.id))
            .map((e) => ({
            id: e.id,
            text: e.text,
            description: e.description ?? firstSentence(e.text),
            ...parseNoteTimestamp(e.ts),
            ...(e.name ? { name: e.name } : {}),
            ...(e.type ? { type: e.type } : {}),
            ...(e.consolidationGenerated ? { consolidationGenerated: true } : {}),
        }));
    }
    searchScored(scope, query, limit = 20) {
        const state = this.scopeState(scope);
        if (state.entries.length === 0)
            return [];
        if (!this.embedder)
            return lexicalScored(state.entries, query, limit);
        const emb = this.embedder;
        return Promise.race([
            Promise.resolve().then(() => emb.embed(query)),
            new Promise((r) => setTimeout(() => r(null), this.embedTimeoutMs).unref?.()),
        ]).then((qv) => (Array.isArray(qv) && qv.length === emb.dimensions ? vectorScored(state, qv, query, emb.dimensions, limit) : lexicalScored(state.entries, query, limit)), () => lexicalScored(state.entries, query, limit));
    }
    update(scope, id, text) {
        const clean = text.trim();
        if (!clean)
            return;
        const state = this.scopeState(scope);
        const entry = state.entries.find((e) => e.id === id);
        if (!entry)
            return;
        const newBody = clean.replace(/^#+\s*/, "");
        const newDesc = entry.description !== undefined && entry.description === firstSentence(entry.text)
            ? firstSentence(newBody)
            : entry.description;
        const ts = nowTs();
        const ev = { op: "update", id, text: newBody, ts };
        if (newDesc !== undefined)
            ev.description = newDesc;
        this.writeLog(scope, state).append(ev, true);
        entry.text = newBody;
        entry.ts = ts;
        entry.description = newDesc;
        this.writeProjection(state);
        state.vectors.delete(id);
        this.queueEmbed(scope, state, id, newBody);
    }
    delete(scope, id) {
        const state = this.scopeState(scope);
        const i = state.entries.findIndex((e) => e.id === id);
        if (i === -1)
            return;
        this.writeLog(scope, state).append({ op: "delete", id }, true);
        state.entries.splice(i, 1);
        this.writeProjection(state);
    }
    close() {
        if (this.closed)
            return;
        this.closed = true;
        this.sharedScopes.refs -= 1;
        if (this.sharedScopes.refs > 0)
            return;
        if (sharedMemoryDirs.get(this.scopeKey) === this.sharedScopes)
            sharedMemoryDirs.delete(this.scopeKey);
        for (const s of this.scopes.values()) {
            s.log.close();
            s.vlog?.close();
        }
        this.sharedScopes.openLogs.clear();
    }
}
function renderBullet(e) {
    return `- (${e.ts} UTC) ${e.text}`;
}
function lexicalScored(entries, query, limit) {
    const q = termSet(query);
    if (q.size === 0)
        return [];
    const scored = [];
    for (const e of entries) {
        const d = jaccardDistance(q, e.text);
        if (d === null)
            continue;
        scored.push({ id: e.id, text: e.text, score: d, ...(e.consolidationGenerated ? { consolidationGenerated: true } : {}) });
    }
    scored.sort((a, b) => a.score - b.score);
    return scored.slice(0, limit);
}
function vectorScored(state, qv, query, dims, limit) {
    const qTerms = termSet(query);
    const scored = [];
    for (const e of state.entries) {
        const ev = state.vectors.get(e.id);
        const usable = ev && ev.embedding.length === dims && ev.h === hashBody(e.text);
        const score = usable ? cosineDistance(qv, ev.embedding) : jaccardDistance(qTerms, e.text);
        if (score === null)
            continue;
        scored.push({ id: e.id, text: e.text, score, ...(e.consolidationGenerated ? { consolidationGenerated: true } : {}) });
    }
    scored.sort((a, b) => a.score - b.score);
    return scored.slice(0, limit);
}
function hashBody(s) {
    let h = 0x811c9dc5;
    for (let i = 0; i < s.length; i++) {
        h ^= s.charCodeAt(i);
        h = Math.imul(h, 0x01000193);
    }
    return (h >>> 0).toString(16);
}
function nowTs() {
    return new Date().toISOString().replace("T", " ").slice(0, 16);
}