open-memex/dist/store/markdown.js

Local-first memory layer and protocol for AI coding agents. Markdown source of truth, SQLite FTS5 index, zero cloud.

import fs from "node:fs";
import path from "node:path";
import { randomBytes } from "node:crypto";
import yaml from "js-yaml";
import { memoriesDirFor, memoriesDirPath, inRepoMemoriesDirPath, } from "../paths.js";
/** v2 content-kind taxonomy (V2-DESIGN §3.1). `type` = what the memory IS
 *  (single-valued, drives behavior). D41: 11 types; `warning`→`gotcha`,
 *  `workflow`→`howto`, `incident`→`lesson`, `architecture`→`knowledge`. */
export const MEMORY_TYPE_TAXONOMY = [
    "fact",
    "preference",
    "decision",
    "constraint",
    "todo",
    "knowledge",
    "howto",
    "gotcha",
    "lesson",
    "observation",
    "reference",
];
const TAXONOMY = new Set(MEMORY_TYPE_TAXONOMY);
/** Defensive parse: malformed entries are dropped, never fatal. */
export function asReviewHistory(v) {
    if (!Array.isArray(v))
        return [];
    const out = [];
    for (const e of v) {
        if (!e || typeof e !== "object")
            continue;
        const r = e;
        const from = asReviewState(r.from);
        const to = asReviewState(r.to);
        const at = typeof r.at === "string" && r.at ? r.at : null;
        const by = typeof r.by === "string" && r.by ? r.by : null;
        if (!at || !by)
            continue;
        out.push({
            at,
            by,
            from,
            to,
            note: typeof r.note === "string" && r.note ? r.note : null,
        });
    }
    return out;
}
const REVIEW_STATES = ["draft", "proposed", "approved", "rejected", "published"];
export function asReviewState(v) {
    return typeof v === "string" && REVIEW_STATES.includes(v)
        ? v
        : "draft";
}
const CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
/** ULID-ish: 10 chars of base32 timestamp + 16 chars of base32 randomness. */
export function ulid() {
    let ts = "";
    let n = Date.now();
    for (let i = 0; i < 10; i++) {
        ts = CROCKFORD[n % 32] + ts;
        n = Math.floor(n / 32);
    }
    const bytes = randomBytes(16);
    let rand = "";
    for (const b of bytes)
        rand += CROCKFORD[b % 32];
    return ts + rand;
}
/** epoch ms → RFC 3339 (v2 times). */
export function msToRfc3339(ms) {
    return new Date(ms).toISOString().replace(/\.000Z$/, "Z");
}
/** RFC 3339 (or epoch ms) → epoch ms. Falls back to now on garbage. */
export function timeToMs(v) {
    if (typeof v === "number" && Number.isFinite(v))
        return Math.round(v);
    if (typeof v === "string") {
        const t = Date.parse(v);
        if (Number.isFinite(t))
            return t;
    }
    return Date.now();
}
function asScopeKind(v, scopeKind) {
    if (v === "personal" || v === "project" || v === "org")
        return v;
    if (scopeKind === "user")
        return "personal"; // v1 → v2 (§19)
    if (scopeKind === "project")
        return "project";
    return "personal";
}
function asVisibility(v, scope) {
    if (v === "private" || v === "internal" || v === "shared")
        return v;
    // v2 defaults (§4): personal stays local, project is internal.
    return scope === "personal" ? "private" : "internal";
}
function asRole(v) {
    if (v === "knowledge" || v === "instruction")
        return v;
    return "knowledge";
}
function asImportance(v, priority) {
    if (v === "low" || v === "normal" || v === "high")
        return v;
    // v1 `priority: N` → importance (§19): 1–3 low, 8–10 high, else normal.
    if (typeof priority === "number" && Number.isFinite(priority)) {
        if (priority <= 3)
            return "low";
        if (priority >= 8)
            return "high";
    }
    return "normal";
}
function asStatus(v) {
    if (v === "active" ||
        v === "superseded" ||
        v === "deprecated" ||
        v === "retracted" ||
        v === "archived")
        return v;
    return "active";
}
/**
 * Normalize raw (possibly v1) frontmatter into v2 shape. Lenient on read:
 * v1 files (epoch times, scope_kind, priority, type: instruction) are mapped
 * per §19 so old files keep working even before `migrate --to-v2` rewrites
 * them. Use `planConversion` in v2migrate.ts for the explicit, reporting
 * migration path.
 */
export function normalizeFrontmatter(raw) {
    const scope = asScopeKind(raw.scope, raw.scope_kind);
    let scopeKey = typeof raw.scope_key === "string" && raw.scope_key ? raw.scope_key : scope;
    if (scopeKey === "user")
        scopeKey = "personal"; // v1 storage dir → v2
    let type = typeof raw.type === "string" && raw.type ? raw.type : "fact";
    let role = asRole(raw.role);
    if (type === "instruction") {
        // §19: v1 `type: instruction` → content-kind + role split (D11).
        type = "knowledge";
        role = "instruction";
    }
    return {
        id: String(raw.id),
        schema_version: 2,
        scope_key: scopeKey,
        scope,
        visibility: asVisibility(raw.visibility, scope),
        project_name: typeof raw.project_name === "string" ? raw.project_name : scopeKey,
        type,
        role,
        importance: asImportance(raw.importance, raw.priority),
        status: asStatus(raw.status),
        tags: Array.isArray(raw.tags)
            ? raw.tags.filter((t) => typeof t === "string")
            : [],
        ...(Array.isArray(raw.aliases)
            ? (() => {
                const aliases = normalizeAliases(raw.aliases.filter((a) => typeof a === "string"));
                return aliases.length > 0 ? { aliases } : {};
            })()
            : {}),
        source: typeof raw.source === "string" ? raw.source : "",
        created_at: msToRfc3339(timeToMs(raw.created_at)),
        updated_at: msToRfc3339(timeToMs(raw.updated_at)),
        supersedes: typeof raw.supersedes === "string" && raw.supersedes ? raw.supersedes : null,
        superseded_by: typeof raw.superseded_by === "string" && raw.superseded_by
            ? raw.superseded_by
            : null,
        review_state: asReviewState(raw.review_state),
        proposed_by: typeof raw.proposed_by === "string" && raw.proposed_by ? raw.proposed_by : null,
        approved_by: typeof raw.approved_by === "string" && raw.approved_by ? raw.approved_by : null,
        derived_from: typeof raw.derived_from === "string" && raw.derived_from ? raw.derived_from : null,
        review_note: typeof raw.review_note === "string" && raw.review_note ? raw.review_note : null,
        review_history: asReviewHistory(raw.review_history),
    };
}
/**
 * D61: normalize capture-time aliases — trim, drop empties, dedupe
 * (case-insensitive), cap at 4. Bad aliases are noise, never an error.
 * Shared by the tool ops, the CLI, and frontmatter parsing.
 */
export function normalizeAliases(input) {
    if (!input)
        return [];
    const seen = new Set();
    const out = [];
    for (const a of input) {
        const t = a.trim();
        if (!t)
            continue;
        const k = t.toLowerCase();
        if (seen.has(k))
            continue;
        seen.add(k);
        out.push(t);
        if (out.length >= 4)
            break;
    }
    return out;
}
export function isTaxonomyType(t) {
    return TAXONOMY.has(t);
}
export function serialize(fm, body) {
    // D61: an empty aliases list is noise in the file — drop the key entirely
    // rather than relying on the YAML dumper to skip an undefined value
    // (parse only sets the field when non-empty; keep both ends consistent).
    let out = fm;
    if (fm.aliases && fm.aliases.length === 0) {
        out = { ...fm };
        delete out.aliases;
    }
    const yml = yaml.dump(out, { lineWidth: -1, quotingType: '"' });
    return `---\n${yml}---\n\n${body.trimEnd()}\n`;
}
/** Raw frontmatter parse (no normalization) — for migration tooling. */
export function parseRawFrontmatter(raw) {
    const m = raw.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/);
    if (!m)
        return null;
    try {
        const rawFm = yaml.load(m[1]);
        if (!rawFm || typeof rawFm !== "object" || !rawFm.id)
            return null;
        const body = (m[2] ?? "").replace(/^\n+/, "");
        return { rawFm, body };
    }
    catch {
        return null;
    }
}
export function parse(raw) {
    const parsed = parseRawFrontmatter(raw);
    if (!parsed)
        return null;
    return { fm: normalizeFrontmatter(parsed.rawFm), body: parsed.body };
}
/**
 * Atomic text publish: write a sibling tmp file and rename over the
 * target, so a concurrent reader (another process mid-sync) never sees a
 * torn half-written file. Same-directory rename is atomic on one fs.
 * Every memory-file write/rewrite must go through here (D57 for creates,
 * D64 extended to all rewrites: supersede flips, review transitions,
 * submit landings, status changes).
 */
export function atomicWriteTextSync(filePath, text) {
    const tmpPath = `${filePath}.tmp-${process.pid}`;
    fs.writeFileSync(tmpPath, text, "utf8");
    fs.renameSync(tmpPath, filePath);
}
export function writeMemoryFile(fm, body) {
    // 2B/D26: project drafts live in appdata (the outbox) — the in-repo dir
    // only ever holds submitted memories (proposed/approved/published).
    // Personal stays in appdata and never leaves the machine.
    const dir = memoriesDirFor(fm.scope_key);
    const filePath = path.join(dir, `${fm.id}.md`);
    atomicWriteTextSync(filePath, serialize(fm, body));
    const st = fs.statSync(filePath);
    return { filePath, mtimeMs: st.mtimeMs };
}
/** Yield `*.md` files in the in-repo dir (read path — never creates it). */
export function* iterInRepoMemoryFiles(root, memoryDir) {
    const dir = inRepoMemoriesDirPath(root, memoryDir);
    if (!fs.existsSync(dir))
        return;
    for (const name of fs.readdirSync(dir)) {
        if (name.endsWith(".md"))
            yield path.join(dir, name);
    }
}
export function readMemoryFile(filePath) {
    if (!fs.existsSync(filePath))
        return null;
    const raw = fs.readFileSync(filePath, "utf8");
    const parsed = parse(raw);
    if (!parsed)
        return null;
    const st = fs.statSync(filePath);
    return { fm: parsed.fm, body: parsed.body, filePath, mtimeMs: st.mtimeMs };
}
export function* iterMemoryFiles(scopeKey) {
    const dir = memoriesDirPath(scopeKey);
    if (!fs.existsSync(dir))
        return;
    for (const name of fs.readdirSync(dir)) {
        if (name.endsWith(".md"))
            yield path.join(dir, name);
    }
}