openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
720 lines (716 loc) • 33.6 kB
JavaScript
import { s as formatErrorMessage$1 } from "./error-coercion-W9BtigRm.mjs";
import { b as redactToolPayloadText } from "./redact-BV3wTtTv.mjs";
import { r as canReuseSqliteSchemaInTransaction } from "./sqlite-schema-facts-BVf0E_0z.mjs";
import { a as runSqliteImmediateTransactionSync } from "./sqlite-transaction-DBRLvCDM.mjs";
import { n as assertSqliteSchemaContains, s as createSqliteTableContractReader } from "./sqlite-schema-contract-BXfI47YR.mjs";
import { n as migrateSqliteSchemaToStrictInTransaction, t as migrateSqliteSchemaToStrict } from "./sqlite-strict-B8HPGv3j.mjs";
import { a as MEMORY_INDEX_SOURCES_TABLE, c as dropMemoryChunkFtsTriggers, d as ensureMemoryChunkFtsTriggers, f as ensureMemoryPathFtsSchema, h as rebuildMemoryChunkFts, i as MEMORY_INDEX_PATHS_FTS_TABLE, l as dropMemoryPathFtsTriggers, n as MEMORY_INDEX_CHUNKS_TABLE$1, p as ensureMemoryPathFtsTriggers, r as MEMORY_INDEX_FTS_TABLE, s as dropDisabledMemoryFts, t as MEMORY_CHUNK_FTS_TRIGGER_DEFINITIONS, u as ensureMemoryChunkFtsSchema } from "./memory-schema-fts-B8vS-yZQ.mjs";
import { n as decodeMemoryEmbedding, r as encodeMemoryEmbedding } from "./embedding-vector-DA_9H0FL.mjs";
//#region packages/memory-host-sdk/src/host/memory-schema-provenance.ts
const MEMORY_INDEX_CHUNK_PROVENANCE_TABLE = "memory_index_chunk_provenance";
const MEMORY_INDEX_CHUNK_PROVENANCE_SCHEMA_SQL = `
CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_CHUNK_PROVENANCE_TABLE} (
chunk_id TEXT PRIMARY KEY,
origin_class TEXT NOT NULL CHECK (origin_class IN ('owner', 'agent', 'untrusted', 'system')),
session_kind TEXT NOT NULL CHECK (session_kind IN ('interactive', 'cron', 'heartbeat', 'subagent', 'unknown')),
observed_at INTEGER NOT NULL,
supersedes_key TEXT,
FOREIGN KEY (chunk_id) REFERENCES memory_index_chunks(id) ON DELETE CASCADE
) STRICT;
`;
const missingChunkIdsSql = `
SELECT chunk.id
FROM memory_index_chunks AS chunk
LEFT JOIN ${MEMORY_INDEX_CHUNK_PROVENANCE_TABLE} AS provenance
ON provenance.chunk_id = chunk.id
WHERE provenance.chunk_id IS NULL
`;
function ensureMemoryChunkProvenance(db) {
const ensure = () => {
db.exec("DROP TRIGGER IF EXISTS memory_index_chunk_provenance_after_insert");
db.exec(MEMORY_INDEX_CHUNK_PROVENANCE_SCHEMA_SQL);
db.exec(`
WITH missing AS MATERIALIZED (${missingChunkIdsSql}),
affected AS MATERIALIZED (
SELECT chunk.path, chunk.source
FROM missing
CROSS JOIN memory_index_chunks AS chunk ON chunk.id = missing.id
)
UPDATE memory_index_sources
SET hash = ''
WHERE EXISTS (
SELECT 1
FROM affected
WHERE affected.path = memory_index_sources.path
AND affected.source IS memory_index_sources.source
);
WITH missing AS MATERIALIZED (${missingChunkIdsSql})
INSERT OR IGNORE INTO ${MEMORY_INDEX_CHUNK_PROVENANCE_TABLE} (
chunk_id, origin_class, session_kind, observed_at
)
SELECT chunk.id, 'untrusted', 'unknown', chunk.updated_at
FROM missing
JOIN memory_index_chunks AS chunk ON chunk.id = missing.id;
`);
};
if (db.isTransaction) {
ensure();
return;
}
runSqliteImmediateTransactionSync(db, ensure);
}
//#endregion
//#region packages/memory-host-sdk/src/host/memory-schema-recall.ts
const MEMORY_INDEX_CHUNKS_TABLE = "memory_index_chunks";
const MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE = "memory_index_chunk_recall_metadata";
const MEMORY_INDEX_CHUNK_RECALL_METADATA_SCHEMA_SQL = `
CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE} (
chunk_id TEXT PRIMARY KEY,
importance INTEGER CHECK (importance IS NULL OR importance BETWEEN 1 AND 10),
triggers TEXT,
project_key TEXT,
FOREIGN KEY (chunk_id) REFERENCES ${MEMORY_INDEX_CHUNKS_TABLE}(id) ON DELETE CASCADE
) STRICT;
`;
function readMemoryChunkColumns(db) {
const rows = db.prepare(`PRAGMA table_info(${MEMORY_INDEX_CHUNKS_TABLE})`).all();
return new Set(rows.flatMap((row) => typeof row.name === "string" ? [row.name] : []));
}
function tableExists$1(db, tableName) {
return Boolean(db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ? LIMIT 1").get(tableName));
}
function legacyRecallMetadataColumns(db) {
const columns = readMemoryChunkColumns(db);
return [
"importance",
"triggers",
"project_key"
].filter((column) => columns.has(column));
}
function hasLegacyMemoryRecallMetadataColumns(db) {
return legacyRecallMetadataColumns(db).length > 0;
}
function ensureMemoryRecallMetadataSchema(db) {
if (legacyRecallMetadataColumns(db).length === 0 && tableExists$1(db, "memory_index_chunk_recall_metadata")) return;
const ensure = () => {
db.exec(MEMORY_INDEX_CHUNK_RECALL_METADATA_SCHEMA_SQL);
const columns = new Set(legacyRecallMetadataColumns(db));
if (columns.size === 0) return;
const importance = columns.has("importance") ? "importance" : "NULL";
const triggers = columns.has("triggers") ? "triggers" : "NULL";
const projectKey = columns.has("project_key") ? "project_key" : "NULL";
db.exec(`
INSERT INTO ${MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE} (
chunk_id, importance, triggers, project_key
)
SELECT id, ${importance}, ${triggers}, ${projectKey}
FROM ${MEMORY_INDEX_CHUNKS_TABLE}
WHERE ${importance} IS NOT NULL
OR ${triggers} IS NOT NULL
OR ${projectKey} IS NOT NULL
ON CONFLICT(chunk_id) DO UPDATE SET
importance=excluded.importance,
triggers=excluded.triggers,
project_key=excluded.project_key;
`);
for (const column of [
"project_key",
"triggers",
"importance"
]) if (columns.has(column)) db.exec(`ALTER TABLE ${MEMORY_INDEX_CHUNKS_TABLE} DROP COLUMN ${column}`);
};
if (db.isTransaction) {
ensure();
return;
}
runSqliteImmediateTransactionSync(db, ensure);
}
//#endregion
//#region packages/memory-host-sdk/src/host/error-utils.ts
/** Format memory-host errors through the canonical formatter and redaction policy. */
function formatErrorMessage(err) {
return formatErrorMessage$1(err, { redact: redactToolPayloadText });
}
//#endregion
//#region packages/memory-host-sdk/src/host/memory-schema-base.ts
const MEMORY_INDEX_META_TABLE = "memory_index_meta";
const MEMORY_EMBEDDING_CACHE_TABLE = "memory_embedding_cache";
const MEMORY_INDEX_STATE_TABLE = "memory_index_state";
const MEMORY_INDEX_VECTOR_TABLE = "memory_index_chunks_vec";
const MEMORY_INDEX_CHUNKS_SCHEMA_SQL = `
CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_CHUNKS_TABLE$1} (
chunk_rowid INTEGER PRIMARY KEY,
id TEXT NOT NULL UNIQUE,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding BLOB NOT NULL,
updated_at INTEGER NOT NULL
) STRICT;
`;
function buildMemoryEmbeddingCacheSchema(table) {
return `
CREATE TABLE IF NOT EXISTS ${table} (
provider TEXT NOT NULL,
model TEXT NOT NULL,
provider_key TEXT NOT NULL,
hash TEXT NOT NULL,
embedding BLOB NOT NULL,
dims INTEGER,
updated_at INTEGER NOT NULL,
PRIMARY KEY (provider, model, provider_key, hash)
) STRICT;
`;
}
const MEMORY_INDEX_DERIVED_TABLES = [
MEMORY_INDEX_VECTOR_TABLE,
MEMORY_INDEX_FTS_TABLE,
MEMORY_INDEX_PATHS_FTS_TABLE,
MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE,
MEMORY_INDEX_CHUNK_PROVENANCE_TABLE,
MEMORY_INDEX_CHUNKS_TABLE$1,
MEMORY_INDEX_SOURCES_TABLE,
MEMORY_INDEX_META_TABLE,
MEMORY_EMBEDDING_CACHE_TABLE,
MEMORY_INDEX_STATE_TABLE
];
function buildMemoryIndexStrictSchema(params) {
const embeddingCacheSql = params.includeEmbeddingCache ? buildMemoryEmbeddingCacheSchema(params.embeddingCacheTable) : "";
return `
CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_META_TABLE} (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
) STRICT;
CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_SOURCES_TABLE} (
id INTEGER PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime REAL NOT NULL,
size INTEGER NOT NULL,
UNIQUE (path, source)
) STRICT;
${MEMORY_INDEX_CHUNKS_SCHEMA_SQL}
${MEMORY_INDEX_CHUNK_RECALL_METADATA_SCHEMA_SQL}
${MEMORY_INDEX_CHUNK_PROVENANCE_SCHEMA_SQL}
CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_STATE_TABLE} (
id INTEGER PRIMARY KEY CHECK (id = 1),
revision INTEGER NOT NULL
) STRICT;
${embeddingCacheSql}
`;
}
//#endregion
//#region packages/memory-host-sdk/src/host/memory-schema-storage-migration.ts
const LEGACY_CHUNK_SCHEMA = `
CREATE TABLE IF NOT EXISTS memory_index_chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
) STRICT;
`;
function legacyCacheSchema(table) {
return `CREATE TABLE IF NOT EXISTS ${table} (
provider TEXT NOT NULL,
model TEXT NOT NULL,
provider_key TEXT NOT NULL,
hash TEXT NOT NULL,
embedding TEXT NOT NULL,
dims INTEGER,
updated_at INTEGER NOT NULL,
PRIMARY KEY (provider, model, provider_key, hash)
) STRICT;`;
}
const CHUNK_INDEXES = [
{
name: "idx_memory_index_chunks_path_source",
columns: "path, source"
},
{
name: "idx_memory_index_chunks_path",
columns: "path"
},
{
name: "idx_memory_index_chunks_source",
columns: "source"
}
];
const CHUNK_REVISION_TRIGGERS = [
"insert",
"update",
"delete"
].map((event) => ({
name: `memory_index_chunks_revision_after_${event}`,
sql: `CREATE TRIGGER IF NOT EXISTS memory_index_chunks_revision_after_${event}
AFTER ${event.toUpperCase()} ON memory_index_chunks
BEGIN UPDATE memory_index_state SET revision = revision + 1 WHERE id = 1; END;`
}));
const INLINE_RECALL_COLUMNS = [
["importance", "importance INTEGER CHECK (importance IS NULL OR importance BETWEEN 1 AND 10)"],
["triggers", "triggers TEXT"],
["project_key", "project_key TEXT"]
];
function storageShape(db, table, chunks, readTable) {
const tableColumns = columns(db, table);
if (tableColumns.size === 0) return "absent";
const shape = tableColumns.get("embedding") === "TEXT" ? "legacy" : tableColumns.get("embedding") === "BLOB" ? "binary" : void 0;
if (!shape || chunks && tableColumns.has("chunk_rowid") !== (shape === "binary")) throw new Error(`Unsupported partial memory storage schema: ${table}`);
let schema = chunks ? shape === "legacy" ? LEGACY_CHUNK_SCHEMA : MEMORY_INDEX_CHUNKS_SCHEMA_SQL : shape === "legacy" ? legacyCacheSchema(table) : buildMemoryEmbeddingCacheSchema(table);
if (chunks) {
const inline = INLINE_RECALL_COLUMNS.filter(([name]) => tableColumns.has(name));
if (inline.length > 0) schema = schema.replace("updated_at INTEGER NOT NULL", `updated_at INTEGER NOT NULL,\n${inline.map(([, declaration]) => declaration).join(",\n")}`);
}
if (shape === "legacy" && db.prepare("SELECT strict FROM pragma_table_list WHERE schema = 'main' AND name = ?").get(table)?.strict === 0) schema = schema.replace(") STRICT;", ");");
const indexes = chunks ? CHUNK_INDEXES : [{
name: table === "memory_embedding_cache" ? "idx_memory_embedding_cache_updated_at" : "idx_embedding_cache_updated_at",
columns: "updated_at"
}];
schema += indexes.map((index) => `CREATE INDEX ${index.name} ON ${table}(${index.columns});`).join("\n");
assertSqliteSchemaContains(db, `memory storage ${table}`, schema, {
allowCompatibleAdditiveColumns: shape === "binary",
allowedMissingIndexes: indexes.map((index) => index.name),
optionalCanonicalTriggerGroups: chunks ? [{
tableName: table,
triggers: CHUNK_REVISION_TRIGGERS
}, ...shape === "binary" ? [{
tableName: table,
triggers: MEMORY_CHUNK_FTS_TRIGGER_DEFINITIONS
}] : []] : []
}, readTable);
if (shape === "legacy") assertKnownRebuildDependents(db, table, indexes.map((index) => index.name), chunks ? CHUNK_REVISION_TRIGGERS.map((trigger) => trigger.name) : []);
return shape;
}
function assertKnownRebuildDependents(db, table, indexes, triggers) {
const known = /* @__PURE__ */ new Set([...indexes, ...triggers]);
const dependent = db.prepare(`
SELECT type, name, tbl_name FROM main.sqlite_schema
WHERE (type IN ('trigger', 'index') AND tbl_name = ? AND sql IS NOT NULL)
OR (type IN ('view', 'trigger') AND instr(lower(sql), ?) > 0)
`).all(table, table).find((row) => row.tbl_name !== table || !known.has(String(row.name)));
if (dependent) throw new Error(`Memory storage migration cannot rebuild unknown ${String(dependent.type)} ${String(dependent.name)} on ${table}`);
for (const row of db.prepare("SELECT name FROM main.sqlite_schema WHERE type = 'table'").all()) {
const name = String(row.name);
const references = db.prepare(`PRAGMA main.foreign_key_list("${name.replaceAll("\"", "\"\"")}")`).all().filter((key) => key.table === table);
const knownChild = table === "memory_index_chunks" && ["memory_index_chunk_provenance", "memory_index_chunk_recall_metadata"].includes(name);
if (references.some((key) => !knownChild || key.from !== "chunk_id" || key.to !== "id" || key.on_delete !== "CASCADE" || key.on_update !== "NO ACTION")) throw new Error(`Memory storage migration cannot rebuild ${table} referenced by an unknown foreign key in ${name}`);
}
}
function storageShapes(db, cacheTable) {
const readTable = canReuseSqliteSchemaInTransaction(db) ? createSqliteTableContractReader(db) : void 0;
return {
chunks: storageShape(db, "memory_index_chunks", true, readTable),
cache: storageShape(db, cacheTable, false, readTable)
};
}
function assertBinaryEmbeddings(db, table) {
const rows = db.prepare(`SELECT openclaw_memory_embedding_blob_valid(embedding) AS valid FROM ${table}`);
for (const row of rows.iterate()) if (row.valid === 0) throw new Error(`Memory storage migration found invalid binary embeddings in ${table}`);
}
function copyMemoryStorageRows(db, table, sql) {
const rows = db.prepare(`SELECT rowid AS storage_rowid FROM ${table}`);
rows.setReadBigInts(true);
const copy = db.prepare(sql);
copy.setReadBigInts(true);
for (const row of rows.iterate()) {
if (typeof row.storage_rowid !== "bigint") throw new Error("Invalid memory storage identity during migration");
copy.run(row.storage_rowid);
}
}
function markInvalidMemoryEmbeddings(db, sql) {
const rows = db.prepare(sql);
rows.setReadBigInts(true);
const dirtySource = db.prepare(`
UPDATE main.memory_index_sources SET hash = ''
WHERE (path, source) IN (
SELECT path, source FROM main.memory_index_chunks WHERE rowid = ?
)`);
dirtySource.setReadBigInts(true);
let invalid = false;
for (const row of rows.iterate()) if (row.valid === 0) {
if (typeof row.storage_rowid !== "bigint") throw new Error("Invalid memory storage identity during migration");
dirtySource.run(row.storage_rowid);
invalid = true;
}
if (invalid) db.exec(`
INSERT INTO main.memory_index_meta (key, value)
VALUES ('memory_vector_rebuild_v1', '1')
ON CONFLICT(key) DO UPDATE SET value = excluded.value;
`);
}
function legacyEmbedding(raw) {
if (typeof raw !== "string") return;
try {
const parsed = JSON.parse(raw);
return Array.isArray(parsed) && parsed.every((value) => typeof value === "number" && Number.isFinite(value)) ? parsed : void 0;
} catch {
return;
}
}
function copyLegacyMemoryEmbeddingCache(db, table, replacement, renewAuthority) {
const maxEmbeddingBytes = 1048576;
const rows = db.prepare(`
SELECT rowid AS storage_rowid,
CASE WHEN length(CAST(embedding AS BLOB)) <= ${maxEmbeddingBytes}
THEN embedding ELSE NULL END AS embedding
FROM ${table} WHERE rowid >= ? ORDER BY rowid LIMIT 128
`);
rows.setReadBigInts(true);
const copy = db.prepare(`
INSERT INTO ${replacement} (rowid, provider, model, provider_key, hash, embedding, dims, updated_at)
SELECT rowid, provider, model, provider_key, hash, ?, dims, updated_at
FROM ${table} WHERE rowid = ?
`);
copy.setReadBigInts(true);
let nextRowid = -9223372036854775808n;
let skipped = 0;
const firstSkippedRowids = [];
while (nextRowid <= 9223372036854775807n) {
let count = 0;
for (const row of rows.iterate(nextRowid)) {
if (typeof row.storage_rowid !== "bigint") throw new Error("Invalid memory storage identity during migration");
renewAuthority();
nextRowid = row.storage_rowid + 1n;
count += 1;
const embedding = legacyEmbedding(row.embedding ?? null);
if (embedding === void 0) {
skipped += 1;
if (firstSkippedRowids.length < 8) firstSkippedRowids.push(String(row.storage_rowid));
continue;
}
copy.run(encodeMemoryEmbedding(embedding), row.storage_rowid);
}
if (count < 128) break;
}
return skipped > 0 ? `Skipped ${skipped} ${table} rows with invalid embeddings or embeddings larger than ${maxEmbeddingBytes} bytes (first rowids: ${firstSkippedRowids.join(", ")}). These cache entries will be rebuilt on demand.` : void 0;
}
/** Only migrations/imports interpret the retired JSON representation. */
function registerMemoryEmbeddingMigrationFunctions(db, renewAuthority) {
let renewedAt = Number.NEGATIVE_INFINITY;
const renew = () => {
if (!renewAuthority) return;
const now = performance.now();
if (now - renewedAt >= 1e3) {
renewAuthority();
renewedAt = now;
}
};
db.function("openclaw_memory_embedding_from_json", { deterministic: true }, (raw) => {
renew();
return encodeMemoryEmbedding(legacyEmbedding(raw) ?? []);
});
db.function("openclaw_memory_embedding_json_valid", { deterministic: true }, (raw) => {
renew();
return Number(legacyEmbedding(raw) !== void 0);
});
db.function("openclaw_memory_embedding_blob_valid", { deterministic: true }, (raw) => {
renew();
return Number(raw instanceof Uint8Array && raw.byteLength % 8 === 0 && decodeMemoryEmbedding(raw).length * 8 === raw.byteLength);
});
return renew;
}
function existingStorageObjects(db, table) {
return db.prepare(`SELECT sql FROM main.sqlite_schema
WHERE tbl_name = ? AND type IN ('index', 'trigger') AND sql IS NOT NULL
ORDER BY type, name`).all(table).map((row) => String(row.sql));
}
function replaceStorageTable(db, table, objects) {
db.exec(`
DROP TABLE ${table};
ALTER TABLE ${table}_storage_migration RENAME TO ${table};
`);
for (const sql of objects) db.exec(sql);
}
/** Record regeneration debt after a legacy import has verified its canonical copy. */
function markInvalidImportedMemoryEmbeddings(db, schema) {
if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(schema)) throw new Error("Invalid legacy memory schema identifier");
markInvalidMemoryEmbeddings(db, `
SELECT chunk.rowid AS storage_rowid,
openclaw_memory_embedding_json_valid(legacy.embedding) AS valid
FROM ${schema}.chunks AS legacy
JOIN main.memory_index_chunks AS chunk ON chunk.id = legacy.id
WHERE length(chunk.embedding) = 0
`);
}
function columns(db, table) {
return new Map(db.prepare("SELECT name, type FROM pragma_table_info(?)").all(table).map((row) => [String(row.name), String(row.type).toUpperCase()]));
}
/**
* Convert only memory storage, without provider calls. The agent migration owner
* supplies its admitted transaction with foreign keys disabled before BEGIN.
* Standalone/shadow index owners receive the same atomic conversion here.
*/
function migrateMemoryIndexStorage(db, options = {}) {
const cacheTable = options.embeddingCacheTable ?? "memory_embedding_cache";
if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(cacheTable)) throw new Error("Invalid memory embedding cache table identifier");
const shapes = storageShapes(db, cacheTable);
const migrateChunks = shapes.chunks === "legacy";
const migrateCache = shapes.cache === "legacy";
if (!migrateChunks && !migrateCache) return;
const foreignKeys = Number(db.prepare("PRAGMA foreign_keys").get()?.foreign_keys) !== 0;
if (migrateChunks && foreignKeys && db.isTransaction) throw new Error("Memory storage migration requires foreign keys disabled before BEGIN");
if (migrateChunks && foreignKeys) db.exec("PRAGMA foreign_keys = OFF");
let cacheWarning;
try {
runSqliteImmediateTransactionSync(db, () => {
const current = storageShapes(db, cacheTable);
if (current.chunks !== shapes.chunks || current.cache !== shapes.cache) throw new Error("Memory storage schema changed before migration admission");
const renewAuthority = registerMemoryEmbeddingMigrationFunctions(db, options.renewAuthority);
const chunkObjects = migrateChunks ? existingStorageObjects(db, "memory_index_chunks") : [];
const cacheObjects = migrateCache ? existingStorageObjects(db, cacheTable) : [];
if (current.chunks === "binary") assertBinaryEmbeddings(db, "memory_index_chunks");
if (current.cache === "binary") assertBinaryEmbeddings(db, cacheTable);
if (migrateChunks) {
ensureMemoryRecallMetadataSchema(db);
markInvalidMemoryEmbeddings(db, `
SELECT rowid AS storage_rowid,
openclaw_memory_embedding_json_valid(embedding) AS valid
FROM memory_index_chunks
`);
dropMemoryChunkFtsTriggers(db);
db.exec(MEMORY_INDEX_CHUNKS_SCHEMA_SQL.replace("CREATE TABLE IF NOT EXISTS", "CREATE TABLE").replace("memory_index_chunks", "memory_index_chunks_storage_migration"));
copyMemoryStorageRows(db, "memory_index_chunks", `
INSERT INTO memory_index_chunks_storage_migration (
chunk_rowid, id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
)
SELECT rowid, id, path, source, start_line, end_line, hash, model, text,
openclaw_memory_embedding_from_json(embedding), updated_at
FROM memory_index_chunks WHERE rowid = ?
`);
replaceStorageTable(db, "memory_index_chunks", chunkObjects);
if (db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?").get("memory_index_chunks_fts")) {
rebuildMemoryChunkFts(db, MEMORY_INDEX_FTS_TABLE);
ensureMemoryChunkFtsTriggers(db);
}
}
if (migrateCache) {
const replacement = `${cacheTable}_storage_migration`;
db.exec(buildMemoryEmbeddingCacheSchema(replacement).replace("CREATE TABLE IF NOT EXISTS", "CREATE TABLE"));
cacheWarning = copyLegacyMemoryEmbeddingCache(db, cacheTable, replacement, renewAuthority);
replaceStorageTable(db, cacheTable, cacheObjects);
}
if (migrateChunks) {
for (const table of ["memory_index_chunk_provenance", "memory_index_chunk_recall_metadata"]) if (columns(db, table).size > 0 && db.prepare(`PRAGMA foreign_key_check(${table})`).all().length > 0) throw new Error("Memory storage migration failed foreign key validation");
}
});
} finally {
if (options.renewAuthority) registerMemoryEmbeddingMigrationFunctions(db);
if (migrateChunks && foreignKeys) db.exec("PRAGMA foreign_keys = ON");
}
if (cacheWarning) {
if (options.onWarning) options.onWarning(cacheWarning);
else process.emitWarning(cacheWarning);
}
}
//#endregion
//#region packages/memory-host-sdk/src/host/memory-schema.ts
const LEGACY_MEMORY_INDEX_SOURCE_COLUMNS = [
"path",
"source",
"hash",
"mtime",
"size"
];
const MEMORY_INDEX_SOURCE_COLUMNS = ["id", ...LEGACY_MEMORY_INDEX_SOURCE_COLUMNS];
const MEMORY_INDEX_SOURCE_COLUMN_TYPES = /* @__PURE__ */ new Map([
["id", "INTEGER"],
["path", "TEXT"],
["source", "TEXT"],
["hash", "TEXT"],
["mtime", "REAL"],
["size", "INTEGER"]
]);
function tableColumnInfo(db, tableName) {
return db.prepare(`PRAGMA main.table_xinfo(${tableName})`).all().flatMap((row) => typeof row.name === "string" && typeof row.type === "string" ? [{
name: row.name,
type: row.type.toUpperCase(),
notnull: Number(row.notnull ?? 0),
pk: Number(row.pk ?? 0),
defaultValue: typeof row.dflt_value === "string" ? row.dflt_value : null,
hidden: Number(row.hidden ?? 0)
}] : []);
}
function tableHasExactColumns(db, tableName, expected, preparedColumns) {
const columns = new Set((preparedColumns ?? tableColumnInfo(db, tableName)).map((row) => row.name));
return columns.size === expected.length && expected.every((column) => columns.has(column));
}
function tableHasPrimaryKey(db, tableName, expectedColumns, preparedColumns) {
const columns = (preparedColumns ?? tableColumnInfo(db, tableName)).filter((row) => row.pk > 0).toSorted((left, right) => left.pk - right.pk).map((row) => row.name);
return columns.length === expectedColumns.length && columns.every((column, index) => column === expectedColumns[index]);
}
function tableHasUniqueIndex(db, tableName, expectedColumns) {
const indexes = db.prepare(`SELECT name, partial FROM pragma_index_list(?) WHERE "unique" = 1`).all(tableName);
if (indexes.length !== 1) return false;
return indexes.some((index) => {
if (typeof index.name !== "string" || Number(index.partial ?? 0) !== 0) return false;
const columns = db.prepare(`SELECT cid, name, coll, "desc" AS sort_desc, key FROM pragma_index_xinfo(?) ORDER BY seqno`).all(index.name).filter((row) => Number(row.key ?? 0) === 1);
return columns.length === expectedColumns.length && columns.every((column, columnIndex) => Number(column.cid ?? -1) >= 0 && column.name === expectedColumns[columnIndex] && column.coll === "BINARY" && Number(column.sort_desc ?? 0) === 0);
});
}
function tableHasNoDeclaredCollations(db, tableName) {
const row = db.prepare(`SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?`).get(tableName);
return typeof row?.sql === "string" && !/\bCOLLATE\b/iu.test(row.sql);
}
function tableHasSourceColumnContract(db, nullableColumn, preparedColumns) {
return (preparedColumns ?? tableColumnInfo(db, "memory_index_sources")).every((column) => (column.type === MEMORY_INDEX_SOURCE_COLUMN_TYPES.get(column.name) || column.name === "mtime" && column.type === "INTEGER") && column.defaultValue === (column.name === "source" ? "'memory'" : null) && column.hidden === 0 && (column.name === nullableColumn || column.notnull === 1));
}
function tableHasIntegerRowIdPrimaryKey(db, preparedColumns) {
if ((preparedColumns ?? tableColumnInfo(db, "memory_index_sources")).find((column) => column.name === "id")?.type !== "INTEGER" || !tableHasPrimaryKey(db, "memory_index_sources", ["id"], preparedColumns)) return false;
return db.prepare(`SELECT 1 AS found FROM pragma_index_list(?) WHERE origin = 'pk' LIMIT 1`).get(MEMORY_INDEX_SOURCES_TABLE)?.found !== 1;
}
function tableExists(db, tableName) {
return db.prepare(`SELECT 1 AS found FROM sqlite_master WHERE type = 'table' AND name = ?`).get(tableName)?.found === 1;
}
/** Upgrade canonical memory sources to stable integer identities. */
function migrateMemoryIndexSourcesIdentity(db) {
if (!tableExists(db, "memory_index_sources")) return;
const columns = canReuseSqliteSchemaInTransaction(db) ? tableColumnInfo(db, MEMORY_INDEX_SOURCES_TABLE) : void 0;
if (tableHasExactColumns(db, "memory_index_sources", MEMORY_INDEX_SOURCE_COLUMNS, columns)) {
if (tableHasSourceColumnContract(db, "id", columns) && tableHasIntegerRowIdPrimaryKey(db, columns) && tableHasNoDeclaredCollations(db, "memory_index_sources") && tableHasUniqueIndex(db, "memory_index_sources", ["path", "source"])) return;
throw new Error("canonical memory source identity schema is invalid");
}
if (!tableHasExactColumns(db, "memory_index_sources", LEGACY_MEMORY_INDEX_SOURCE_COLUMNS, columns)) throw new Error("canonical memory source identity schema is invalid");
const hasPathPrimaryKey = tableHasPrimaryKey(db, MEMORY_INDEX_SOURCES_TABLE, ["path"], columns);
const hasPathSourcePrimaryKey = tableHasPrimaryKey(db, MEMORY_INDEX_SOURCES_TABLE, ["path", "source"], columns);
if (!hasPathPrimaryKey && !hasPathSourcePrimaryKey) throw new Error("canonical memory source identity schema is invalid");
if (!tableHasSourceColumnContract(db, hasPathPrimaryKey ? "path" : void 0, columns)) throw new Error("canonical memory source identity schema is invalid");
const rebuildsPathFts = tableExists(db, MEMORY_INDEX_PATHS_FTS_TABLE);
db.exec("SAVEPOINT migrate_memory_index_sources_identity");
try {
dropMemoryPathFtsTriggers(db);
db.exec(`
DROP TRIGGER IF EXISTS memory_index_sources_revision_after_insert;
DROP TRIGGER IF EXISTS memory_index_sources_revision_after_update;
DROP TRIGGER IF EXISTS memory_index_sources_revision_after_delete;
ALTER TABLE ${MEMORY_INDEX_SOURCES_TABLE}
RENAME TO memory_index_sources_identity_migration;
CREATE TABLE ${MEMORY_INDEX_SOURCES_TABLE} (
id INTEGER PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime REAL NOT NULL,
size INTEGER NOT NULL,
UNIQUE (path, source)
) STRICT;
INSERT INTO ${MEMORY_INDEX_SOURCES_TABLE} (id, path, source, hash, mtime, size)
SELECT rowid, path, source, hash, mtime, size
FROM memory_index_sources_identity_migration;
DROP TABLE memory_index_sources_identity_migration;
`);
if (rebuildsPathFts) {
db.exec(`
DELETE FROM ${MEMORY_INDEX_PATHS_FTS_TABLE};
INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source)
SELECT id, path, source FROM ${MEMORY_INDEX_SOURCES_TABLE};
`);
ensureMemoryPathFtsTriggers(db);
}
db.exec("RELEASE migrate_memory_index_sources_identity");
} catch (err) {
db.exec("ROLLBACK TO migrate_memory_index_sources_identity");
db.exec("RELEASE migrate_memory_index_sources_identity");
throw err;
}
}
/** Ensure canonical memory index tables and the optional FTS table exist. */
function ensureMemoryIndexSchema(params) {
if (tableHasExactColumns(params.db, "meta", ["key", "value"]) && tableHasExactColumns(params.db, "files", [
"path",
"source",
"hash",
"mtime",
"size"
]) && tableHasExactColumns(params.db, "chunks", [
"id",
"path",
"source",
"start_line",
"end_line",
"hash",
"model",
"text",
"embedding",
"updated_at"
])) throw new Error("Retired memory index format detected. Preserve a complete copy of your state and configuration, then use OpenClaw 2026.9.7 to migrate a compatible copy of this index before retrying the upgrade.");
const embeddingCacheTable = params.embeddingCacheTable ?? "memory_embedding_cache";
const ftsTable = params.ftsTable ?? "memory_index_chunks_fts";
params.db.exec(buildMemoryIndexStrictSchema({
embeddingCacheTable,
includeEmbeddingCache: params.cacheEnabled
}));
migrateMemoryIndexStorage(params.db, { embeddingCacheTable });
ensureMemoryRecallMetadataSchema(params.db);
params.db.exec(`
INSERT OR IGNORE INTO ${MEMORY_INDEX_STATE_TABLE} (id, revision) VALUES (1, 0);
`);
migrateMemoryIndexSourcesIdentity(params.db);
params.db.exec(`
${[MEMORY_INDEX_SOURCES_TABLE, MEMORY_INDEX_CHUNKS_TABLE$1].flatMap((table) => [
"insert",
"update",
"delete"
].map((event) => `CREATE TRIGGER IF NOT EXISTS ${table}_revision_after_${event}
AFTER ${event.toUpperCase()} ON ${table}
BEGIN
UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1;
END;`)).join("\n")}
CREATE INDEX IF NOT EXISTS idx_memory_index_sources_source
ON ${MEMORY_INDEX_SOURCES_TABLE}(source);
CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path_source
ON ${MEMORY_INDEX_CHUNKS_TABLE$1}(path, source);
DROP INDEX IF EXISTS idx_memory_index_chunks_path;
CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_source
ON ${MEMORY_INDEX_CHUNKS_TABLE$1}(source);
`);
ensureMemoryChunkProvenance(params.db);
dropDisabledMemoryFts(params.db, ftsTable, params.ftsEnabled);
if (params.cacheEnabled) {
const updatedAtIndex = embeddingCacheTable === "memory_embedding_cache" ? "idx_memory_embedding_cache_updated_at" : "idx_embedding_cache_updated_at";
params.db.exec(`
CREATE INDEX IF NOT EXISTS ${updatedAtIndex}
ON ${embeddingCacheTable}(updated_at);
`);
}
(params.db.isTransaction ? migrateSqliteSchemaToStrictInTransaction : migrateSqliteSchemaToStrict)(params.db, buildMemoryIndexStrictSchema({
embeddingCacheTable,
includeEmbeddingCache: params.cacheEnabled || tableExists(params.db, embeddingCacheTable)
}), { databaseLabel: "memory index" });
let ftsAvailable = false;
let ftsError;
if (params.ftsEnabled) try {
const tokenizeClause = (params.ftsTokenizer ?? "unicode61") === "trigram" ? `, tokenize='trigram case_sensitive 0'` : "";
ensureMemoryChunkFtsSchema({
db: params.db,
ftsTable,
tokenizeClause
});
if (ftsTable === "memory_index_chunks_fts") ensureMemoryPathFtsSchema({
db: params.db,
tokenizeClause
});
ftsAvailable = true;
} catch (err) {
if (ftsTable === "memory_index_chunks_fts") {
dropMemoryChunkFtsTriggers(params.db);
dropMemoryPathFtsTriggers(params.db);
}
ftsError = formatErrorMessage(err);
}
return {
ftsAvailable,
...ftsError ? { ftsError } : {}
};
}
//#endregion
export { registerMemoryEmbeddingMigrationFunctions as a, MEMORY_INDEX_META_TABLE as c, formatErrorMessage as d, MEMORY_INDEX_CHUNK_RECALL_METADATA_TABLE as f, ensureMemoryChunkProvenance as g, MEMORY_INDEX_CHUNK_PROVENANCE_TABLE as h, migrateMemoryIndexStorage as i, MEMORY_INDEX_STATE_TABLE as l, hasLegacyMemoryRecallMetadataColumns as m, migrateMemoryIndexSourcesIdentity as n, MEMORY_EMBEDDING_CACHE_TABLE as o, ensureMemoryRecallMetadataSchema as p, markInvalidImportedMemoryEmbeddings as r, MEMORY_INDEX_DERIVED_TABLES as s, ensureMemoryIndexSchema as t, MEMORY_INDEX_VECTOR_TABLE as u };