openclaw/dist/memory-schema-fts-B8vS-yZQ.mjs

Multi-channel AI gateway with extensible messaging integrations

//#region packages/memory-host-sdk/src/host/memory-schema-fts.ts
const MEMORY_INDEX_SOURCES_TABLE = "memory_index_sources";
const MEMORY_INDEX_CHUNKS_TABLE = "memory_index_chunks";
const MEMORY_INDEX_FTS_TABLE = "memory_index_chunks_fts";
const MEMORY_INDEX_PATHS_FTS_TABLE = "memory_index_paths_fts";
/** Check every persisted FTS column declaration and supported table option. */
function ftsTableMatchesSchema(params) {
	const table = params.db.prepare("SELECT type, sql FROM sqlite_schema WHERE type IN ('table', 'view') AND name = ? COLLATE NOCASE").get(params.tableName);
	if (table?.type === "view") throw new Error(`cannot modify ${params.tableName} because it is a view`);
	if (typeof table?.sql !== "string") return "missing";
	const definition = /^\s*CREATE\s+VIRTUAL\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:"[^"]+"|`[^`]+`|\[[^\]]+\]|[^\s(]+)\s+USING\s+fts5\s*\(([\s\S]*)\)\s*$/iu.exec(table.sql)?.[1];
	if (definition === void 0) return "not-fts";
	const declarations = definition.split(",").map((declaration) => declaration.trim().replace(/\s+/g, " ").toLowerCase());
	const expectedDeclarations = params.expectedColumns.map((column, index) => index === 0 ? column : `${column} unindexed`);
	if (declarations.length < expectedDeclarations.length || expectedDeclarations.some((column, index) => declarations[index] !== column)) return "mismatched";
	const options = declarations.slice(expectedDeclarations.length);
	const tokenizerOption = options[0] ?? "";
	if (!(params.tokenizeClause.includes("trigram") ? options.length === 1 && /^tokenize\s*=\s*(['"])trigram case_sensitive 0\1$/u.test(tokenizerOption) : options.length === 0 || options.length === 1 && /^tokenize\s*=\s*(['"])unicode61\1$/u.test(tokenizerOption))) return "mismatched";
	const columns = params.db.prepare("SELECT name FROM pragma_table_info(?) ORDER BY cid").all(params.tableName);
	return columns.length === params.expectedColumns.length && columns.every((column, index) => column.name === params.expectedColumns[index]) ? "matching" : "mismatched";
}
/** Remove only derived FTS state whose persisted schema no longer matches. */
function dropMismatchedFtsTable(params) {
	const status = ftsTableMatchesSchema(params);
	if (status === "missing" || status === "matching") return;
	if (status === "not-fts" && params.tableName !== "memory_index_chunks_fts" && params.tableName !== "memory_index_paths_fts") throw new Error(`Memory FTS table "${params.tableName}" collides with a non-FTS table`);
	if (params.tableName === "memory_index_paths_fts") dropMemoryPathFtsTriggers(params.db);
	if (params.tableName === "memory_index_chunks_fts") dropMemoryChunkFtsTriggers(params.db);
	params.db.exec(`DROP TABLE ${params.tableName}`);
}
function buildMemoryFtsTriggers(name, table, columns, insert, remove) {
	const trigger = (event, clause, body) => ({
		name: `${name}_after_${event}`,
		sql: `
      CREATE TRIGGER IF NOT EXISTS main.${name}_after_${event}
      AFTER ${clause} ON ${table}
      BEGIN
${body}
      END;
    `
	});
	return [
		trigger("insert", "INSERT", insert),
		trigger("update", `UPDATE OF ${columns}`, `${remove}
${insert}`),
		trigger("delete", "DELETE", remove)
	];
}
/** Optional canonical triggers owned by the derived path FTS index. */
const MEMORY_PATH_FTS_TRIGGER_DEFINITIONS = buildMemoryFtsTriggers(MEMORY_INDEX_PATHS_FTS_TABLE, MEMORY_INDEX_SOURCES_TABLE, "id, path, source", `        INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source)
        VALUES (NEW.id, NEW.path, NEW.source);`, `        DELETE FROM ${MEMORY_INDEX_PATHS_FTS_TABLE}
        WHERE rowid = OLD.id;`);
/** Canonical chunks own the optional body index; every mutation addresses its rowid. */
const MEMORY_CHUNK_FTS_TRIGGER_DEFINITIONS = buildMemoryFtsTriggers(MEMORY_INDEX_FTS_TABLE, MEMORY_INDEX_CHUNKS_TABLE, "chunk_rowid, text, id, path, source, model, start_line, end_line", `        INSERT INTO ${MEMORY_INDEX_FTS_TABLE} (rowid, text, id, path, source, model, start_line, end_line)
        VALUES (NEW.chunk_rowid, NEW.text, NEW.id, NEW.path, NEW.source, NEW.model, NEW.start_line, NEW.end_line);`, `        DELETE FROM ${MEMORY_INDEX_FTS_TABLE} WHERE rowid = OLD.chunk_rowid;`);
function dropMemoryChunkFtsTriggers(db) {
	for (const trigger of MEMORY_CHUNK_FTS_TRIGGER_DEFINITIONS) db.exec(`DROP TRIGGER IF EXISTS main.${trigger.name}`);
}
function ensureMemoryChunkFtsTriggers(db) {
	for (const trigger of MEMORY_CHUNK_FTS_TRIGGER_DEFINITIONS) db.exec(trigger.sql);
}
function rebuildMemoryChunkFts(db, ftsTable) {
	db.exec(`
    DELETE FROM ${ftsTable};
    INSERT INTO ${ftsTable} (
      rowid, text, id, path, source, model, start_line, end_line
    )
    SELECT chunk_rowid, text, id, path, source, model, start_line, end_line
    FROM ${MEMORY_INDEX_CHUNKS_TABLE};
  `);
}
function buildMemoryChunkFtsSchemaSql(ftsTable, tokenizeClause) {
	return `CREATE VIRTUAL TABLE IF NOT EXISTS ${ftsTable} USING fts5(\n  text,\n  id UNINDEXED,\n  path UNINDEXED,\n  source UNINDEXED,\n  model UNINDEXED,\n  start_line UNINDEXED,\n  end_line UNINDEXED\n${tokenizeClause});`;
}
function buildMemoryPathFtsSchemaSql(tokenizeClause) {
	return `CREATE VIRTUAL TABLE IF NOT EXISTS ${MEMORY_INDEX_PATHS_FTS_TABLE} USING fts5(
        path,
        source UNINDEXED
        ${tokenizeClause}
      );`;
}
/** Canonical FTS definitions, including the deployed explicit unicode61 spellings. */
function getMemoryFtsSchemaSqlVariants() {
	return [
		"",
		", tokenize='unicode61'",
		", tokenize=\"unicode61\"",
		", tokenize='trigram case_sensitive 0'"
	].flatMap((tokenizeClause) => [buildMemoryChunkFtsSchemaSql(MEMORY_INDEX_FTS_TABLE, tokenizeClause), buildMemoryPathFtsSchemaSql(tokenizeClause)]);
}
/** Reconcile and backfill the derived body index in one atomic savepoint. */
function ensureMemoryChunkFtsSchema(params) {
	if (params.ftsTable.toLowerCase() === "memory_index_paths_fts".toLowerCase()) throw new Error(`Memory body FTS table "${params.ftsTable}" collides with the path FTS index`);
	params.db.exec("SAVEPOINT ensure_memory_index_chunks_fts");
	try {
		dropMismatchedFtsTable({
			db: params.db,
			tableName: params.ftsTable,
			expectedColumns: [
				"text",
				"id",
				"path",
				"source",
				"model",
				"start_line",
				"end_line"
			],
			tokenizeClause: params.tokenizeClause
		});
		params.db.exec(buildMemoryChunkFtsSchemaSql(params.ftsTable, params.tokenizeClause));
		const rowCounts = params.db.prepare(`SELECT
           (SELECT COUNT(*) FROM ${MEMORY_INDEX_CHUNKS_TABLE}) AS canonical_count,
           (SELECT COUNT(*) FROM ${params.ftsTable}) AS derived_count`).get();
		const canonical = params.ftsTable === MEMORY_INDEX_FTS_TABLE;
		if (!(!canonical || MEMORY_CHUNK_FTS_TRIGGER_DEFINITIONS.every((trigger) => params.db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'trigger' AND name = ?").get(trigger.name))) || rowCounts.canonical_count !== rowCounts.derived_count) rebuildMemoryChunkFts(params.db, params.ftsTable);
		if (canonical) ensureMemoryChunkFtsTriggers(params.db);
		params.db.exec("RELEASE ensure_memory_index_chunks_fts");
	} catch (err) {
		params.db.exec("ROLLBACK TO ensure_memory_index_chunks_fts");
		params.db.exec("RELEASE ensure_memory_index_chunks_fts");
		throw err;
	}
}
function dropDisabledMemoryFts(db, ftsTable, enabled) {
	if (enabled) return;
	dropMemoryPathFtsTriggers(db);
	db.exec(`DROP TABLE IF EXISTS ${MEMORY_INDEX_PATHS_FTS_TABLE}`);
	if (ftsTable === "memory_index_chunks_fts") {
		dropMemoryChunkFtsTriggers(db);
		db.exec(`DROP TABLE IF EXISTS ${ftsTable}`);
	}
}
/** Drop the canonical source-to-path-FTS maintenance triggers. */
function dropMemoryPathFtsTriggers(db) {
	for (const trigger of MEMORY_PATH_FTS_TRIGGER_DEFINITIONS) db.exec(`DROP TRIGGER IF EXISTS main.${trigger.name}`);
}
/** Install the canonical source-to-path-FTS maintenance triggers. */
function ensureMemoryPathFtsTriggers(db) {
	for (const trigger of MEMORY_PATH_FTS_TRIGGER_DEFINITIONS) db.exec(trigger.sql);
}
function ensureMemoryPathFtsSchema(params) {
	params.db.exec("SAVEPOINT ensure_memory_index_paths_fts");
	try {
		dropMismatchedFtsTable({
			db: params.db,
			tableName: MEMORY_INDEX_PATHS_FTS_TABLE,
			expectedColumns: ["path", "source"],
			tokenizeClause: params.tokenizeClause
		});
		params.db.exec(`
      ${buildMemoryPathFtsSchemaSql(params.tokenizeClause)}
      -- The initial copy and trigger installation share this savepoint. Once
      -- populated, the triggers own completeness; per-row FTS probes are too costly.
      INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source)
      SELECT id, path, source
      FROM ${MEMORY_INDEX_SOURCES_TABLE}
      WHERE NOT EXISTS (SELECT 1 FROM ${MEMORY_INDEX_PATHS_FTS_TABLE} LIMIT 1);
    `);
		ensureMemoryPathFtsTriggers(params.db);
		params.db.exec("RELEASE ensure_memory_index_paths_fts");
	} catch (err) {
		params.db.exec("ROLLBACK TO ensure_memory_index_paths_fts");
		params.db.exec("RELEASE ensure_memory_index_paths_fts");
		throw err;
	}
}
//#endregion
export { MEMORY_INDEX_SOURCES_TABLE as a, dropMemoryChunkFtsTriggers as c, ensureMemoryChunkFtsTriggers as d, ensureMemoryPathFtsSchema as f, rebuildMemoryChunkFts as h, MEMORY_INDEX_PATHS_FTS_TABLE as i, dropMemoryPathFtsTriggers as l, getMemoryFtsSchemaSqlVariants as m, MEMORY_INDEX_CHUNKS_TABLE as n, MEMORY_PATH_FTS_TRIGGER_DEFINITIONS as o, ensureMemoryPathFtsTriggers as p, MEMORY_INDEX_FTS_TABLE as r, dropDisabledMemoryFts as s, MEMORY_CHUNK_FTS_TRIGGER_DEFINITIONS as t, ensureMemoryChunkFtsSchema as u };