openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
509 lines (508 loc) • 26.8 kB
JavaScript
import { d as normalizeStringEntries, f as normalizeStringEntriesLower, y as uniqueStrings } from "./string-normalization-CECOqf64.mjs";
import { t as escapeRegExp } from "./regexp-BZyMFTlj.mjs";
import { a as getNodeSqliteKysely, r as executeSqliteQuerySync, u as sqliteStringSet } from "./kysely-sync-K7ftY-g4.mjs";
import { u as MEMORY_INDEX_VECTOR_TABLE } from "./memory-schema-BA04oCjG.mjs";
import { n as decodeMemoryEmbedding, t as cosineSimilarity } from "./embedding-vector-DA_9H0FL.mjs";
import "./memory-core-host-engine-knn-DTjLOT03.mjs";
import { n as projectMemorySearchRow, r as resolveSnippetProjection, t as buildMemoryModelFilter } from "./manager-search-shared-DI3UduOd.mjs";
import "./string-coerce-runtime-50qKeWeL.mjs";
import { s as readMemoryRecallMetadata } from "./engine-storage-yT207ZoA.mjs";
import "./memory-core-host-engine-storage-g5_Iw_VB.mjs";
import { s as buildFileEntry, u as listMemoryFiles } from "./read-file-DsXzvp1B.mjs";
import { t as runWithConcurrency } from "./concurrency-BvONODc2.mjs";
import "./sqlite-runtime-Ch9K5Xik.mjs";
import "./text-utility-runtime-BDs7x8Fk.mjs";
import { a as resolvePersistedMemoryVectorIndexState, r as memoryTableExists } from "./manager-vector-rebuild-state-DWrs_Mbx.mjs";
import { setImmediate } from "node:timers/promises";
//#region extensions/memory-core/src/memory/keyword-query.ts
function buildFtsQuery(raw) {
return buildMatchQueryFromTerms(normalizeStringEntries(raw.match(/[\p{L}\p{N}_]+/gu) ?? []));
}
function buildMatchQueryFromTerms(terms) {
if (terms.length === 0) return null;
return terms.map((term) => `"${term.replaceAll("\"", "")}"`).join(" AND ");
}
function bm25RankToScore(rank) {
if (!Number.isFinite(rank)) return 1 / 1e3;
if (rank < 0) {
const relevance = -rank;
return relevance / (1 + relevance);
}
return 1 / (1 + rank);
}
//#endregion
//#region extensions/memory-core/src/memory/manager-source-state.ts
/** Resolve exactly the entries eligible for indexing, including validated multimodal files. */
async function resolveMemorySourceFileEntries(params) {
const files = await (params.files?.listFiles ?? listMemoryFiles)(params.workspaceDir, params.settings.extraPaths, params.settings.multimodal, params.onSkippedSymlinkRoot);
return (await runWithConcurrency(files.map((file) => async () => await (params.files?.inspectFile ?? buildFileEntry)(file, params.workspaceDir, params.settings.multimodal)), params.concurrency)).filter((entry) => entry !== null);
}
async function inspectMemorySourceState(params) {
const skippedRoots = /* @__PURE__ */ new Set();
const entries = await resolveMemorySourceFileEntries({
...params,
onSkippedSymlinkRoot: (root) => skippedRoots.add(root)
});
const indexedByPath = new Map(loadMemorySourceFileState({
db: params.db,
source: "memory"
}).map((row) => [row.path, row.hash]));
return {
source: "memory",
dirty: indexedByPath.size !== entries.length || entries.some((entry) => indexedByPath.get(entry.path) !== entry.hash),
eligible: entries.length,
issues: [...entries.length === 0 ? ["no eligible memory files found"] : [], ...Array.from(skippedRoots, (root) => `extra path "${root}" is a symlink root; symlinked roots are not traversed, so configure its canonical absolute directory instead`)]
};
}
function loadMemorySourceFileState(params) {
let query = getNodeSqliteKysely(params.db).selectFrom("memory_index_sources").select([
"path",
"hash",
"mtime",
"size"
]).where("source", "=", params.source);
if (params.paths) query = query.where("path", "in", sqliteStringSet(params.paths));
return executeSqliteQuerySync(params.db, query).rows;
}
//#endregion
//#region extensions/memory-core/src/memory/manager-retrieval-read.ts
const MEMORY_INDEX_META_KEY = "memory_index_meta_v1";
function readMemoryIndexMetadata(db) {
const row = db.prepare("SELECT value FROM memory_index_meta WHERE key = ?").get(MEMORY_INDEX_META_KEY);
if (typeof row?.value !== "string" || !row.value) return {
meta: null,
serialized: null
};
try {
return {
meta: JSON.parse(row.value),
serialized: row.value
};
} catch {
return {
meta: null,
serialized: null
};
}
}
function readMemoryRetrievalIndexState(db) {
const { meta } = readMemoryIndexMetadata(db);
const hasIndexedChunks = db.prepare("SELECT 1 FROM memory_index_chunks LIMIT 1").get() !== void 0;
return {
meta,
hasIndexedChunks,
hasFtsContent: !hasIndexedChunks && memoryTableExists(db, "memory_index_chunks_fts") && db.prepare(`SELECT 1 FROM memory_index_chunks_fts LIMIT 1`).get() !== void 0,
vectorState: meta && meta.provider !== "none" ? resolvePersistedMemoryVectorIndexState({
db,
vectorTable: MEMORY_INDEX_VECTOR_TABLE,
metaVectorDims: meta.vectorDims,
hasSemanticChunks: db.prepare("SELECT 1 FROM memory_index_chunks WHERE model != 'fts-only' LIMIT 1").get() !== void 0
}) : { state: "empty" }
};
}
function readMemoryRecallData(db, request) {
const rows = readMemoryRecallMetadata(db, request.candidates.map((entry) => entry.id));
const sourceMtimes = {
memory: /* @__PURE__ */ new Map(),
sessions: /* @__PURE__ */ new Map()
};
for (const source of ["sessions", "memory"]) {
if (source === "memory" && !request.includeMemoryMtimes) continue;
const paths = Array.from(new Set(request.candidates.filter((entry) => entry.source === source && rows.has(entry.id)).map((entry) => entry.path)));
if (paths.length > 0) sourceMtimes[source] = new Map(loadMemorySourceFileState({
db,
source,
paths
}).map((row) => [row.path, row.mtime]));
}
return {
rows,
sourceMtimes
};
}
//#endregion
//#region extensions/memory-core/src/memory/manager-search-vector.ts
const FALLBACK_VECTOR_BATCH_SIZE = 256;
function resolveProviderModels(primary, aliases) {
return Array.from(/* @__PURE__ */ new Set([primary, ...(aliases ?? []).filter(Boolean)]));
}
async function searchVector(params) {
if (params.queryVec.length === 0 || params.limit <= 0) return [];
params.signal?.throwIfAborted();
const providerModels = resolveProviderModels(params.providerModel, params.providerModelAliases);
const vectorReady = await params.ensureVectorReady(params.queryVec.length);
params.signal?.throwIfAborted();
if (vectorReady) {
if (!params.runVectorKnn) throw new Error("memory vector KNN subprocess is unavailable");
const response = await params.runVectorKnn({
vectorTable: params.vectorTable,
providerModels,
queryVec: params.queryVec,
limit: params.limit,
snippetMaxChars: params.snippetMaxChars,
sourceFilter: params.sourceFilterVec
}, params.signal);
if (response.fallbackScanRequired) return await params.runFallback();
return response.rows.map((row) => projectMemorySearchRow(row, params.snippetMaxChars, 1 - row.dist));
}
return await params.runFallback();
}
async function searchChunksByEmbedding(params) {
if (params.limit <= 0) return [];
const providerModels = resolveProviderModels(params.providerModel, params.providerModelAliases);
const projection = `SELECT rowid AS rowid, embedding
FROM memory_index_chunks
WHERE ${buildMemoryModelFilter("model", providerModels)}`;
const ordering = `${params.sourceFilter.sql}\n ORDER BY rowid ASC\n LIMIT ?`;
const firstStmt = params.db.prepare(`${projection}${ordering}`);
const stmt = params.db.prepare(`${projection} AND rowid > ?${ordering}`);
firstStmt.setReadBigInts(true);
stmt.setReadBigInts(true);
const snippet = resolveSnippetProjection("text", params.snippetMaxChars);
const payloadStmt = params.db.prepare(`SELECT id, path, start_line, end_line, ${snippet.sql} AS text, source FROM memory_index_chunks WHERE rowid = ?`);
const topResults = [];
let lastRowid;
while (true) {
const batch = lastRowid === void 0 ? firstStmt.iterate(...providerModels, ...params.sourceFilter.params, FALLBACK_VECTOR_BATCH_SIZE) : stmt.iterate(...providerModels, lastRowid, ...params.sourceFilter.params, FALLBACK_VECTOR_BATCH_SIZE);
let batchSize = 0;
for (const row of batch) {
batchSize += 1;
lastRowid = row.rowid;
const score = cosineSimilarity(params.queryVec, decodeMemoryEmbedding(row.embedding));
const lowest = topResults.at(-1);
if (Number.isFinite(score) && (topResults.length < params.limit || lowest && score > lowest.score)) {
const payload = payloadStmt.get(...snippet.params, row.rowid);
const result = projectMemorySearchRow(payload, params.snippetMaxChars, score);
if (topResults.length < params.limit) {
topResults.push(result);
if (topResults.length === params.limit) topResults.sort((a, b) => b.score - a.score);
} else {
topResults[topResults.length - 1] = result;
topResults.sort((a, b) => b.score - a.score);
}
}
}
if (batchSize < FALLBACK_VECTOR_BATCH_SIZE) break;
await setImmediate();
params.signal?.throwIfAborted();
}
topResults.sort((a, b) => b.score - a.score);
return topResults;
}
//#endregion
//#region extensions/memory-core/src/memory/manager-search.ts
const FTS_QUERY_TOKEN_RE = /[\p{L}\p{N}_]+/gu;
const EXACT_PATH_SPECIFICITY_SQL_FUNCTION = "openclaw_memory_exact_path_specificity";
const NORMALIZED_CONTAINS_SQL_FUNCTION = "openclaw_memory_normalized_contains";
function loadKeywordRowsWithFallback(plan, loadRows, fallbackTokens, warning) {
if (plan.matchQuery) try {
return {
rows: loadRows(plan.matchQuery, plan.substringTerms),
usedMatch: true
};
} catch (error) {
console.warn(`${warning}: ${String(error)}`);
return {
rows: loadRows(null, uniqueStrings([...fallbackTokens(), ...plan.substringTerms])),
usedMatch: false
};
}
return {
rows: loadRows(null, plan.substringTerms),
usedMatch: false
};
}
function comparePathKeywordSearchResults(left, right) {
const specificityDelta = right.exactPathSpecificity - left.exactPathSpecificity;
if (specificityDelta !== 0) return specificityDelta;
if (left.exactPathSpecificity === 0) {
const pathDelta = right.pathScore - left.pathScore;
if (pathDelta !== 0) return pathDelta;
}
return left.path.localeCompare(right.path) || left.startLine - right.startLine || left.id.localeCompare(right.id);
}
function normalizeSearchTokens(raw) {
return normalizeStringEntriesLower(raw.normalize("NFC").match(FTS_QUERY_TOKEN_RE) ?? []);
}
function literalSearchMatcher(value, whole = false) {
const literal = escapeRegExp(value.normalize("NFC"));
return new RegExp(whole ? `^(?:${literal})$` : literal, "iu");
}
function scoreFallbackKeywordResult(params) {
const { queryMatchers } = params;
if (queryMatchers.length === 0) return params.ftsScore;
const textTokens = normalizeSearchTokens(params.text);
const textTokenSet = new Set(textTokens);
const overlap = queryMatchers.filter(({ word }) => textTokens.some((token) => word.test(token))).length;
const uniqueQueryOverlap = overlap / queryMatchers.length;
const density = overlap / Math.max(textTokenSet.size, 1);
const normalizedPath = params.path.normalize("NFC");
const pathBoost = queryMatchers.reduce((score, { substring }) => score + (substring.test(normalizedPath) ? .18 : 0), 0);
const textLengthBoost = Math.min(params.text.length / 160, .18);
const lexicalBoost = uniqueQueryOverlap * .45 + density * .2 + pathBoost + textLengthBoost;
return Math.min(1, params.ftsScore + lexicalBoost);
}
function escapeLikePattern(term) {
return term.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
}
function isAscii(value) {
return !/[^\p{ASCII}]/u.test(value);
}
function resolveUnicodeCandidateAnchors(value) {
const firstNonAsciiCodePoint = Array.from(value).find((codePoint) => !isAscii(codePoint));
if (!firstNonAsciiCodePoint) return [];
return [.../* @__PURE__ */ new Set([
firstNonAsciiCodePoint,
firstNonAsciiCodePoint.toLowerCase(),
firstNonAsciiCodePoint.toUpperCase()
])];
}
function normalizePathIdentifier(value) {
return value.trim().replaceAll("\\", "/").replace(/^\.\//, "").normalize("NFC").toLowerCase();
}
function prepareExactPathMatcher(query) {
const normalizedQuery = normalizePathIdentifier(query);
if (!normalizedQuery || normalizedQuery === ".") return () => 0;
const hasDirectory = normalizedQuery.includes("/");
return (candidatePath) => {
const normalizedPath = normalizePathIdentifier(candidatePath);
if (normalizedQuery === normalizedPath) return 3;
if (hasDirectory) return 0;
const basename = normalizedPath.split("/").at(-1) ?? normalizedPath;
if (normalizedQuery === basename) return 2;
const extensionIndex = basename.lastIndexOf(".");
const stem = extensionIndex > 0 ? basename.slice(0, extensionIndex) : basename;
return normalizedQuery === stem ? 1 : 0;
};
}
function registerSubstringSqlFunction(db, terms) {
const matchers = new Map(terms.map((term) => [term, literalSearchMatcher(term)]));
db.function(NORMALIZED_CONTAINS_SQL_FUNCTION, { deterministic: true }, (value, query) => typeof value === "string" && typeof query === "string" ? Number(matchers.get(query)?.test(value.normalize("NFC")) === true) : 0);
}
function buildSubstringFilter(terms, column) {
return terms.map(() => ` AND ${NORMALIZED_CONTAINS_SQL_FUNCTION}(${column}, ?) = 1`).join("");
}
function buildExactPathCandidatePatterns(query) {
const normalized = query.trim().replaceAll("\\", "/").replace(/^\.\//, "");
if (!normalized || normalized === ".") return [];
const canonicalForms = [normalized.normalize("NFC"), normalized.normalize("NFD")];
const forms = new Set(canonicalForms);
if (!isAscii(normalized)) for (const form of canonicalForms) {
forms.add(form.toLowerCase());
forms.add(form.toUpperCase());
}
const patterns = /* @__PURE__ */ new Set();
for (const form of forms) {
const escaped = escapeLikePattern(form);
if (normalized.includes("/")) {
patterns.add(escaped);
continue;
}
patterns.add(escaped);
patterns.add(`${escaped}.%`);
patterns.add(`%/${escaped}`);
patterns.add(`%/${escaped}.%`);
}
if (!isAscii(normalized)) {
const asciiAnchor = normalized.normalize("NFD").toLowerCase().match(/[a-z0-9_]+/g)?.toSorted((left, right) => right.length - left.length)[0];
if (asciiAnchor) patterns.add(`%${escapeLikePattern(asciiAnchor)}%`);
if (normalized.toLowerCase() !== normalized.toUpperCase()) for (const anchor of resolveUnicodeCandidateAnchors(normalized)) patterns.add(`%${escapeLikePattern(anchor)}%`);
}
return [...patterns];
}
function planKeywordSearch(params) {
if (params.ftsTokenizer !== "trigram") return {
matchQuery: params.buildFtsQuery(params.query),
substringTerms: []
};
const tokenPattern = params.includeCombiningMarks ? /[\p{L}\p{M}\p{N}_]+/gu : FTS_QUERY_TOKEN_RE;
const tokens = normalizeStringEntries(params.query.match(tokenPattern) ?? []);
if (tokens.length === 0) return {
matchQuery: null,
substringTerms: []
};
const matchTerms = [];
const substringTerms = [];
for (const token of tokens) {
if (Array.from(token).length < 3) {
substringTerms.push(token);
continue;
}
matchTerms.push(token);
}
return {
matchQuery: buildMatchQueryFromTerms(matchTerms),
substringTerms
};
}
function planPathKeywordSearch(params) {
const forms = params.ftsTokenizer === "trigram" ? /* @__PURE__ */ new Set([params.query.normalize("NFC"), params.query.normalize("NFD")]) : /* @__PURE__ */ new Set([params.query]);
const seen = /* @__PURE__ */ new Set();
const plans = [];
const addPlan = (query, plan) => {
const key = JSON.stringify([plan.matchQuery, plan.substringTerms]);
if (!seen.has(key)) {
seen.add(key);
plans.push({
query,
...plan
});
}
};
for (const query of forms) addPlan(query, planKeywordSearch({
...params,
query,
includeCombiningMarks: true
}));
if (params.ftsTokenizer !== "trigram") for (const query of /* @__PURE__ */ new Set([params.query.normalize("NFC"), params.query.normalize("NFD")])) {
const tokens = normalizeStringEntries(query.match(/[\p{L}\p{M}\p{N}_]+/gu) ?? []);
const substringTerms = tokens.filter((token) => !isAscii(token));
if (substringTerms.length > 0) addPlan(query, {
matchQuery: buildMatchQueryFromTerms(tokens.filter(isAscii)),
substringTerms
});
}
return plans;
}
async function searchKeyword(params) {
if (params.limit <= 0) return [];
const plan = planKeywordSearch({
query: params.query,
ftsTokenizer: params.ftsTokenizer,
buildFtsQuery: params.buildFtsQuery
});
if (!plan.matchQuery && plan.substringTerms.length === 0) return [];
const liveChunkClause = ` AND EXISTS (SELECT 1 FROM memory_index_chunks c WHERE c.id = ${params.ftsTable}.id)`;
const loadRows = (matchQuery, terms) => {
const filter = buildSubstringFilter(terms, "text");
if (terms.length > 0) registerSubstringSqlFunction(params.db, terms);
const matchClause = matchQuery ? `${params.ftsTable} MATCH ? AND ${params.ftsTable}.rank MATCH 'bm25()'` : "1=1";
return params.db.prepare(`SELECT id, path, source, start_line, end_line, text,\n ${matchQuery ? `${params.ftsTable}.rank` : "0"} AS rank\n FROM ${params.ftsTable}\n WHERE ${matchClause}${filter}${liveChunkClause}${params.sourceFilter.sql}\n` + (matchQuery ? ` ORDER BY rank ASC\n` : "") + ` LIMIT ?`).all(...matchQuery ? [matchQuery] : [], ...terms, ...params.sourceFilter.params, params.limit);
};
const { rows, usedMatch } = loadKeywordRowsWithFallback(plan, loadRows, () => normalizeStringEntries(params.query.match(FTS_QUERY_TOKEN_RE) ?? []), "memory search: FTS5 MATCH failed, falling back to substring search");
const queryMatchers = params.boostFallbackRanking ? uniqueStrings(normalizeSearchTokens(params.rankingQuery ?? params.query)).map((token) => ({
word: literalSearchMatcher(token, true),
substring: literalSearchMatcher(token)
})) : [];
return rows.map((row) => {
const textScore = usedMatch ? params.bm25RankToScore(row.rank) : 0;
const score = params.boostFallbackRanking ? scoreFallbackKeywordResult({
queryMatchers,
path: row.path,
text: row.text,
ftsScore: textScore
}) : textScore;
return Object.assign(projectMemorySearchRow(row, params.snippetMaxChars, score), {
textScore,
hasBodyMatch: true
});
});
}
async function searchPathKeyword(params) {
if (params.limit <= 0) return [];
const snippet = resolveSnippetProjection("c.text", params.snippetMaxChars);
const pathColumn = `${params.pathFtsTable}.path`;
const pathPlans = planPathKeywordSearch({
query: params.query,
ftsTokenizer: params.ftsTokenizer,
buildFtsQuery: params.buildFtsQuery
});
const plan = pathPlans[0] ?? {
query: params.query,
matchQuery: null,
substringTerms: []
};
const planSubstringFilter = buildSubstringFilter(plan.substringTerms, pathColumn);
registerSubstringSqlFunction(params.db, plan.substringTerms);
const exactPathQuery = params.exactPathQuery ?? params.query;
const matchExactPath = prepareExactPathMatcher(exactPathQuery);
params.db.function(EXACT_PATH_SPECIFICITY_SQL_FUNCTION, { deterministic: true }, (candidatePath) => typeof candidatePath === "string" ? matchExactPath(candidatePath) : 0);
const hasExplicitExactPathHeadroom = params.exactPathLimit !== void 0;
const exactPathLimit = Math.max(0, Math.floor(params.exactPathLimit ?? params.limit));
const exactCandidatePatterns = buildExactPathCandidatePatterns(exactPathQuery);
const loadExactRows = (useLexicalCandidates) => {
const qualifiedPatternClause = exactCandidatePatterns.map(() => `${pathColumn} LIKE ? ESCAPE '\\'`).join(" OR ");
const candidateCtes = useLexicalCandidates ? `candidates AS MATERIALIZED (\n SELECT ${params.pathFtsTable}.path, ${params.pathFtsTable}.source\n FROM ${params.pathFtsTable}\n WHERE ${plan.matchQuery ? `${params.pathFtsTable} MATCH ?` : "1=1"}${planSubstringFilter}${params.sourceFilter.sql}\n), pattern_candidates AS MATERIALIZED (\n SELECT path, source FROM candidates\n WHERE (${exactCandidatePatterns.map(() => "path LIKE ? ESCAPE '\\'").join(" OR ")})\n)` : `pattern_candidates AS MATERIALIZED (\n SELECT ${params.pathFtsTable}.path, ${params.pathFtsTable}.source\n FROM ${params.pathFtsTable}\n WHERE (${qualifiedPatternClause})${params.sourceFilter.sql}\n)`;
const candidateParams = useLexicalCandidates ? [
...plan.matchQuery ? [plan.matchQuery] : [],
...plan.substringTerms,
...params.sourceFilter.params,
...exactCandidatePatterns
] : [...exactCandidatePatterns, ...params.sourceFilter.params];
return params.db.prepare(`WITH ${candidateCtes}, scored_paths AS MATERIALIZED (\n SELECT path, source,\n ${EXACT_PATH_SPECIFICITY_SQL_FUNCTION}(path) AS exact_path_specificity\n FROM pattern_candidates\n), exact_paths AS MATERIALIZED (\n SELECT path, source, exact_path_specificity FROM scored_paths\n WHERE exact_path_specificity > 0\n AND EXISTS (SELECT 1 FROM memory_index_chunks live\n WHERE live.path = scored_paths.path\n AND live.source = scored_paths.source)\n ORDER BY exact_path_specificity DESC, path ASC, source ASC\n LIMIT ?\n)\nSELECT c.id, exact_paths.path, exact_paths.source,\n c.start_line, c.end_line, ${snippet.sql} AS text, exact_paths.exact_path_specificity\n FROM exact_paths\n CROSS JOIN memory_index_chunks c INDEXED BY sqlite_autoindex_memory_index_chunks_1 ON c.id = (\n SELECT candidate.id FROM memory_index_chunks candidate INDEXED BY idx_memory_index_chunks_path_source\n WHERE candidate.path = exact_paths.path\n AND candidate.source = exact_paths.source\n ORDER BY candidate.start_line, candidate.end_line, candidate.id\n LIMIT 1\n )\n ORDER BY exact_paths.exact_path_specificity DESC,\n exact_paths.path ASC, exact_paths.source ASC`).all(...candidateParams, exactPathLimit, ...snippet.params);
};
const useLexicalExactCandidates = isAscii(exactPathQuery) && (plan.matchQuery !== null || plan.substringTerms.length > 0);
let exactRows = [];
if (exactCandidatePatterns.length > 0 && exactPathLimit > 0) try {
exactRows = loadExactRows(useLexicalExactCandidates);
} catch (err) {
if (!useLexicalExactCandidates) throw err;
exactRows = loadExactRows(false);
}
const exactResults = exactRows.map((row) => Object.assign(projectMemorySearchRow(row, params.snippetMaxChars, 0), {
textScore: 0,
pathScore: 0,
exactPathSpecificity: row.exact_path_specificity,
hasBodyMatch: false
}));
if (!pathPlans.some((entry) => entry.matchQuery || entry.substringTerms.length > 0)) return exactResults;
const loadFilteredLexicalRows = (matchQuery, terms, specificity, resultLimit) => {
const filter = buildSubstringFilter(terms, pathColumn);
const qualifiedSpecificityClause = ` AND ${EXACT_PATH_SPECIFICITY_SQL_FUNCTION}(${pathColumn}) ${specificity === "exact" ? ">" : "="} 0`;
const queryParams = [
...matchQuery ? [matchQuery] : [],
...terms,
...params.sourceFilter.params
];
return params.db.prepare(`WITH retained_paths AS MATERIALIZED (\n SELECT ${params.pathFtsTable}.path, ${params.pathFtsTable}.source,\n ${matchQuery ? `bm25(${params.pathFtsTable})` : "0"} AS rank\n FROM ${params.pathFtsTable}\n WHERE ${matchQuery ? `${params.pathFtsTable} MATCH ?` : "1=1"}${filter}${params.sourceFilter.sql}${qualifiedSpecificityClause}\n AND EXISTS (SELECT 1 FROM memory_index_chunks live\n WHERE live.path = ${params.pathFtsTable}.path\n AND live.source = ${params.pathFtsTable}.source)\n ORDER BY rank ASC, ${params.pathFtsTable}.path ASC, ${params.pathFtsTable}.source ASC\n LIMIT ?\n)\nSELECT c.id, retained_paths.path, retained_paths.source,\n c.start_line, c.end_line, ${snippet.sql} AS text, retained_paths.rank\n FROM retained_paths\n CROSS JOIN memory_index_chunks c INDEXED BY sqlite_autoindex_memory_index_chunks_1 ON c.id = (\n SELECT candidate.id FROM memory_index_chunks candidate INDEXED BY idx_memory_index_chunks_path_source\n WHERE candidate.path = retained_paths.path\n AND candidate.source = retained_paths.source\n ORDER BY candidate.start_line, candidate.end_line, candidate.id\n LIMIT 1\n )\n ORDER BY retained_paths.rank ASC, retained_paths.path ASC, retained_paths.source ASC`).all(...queryParams, resultLimit, ...snippet.params);
};
const loadLexicalRows = (lexicalPlan) => {
const loadPartitions = (matchQuery, terms) => {
registerSubstringSqlFunction(params.db, terms);
return [...exactPathLimit > 0 ? loadFilteredLexicalRows(matchQuery, terms, "exact", exactPathLimit) : [], ...loadFilteredLexicalRows(matchQuery, terms, "non-exact", params.limit)];
};
return loadKeywordRowsWithFallback(lexicalPlan, loadPartitions, () => normalizeStringEntries(lexicalPlan.query.match(/[\p{L}\p{M}\p{N}_]+/gu) ?? []), "memory search: path FTS5 MATCH failed, falling back to substring search");
};
const lexicalById = /* @__PURE__ */ new Map();
for (const lexicalPlan of pathPlans) {
if (!lexicalPlan.matchQuery && lexicalPlan.substringTerms.length === 0) continue;
const { rows, usedMatch } = loadLexicalRows(lexicalPlan);
for (const row of rows) {
const pathScore = usedMatch ? params.bm25RankToScore(row.rank) : 1;
const exactPathSpecificity = matchExactPath(row.path);
const result = Object.assign(projectMemorySearchRow(row, params.snippetMaxChars, pathScore), {
textScore: 0,
pathScore,
exactPathSpecificity,
hasBodyMatch: false
});
const existing = lexicalById.get(result.id);
if (!existing) {
lexicalById.set(result.id, result);
continue;
}
existing.pathScore = Math.max(existing.pathScore, result.pathScore);
existing.score = Math.max(existing.score, result.score);
existing.exactPathSpecificity = Math.max(existing.exactPathSpecificity, result.exactPathSpecificity);
}
}
const byId = new Map(exactResults.map((entry) => [entry.id, entry]));
let nonExactCount = 0;
for (const entry of [...lexicalById.values()].toSorted(comparePathKeywordSearchResults)) {
const exact = byId.get(entry.id);
if (entry.exactPathSpecificity > 0) {
if (!exact) continue;
exact.pathScore = Math.max(exact.pathScore, entry.pathScore);
exact.score = Math.max(exact.score, entry.score);
exact.exactPathSpecificity = Math.max(exact.exactPathSpecificity, entry.exactPathSpecificity);
continue;
}
if (nonExactCount >= params.limit) continue;
byId.set(entry.id, entry);
nonExactCount += 1;
}
const resultLimit = hasExplicitExactPathHeadroom ? exactPathLimit + params.limit : params.limit;
return [...byId.values()].toSorted(comparePathKeywordSearchResults).slice(0, resultLimit);
}
//#endregion
export { searchVector as a, readMemoryRecallData as c, loadMemorySourceFileState as d, resolveMemorySourceFileEntries as f, searchChunksByEmbedding as i, readMemoryRetrievalIndexState as l, buildFtsQuery as m, searchKeyword as n, MEMORY_INDEX_META_KEY as o, bm25RankToScore as p, searchPathKeyword as r, readMemoryIndexMetadata as s, prepareExactPathMatcher as t, inspectMemorySourceState as u };