openclaw
Version:
Multi-channel AI gateway with extensible messaging integrations
504 lines (503 loc) • 26.8 kB
JavaScript
import { d as normalizeStringEntries, f as normalizeStringEntriesLower, y as uniqueStrings } from "./string-normalization-D6DKCpwE.mjs";
import { t as escapeRegExp } from "./regexp-BZyMFTlj.mjs";
import { u as MEMORY_INDEX_VECTOR_TABLE } from "./memory-schema-h8l17Nvx.mjs";
import "./memory-schema-fts-B8vS-yZQ.mjs";
import { n as decodeMemoryEmbedding, t as cosineSimilarity } from "./embedding-vector-DA_9H0FL.mjs";
import "./memory-core-host-engine-knn-kB7IacL8.mjs";
import { n as projectMemorySearchRow, r as resolveSnippetProjection, t as buildMemoryModelFilter } from "./manager-search-shared-CWLBKhh0.mjs";
import "./string-coerce-runtime-BbIJhy7G.mjs";
import { s as readMemoryRecallMetadata } from "./engine-storage-DwRRcchW.mjs";
import "./memory-core-host-engine-storage-D6wIwcVO.mjs";
import "./text-utility-runtime-BB0hWeEg.mjs";
import { n as loadMemorySourceFileState } from "./manager-source-state-B1ZG-rB2.mjs";
import { a as resolvePersistedMemoryVectorIndexState, r as memoryTableExists } from "./manager-vector-rebuild-state-DnYGbs4C.mjs";
import { setImmediate } from "node:timers/promises";
//#region extensions/memory-core/src/memory/manager-retrieval-read.ts
const MEMORY_INDEX_META_KEY = "memory_index_meta_v1";
function readMemoryIndexMetadata(db) {
const row = db.prepare("SELECT value FROM memory_index_meta WHERE key = ?").get(MEMORY_INDEX_META_KEY);
if (typeof row?.value !== "string" || !row.value) return {
meta: null,
serialized: null
};
try {
return {
meta: JSON.parse(row.value),
serialized: row.value
};
} catch {
return {
meta: null,
serialized: null
};
}
}
function readMemoryRetrievalIndexState(db) {
const { meta } = readMemoryIndexMetadata(db);
const hasIndexedChunks = db.prepare("SELECT 1 FROM memory_index_chunks LIMIT 1").get() !== void 0;
return {
meta,
hasIndexedChunks,
hasFtsContent: !hasIndexedChunks && memoryTableExists(db, "memory_index_chunks_fts") && db.prepare(`SELECT 1 FROM memory_index_chunks_fts LIMIT 1`).get() !== void 0,
vectorState: meta && meta.provider !== "none" ? resolvePersistedMemoryVectorIndexState({
db,
vectorTable: MEMORY_INDEX_VECTOR_TABLE,
metaVectorDims: meta.vectorDims,
hasSemanticChunks: db.prepare("SELECT 1 FROM memory_index_chunks WHERE model != 'fts-only' LIMIT 1").get() !== void 0
}) : { state: "empty" }
};
}
function readMemoryRecallData(db, request) {
const rows = readMemoryRecallMetadata(db, request.candidates.map((entry) => entry.id));
const sourceMtimes = {
memory: /* @__PURE__ */ new Map(),
sessions: /* @__PURE__ */ new Map()
};
for (const source of ["sessions", "memory"]) {
if (source === "memory" && !request.includeMemoryMtimes) continue;
const paths = Array.from(new Set(request.candidates.filter((entry) => entry.source === source && rows.has(entry.id)).map((entry) => entry.path)));
if (paths.length > 0) sourceMtimes[source] = new Map(loadMemorySourceFileState({
db,
source,
paths
}).map((row) => [row.path, row.mtime]));
}
return {
rows,
sourceMtimes
};
}
//#endregion
//#region extensions/memory-core/src/memory/manager-search-vector.ts
const FALLBACK_VECTOR_BATCH_SIZE = 256;
function resolveProviderModels(primary, aliases) {
return Array.from(/* @__PURE__ */ new Set([primary, ...(aliases ?? []).filter(Boolean)]));
}
async function searchVector(params) {
if (params.queryVec.length === 0 || params.limit <= 0) return [];
params.signal?.throwIfAborted();
const providerModels = resolveProviderModels(params.providerModel, params.providerModelAliases);
const vectorReady = await params.ensureVectorReady(params.queryVec.length);
params.signal?.throwIfAborted();
if (vectorReady) {
if (!params.runVectorKnn) throw new Error("memory vector KNN subprocess is unavailable");
const response = await params.runVectorKnn({
vectorTable: params.vectorTable,
providerModels,
queryVec: params.queryVec,
limit: params.limit,
snippetMaxChars: params.snippetMaxChars,
sourceFilter: params.sourceFilterVec
}, params.signal);
if (!response.fallbackScanRequired) return response.rows.map((row) => projectMemorySearchRow(row, params.snippetMaxChars, 1 - row.dist));
}
return await params.runFallback();
}
async function searchChunksByEmbedding(params) {
if (params.limit <= 0) return [];
const providerModels = resolveProviderModels(params.providerModel, params.providerModelAliases);
const projection = `SELECT rowid AS rowid, embedding
FROM memory_index_chunks
WHERE ${buildMemoryModelFilter("model", providerModels)}`;
const ordering = `${params.sourceFilter.sql}\n ORDER BY rowid ASC\n LIMIT ?`;
const firstStmt = params.db.prepare(`${projection}${ordering}`);
const stmt = params.db.prepare(`${projection} AND rowid > ?${ordering}`);
firstStmt.setReadBigInts(true);
stmt.setReadBigInts(true);
const snippet = resolveSnippetProjection("text", params.snippetMaxChars);
const payloadStmt = params.db.prepare(`SELECT id, path, start_line, end_line, ${snippet.sql} AS text, source FROM memory_index_chunks WHERE rowid = ?`);
const topResults = [];
let lastRowid;
while (true) {
const batch = (lastRowid === void 0 ? firstStmt : stmt).iterate(...providerModels, ...lastRowid === void 0 ? [] : [lastRowid], ...params.sourceFilter.params, FALLBACK_VECTOR_BATCH_SIZE);
let batchSize = 0;
for (const row of batch) {
batchSize += 1;
lastRowid = row.rowid;
const score = cosineSimilarity(params.queryVec, decodeMemoryEmbedding(row.embedding));
const lowest = topResults.at(-1);
if (Number.isFinite(score) && (topResults.length < params.limit || lowest && score > lowest.score)) {
const payload = payloadStmt.get(...snippet.params, row.rowid);
const result = projectMemorySearchRow(payload, params.snippetMaxChars, score);
if (topResults.length < params.limit) {
topResults.push(result);
if (topResults.length === params.limit) topResults.sort((a, b) => b.score - a.score);
} else {
topResults[topResults.length - 1] = result;
topResults.sort((a, b) => b.score - a.score);
}
}
}
if (batchSize < FALLBACK_VECTOR_BATCH_SIZE) break;
await setImmediate();
params.signal?.throwIfAborted();
}
topResults.sort((a, b) => b.score - a.score);
return topResults;
}
//#endregion
//#region extensions/memory-core/src/memory/keyword-query.ts
function tokenizeFtsQuery(raw, includeLeadingMarks = false) {
const pattern = includeLeadingMarks ? /[\p{L}\p{M}\p{N}_]+/gu : /[\p{L}\p{N}_][\p{L}\p{M}\p{N}_]*/gu;
return normalizeStringEntries(raw.match(pattern) ?? []);
}
function buildFtsQuery(raw, canonicalTokenizer) {
return buildMatchQueryFromTerms(tokenizeFtsQuery(raw), canonicalTokenizer);
}
function simpleCaseFold(character) {
const upper = character.toUpperCase();
const folded = Array.from(upper).length === 1 ? upper.toLowerCase() : character.toLowerCase();
return Array.from(folded).length === 1 ? folded : character;
}
const UNICODE61_DIACRITIC = /[\u0300-\u0304\u0306-\u030c\u030f\u0311\u031b\u0323-\u0328\u032d\u032e\u0330\u0331]/u;
const UNICODE61_MARK = /* @__PURE__ */ new RegExp("[̀-ͯ҃-҉֑-ֽֿׁ-ׂׄ-ׇׅؐ-ًؚ-ٰٟۖ-ۜ۟-ۤۧ-۪ۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣤ-ࣾऀ-ःऺ-़ा-ॏ॑-ॗॢ-ॣঁ-ঃ়া-ৄে-ৈো-্ৗৢ-ৣਁ-ਃ਼ਾ-ੂੇ-ੈੋ-੍ੑੰ-ੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢ-ૣଁ-ଃ଼ା-ୄେ-ୈୋ-୍ୖ-ୗୢ-ୣஂா-ூெ-ைொ-்ௗఁ-ఃా-ౄె-ైొ-్ౕ-ౖౢ-ౣಂ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕ-ೖೢ-ೣം-ഃാ-ൄെ-ൈൊ-്ൗൢ-ൣං-ඃ්ා-ුූෘ-ෟෲ-ෳัิ-ฺ็-๎ັິ-ູົ-ຼ່-ໍ༘-༹༙༵༷༾-༿ཱ-྄྆-྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒ-ᝓᝲ-ᝳ឴-៓៝᠋-᠍ᢩᤠ-ᤫᤰ-᤻ᦰ-ᧀᧈ-ᧉᨗ-ᨛᩕ-ᩞ᩠-᩿᩼ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᷀-ᷦ᷼-᷿⃐-⃰⳯-⵿⳱ⷠ-〪ⷿ-゙〯-゚꙯-꙲ꙴ-꙽ꚟ꛰-꛱ꠂ꠆ꠋꠣ-ꠧꢀ-ꢁꢴ-꣄꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꨩ-ꨶꩃꩌ-ꩍꩻꪰꪲ-ꪴꪷ-ꪸꪾ-꪿꫁ꫫ-ꫯꫵ-꫶ꯣ-ꯪ꯬-꯭ﬞ︀-️︠-𐇽︦𐨁-𐨃𐨅-𐨆𐨌-𐨏𐨸-𐨿𐨺𑀀-𑀂𑀸-𑁆𑂀-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑄴𑆀-𑆂𑆳-𑇀𑚫-𑚷𖽑-𖽾𖾏-𖾒𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄󠄀-󠇯]", "u");
function unicode61TokenizerKey(term) {
let key = "";
for (const character of term) {
if (UNICODE61_DIACRITIC.test(character)) continue;
if (UNICODE61_MARK.test(character)) {
key += " ";
continue;
}
const decomposed = Array.from(character.normalize("NFD"));
const base = decomposed[0];
const foldedBase = base ? simpleCaseFold(base) : void 0;
if (decomposed.length === 2 && UNICODE61_DIACRITIC.test(decomposed[1] ?? "") && foldedBase && /^[a-z]$/u.test(foldedBase)) {
key += foldedBase;
continue;
}
key += simpleCaseFold(character);
}
return key.replace(/ +/g, " ").trim();
}
function canonicalTermForms(term, tokenizer) {
if (!tokenizer) return [term];
const seen = /* @__PURE__ */ new Set();
return [
term,
term.normalize("NFC"),
term.normalize("NFD")
].filter((form) => {
const key = tokenizer === "unicode61" ? unicode61TokenizerKey(form) : Array.from(form, simpleCaseFold).join("");
if (seen.has(key)) return false;
seen.add(key);
return true;
});
}
function buildMatchQueryFromTerms(terms, canonicalTokenizer) {
if (terms.length === 0) return null;
return uniqueStrings(terms).map((term) => {
const alternatives = canonicalTermForms(term, canonicalTokenizer).map((form) => `"${form.replaceAll("\"", "")}"`);
return alternatives.length === 1 ? alternatives[0] : `(${alternatives.join(" OR ")})`;
}).join(" AND ");
}
function planKeywordSearch(params) {
const canonicalTokenizer = params.canonicalVariants ? params.ftsTokenizer ?? "unicode61" : void 0;
if (params.ftsTokenizer !== "trigram") return {
matchQuery: buildFtsQuery(params.query, canonicalTokenizer),
substringTerms: []
};
const tokens = tokenizeFtsQuery(params.query, params.includeLeadingMarks);
const matchTerms = [];
const substringTerms = [];
for (const token of tokens) if (canonicalTermForms(token, canonicalTokenizer).some((form) => Array.from(form).length < 3)) substringTerms.push(token);
else matchTerms.push(token);
return {
matchQuery: buildMatchQueryFromTerms(matchTerms, canonicalTokenizer),
substringTerms
};
}
function bm25RankToScore(rank) {
if (!Number.isFinite(rank)) return 1 / 1e3;
if (rank < 0) {
const relevance = -rank;
return relevance / (1 + relevance);
}
return 1 / (1 + rank);
}
//#endregion
//#region extensions/memory-core/src/memory/manager-search.ts
const EXACT_PATH_SPECIFICITY_SQL_FUNCTION = "openclaw_memory_exact_path_specificity";
const NORMALIZED_CONTAINS_SQL_FUNCTION = "openclaw_memory_normalized_contains";
function loadKeywordRowsWithFallback(plan, loadRows, fallbackTokens, warning) {
if (plan.matchQuery) try {
return {
rows: loadRows(plan.matchQuery, plan.substringTerms),
usedMatch: true
};
} catch (error) {
console.warn(`${warning}: ${String(error)}`);
return {
rows: loadRows(null, uniqueStrings([...fallbackTokens(), ...plan.substringTerms])),
usedMatch: false
};
}
return {
rows: loadRows(null, plan.substringTerms),
usedMatch: false
};
}
function mergePathKeywordScores(target, hit) {
target.pathScore = Math.max(target.pathScore, hit.pathScore);
target.score = Math.max(target.score, hit.score);
target.exactPathSpecificity = Math.max(target.exactPathSpecificity, hit.exactPathSpecificity);
}
function comparePathKeywordSearchResults(left, right) {
const specificityDelta = right.exactPathSpecificity - left.exactPathSpecificity;
if (specificityDelta !== 0) return specificityDelta;
if (left.exactPathSpecificity === 0) {
const pathDelta = right.pathScore - left.pathScore;
if (pathDelta !== 0) return pathDelta;
}
return left.path.localeCompare(right.path) || left.startLine - right.startLine || left.id.localeCompare(right.id);
}
function normalizeSearchTokens(raw) {
return normalizeStringEntriesLower(tokenizeFtsQuery(raw.normalize("NFC")));
}
function literalSearchMatcher(value, whole = false) {
const literal = escapeRegExp(value.normalize("NFC"));
return new RegExp(whole ? `^(?:${literal})$` : literal, "iu");
}
function scoreFallbackKeywordResult(params) {
const { queryMatchers } = params;
if (queryMatchers.length === 0) return params.ftsScore;
const textTokens = normalizeSearchTokens(params.text);
const textTokenSet = new Set(textTokens);
const overlap = queryMatchers.filter(({ word }) => textTokens.some((token) => word.test(token))).length;
const uniqueQueryOverlap = overlap / queryMatchers.length;
const density = overlap / Math.max(textTokenSet.size, 1);
const normalizedPath = params.path.normalize("NFC");
const pathBoost = queryMatchers.reduce((score, { substring }) => score + (substring.test(normalizedPath) ? .18 : 0), 0);
const textLengthBoost = Math.min(params.text.length / 160, .18);
const lexicalBoost = uniqueQueryOverlap * .45 + density * .2 + pathBoost + textLengthBoost;
return Math.min(1, params.ftsScore + lexicalBoost);
}
function escapeLikePattern(term) {
return term.replaceAll("\\", "\\\\").replaceAll("%", "\\%").replaceAll("_", "\\_");
}
function isAscii(value) {
return !/[^\p{ASCII}]/u.test(value);
}
function resolveUnicodeCandidateAnchors(value) {
const firstNonAsciiCodePoint = Array.from(value).find((codePoint) => !isAscii(codePoint));
if (!firstNonAsciiCodePoint) return [];
return [.../* @__PURE__ */ new Set([
firstNonAsciiCodePoint,
firstNonAsciiCodePoint.toLowerCase(),
firstNonAsciiCodePoint.toUpperCase()
])];
}
function normalizePathIdentifier(value) {
return value.trim().replaceAll("\\", "/").replace(/^\.\//, "").normalize("NFC").toLowerCase();
}
function prepareExactPathMatcher(query) {
const normalizedQuery = normalizePathIdentifier(query);
if (!normalizedQuery || normalizedQuery === ".") return () => 0;
const hasDirectory = normalizedQuery.includes("/");
return (candidatePath) => {
const normalizedPath = normalizePathIdentifier(candidatePath);
if (normalizedQuery === normalizedPath) return 3;
if (hasDirectory) return 0;
const basename = normalizedPath.split("/").at(-1) ?? normalizedPath;
if (normalizedQuery === basename) return 2;
const extensionIndex = basename.lastIndexOf(".");
const stem = extensionIndex > 0 ? basename.slice(0, extensionIndex) : basename;
return normalizedQuery === stem ? 1 : 0;
};
}
function registerSubstringSqlFunction(db, terms) {
const matchers = new Map(terms.map((term) => [term, literalSearchMatcher(term)]));
db.function(NORMALIZED_CONTAINS_SQL_FUNCTION, { deterministic: true }, (value, query) => typeof value === "string" && typeof query === "string" ? Number(matchers.get(query)?.test(value.normalize("NFC")) === true) : 0);
}
function buildSubstringFilter(terms, column) {
return terms.map(() => ` AND ${NORMALIZED_CONTAINS_SQL_FUNCTION}(${column}, ?) = 1`).join("");
}
function buildExactPathCandidatePatterns(query) {
const normalized = query.trim().replaceAll("\\", "/").replace(/^\.\//, "");
if (!normalized || normalized === ".") return [];
const canonicalForms = [normalized.normalize("NFC"), normalized.normalize("NFD")];
const forms = new Set(canonicalForms);
if (!isAscii(normalized)) for (const form of canonicalForms) {
forms.add(form.toLowerCase());
forms.add(form.toUpperCase());
}
const patterns = /* @__PURE__ */ new Set();
for (const form of forms) {
const escaped = escapeLikePattern(form);
if (normalized.includes("/")) {
patterns.add(escaped);
continue;
}
patterns.add(escaped);
patterns.add(`${escaped}.%`);
patterns.add(`%/${escaped}`);
patterns.add(`%/${escaped}.%`);
}
if (!isAscii(normalized)) {
const asciiAnchor = normalized.normalize("NFD").toLowerCase().match(/[a-z0-9_]+/g)?.toSorted((left, right) => right.length - left.length)[0];
if (asciiAnchor) patterns.add(`%${escapeLikePattern(asciiAnchor)}%`);
if (normalized.toLowerCase() !== normalized.toUpperCase()) for (const anchor of resolveUnicodeCandidateAnchors(normalized)) patterns.add(`%${escapeLikePattern(anchor)}%`);
}
return [...patterns];
}
function planPathKeywordSearch(params) {
const forms = params.ftsTokenizer === "trigram" ? /* @__PURE__ */ new Set([params.query.normalize("NFC"), params.query.normalize("NFD")]) : /* @__PURE__ */ new Set([params.query]);
const seen = /* @__PURE__ */ new Set();
const plans = [];
const addPlan = (query, plan) => {
const key = JSON.stringify([plan.matchQuery, plan.substringTerms]);
if (!seen.has(key)) {
seen.add(key);
plans.push({
query,
...plan
});
}
};
for (const query of forms) addPlan(query, planKeywordSearch({
...params,
query,
includeLeadingMarks: true
}));
if (params.ftsTokenizer !== "trigram") for (const query of /* @__PURE__ */ new Set([params.query.normalize("NFC"), params.query.normalize("NFD")])) {
const tokens = tokenizeFtsQuery(query, true);
const substringTerms = tokens.filter((token) => !isAscii(token));
if (substringTerms.length > 0) addPlan(query, {
matchQuery: buildMatchQueryFromTerms(tokens.filter(isAscii)),
substringTerms
});
}
return plans;
}
async function searchKeyword(params) {
if (params.limit <= 0) return [];
const plan = planKeywordSearch({
query: params.query,
ftsTokenizer: params.ftsTokenizer,
canonicalVariants: true
});
if (!plan.matchQuery && plan.substringTerms.length === 0) return [];
const liveChunkClause = ` AND EXISTS (SELECT 1 FROM memory_index_chunks c WHERE c.id = ${params.ftsTable}.id)`;
const loadRows = (matchQuery, terms) => {
const filter = buildSubstringFilter(terms, "text");
if (terms.length > 0) registerSubstringSqlFunction(params.db, terms);
const matchClause = matchQuery ? `${params.ftsTable} MATCH ? AND ${params.ftsTable}.rank MATCH 'bm25()'` : "1=1";
return params.db.prepare(`SELECT id, path, source, start_line, end_line, text,\n ${matchQuery ? `${params.ftsTable}.rank` : "0"} AS rank\n FROM ${params.ftsTable}\n WHERE ${matchClause}${filter}${liveChunkClause}${params.sourceFilter.sql}\n` + (matchQuery ? ` ORDER BY rank ASC\n` : "") + ` LIMIT ?`).all(...matchQuery ? [matchQuery] : [], ...terms, ...params.sourceFilter.params, params.limit);
};
const { rows, usedMatch } = loadKeywordRowsWithFallback(plan, loadRows, () => tokenizeFtsQuery(params.query), "memory search: FTS5 MATCH failed, falling back to substring search");
const queryMatchers = params.boostFallbackRanking ? uniqueStrings(normalizeSearchTokens(params.rankingQuery ?? params.query)).map((token) => ({
word: literalSearchMatcher(token, true),
substring: literalSearchMatcher(token)
})) : [];
return rows.map((row) => {
const textScore = usedMatch ? bm25RankToScore(row.rank) : 0;
const score = params.boostFallbackRanking ? scoreFallbackKeywordResult({
queryMatchers,
path: row.path,
text: row.text,
ftsScore: textScore
}) : textScore;
return Object.assign(projectMemorySearchRow(row, params.snippetMaxChars, score), {
textScore,
hasBodyMatch: true
});
});
}
async function searchPathKeyword(params) {
if (params.limit <= 0) return [];
const snippet = resolveSnippetProjection("c.text", params.snippetMaxChars);
const pathColumn = `${params.pathFtsTable}.path`;
const pathPlans = planPathKeywordSearch({
query: params.query,
ftsTokenizer: params.ftsTokenizer
});
const plan = pathPlans[0] ?? {
query: params.query,
matchQuery: null,
substringTerms: []
};
const planSubstringFilter = buildSubstringFilter(plan.substringTerms, pathColumn);
registerSubstringSqlFunction(params.db, plan.substringTerms);
const exactPathQuery = params.exactPathQuery ?? params.query;
const matchExactPath = prepareExactPathMatcher(exactPathQuery);
params.db.function(EXACT_PATH_SPECIFICITY_SQL_FUNCTION, { deterministic: true }, (candidatePath) => typeof candidatePath === "string" ? matchExactPath(candidatePath) : 0);
const projectPathRow = (row, pathScore, exactPathSpecificity = matchExactPath(row.path)) => Object.assign(projectMemorySearchRow(row, params.snippetMaxChars, pathScore), {
textScore: 0,
pathScore,
exactPathSpecificity,
hasBodyMatch: false
});
const projectFirstChunks = (paths, score, direction) => `SELECT c.id, ${paths}.path, ${paths}.source,\n c.start_line, c.end_line, ${snippet.sql} AS text, ${paths}.${score}\n FROM ${paths}\n CROSS JOIN memory_index_chunks c INDEXED BY sqlite_autoindex_memory_index_chunks_1 ON c.id = (\n SELECT candidate.id FROM memory_index_chunks candidate INDEXED BY idx_memory_index_chunks_path_source\n WHERE candidate.path = ${paths}.path\n AND candidate.source = ${paths}.source\n ORDER BY candidate.start_line, candidate.end_line, candidate.id\n LIMIT 1\n )\n ORDER BY ${paths}.${score} ${direction}, ${paths}.path ASC, ${paths}.source ASC`;
const hasExplicitExactPathHeadroom = params.exactPathLimit !== void 0;
const exactPathLimit = Math.max(0, Math.floor(params.exactPathLimit ?? params.limit));
const exactCandidatePatterns = buildExactPathCandidatePatterns(exactPathQuery);
const loadExactRows = (useLexicalCandidates) => {
const qualifiedPatternClause = exactCandidatePatterns.map(() => `${pathColumn} LIKE ? ESCAPE '\\'`).join(" OR ");
const candidateCtes = useLexicalCandidates ? `candidates AS MATERIALIZED (\n SELECT ${params.pathFtsTable}.path, ${params.pathFtsTable}.source\n FROM ${params.pathFtsTable}\n WHERE ${plan.matchQuery ? `${params.pathFtsTable} MATCH ?` : "1=1"}${planSubstringFilter}${params.sourceFilter.sql}\n), pattern_candidates AS MATERIALIZED (\n SELECT path, source FROM candidates\n WHERE (${exactCandidatePatterns.map(() => "path LIKE ? ESCAPE '\\'").join(" OR ")})\n)` : `pattern_candidates AS MATERIALIZED (\n SELECT ${params.pathFtsTable}.path, ${params.pathFtsTable}.source\n FROM ${params.pathFtsTable}\n WHERE (${qualifiedPatternClause})${params.sourceFilter.sql}\n)`;
const candidateParams = useLexicalCandidates ? [
...plan.matchQuery ? [plan.matchQuery] : [],
...plan.substringTerms,
...params.sourceFilter.params,
...exactCandidatePatterns
] : [...exactCandidatePatterns, ...params.sourceFilter.params];
return params.db.prepare(`WITH ${candidateCtes}, scored_paths AS MATERIALIZED (\n SELECT path, source,\n ${EXACT_PATH_SPECIFICITY_SQL_FUNCTION}(path) AS exact_path_specificity\n FROM pattern_candidates\n), exact_paths AS MATERIALIZED (\n SELECT path, source, exact_path_specificity FROM scored_paths\n WHERE exact_path_specificity > 0\n AND EXISTS (SELECT 1 FROM memory_index_chunks live\n WHERE live.path = scored_paths.path\n AND live.source = scored_paths.source)\n ORDER BY exact_path_specificity DESC, path ASC, source ASC\n LIMIT ?\n)\n` + projectFirstChunks("exact_paths", "exact_path_specificity", "DESC")).all(...candidateParams, exactPathLimit, ...snippet.params);
};
const useLexicalExactCandidates = isAscii(exactPathQuery) && (plan.matchQuery !== null || plan.substringTerms.length > 0);
let exactRows = [];
if (exactCandidatePatterns.length > 0 && exactPathLimit > 0) try {
exactRows = loadExactRows(useLexicalExactCandidates);
} catch (err) {
if (!useLexicalExactCandidates) throw err;
exactRows = loadExactRows(false);
}
const exactResults = exactRows.map((row) => projectPathRow(row, 0, row.exact_path_specificity));
if (!pathPlans.some((entry) => entry.matchQuery || entry.substringTerms.length > 0)) return exactResults;
const loadFilteredLexicalRows = (matchQuery, terms, specificity, resultLimit) => {
const filter = buildSubstringFilter(terms, pathColumn);
const qualifiedSpecificityClause = ` AND ${EXACT_PATH_SPECIFICITY_SQL_FUNCTION}(${pathColumn}) ${specificity === "exact" ? ">" : "="} 0`;
const queryParams = [
...matchQuery ? [matchQuery] : [],
...terms,
...params.sourceFilter.params
];
return params.db.prepare(`WITH retained_paths AS MATERIALIZED (\n SELECT ${params.pathFtsTable}.path, ${params.pathFtsTable}.source,\n ${matchQuery ? `bm25(${params.pathFtsTable})` : "0"} AS rank\n FROM ${params.pathFtsTable}\n WHERE ${matchQuery ? `${params.pathFtsTable} MATCH ?` : "1=1"}${filter}${params.sourceFilter.sql}${qualifiedSpecificityClause}\n AND EXISTS (SELECT 1 FROM memory_index_chunks live\n WHERE live.path = ${params.pathFtsTable}.path\n AND live.source = ${params.pathFtsTable}.source)\n ORDER BY rank ASC, ${params.pathFtsTable}.path ASC, ${params.pathFtsTable}.source ASC\n LIMIT ?\n)\n` + projectFirstChunks("retained_paths", "rank", "ASC")).all(...queryParams, resultLimit, ...snippet.params);
};
const loadLexicalRows = (lexicalPlan) => {
const loadPartitions = (matchQuery, terms) => {
registerSubstringSqlFunction(params.db, terms);
return [...exactPathLimit > 0 ? loadFilteredLexicalRows(matchQuery, terms, "exact", exactPathLimit) : [], ...loadFilteredLexicalRows(matchQuery, terms, "non-exact", params.limit)];
};
return loadKeywordRowsWithFallback(lexicalPlan, loadPartitions, () => tokenizeFtsQuery(lexicalPlan.query, true), "memory search: path FTS5 MATCH failed, falling back to substring search");
};
const lexicalById = /* @__PURE__ */ new Map();
for (const lexicalPlan of pathPlans) {
if (!lexicalPlan.matchQuery && lexicalPlan.substringTerms.length === 0) continue;
const { rows, usedMatch } = loadLexicalRows(lexicalPlan);
for (const row of rows) {
const result = projectPathRow(row, usedMatch ? bm25RankToScore(row.rank) : 1);
const existing = lexicalById.get(result.id);
if (!existing) {
lexicalById.set(result.id, result);
continue;
}
mergePathKeywordScores(existing, result);
}
}
const byId = new Map(exactResults.map((entry) => [entry.id, entry]));
let nonExactCount = 0;
for (const entry of [...lexicalById.values()].toSorted(comparePathKeywordSearchResults)) {
const exact = byId.get(entry.id);
if (entry.exactPathSpecificity > 0) {
if (!exact) continue;
mergePathKeywordScores(exact, entry);
continue;
}
if (nonExactCount >= params.limit) continue;
byId.set(entry.id, entry);
nonExactCount += 1;
}
const resultLimit = hasExplicitExactPathHeadroom ? exactPathLimit + params.limit : params.limit;
return [...byId.values()].toSorted(comparePathKeywordSearchResults).slice(0, resultLimit);
}
//#endregion
export { searchChunksByEmbedding as a, readMemoryIndexMetadata as c, buildFtsQuery as i, readMemoryRecallData as l, searchKeyword as n, searchVector as o, searchPathKeyword as r, MEMORY_INDEX_META_KEY as s, prepareExactPathMatcher as t, readMemoryRetrievalIndexState as u };