@gguf/claw
Version:
WhatsApp gateway CLI (Baileys web) with Pi RPC agent
1,321 lines (1,308 loc) • 106 kB
JavaScript
import { s as resolveStateDir } from "./paths-VslOJiD2.js";
import { n as resolveAgentConfig, r as resolveAgentDir, s as resolveAgentWorkspaceDir } from "./agent-scope-COnICB_7.js";
import { I as sleep, N as resolveUserPath, T as clampNumber$1, l as createSubsystemLogger, w as clampInt, z as truncateUtf16Safe } from "./exec-B7WKla_0.js";
import { D as isTruthyEnvValue, v as requireApiKey, y as resolveApiKeyForProvider } from "./model-selection-Cs1y6OBv.js";
import { i as resolveSessionTranscriptsDirForAgent } from "./paths-xPuk88Yf.js";
import { n as onSessionTranscriptUpdate } from "./transcript-events-DW_H__a1.js";
import { a as ensureDir, c as listMemoryFiles, i as cosineSimilarity, l as normalizeExtraMemoryPaths, n as buildFileEntry, o as hashText, r as chunkMarkdown, s as isMemoryPath, t as requireNodeSqlite, u as parseEmbedding } from "./sqlite-C59YNxdL.js";
import os from "node:os";
import path from "node:path";
import fs from "node:fs";
import fs$1 from "node:fs/promises";
import { randomUUID } from "node:crypto";
import chokidar from "chokidar";
//#region src/agents/memory-search.ts
const DEFAULT_OPENAI_MODEL = "text-embedding-3-small";
const DEFAULT_GEMINI_MODEL = "gemini-embedding-001";
const DEFAULT_CHUNK_TOKENS = 400;
const DEFAULT_CHUNK_OVERLAP = 80;
const DEFAULT_WATCH_DEBOUNCE_MS = 1500;
const DEFAULT_SESSION_DELTA_BYTES = 1e5;
const DEFAULT_SESSION_DELTA_MESSAGES = 50;
const DEFAULT_MAX_RESULTS = 6;
const DEFAULT_MIN_SCORE = .35;
const DEFAULT_HYBRID_ENABLED = true;
const DEFAULT_HYBRID_VECTOR_WEIGHT = .7;
const DEFAULT_HYBRID_TEXT_WEIGHT = .3;
const DEFAULT_HYBRID_CANDIDATE_MULTIPLIER = 4;
const DEFAULT_CACHE_ENABLED = true;
const DEFAULT_SOURCES = ["memory"];
function normalizeSources(sources, sessionMemoryEnabled) {
const normalized = /* @__PURE__ */ new Set();
const input = sources?.length ? sources : DEFAULT_SOURCES;
for (const source of input) {
if (source === "memory") normalized.add("memory");
if (source === "sessions" && sessionMemoryEnabled) normalized.add("sessions");
}
if (normalized.size === 0) normalized.add("memory");
return Array.from(normalized);
}
function resolveStorePath(agentId, raw) {
const stateDir = resolveStateDir(process.env, os.homedir);
const fallback = path.join(stateDir, "memory", `${agentId}.sqlite`);
if (!raw) return fallback;
return resolveUserPath(raw.includes("{agentId}") ? raw.replaceAll("{agentId}", agentId) : raw);
}
function mergeConfig(defaults, overrides, agentId) {
const enabled = overrides?.enabled ?? defaults?.enabled ?? true;
const sessionMemory = overrides?.experimental?.sessionMemory ?? defaults?.experimental?.sessionMemory ?? false;
const provider = overrides?.provider ?? defaults?.provider ?? "auto";
const defaultRemote = defaults?.remote;
const overrideRemote = overrides?.remote;
const includeRemote = Boolean(overrideRemote?.baseUrl || overrideRemote?.apiKey || overrideRemote?.headers || defaultRemote?.baseUrl || defaultRemote?.apiKey || defaultRemote?.headers) || provider === "openai" || provider === "gemini" || provider === "auto";
const batch = {
enabled: overrideRemote?.batch?.enabled ?? defaultRemote?.batch?.enabled ?? true,
wait: overrideRemote?.batch?.wait ?? defaultRemote?.batch?.wait ?? true,
concurrency: Math.max(1, overrideRemote?.batch?.concurrency ?? defaultRemote?.batch?.concurrency ?? 2),
pollIntervalMs: overrideRemote?.batch?.pollIntervalMs ?? defaultRemote?.batch?.pollIntervalMs ?? 2e3,
timeoutMinutes: overrideRemote?.batch?.timeoutMinutes ?? defaultRemote?.batch?.timeoutMinutes ?? 60
};
const remote = includeRemote ? {
baseUrl: overrideRemote?.baseUrl ?? defaultRemote?.baseUrl,
apiKey: overrideRemote?.apiKey ?? defaultRemote?.apiKey,
headers: overrideRemote?.headers ?? defaultRemote?.headers,
batch
} : void 0;
const fallback = overrides?.fallback ?? defaults?.fallback ?? "none";
const modelDefault = provider === "gemini" ? DEFAULT_GEMINI_MODEL : provider === "openai" ? DEFAULT_OPENAI_MODEL : void 0;
const model = overrides?.model ?? defaults?.model ?? modelDefault ?? "";
const local = {
modelPath: overrides?.local?.modelPath ?? defaults?.local?.modelPath,
modelCacheDir: overrides?.local?.modelCacheDir ?? defaults?.local?.modelCacheDir
};
const sources = normalizeSources(overrides?.sources ?? defaults?.sources, sessionMemory);
const rawPaths = [...defaults?.extraPaths ?? [], ...overrides?.extraPaths ?? []].map((value) => value.trim()).filter(Boolean);
const extraPaths = Array.from(new Set(rawPaths));
const vector = {
enabled: overrides?.store?.vector?.enabled ?? defaults?.store?.vector?.enabled ?? true,
extensionPath: overrides?.store?.vector?.extensionPath ?? defaults?.store?.vector?.extensionPath
};
const store = {
driver: overrides?.store?.driver ?? defaults?.store?.driver ?? "sqlite",
path: resolveStorePath(agentId, overrides?.store?.path ?? defaults?.store?.path),
vector
};
const chunking = {
tokens: overrides?.chunking?.tokens ?? defaults?.chunking?.tokens ?? DEFAULT_CHUNK_TOKENS,
overlap: overrides?.chunking?.overlap ?? defaults?.chunking?.overlap ?? DEFAULT_CHUNK_OVERLAP
};
const sync = {
onSessionStart: overrides?.sync?.onSessionStart ?? defaults?.sync?.onSessionStart ?? true,
onSearch: overrides?.sync?.onSearch ?? defaults?.sync?.onSearch ?? true,
watch: overrides?.sync?.watch ?? defaults?.sync?.watch ?? true,
watchDebounceMs: overrides?.sync?.watchDebounceMs ?? defaults?.sync?.watchDebounceMs ?? DEFAULT_WATCH_DEBOUNCE_MS,
intervalMinutes: overrides?.sync?.intervalMinutes ?? defaults?.sync?.intervalMinutes ?? 0,
sessions: {
deltaBytes: overrides?.sync?.sessions?.deltaBytes ?? defaults?.sync?.sessions?.deltaBytes ?? DEFAULT_SESSION_DELTA_BYTES,
deltaMessages: overrides?.sync?.sessions?.deltaMessages ?? defaults?.sync?.sessions?.deltaMessages ?? DEFAULT_SESSION_DELTA_MESSAGES
}
};
const query = {
maxResults: overrides?.query?.maxResults ?? defaults?.query?.maxResults ?? DEFAULT_MAX_RESULTS,
minScore: overrides?.query?.minScore ?? defaults?.query?.minScore ?? DEFAULT_MIN_SCORE
};
const hybrid = {
enabled: overrides?.query?.hybrid?.enabled ?? defaults?.query?.hybrid?.enabled ?? DEFAULT_HYBRID_ENABLED,
vectorWeight: overrides?.query?.hybrid?.vectorWeight ?? defaults?.query?.hybrid?.vectorWeight ?? DEFAULT_HYBRID_VECTOR_WEIGHT,
textWeight: overrides?.query?.hybrid?.textWeight ?? defaults?.query?.hybrid?.textWeight ?? DEFAULT_HYBRID_TEXT_WEIGHT,
candidateMultiplier: overrides?.query?.hybrid?.candidateMultiplier ?? defaults?.query?.hybrid?.candidateMultiplier ?? DEFAULT_HYBRID_CANDIDATE_MULTIPLIER
};
const cache = {
enabled: overrides?.cache?.enabled ?? defaults?.cache?.enabled ?? DEFAULT_CACHE_ENABLED,
maxEntries: overrides?.cache?.maxEntries ?? defaults?.cache?.maxEntries
};
const overlap = clampNumber$1(chunking.overlap, 0, Math.max(0, chunking.tokens - 1));
const minScore = clampNumber$1(query.minScore, 0, 1);
const vectorWeight = clampNumber$1(hybrid.vectorWeight, 0, 1);
const textWeight = clampNumber$1(hybrid.textWeight, 0, 1);
const sum = vectorWeight + textWeight;
const normalizedVectorWeight = sum > 0 ? vectorWeight / sum : DEFAULT_HYBRID_VECTOR_WEIGHT;
const normalizedTextWeight = sum > 0 ? textWeight / sum : DEFAULT_HYBRID_TEXT_WEIGHT;
const candidateMultiplier = clampInt(hybrid.candidateMultiplier, 1, 20);
const deltaBytes = clampInt(sync.sessions.deltaBytes, 0, Number.MAX_SAFE_INTEGER);
const deltaMessages = clampInt(sync.sessions.deltaMessages, 0, Number.MAX_SAFE_INTEGER);
return {
enabled,
sources,
extraPaths,
provider,
remote,
experimental: { sessionMemory },
fallback,
model,
local,
store,
chunking: {
tokens: Math.max(1, chunking.tokens),
overlap
},
sync: {
...sync,
sessions: {
deltaBytes,
deltaMessages
}
},
query: {
...query,
minScore,
hybrid: {
enabled: Boolean(hybrid.enabled),
vectorWeight: normalizedVectorWeight,
textWeight: normalizedTextWeight,
candidateMultiplier
}
},
cache: {
enabled: Boolean(cache.enabled),
maxEntries: typeof cache.maxEntries === "number" && Number.isFinite(cache.maxEntries) ? Math.max(1, Math.floor(cache.maxEntries)) : void 0
}
};
}
function resolveMemorySearchConfig(cfg, agentId) {
const defaults = cfg.agents?.defaults?.memorySearch;
const overrides = resolveAgentConfig(cfg, agentId)?.memorySearch;
const resolved = mergeConfig(defaults, overrides, agentId);
if (!resolved.enabled) return null;
return resolved;
}
//#endregion
//#region src/memory/batch-gemini.ts
const GEMINI_BATCH_MAX_REQUESTS = 5e4;
const debugEmbeddings$1 = isTruthyEnvValue(process.env.OPENCLAW_DEBUG_MEMORY_EMBEDDINGS);
const log$2 = createSubsystemLogger("memory/embeddings");
const debugLog$1 = (message, meta) => {
if (!debugEmbeddings$1) return;
const suffix = meta ? ` ${JSON.stringify(meta)}` : "";
log$2.raw(`${message}${suffix}`);
};
function getGeminiBaseUrl(gemini) {
return gemini.baseUrl?.replace(/\/$/, "") ?? "";
}
function getGeminiHeaders(gemini, params) {
const headers = gemini.headers ? { ...gemini.headers } : {};
if (params.json) {
if (!headers["Content-Type"] && !headers["content-type"]) headers["Content-Type"] = "application/json";
} else {
delete headers["Content-Type"];
delete headers["content-type"];
}
return headers;
}
function getGeminiUploadUrl(baseUrl) {
if (baseUrl.includes("/v1beta")) return baseUrl.replace(/\/v1beta\/?$/, "/upload/v1beta");
return `${baseUrl.replace(/\/$/, "")}/upload`;
}
function splitGeminiBatchRequests(requests) {
if (requests.length <= GEMINI_BATCH_MAX_REQUESTS) return [requests];
const groups = [];
for (let i = 0; i < requests.length; i += GEMINI_BATCH_MAX_REQUESTS) groups.push(requests.slice(i, i + GEMINI_BATCH_MAX_REQUESTS));
return groups;
}
function buildGeminiUploadBody(params) {
const boundary = `openclaw-${hashText(params.displayName)}`;
const jsonPart = JSON.stringify({ file: {
displayName: params.displayName,
mimeType: "application/jsonl"
} });
const delimiter = `--${boundary}\r\n`;
const closeDelimiter = `--${boundary}--\r\n`;
const parts = [
`${delimiter}Content-Type: application/json; charset=UTF-8\r\n\r\n${jsonPart}\r\n`,
`${delimiter}Content-Type: application/jsonl; charset=UTF-8\r\n\r\n${params.jsonl}\r\n`,
closeDelimiter
];
return {
body: new Blob([parts.join("")], { type: "multipart/related" }),
contentType: `multipart/related; boundary=${boundary}`
};
}
async function submitGeminiBatch(params) {
const baseUrl = getGeminiBaseUrl(params.gemini);
const uploadPayload = buildGeminiUploadBody({
jsonl: params.requests.map((request) => JSON.stringify({
key: request.custom_id,
request: {
content: request.content,
task_type: request.taskType
}
})).join("\n"),
displayName: `memory-embeddings-${hashText(String(Date.now()))}`
});
const uploadUrl = `${getGeminiUploadUrl(baseUrl)}/files?uploadType=multipart`;
debugLog$1("memory embeddings: gemini batch upload", {
uploadUrl,
baseUrl,
requests: params.requests.length
});
const fileRes = await fetch(uploadUrl, {
method: "POST",
headers: {
...getGeminiHeaders(params.gemini, { json: false }),
"Content-Type": uploadPayload.contentType
},
body: uploadPayload.body
});
if (!fileRes.ok) {
const text = await fileRes.text();
throw new Error(`gemini batch file upload failed: ${fileRes.status} ${text}`);
}
const filePayload = await fileRes.json();
const fileId = filePayload.name ?? filePayload.file?.name;
if (!fileId) throw new Error("gemini batch file upload failed: missing file id");
const batchBody = { batch: {
displayName: `memory-embeddings-${params.agentId}`,
inputConfig: { file_name: fileId }
} };
const batchEndpoint = `${baseUrl}/${params.gemini.modelPath}:asyncBatchEmbedContent`;
debugLog$1("memory embeddings: gemini batch create", {
batchEndpoint,
fileId
});
const batchRes = await fetch(batchEndpoint, {
method: "POST",
headers: getGeminiHeaders(params.gemini, { json: true }),
body: JSON.stringify(batchBody)
});
if (batchRes.ok) return await batchRes.json();
const text = await batchRes.text();
if (batchRes.status === 404) throw new Error("gemini batch create failed: 404 (asyncBatchEmbedContent not available for this model/baseUrl). Disable remote.batch.enabled or switch providers.");
throw new Error(`gemini batch create failed: ${batchRes.status} ${text}`);
}
async function fetchGeminiBatchStatus(params) {
const statusUrl = `${getGeminiBaseUrl(params.gemini)}/${params.batchName.startsWith("batches/") ? params.batchName : `batches/${params.batchName}`}`;
debugLog$1("memory embeddings: gemini batch status", { statusUrl });
const res = await fetch(statusUrl, { headers: getGeminiHeaders(params.gemini, { json: true }) });
if (!res.ok) {
const text = await res.text();
throw new Error(`gemini batch status failed: ${res.status} ${text}`);
}
return await res.json();
}
async function fetchGeminiFileContent(params) {
const downloadUrl = `${getGeminiBaseUrl(params.gemini)}/${params.fileId.startsWith("files/") ? params.fileId : `files/${params.fileId}`}:download`;
debugLog$1("memory embeddings: gemini batch download", { downloadUrl });
const res = await fetch(downloadUrl, { headers: getGeminiHeaders(params.gemini, { json: true }) });
if (!res.ok) {
const text = await res.text();
throw new Error(`gemini batch file content failed: ${res.status} ${text}`);
}
return await res.text();
}
function parseGeminiBatchOutput(text) {
if (!text.trim()) return [];
return text.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line));
}
async function waitForGeminiBatch(params) {
const start = Date.now();
let current = params.initial;
while (true) {
const status = current ?? await fetchGeminiBatchStatus({
gemini: params.gemini,
batchName: params.batchName
});
const state = status.state ?? "UNKNOWN";
if ([
"SUCCEEDED",
"COMPLETED",
"DONE"
].includes(state)) {
const outputFileId = status.outputConfig?.file ?? status.outputConfig?.fileId ?? status.metadata?.output?.responsesFile;
if (!outputFileId) throw new Error(`gemini batch ${params.batchName} completed without output file`);
return { outputFileId };
}
if ([
"FAILED",
"CANCELLED",
"CANCELED",
"EXPIRED"
].includes(state)) {
const message = status.error?.message ?? "unknown error";
throw new Error(`gemini batch ${params.batchName} ${state}: ${message}`);
}
if (!params.wait) throw new Error(`gemini batch ${params.batchName} still ${state}; wait disabled`);
if (Date.now() - start > params.timeoutMs) throw new Error(`gemini batch ${params.batchName} timed out after ${params.timeoutMs}ms`);
params.debug?.(`gemini batch ${params.batchName} ${state}; waiting ${params.pollIntervalMs}ms`);
await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs));
current = void 0;
}
}
async function runWithConcurrency$1(tasks, limit) {
if (tasks.length === 0) return [];
const resolvedLimit = Math.max(1, Math.min(limit, tasks.length));
const results = Array.from({ length: tasks.length });
let next = 0;
let firstError = null;
const workers = Array.from({ length: resolvedLimit }, async () => {
while (true) {
if (firstError) return;
const index = next;
next += 1;
if (index >= tasks.length) return;
try {
results[index] = await tasks[index]();
} catch (err) {
firstError = err;
return;
}
}
});
await Promise.allSettled(workers);
if (firstError) throw firstError;
return results;
}
async function runGeminiEmbeddingBatches(params) {
if (params.requests.length === 0) return /* @__PURE__ */ new Map();
const groups = splitGeminiBatchRequests(params.requests);
const byCustomId = /* @__PURE__ */ new Map();
const tasks = groups.map((group, groupIndex) => async () => {
const batchInfo = await submitGeminiBatch({
gemini: params.gemini,
requests: group,
agentId: params.agentId
});
const batchName = batchInfo.name ?? "";
if (!batchName) throw new Error("gemini batch create failed: missing batch name");
params.debug?.("memory embeddings: gemini batch created", {
batchName,
state: batchInfo.state,
group: groupIndex + 1,
groups: groups.length,
requests: group.length
});
if (!params.wait && batchInfo.state && ![
"SUCCEEDED",
"COMPLETED",
"DONE"
].includes(batchInfo.state)) throw new Error(`gemini batch ${batchName} submitted; enable remote.batch.wait to await completion`);
const completed = batchInfo.state && [
"SUCCEEDED",
"COMPLETED",
"DONE"
].includes(batchInfo.state) ? { outputFileId: batchInfo.outputConfig?.file ?? batchInfo.outputConfig?.fileId ?? batchInfo.metadata?.output?.responsesFile ?? "" } : await waitForGeminiBatch({
gemini: params.gemini,
batchName,
wait: params.wait,
pollIntervalMs: params.pollIntervalMs,
timeoutMs: params.timeoutMs,
debug: params.debug,
initial: batchInfo
});
if (!completed.outputFileId) throw new Error(`gemini batch ${batchName} completed without output file`);
const outputLines = parseGeminiBatchOutput(await fetchGeminiFileContent({
gemini: params.gemini,
fileId: completed.outputFileId
}));
const errors = [];
const remaining = new Set(group.map((request) => request.custom_id));
for (const line of outputLines) {
const customId = line.key ?? line.custom_id ?? line.request_id;
if (!customId) continue;
remaining.delete(customId);
if (line.error?.message) {
errors.push(`${customId}: ${line.error.message}`);
continue;
}
if (line.response?.error?.message) {
errors.push(`${customId}: ${line.response.error.message}`);
continue;
}
const embedding = line.embedding?.values ?? line.response?.embedding?.values ?? [];
if (embedding.length === 0) {
errors.push(`${customId}: empty embedding`);
continue;
}
byCustomId.set(customId, embedding);
}
if (errors.length > 0) throw new Error(`gemini batch ${batchName} failed: ${errors.join("; ")}`);
if (remaining.size > 0) throw new Error(`gemini batch ${batchName} missing ${remaining.size} embedding responses`);
});
params.debug?.("memory embeddings: gemini batch submit", {
requests: params.requests.length,
groups: groups.length,
wait: params.wait,
concurrency: params.concurrency,
pollIntervalMs: params.pollIntervalMs,
timeoutMs: params.timeoutMs
});
await runWithConcurrency$1(tasks, params.concurrency);
return byCustomId;
}
//#endregion
//#region src/infra/retry.ts
const DEFAULT_RETRY_CONFIG = {
attempts: 3,
minDelayMs: 300,
maxDelayMs: 3e4,
jitter: 0
};
const asFiniteNumber = (value) => typeof value === "number" && Number.isFinite(value) ? value : void 0;
const clampNumber = (value, fallback, min, max) => {
const next = asFiniteNumber(value);
if (next === void 0) return fallback;
const floor = typeof min === "number" ? min : Number.NEGATIVE_INFINITY;
const ceiling = typeof max === "number" ? max : Number.POSITIVE_INFINITY;
return Math.min(Math.max(next, floor), ceiling);
};
function resolveRetryConfig(defaults = DEFAULT_RETRY_CONFIG, overrides) {
const attempts = Math.max(1, Math.round(clampNumber(overrides?.attempts, defaults.attempts, 1)));
const minDelayMs = Math.max(0, Math.round(clampNumber(overrides?.minDelayMs, defaults.minDelayMs, 0)));
return {
attempts,
minDelayMs,
maxDelayMs: Math.max(minDelayMs, Math.round(clampNumber(overrides?.maxDelayMs, defaults.maxDelayMs, 0))),
jitter: clampNumber(overrides?.jitter, defaults.jitter, 0, 1)
};
}
function applyJitter(delayMs, jitter) {
if (jitter <= 0) return delayMs;
const offset = (Math.random() * 2 - 1) * jitter;
return Math.max(0, Math.round(delayMs * (1 + offset)));
}
async function retryAsync(fn, attemptsOrOptions = 3, initialDelayMs = 300) {
if (typeof attemptsOrOptions === "number") {
const attempts = Math.max(1, Math.round(attemptsOrOptions));
let lastErr;
for (let i = 0; i < attempts; i += 1) try {
return await fn();
} catch (err) {
lastErr = err;
if (i === attempts - 1) break;
await sleep(initialDelayMs * 2 ** i);
}
throw lastErr ?? /* @__PURE__ */ new Error("Retry failed");
}
const options = attemptsOrOptions;
const resolved = resolveRetryConfig(DEFAULT_RETRY_CONFIG, options);
const maxAttempts = resolved.attempts;
const minDelayMs = resolved.minDelayMs;
const maxDelayMs = Number.isFinite(resolved.maxDelayMs) && resolved.maxDelayMs > 0 ? resolved.maxDelayMs : Number.POSITIVE_INFINITY;
const jitter = resolved.jitter;
const shouldRetry = options.shouldRetry ?? (() => true);
let lastErr;
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) try {
return await fn();
} catch (err) {
lastErr = err;
if (attempt >= maxAttempts || !shouldRetry(err, attempt)) break;
const retryAfterMs = options.retryAfterMs?.(err);
const baseDelay = typeof retryAfterMs === "number" && Number.isFinite(retryAfterMs) ? Math.max(retryAfterMs, minDelayMs) : minDelayMs * 2 ** (attempt - 1);
let delay = Math.min(baseDelay, maxDelayMs);
delay = applyJitter(delay, jitter);
delay = Math.min(Math.max(delay, minDelayMs), maxDelayMs);
options.onRetry?.({
attempt,
maxAttempts,
delayMs: delay,
err,
label: options.label
});
await sleep(delay);
}
throw lastErr ?? /* @__PURE__ */ new Error("Retry failed");
}
//#endregion
//#region src/memory/batch-openai.ts
const OPENAI_BATCH_ENDPOINT = "/v1/embeddings";
const OPENAI_BATCH_COMPLETION_WINDOW = "24h";
const OPENAI_BATCH_MAX_REQUESTS = 5e4;
function getOpenAiBaseUrl(openAi) {
return openAi.baseUrl?.replace(/\/$/, "") ?? "";
}
function getOpenAiHeaders(openAi, params) {
const headers = openAi.headers ? { ...openAi.headers } : {};
if (params.json) {
if (!headers["Content-Type"] && !headers["content-type"]) headers["Content-Type"] = "application/json";
} else {
delete headers["Content-Type"];
delete headers["content-type"];
}
return headers;
}
function splitOpenAiBatchRequests(requests) {
if (requests.length <= OPENAI_BATCH_MAX_REQUESTS) return [requests];
const groups = [];
for (let i = 0; i < requests.length; i += OPENAI_BATCH_MAX_REQUESTS) groups.push(requests.slice(i, i + OPENAI_BATCH_MAX_REQUESTS));
return groups;
}
async function submitOpenAiBatch(params) {
const baseUrl = getOpenAiBaseUrl(params.openAi);
const jsonl = params.requests.map((request) => JSON.stringify(request)).join("\n");
const form = new FormData();
form.append("purpose", "batch");
form.append("file", new Blob([jsonl], { type: "application/jsonl" }), `memory-embeddings.${hashText(String(Date.now()))}.jsonl`);
const fileRes = await fetch(`${baseUrl}/files`, {
method: "POST",
headers: getOpenAiHeaders(params.openAi, { json: false }),
body: form
});
if (!fileRes.ok) {
const text = await fileRes.text();
throw new Error(`openai batch file upload failed: ${fileRes.status} ${text}`);
}
const filePayload = await fileRes.json();
if (!filePayload.id) throw new Error("openai batch file upload failed: missing file id");
return await (await retryAsync(async () => {
const res = await fetch(`${baseUrl}/batches`, {
method: "POST",
headers: getOpenAiHeaders(params.openAi, { json: true }),
body: JSON.stringify({
input_file_id: filePayload.id,
endpoint: OPENAI_BATCH_ENDPOINT,
completion_window: OPENAI_BATCH_COMPLETION_WINDOW,
metadata: {
source: "openclaw-memory",
agent: params.agentId
}
})
});
if (!res.ok) {
const text = await res.text();
const err = /* @__PURE__ */ new Error(`openai batch create failed: ${res.status} ${text}`);
err.status = res.status;
throw err;
}
return res;
}, {
attempts: 3,
minDelayMs: 300,
maxDelayMs: 2e3,
jitter: .2,
shouldRetry: (err) => {
const status = err.status;
return status === 429 || typeof status === "number" && status >= 500;
}
})).json();
}
async function fetchOpenAiBatchStatus(params) {
const baseUrl = getOpenAiBaseUrl(params.openAi);
const res = await fetch(`${baseUrl}/batches/${params.batchId}`, { headers: getOpenAiHeaders(params.openAi, { json: true }) });
if (!res.ok) {
const text = await res.text();
throw new Error(`openai batch status failed: ${res.status} ${text}`);
}
return await res.json();
}
async function fetchOpenAiFileContent(params) {
const baseUrl = getOpenAiBaseUrl(params.openAi);
const res = await fetch(`${baseUrl}/files/${params.fileId}/content`, { headers: getOpenAiHeaders(params.openAi, { json: true }) });
if (!res.ok) {
const text = await res.text();
throw new Error(`openai batch file content failed: ${res.status} ${text}`);
}
return await res.text();
}
function parseOpenAiBatchOutput(text) {
if (!text.trim()) return [];
return text.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line));
}
async function readOpenAiBatchError(params) {
try {
const first = parseOpenAiBatchOutput(await fetchOpenAiFileContent({
openAi: params.openAi,
fileId: params.errorFileId
})).find((line) => line.error?.message || line.response?.body?.error);
return first?.error?.message ?? (typeof first?.response?.body?.error?.message === "string" ? first?.response?.body?.error?.message : void 0);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
return message ? `error file unavailable: ${message}` : void 0;
}
}
async function waitForOpenAiBatch(params) {
const start = Date.now();
let current = params.initial;
while (true) {
const status = current ?? await fetchOpenAiBatchStatus({
openAi: params.openAi,
batchId: params.batchId
});
const state = status.status ?? "unknown";
if (state === "completed") {
if (!status.output_file_id) throw new Error(`openai batch ${params.batchId} completed without output file`);
return {
outputFileId: status.output_file_id,
errorFileId: status.error_file_id ?? void 0
};
}
if ([
"failed",
"expired",
"cancelled",
"canceled"
].includes(state)) {
const detail = status.error_file_id ? await readOpenAiBatchError({
openAi: params.openAi,
errorFileId: status.error_file_id
}) : void 0;
const suffix = detail ? `: ${detail}` : "";
throw new Error(`openai batch ${params.batchId} ${state}${suffix}`);
}
if (!params.wait) throw new Error(`openai batch ${params.batchId} still ${state}; wait disabled`);
if (Date.now() - start > params.timeoutMs) throw new Error(`openai batch ${params.batchId} timed out after ${params.timeoutMs}ms`);
params.debug?.(`openai batch ${params.batchId} ${state}; waiting ${params.pollIntervalMs}ms`);
await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs));
current = void 0;
}
}
async function runWithConcurrency(tasks, limit) {
if (tasks.length === 0) return [];
const resolvedLimit = Math.max(1, Math.min(limit, tasks.length));
const results = Array.from({ length: tasks.length });
let next = 0;
let firstError = null;
const workers = Array.from({ length: resolvedLimit }, async () => {
while (true) {
if (firstError) return;
const index = next;
next += 1;
if (index >= tasks.length) return;
try {
results[index] = await tasks[index]();
} catch (err) {
firstError = err;
return;
}
}
});
await Promise.allSettled(workers);
if (firstError) throw firstError;
return results;
}
async function runOpenAiEmbeddingBatches(params) {
if (params.requests.length === 0) return /* @__PURE__ */ new Map();
const groups = splitOpenAiBatchRequests(params.requests);
const byCustomId = /* @__PURE__ */ new Map();
const tasks = groups.map((group, groupIndex) => async () => {
const batchInfo = await submitOpenAiBatch({
openAi: params.openAi,
requests: group,
agentId: params.agentId
});
if (!batchInfo.id) throw new Error("openai batch create failed: missing batch id");
params.debug?.("memory embeddings: openai batch created", {
batchId: batchInfo.id,
status: batchInfo.status,
group: groupIndex + 1,
groups: groups.length,
requests: group.length
});
if (!params.wait && batchInfo.status !== "completed") throw new Error(`openai batch ${batchInfo.id} submitted; enable remote.batch.wait to await completion`);
const completed = batchInfo.status === "completed" ? {
outputFileId: batchInfo.output_file_id ?? "",
errorFileId: batchInfo.error_file_id ?? void 0
} : await waitForOpenAiBatch({
openAi: params.openAi,
batchId: batchInfo.id,
wait: params.wait,
pollIntervalMs: params.pollIntervalMs,
timeoutMs: params.timeoutMs,
debug: params.debug,
initial: batchInfo
});
if (!completed.outputFileId) throw new Error(`openai batch ${batchInfo.id} completed without output file`);
const outputLines = parseOpenAiBatchOutput(await fetchOpenAiFileContent({
openAi: params.openAi,
fileId: completed.outputFileId
}));
const errors = [];
const remaining = new Set(group.map((request) => request.custom_id));
for (const line of outputLines) {
const customId = line.custom_id;
if (!customId) continue;
remaining.delete(customId);
if (line.error?.message) {
errors.push(`${customId}: ${line.error.message}`);
continue;
}
const response = line.response;
if ((response?.status_code ?? 0) >= 400) {
const message = response?.body?.error?.message ?? (typeof response?.body === "string" ? response.body : void 0) ?? "unknown error";
errors.push(`${customId}: ${message}`);
continue;
}
const embedding = (response?.body?.data ?? [])[0]?.embedding ?? [];
if (embedding.length === 0) {
errors.push(`${customId}: empty embedding`);
continue;
}
byCustomId.set(customId, embedding);
}
if (errors.length > 0) throw new Error(`openai batch ${batchInfo.id} failed: ${errors.join("; ")}`);
if (remaining.size > 0) throw new Error(`openai batch ${batchInfo.id} missing ${remaining.size} embedding responses`);
});
params.debug?.("memory embeddings: openai batch submit", {
requests: params.requests.length,
groups: groups.length,
wait: params.wait,
concurrency: params.concurrency,
pollIntervalMs: params.pollIntervalMs,
timeoutMs: params.timeoutMs
});
await runWithConcurrency(tasks, params.concurrency);
return byCustomId;
}
//#endregion
//#region src/memory/embeddings-gemini.ts
const DEFAULT_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
const DEFAULT_GEMINI_EMBEDDING_MODEL = "gemini-embedding-001";
const debugEmbeddings = isTruthyEnvValue(process.env.OPENCLAW_DEBUG_MEMORY_EMBEDDINGS);
const log$1 = createSubsystemLogger("memory/embeddings");
const debugLog = (message, meta) => {
if (!debugEmbeddings) return;
const suffix = meta ? ` ${JSON.stringify(meta)}` : "";
log$1.raw(`${message}${suffix}`);
};
function resolveRemoteApiKey(remoteApiKey) {
const trimmed = remoteApiKey?.trim();
if (!trimmed) return;
if (trimmed === "GOOGLE_API_KEY" || trimmed === "GEMINI_API_KEY") return process.env[trimmed]?.trim();
return trimmed;
}
function normalizeGeminiModel(model) {
const trimmed = model.trim();
if (!trimmed) return DEFAULT_GEMINI_EMBEDDING_MODEL;
const withoutPrefix = trimmed.replace(/^models\//, "");
if (withoutPrefix.startsWith("gemini/")) return withoutPrefix.slice(7);
if (withoutPrefix.startsWith("google/")) return withoutPrefix.slice(7);
return withoutPrefix;
}
function normalizeGeminiBaseUrl(raw) {
const trimmed = raw.replace(/\/+$/, "");
const openAiIndex = trimmed.indexOf("/openai");
if (openAiIndex > -1) return trimmed.slice(0, openAiIndex);
return trimmed;
}
function buildGeminiModelPath(model) {
return model.startsWith("models/") ? model : `models/${model}`;
}
async function createGeminiEmbeddingProvider(options) {
const client = await resolveGeminiEmbeddingClient(options);
const baseUrl = client.baseUrl.replace(/\/$/, "");
const embedUrl = `${baseUrl}/${client.modelPath}:embedContent`;
const batchUrl = `${baseUrl}/${client.modelPath}:batchEmbedContents`;
const embedQuery = async (text) => {
if (!text.trim()) return [];
const res = await fetch(embedUrl, {
method: "POST",
headers: client.headers,
body: JSON.stringify({
content: { parts: [{ text }] },
taskType: "RETRIEVAL_QUERY"
})
});
if (!res.ok) {
const payload = await res.text();
throw new Error(`gemini embeddings failed: ${res.status} ${payload}`);
}
return (await res.json()).embedding?.values ?? [];
};
const embedBatch = async (texts) => {
if (texts.length === 0) return [];
const requests = texts.map((text) => ({
model: client.modelPath,
content: { parts: [{ text }] },
taskType: "RETRIEVAL_DOCUMENT"
}));
const res = await fetch(batchUrl, {
method: "POST",
headers: client.headers,
body: JSON.stringify({ requests })
});
if (!res.ok) {
const payload = await res.text();
throw new Error(`gemini embeddings failed: ${res.status} ${payload}`);
}
const payload = await res.json();
const embeddings = Array.isArray(payload.embeddings) ? payload.embeddings : [];
return texts.map((_, index) => embeddings[index]?.values ?? []);
};
return {
provider: {
id: "gemini",
model: client.model,
embedQuery,
embedBatch
},
client
};
}
async function resolveGeminiEmbeddingClient(options) {
const remote = options.remote;
const remoteApiKey = resolveRemoteApiKey(remote?.apiKey);
const remoteBaseUrl = remote?.baseUrl?.trim();
const apiKey = remoteApiKey ? remoteApiKey : requireApiKey(await resolveApiKeyForProvider({
provider: "google",
cfg: options.config,
agentDir: options.agentDir
}), "google");
const providerConfig = options.config.models?.providers?.google;
const rawBaseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || DEFAULT_GEMINI_BASE_URL;
const baseUrl = normalizeGeminiBaseUrl(rawBaseUrl);
const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers);
const headers = {
"Content-Type": "application/json",
"x-goog-api-key": apiKey,
...headerOverrides
};
const model = normalizeGeminiModel(options.model);
const modelPath = buildGeminiModelPath(model);
debugLog("memory embeddings: gemini client", {
rawBaseUrl,
baseUrl,
model,
modelPath,
embedEndpoint: `${baseUrl}/${modelPath}:embedContent`,
batchEndpoint: `${baseUrl}/${modelPath}:batchEmbedContents`
});
return {
baseUrl,
headers,
model,
modelPath
};
}
//#endregion
//#region src/memory/embeddings-openai.ts
const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
function normalizeOpenAiModel(model) {
const trimmed = model.trim();
if (!trimmed) return DEFAULT_OPENAI_EMBEDDING_MODEL;
if (trimmed.startsWith("openai/")) return trimmed.slice(7);
return trimmed;
}
async function createOpenAiEmbeddingProvider(options) {
const client = await resolveOpenAiEmbeddingClient(options);
const url = `${client.baseUrl.replace(/\/$/, "")}/embeddings`;
const embed = async (input) => {
if (input.length === 0) return [];
const res = await fetch(url, {
method: "POST",
headers: client.headers,
body: JSON.stringify({
model: client.model,
input
})
});
if (!res.ok) {
const text = await res.text();
throw new Error(`openai embeddings failed: ${res.status} ${text}`);
}
return ((await res.json()).data ?? []).map((entry) => entry.embedding ?? []);
};
return {
provider: {
id: "openai",
model: client.model,
embedQuery: async (text) => {
const [vec] = await embed([text]);
return vec ?? [];
},
embedBatch: embed
},
client
};
}
async function resolveOpenAiEmbeddingClient(options) {
const remote = options.remote;
const remoteApiKey = remote?.apiKey?.trim();
const remoteBaseUrl = remote?.baseUrl?.trim();
const apiKey = remoteApiKey ? remoteApiKey : requireApiKey(await resolveApiKeyForProvider({
provider: "openai",
cfg: options.config,
agentDir: options.agentDir
}), "openai");
const providerConfig = options.config.models?.providers?.openai;
const baseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || DEFAULT_OPENAI_BASE_URL;
const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers);
return {
baseUrl,
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${apiKey}`,
...headerOverrides
},
model: normalizeOpenAiModel(options.model)
};
}
//#endregion
//#region src/memory/node-llama.ts
async function importNodeLlamaCpp() {
return import("node-llama-cpp");
}
//#endregion
//#region src/memory/embeddings.ts
function sanitizeAndNormalizeEmbedding(vec) {
const sanitized = vec.map((value) => Number.isFinite(value) ? value : 0);
const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0));
if (magnitude < 1e-10) return sanitized;
return sanitized.map((value) => value / magnitude);
}
const DEFAULT_LOCAL_MODEL = "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf";
function canAutoSelectLocal(options) {
const modelPath = options.local?.modelPath?.trim();
if (!modelPath) return false;
if (/^(hf:|https?:)/i.test(modelPath)) return false;
const resolved = resolveUserPath(modelPath);
try {
return fs.statSync(resolved).isFile();
} catch {
return false;
}
}
function isMissingApiKeyError(err) {
return formatError(err).includes("No API key found for provider");
}
async function createLocalEmbeddingProvider(options) {
const modelPath = options.local?.modelPath?.trim() || DEFAULT_LOCAL_MODEL;
const modelCacheDir = options.local?.modelCacheDir?.trim();
const { getLlama, resolveModelFile, LlamaLogLevel } = await importNodeLlamaCpp();
let llama = null;
let embeddingModel = null;
let embeddingContext = null;
const ensureContext = async () => {
if (!llama) llama = await getLlama({ logLevel: LlamaLogLevel.error });
if (!embeddingModel) {
const resolved = await resolveModelFile(modelPath, modelCacheDir || void 0);
embeddingModel = await llama.loadModel({ modelPath: resolved });
}
if (!embeddingContext) embeddingContext = await embeddingModel.createEmbeddingContext();
return embeddingContext;
};
return {
id: "local",
model: modelPath,
embedQuery: async (text) => {
const embedding = await (await ensureContext()).getEmbeddingFor(text);
return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector));
},
embedBatch: async (texts) => {
const ctx = await ensureContext();
return await Promise.all(texts.map(async (text) => {
const embedding = await ctx.getEmbeddingFor(text);
return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector));
}));
}
};
}
async function createEmbeddingProvider(options) {
const requestedProvider = options.provider;
const fallback = options.fallback;
const createProvider = async (id) => {
if (id === "local") return { provider: await createLocalEmbeddingProvider(options) };
if (id === "gemini") {
const { provider, client } = await createGeminiEmbeddingProvider(options);
return {
provider,
gemini: client
};
}
const { provider, client } = await createOpenAiEmbeddingProvider(options);
return {
provider,
openAi: client
};
};
const formatPrimaryError = (err, provider) => provider === "local" ? formatLocalSetupError(err) : formatError(err);
if (requestedProvider === "auto") {
const missingKeyErrors = [];
let localError = null;
if (canAutoSelectLocal(options)) try {
return {
...await createProvider("local"),
requestedProvider
};
} catch (err) {
localError = formatLocalSetupError(err);
}
for (const provider of ["openai", "gemini"]) try {
return {
...await createProvider(provider),
requestedProvider
};
} catch (err) {
const message = formatPrimaryError(err, provider);
if (isMissingApiKeyError(err)) {
missingKeyErrors.push(message);
continue;
}
throw new Error(message, { cause: err });
}
const details = [...missingKeyErrors, localError].filter(Boolean);
if (details.length > 0) throw new Error(details.join("\n\n"));
throw new Error("No embeddings provider available.");
}
try {
return {
...await createProvider(requestedProvider),
requestedProvider
};
} catch (primaryErr) {
const reason = formatPrimaryError(primaryErr, requestedProvider);
if (fallback && fallback !== "none" && fallback !== requestedProvider) try {
return {
...await createProvider(fallback),
requestedProvider,
fallbackFrom: requestedProvider,
fallbackReason: reason
};
} catch (fallbackErr) {
throw new Error(`${reason}\n\nFallback to ${fallback} failed: ${formatError(fallbackErr)}`, { cause: fallbackErr });
}
throw new Error(reason, { cause: primaryErr });
}
}
function formatError(err) {
if (err instanceof Error) return err.message;
return String(err);
}
function isNodeLlamaCppMissing(err) {
if (!(err instanceof Error)) return false;
if (err.code === "ERR_MODULE_NOT_FOUND") return err.message.includes("node-llama-cpp");
return false;
}
function formatLocalSetupError(err) {
const detail = formatError(err);
const missing = isNodeLlamaCppMissing(err);
return [
"Local embeddings unavailable.",
missing ? "Reason: optional dependency node-llama-cpp is missing (or failed to install)." : detail ? `Reason: ${detail}` : void 0,
missing && detail ? `Detail: ${detail}` : null,
"To enable local embeddings:",
"1) Use Node 22 LTS (recommended for installs/updates)",
missing ? "2) Reinstall OpenClaw (this should install node-llama-cpp): npm i -g openclaw@latest" : null,
"3) If you use pnpm: pnpm approve-builds (select node-llama-cpp), then pnpm rebuild node-llama-cpp",
"Or set agents.defaults.memorySearch.provider = \"openai\" (remote)."
].filter(Boolean).join("\n");
}
//#endregion
//#region src/memory/hybrid.ts
function buildFtsQuery(raw) {
const tokens = raw.match(/[A-Za-z0-9_]+/g)?.map((t) => t.trim()).filter(Boolean) ?? [];
if (tokens.length === 0) return null;
return tokens.map((t) => `"${t.replaceAll("\"", "")}"`).join(" AND ");
}
function bm25RankToScore(rank) {
return 1 / (1 + (Number.isFinite(rank) ? Math.max(0, rank) : 999));
}
function mergeHybridResults(params) {
const byId = /* @__PURE__ */ new Map();
for (const r of params.vector) byId.set(r.id, {
id: r.id,
path: r.path,
startLine: r.startLine,
endLine: r.endLine,
source: r.source,
snippet: r.snippet,
vectorScore: r.vectorScore,
textScore: 0
});
for (const r of params.keyword) {
const existing = byId.get(r.id);
if (existing) {
existing.textScore = r.textScore;
if (r.snippet && r.snippet.length > 0) existing.snippet = r.snippet;
} else byId.set(r.id, {
id: r.id,
path: r.path,
startLine: r.startLine,
endLine: r.endLine,
source: r.source,
snippet: r.snippet,
vectorScore: 0,
textScore: r.textScore
});
}
return Array.from(byId.values()).map((entry) => {
const score = params.vectorWeight * entry.vectorScore + params.textWeight * entry.textScore;
return {
path: entry.path,
startLine: entry.startLine,
endLine: entry.endLine,
score,
snippet: entry.snippet,
source: entry.source
};
}).toSorted((a, b) => b.score - a.score);
}
//#endregion
//#region src/memory/manager-search.ts
const vectorToBlob$1 = (embedding) => Buffer.from(new Float32Array(embedding).buffer);
async function searchVector(params) {
if (params.queryVec.length === 0 || params.limit <= 0) return [];
if (await params.ensureVectorReady(params.queryVec.length)) return params.db.prepare(`SELECT c.id, c.path, c.start_line, c.end_line, c.text,
c.source,
vec_distance_cosine(v.embedding, ?) AS dist
FROM ${params.vectorTable} v\n JOIN chunks c ON c.id = v.id\n WHERE c.model = ?${params.sourceFilterVec.sql}\n ORDER BY dist ASC\n LIMIT ?`).all(vectorToBlob$1(params.queryVec), params.providerModel, ...params.sourceFilterVec.params, params.limit).map((row) => ({
id: row.id,
path: row.path,
startLine: row.start_line,
endLine: row.end_line,
score: 1 - row.dist,
snippet: truncateUtf16Safe(row.text, params.snippetMaxChars),
source: row.source
}));
return listChunks({
db: params.db,
providerModel: params.providerModel,
sourceFilter: params.sourceFilterChunks
}).map((chunk) => ({
chunk,
score: cosineSimilarity(params.queryVec, chunk.embedding)
})).filter((entry) => Number.isFinite(entry.score)).toSorted((a, b) => b.score - a.score).slice(0, params.limit).map((entry) => ({
id: entry.chunk.id,
path: entry.chunk.path,
startLine: entry.chunk.startLine,
endLine: entry.chunk.endLine,
score: entry.score,
snippet: truncateUtf16Safe(entry.chunk.text, params.snippetMaxChars),
source: entry.chunk.source
}));
}
function listChunks(params) {
return params.db.prepare(`SELECT id, path, start_line, end_line, text, embedding, source
FROM chunks
WHERE model = ?${params.sourceFilter.sql}`).all(params.providerModel, ...params.sourceFilter.params).map((row) => ({
id: row.id,
path: row.path,
startLine: row.start_line,
endLine: row.end_line,
text: row.text,
embedding: parseEmbedding(row.embedding),
source: row.source
}));
}
async function searchKeyword(params) {
if (params.limit <= 0) return [];
const ftsQuery = params.buildFtsQuery(params.query);
if (!ftsQuery) return [];
return params.db.prepare(`SELECT id, path, source, start_line, end_line, text,\n bm25(${params.ftsTable}) AS rank\n FROM ${params.ftsTable}\n WHERE ${params.ftsTable} MATCH ? AND model = ?${params.sourceFilter.sql}\n ORDER BY rank ASC\n LIMIT ?`).all(ftsQuery, params.providerModel, ...params.sourceFilter.params, params.limit).map((row) => {
const textScore = params.bm25RankToScore(row.rank);
return {
id: row.id,
path: row.path,
startLine: row.start_line,
endLine: row.end_line,
score: textScore,
textScore,
snippet: truncateUtf16Safe(row.text, params.snippetMaxChars),
source: row.source
};
});
}
//#endregion
//#region src/memory/memory-schema.ts
function ensureMemoryIndexSchema(params) {
params.db.exec(`
CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
`);
params.db.exec(`
CREATE TABLE IF NOT EXISTS files (
path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
`);
params.db.exec(`
CREATE TABLE IF NOT EXISTS chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
`);
params.db.exec(`
CREATE TABLE IF NOT EXISTS ${params.embeddingCacheTable} (
provider TEXT NOT NULL,
model TEXT NOT NULL,
provider_key TEXT NOT NULL,
hash TEXT NOT NULL,
embedding TEXT NOT NULL,
dims INTEGER,
updated_at INTEGER NOT NULL,
PRIMARY KEY (provider, model, provider_key, hash)
);
`);
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_embedding_cache_updated_at ON ${params.embeddingCacheTable}(updated_at);`);
let ftsAvailable = false;
let ftsError;
if (params.ftsEnabled) try {
params.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${params.ftsTable} USING fts5(\n text,\n id UNINDEXED,\n path UNINDEXED,\n source UNINDEXED,\n model UNINDEXED,\n start_line UNINDEXED,\n end_line UNINDEXED\n);`);
ftsAvailable = true;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
ftsAvailable = false;
ftsError = message;
}
ensureColumn(params.db, "files", "source", "TEXT NOT NULL DEFAULT 'memory'");
ensureColumn(params.db, "chunks", "source", "TEXT NOT NULL DEFAULT 'memory'");
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);`);
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);`);
return {
ftsAvailable,
...ftsError ? { ftsError } : {}
};
}
function ensureColumn(db, table, column, definition) {
if (db.prepare(`PRAGMA table_info(${table})`).all().some((row) => row.name === column)) return;
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
}
//#endregion
//#region src/memory/sqlite-vec.ts
async function loadSqliteVecExtension(params) {
try {
const sqliteVec = await import("sqlite-vec");
const resolvedPath = params.extensionPath?.trim() ? params.extensionPath.trim() : void 0;
const extensionPath = resolvedPath ?? sqliteVec.getLoadablePath();
params.db.enableLoadExtension(true);
if (resolvedPath) params.db.loadExtension(extensionPath);
else sqliteVec.load(params.db);
return {
ok: true,
extensionPath
};
} catch (err) {
return {
ok: false,
error: err instanceof Error ? err.message : String(err)
};
}
}
//#endregion
//#region src/memory/manager.ts
const META_KEY = "memory_index_meta_v1";
const SNIPPET_MAX_CHARS = 700;
const VECTOR_TABLE = "chunks_vec";
const FTS_TABLE = "chunks_fts";
const EMBEDDING_CACHE_TABLE = "embedding_cache";
const SESSION_DIRTY_DEBOUNCE_MS = 5e3;
const EMBEDDING_BATCH_MAX_TOKENS = 8e3;
const EMBEDDING_APPROX_CHARS_PER_TOKEN = 1;
const EMBEDDING_INDEX_CONCURRENCY = 4;
const EMBEDDING_RETRY_MAX_ATTEMPTS = 3;
const EMBEDDING_RETRY_BASE_DELAY_MS = 500;
const EMBEDDING_RETRY_MAX_DELAY_MS = 8e3;
const BATCH_FAILURE_LIMIT = 2;
const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024;
const VECTOR_LOAD_TIMEOUT_MS = 3e4;
const EMBEDDING_QUERY_TIMEOUT_REMOTE_MS = 6e4;
const EMBEDDING_QUERY_TIMEOUT_LOCAL_MS = 5 * 6e4;
const EMBEDDING_BATCH_TIMEOUT_REMOTE_MS = 2 * 6e4;
const EMBEDDING_BATCH_TIMEOUT_LOCAL_MS = 10 * 6e4;
const log = createSubsystemLogger("memory");
const INDEX_CACHE = /* @__PURE__ */ new Map();
const vectorToBlob = (embedding) => Buffer.from(new Float32Array(embedding).buffer);
var MemoryIndexManager = class MemoryIndexManager {
static async get(params) {
co