UNPKG

@gguf/claw

Version:

WhatsApp gateway CLI (Baileys web) with Pi RPC agent

1,321 lines (1,308 loc) 106 kB
import { s as resolveStateDir } from "./paths-VslOJiD2.js"; import { n as resolveAgentConfig, r as resolveAgentDir, s as resolveAgentWorkspaceDir } from "./agent-scope-COnICB_7.js"; import { I as sleep, N as resolveUserPath, T as clampNumber$1, l as createSubsystemLogger, w as clampInt, z as truncateUtf16Safe } from "./exec-B7WKla_0.js"; import { D as isTruthyEnvValue, v as requireApiKey, y as resolveApiKeyForProvider } from "./model-selection-Cs1y6OBv.js"; import { i as resolveSessionTranscriptsDirForAgent } from "./paths-xPuk88Yf.js"; import { n as onSessionTranscriptUpdate } from "./transcript-events-DW_H__a1.js"; import { a as ensureDir, c as listMemoryFiles, i as cosineSimilarity, l as normalizeExtraMemoryPaths, n as buildFileEntry, o as hashText, r as chunkMarkdown, s as isMemoryPath, t as requireNodeSqlite, u as parseEmbedding } from "./sqlite-C59YNxdL.js"; import os from "node:os"; import path from "node:path"; import fs from "node:fs"; import fs$1 from "node:fs/promises"; import { randomUUID } from "node:crypto"; import chokidar from "chokidar"; //#region src/agents/memory-search.ts const DEFAULT_OPENAI_MODEL = "text-embedding-3-small"; const DEFAULT_GEMINI_MODEL = "gemini-embedding-001"; const DEFAULT_CHUNK_TOKENS = 400; const DEFAULT_CHUNK_OVERLAP = 80; const DEFAULT_WATCH_DEBOUNCE_MS = 1500; const DEFAULT_SESSION_DELTA_BYTES = 1e5; const DEFAULT_SESSION_DELTA_MESSAGES = 50; const DEFAULT_MAX_RESULTS = 6; const DEFAULT_MIN_SCORE = .35; const DEFAULT_HYBRID_ENABLED = true; const DEFAULT_HYBRID_VECTOR_WEIGHT = .7; const DEFAULT_HYBRID_TEXT_WEIGHT = .3; const DEFAULT_HYBRID_CANDIDATE_MULTIPLIER = 4; const DEFAULT_CACHE_ENABLED = true; const DEFAULT_SOURCES = ["memory"]; function normalizeSources(sources, sessionMemoryEnabled) { const normalized = /* @__PURE__ */ new Set(); const input = sources?.length ? sources : DEFAULT_SOURCES; for (const source of input) { if (source === "memory") normalized.add("memory"); if (source === "sessions" && sessionMemoryEnabled) normalized.add("sessions"); } if (normalized.size === 0) normalized.add("memory"); return Array.from(normalized); } function resolveStorePath(agentId, raw) { const stateDir = resolveStateDir(process.env, os.homedir); const fallback = path.join(stateDir, "memory", `${agentId}.sqlite`); if (!raw) return fallback; return resolveUserPath(raw.includes("{agentId}") ? raw.replaceAll("{agentId}", agentId) : raw); } function mergeConfig(defaults, overrides, agentId) { const enabled = overrides?.enabled ?? defaults?.enabled ?? true; const sessionMemory = overrides?.experimental?.sessionMemory ?? defaults?.experimental?.sessionMemory ?? false; const provider = overrides?.provider ?? defaults?.provider ?? "auto"; const defaultRemote = defaults?.remote; const overrideRemote = overrides?.remote; const includeRemote = Boolean(overrideRemote?.baseUrl || overrideRemote?.apiKey || overrideRemote?.headers || defaultRemote?.baseUrl || defaultRemote?.apiKey || defaultRemote?.headers) || provider === "openai" || provider === "gemini" || provider === "auto"; const batch = { enabled: overrideRemote?.batch?.enabled ?? defaultRemote?.batch?.enabled ?? true, wait: overrideRemote?.batch?.wait ?? defaultRemote?.batch?.wait ?? true, concurrency: Math.max(1, overrideRemote?.batch?.concurrency ?? defaultRemote?.batch?.concurrency ?? 2), pollIntervalMs: overrideRemote?.batch?.pollIntervalMs ?? defaultRemote?.batch?.pollIntervalMs ?? 2e3, timeoutMinutes: overrideRemote?.batch?.timeoutMinutes ?? defaultRemote?.batch?.timeoutMinutes ?? 60 }; const remote = includeRemote ? { baseUrl: overrideRemote?.baseUrl ?? defaultRemote?.baseUrl, apiKey: overrideRemote?.apiKey ?? defaultRemote?.apiKey, headers: overrideRemote?.headers ?? defaultRemote?.headers, batch } : void 0; const fallback = overrides?.fallback ?? defaults?.fallback ?? "none"; const modelDefault = provider === "gemini" ? DEFAULT_GEMINI_MODEL : provider === "openai" ? DEFAULT_OPENAI_MODEL : void 0; const model = overrides?.model ?? defaults?.model ?? modelDefault ?? ""; const local = { modelPath: overrides?.local?.modelPath ?? defaults?.local?.modelPath, modelCacheDir: overrides?.local?.modelCacheDir ?? defaults?.local?.modelCacheDir }; const sources = normalizeSources(overrides?.sources ?? defaults?.sources, sessionMemory); const rawPaths = [...defaults?.extraPaths ?? [], ...overrides?.extraPaths ?? []].map((value) => value.trim()).filter(Boolean); const extraPaths = Array.from(new Set(rawPaths)); const vector = { enabled: overrides?.store?.vector?.enabled ?? defaults?.store?.vector?.enabled ?? true, extensionPath: overrides?.store?.vector?.extensionPath ?? defaults?.store?.vector?.extensionPath }; const store = { driver: overrides?.store?.driver ?? defaults?.store?.driver ?? "sqlite", path: resolveStorePath(agentId, overrides?.store?.path ?? defaults?.store?.path), vector }; const chunking = { tokens: overrides?.chunking?.tokens ?? defaults?.chunking?.tokens ?? DEFAULT_CHUNK_TOKENS, overlap: overrides?.chunking?.overlap ?? defaults?.chunking?.overlap ?? DEFAULT_CHUNK_OVERLAP }; const sync = { onSessionStart: overrides?.sync?.onSessionStart ?? defaults?.sync?.onSessionStart ?? true, onSearch: overrides?.sync?.onSearch ?? defaults?.sync?.onSearch ?? true, watch: overrides?.sync?.watch ?? defaults?.sync?.watch ?? true, watchDebounceMs: overrides?.sync?.watchDebounceMs ?? defaults?.sync?.watchDebounceMs ?? DEFAULT_WATCH_DEBOUNCE_MS, intervalMinutes: overrides?.sync?.intervalMinutes ?? defaults?.sync?.intervalMinutes ?? 0, sessions: { deltaBytes: overrides?.sync?.sessions?.deltaBytes ?? defaults?.sync?.sessions?.deltaBytes ?? DEFAULT_SESSION_DELTA_BYTES, deltaMessages: overrides?.sync?.sessions?.deltaMessages ?? defaults?.sync?.sessions?.deltaMessages ?? DEFAULT_SESSION_DELTA_MESSAGES } }; const query = { maxResults: overrides?.query?.maxResults ?? defaults?.query?.maxResults ?? DEFAULT_MAX_RESULTS, minScore: overrides?.query?.minScore ?? defaults?.query?.minScore ?? DEFAULT_MIN_SCORE }; const hybrid = { enabled: overrides?.query?.hybrid?.enabled ?? defaults?.query?.hybrid?.enabled ?? DEFAULT_HYBRID_ENABLED, vectorWeight: overrides?.query?.hybrid?.vectorWeight ?? defaults?.query?.hybrid?.vectorWeight ?? DEFAULT_HYBRID_VECTOR_WEIGHT, textWeight: overrides?.query?.hybrid?.textWeight ?? defaults?.query?.hybrid?.textWeight ?? DEFAULT_HYBRID_TEXT_WEIGHT, candidateMultiplier: overrides?.query?.hybrid?.candidateMultiplier ?? defaults?.query?.hybrid?.candidateMultiplier ?? DEFAULT_HYBRID_CANDIDATE_MULTIPLIER }; const cache = { enabled: overrides?.cache?.enabled ?? defaults?.cache?.enabled ?? DEFAULT_CACHE_ENABLED, maxEntries: overrides?.cache?.maxEntries ?? defaults?.cache?.maxEntries }; const overlap = clampNumber$1(chunking.overlap, 0, Math.max(0, chunking.tokens - 1)); const minScore = clampNumber$1(query.minScore, 0, 1); const vectorWeight = clampNumber$1(hybrid.vectorWeight, 0, 1); const textWeight = clampNumber$1(hybrid.textWeight, 0, 1); const sum = vectorWeight + textWeight; const normalizedVectorWeight = sum > 0 ? vectorWeight / sum : DEFAULT_HYBRID_VECTOR_WEIGHT; const normalizedTextWeight = sum > 0 ? textWeight / sum : DEFAULT_HYBRID_TEXT_WEIGHT; const candidateMultiplier = clampInt(hybrid.candidateMultiplier, 1, 20); const deltaBytes = clampInt(sync.sessions.deltaBytes, 0, Number.MAX_SAFE_INTEGER); const deltaMessages = clampInt(sync.sessions.deltaMessages, 0, Number.MAX_SAFE_INTEGER); return { enabled, sources, extraPaths, provider, remote, experimental: { sessionMemory }, fallback, model, local, store, chunking: { tokens: Math.max(1, chunking.tokens), overlap }, sync: { ...sync, sessions: { deltaBytes, deltaMessages } }, query: { ...query, minScore, hybrid: { enabled: Boolean(hybrid.enabled), vectorWeight: normalizedVectorWeight, textWeight: normalizedTextWeight, candidateMultiplier } }, cache: { enabled: Boolean(cache.enabled), maxEntries: typeof cache.maxEntries === "number" && Number.isFinite(cache.maxEntries) ? Math.max(1, Math.floor(cache.maxEntries)) : void 0 } }; } function resolveMemorySearchConfig(cfg, agentId) { const defaults = cfg.agents?.defaults?.memorySearch; const overrides = resolveAgentConfig(cfg, agentId)?.memorySearch; const resolved = mergeConfig(defaults, overrides, agentId); if (!resolved.enabled) return null; return resolved; } //#endregion //#region src/memory/batch-gemini.ts const GEMINI_BATCH_MAX_REQUESTS = 5e4; const debugEmbeddings$1 = isTruthyEnvValue(process.env.OPENCLAW_DEBUG_MEMORY_EMBEDDINGS); const log$2 = createSubsystemLogger("memory/embeddings"); const debugLog$1 = (message, meta) => { if (!debugEmbeddings$1) return; const suffix = meta ? ` ${JSON.stringify(meta)}` : ""; log$2.raw(`${message}${suffix}`); }; function getGeminiBaseUrl(gemini) { return gemini.baseUrl?.replace(/\/$/, "") ?? ""; } function getGeminiHeaders(gemini, params) { const headers = gemini.headers ? { ...gemini.headers } : {}; if (params.json) { if (!headers["Content-Type"] && !headers["content-type"]) headers["Content-Type"] = "application/json"; } else { delete headers["Content-Type"]; delete headers["content-type"]; } return headers; } function getGeminiUploadUrl(baseUrl) { if (baseUrl.includes("/v1beta")) return baseUrl.replace(/\/v1beta\/?$/, "/upload/v1beta"); return `${baseUrl.replace(/\/$/, "")}/upload`; } function splitGeminiBatchRequests(requests) { if (requests.length <= GEMINI_BATCH_MAX_REQUESTS) return [requests]; const groups = []; for (let i = 0; i < requests.length; i += GEMINI_BATCH_MAX_REQUESTS) groups.push(requests.slice(i, i + GEMINI_BATCH_MAX_REQUESTS)); return groups; } function buildGeminiUploadBody(params) { const boundary = `openclaw-${hashText(params.displayName)}`; const jsonPart = JSON.stringify({ file: { displayName: params.displayName, mimeType: "application/jsonl" } }); const delimiter = `--${boundary}\r\n`; const closeDelimiter = `--${boundary}--\r\n`; const parts = [ `${delimiter}Content-Type: application/json; charset=UTF-8\r\n\r\n${jsonPart}\r\n`, `${delimiter}Content-Type: application/jsonl; charset=UTF-8\r\n\r\n${params.jsonl}\r\n`, closeDelimiter ]; return { body: new Blob([parts.join("")], { type: "multipart/related" }), contentType: `multipart/related; boundary=${boundary}` }; } async function submitGeminiBatch(params) { const baseUrl = getGeminiBaseUrl(params.gemini); const uploadPayload = buildGeminiUploadBody({ jsonl: params.requests.map((request) => JSON.stringify({ key: request.custom_id, request: { content: request.content, task_type: request.taskType } })).join("\n"), displayName: `memory-embeddings-${hashText(String(Date.now()))}` }); const uploadUrl = `${getGeminiUploadUrl(baseUrl)}/files?uploadType=multipart`; debugLog$1("memory embeddings: gemini batch upload", { uploadUrl, baseUrl, requests: params.requests.length }); const fileRes = await fetch(uploadUrl, { method: "POST", headers: { ...getGeminiHeaders(params.gemini, { json: false }), "Content-Type": uploadPayload.contentType }, body: uploadPayload.body }); if (!fileRes.ok) { const text = await fileRes.text(); throw new Error(`gemini batch file upload failed: ${fileRes.status} ${text}`); } const filePayload = await fileRes.json(); const fileId = filePayload.name ?? filePayload.file?.name; if (!fileId) throw new Error("gemini batch file upload failed: missing file id"); const batchBody = { batch: { displayName: `memory-embeddings-${params.agentId}`, inputConfig: { file_name: fileId } } }; const batchEndpoint = `${baseUrl}/${params.gemini.modelPath}:asyncBatchEmbedContent`; debugLog$1("memory embeddings: gemini batch create", { batchEndpoint, fileId }); const batchRes = await fetch(batchEndpoint, { method: "POST", headers: getGeminiHeaders(params.gemini, { json: true }), body: JSON.stringify(batchBody) }); if (batchRes.ok) return await batchRes.json(); const text = await batchRes.text(); if (batchRes.status === 404) throw new Error("gemini batch create failed: 404 (asyncBatchEmbedContent not available for this model/baseUrl). Disable remote.batch.enabled or switch providers."); throw new Error(`gemini batch create failed: ${batchRes.status} ${text}`); } async function fetchGeminiBatchStatus(params) { const statusUrl = `${getGeminiBaseUrl(params.gemini)}/${params.batchName.startsWith("batches/") ? params.batchName : `batches/${params.batchName}`}`; debugLog$1("memory embeddings: gemini batch status", { statusUrl }); const res = await fetch(statusUrl, { headers: getGeminiHeaders(params.gemini, { json: true }) }); if (!res.ok) { const text = await res.text(); throw new Error(`gemini batch status failed: ${res.status} ${text}`); } return await res.json(); } async function fetchGeminiFileContent(params) { const downloadUrl = `${getGeminiBaseUrl(params.gemini)}/${params.fileId.startsWith("files/") ? params.fileId : `files/${params.fileId}`}:download`; debugLog$1("memory embeddings: gemini batch download", { downloadUrl }); const res = await fetch(downloadUrl, { headers: getGeminiHeaders(params.gemini, { json: true }) }); if (!res.ok) { const text = await res.text(); throw new Error(`gemini batch file content failed: ${res.status} ${text}`); } return await res.text(); } function parseGeminiBatchOutput(text) { if (!text.trim()) return []; return text.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line)); } async function waitForGeminiBatch(params) { const start = Date.now(); let current = params.initial; while (true) { const status = current ?? await fetchGeminiBatchStatus({ gemini: params.gemini, batchName: params.batchName }); const state = status.state ?? "UNKNOWN"; if ([ "SUCCEEDED", "COMPLETED", "DONE" ].includes(state)) { const outputFileId = status.outputConfig?.file ?? status.outputConfig?.fileId ?? status.metadata?.output?.responsesFile; if (!outputFileId) throw new Error(`gemini batch ${params.batchName} completed without output file`); return { outputFileId }; } if ([ "FAILED", "CANCELLED", "CANCELED", "EXPIRED" ].includes(state)) { const message = status.error?.message ?? "unknown error"; throw new Error(`gemini batch ${params.batchName} ${state}: ${message}`); } if (!params.wait) throw new Error(`gemini batch ${params.batchName} still ${state}; wait disabled`); if (Date.now() - start > params.timeoutMs) throw new Error(`gemini batch ${params.batchName} timed out after ${params.timeoutMs}ms`); params.debug?.(`gemini batch ${params.batchName} ${state}; waiting ${params.pollIntervalMs}ms`); await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs)); current = void 0; } } async function runWithConcurrency$1(tasks, limit) { if (tasks.length === 0) return []; const resolvedLimit = Math.max(1, Math.min(limit, tasks.length)); const results = Array.from({ length: tasks.length }); let next = 0; let firstError = null; const workers = Array.from({ length: resolvedLimit }, async () => { while (true) { if (firstError) return; const index = next; next += 1; if (index >= tasks.length) return; try { results[index] = await tasks[index](); } catch (err) { firstError = err; return; } } }); await Promise.allSettled(workers); if (firstError) throw firstError; return results; } async function runGeminiEmbeddingBatches(params) { if (params.requests.length === 0) return /* @__PURE__ */ new Map(); const groups = splitGeminiBatchRequests(params.requests); const byCustomId = /* @__PURE__ */ new Map(); const tasks = groups.map((group, groupIndex) => async () => { const batchInfo = await submitGeminiBatch({ gemini: params.gemini, requests: group, agentId: params.agentId }); const batchName = batchInfo.name ?? ""; if (!batchName) throw new Error("gemini batch create failed: missing batch name"); params.debug?.("memory embeddings: gemini batch created", { batchName, state: batchInfo.state, group: groupIndex + 1, groups: groups.length, requests: group.length }); if (!params.wait && batchInfo.state && ![ "SUCCEEDED", "COMPLETED", "DONE" ].includes(batchInfo.state)) throw new Error(`gemini batch ${batchName} submitted; enable remote.batch.wait to await completion`); const completed = batchInfo.state && [ "SUCCEEDED", "COMPLETED", "DONE" ].includes(batchInfo.state) ? { outputFileId: batchInfo.outputConfig?.file ?? batchInfo.outputConfig?.fileId ?? batchInfo.metadata?.output?.responsesFile ?? "" } : await waitForGeminiBatch({ gemini: params.gemini, batchName, wait: params.wait, pollIntervalMs: params.pollIntervalMs, timeoutMs: params.timeoutMs, debug: params.debug, initial: batchInfo }); if (!completed.outputFileId) throw new Error(`gemini batch ${batchName} completed without output file`); const outputLines = parseGeminiBatchOutput(await fetchGeminiFileContent({ gemini: params.gemini, fileId: completed.outputFileId })); const errors = []; const remaining = new Set(group.map((request) => request.custom_id)); for (const line of outputLines) { const customId = line.key ?? line.custom_id ?? line.request_id; if (!customId) continue; remaining.delete(customId); if (line.error?.message) { errors.push(`${customId}: ${line.error.message}`); continue; } if (line.response?.error?.message) { errors.push(`${customId}: ${line.response.error.message}`); continue; } const embedding = line.embedding?.values ?? line.response?.embedding?.values ?? []; if (embedding.length === 0) { errors.push(`${customId}: empty embedding`); continue; } byCustomId.set(customId, embedding); } if (errors.length > 0) throw new Error(`gemini batch ${batchName} failed: ${errors.join("; ")}`); if (remaining.size > 0) throw new Error(`gemini batch ${batchName} missing ${remaining.size} embedding responses`); }); params.debug?.("memory embeddings: gemini batch submit", { requests: params.requests.length, groups: groups.length, wait: params.wait, concurrency: params.concurrency, pollIntervalMs: params.pollIntervalMs, timeoutMs: params.timeoutMs }); await runWithConcurrency$1(tasks, params.concurrency); return byCustomId; } //#endregion //#region src/infra/retry.ts const DEFAULT_RETRY_CONFIG = { attempts: 3, minDelayMs: 300, maxDelayMs: 3e4, jitter: 0 }; const asFiniteNumber = (value) => typeof value === "number" && Number.isFinite(value) ? value : void 0; const clampNumber = (value, fallback, min, max) => { const next = asFiniteNumber(value); if (next === void 0) return fallback; const floor = typeof min === "number" ? min : Number.NEGATIVE_INFINITY; const ceiling = typeof max === "number" ? max : Number.POSITIVE_INFINITY; return Math.min(Math.max(next, floor), ceiling); }; function resolveRetryConfig(defaults = DEFAULT_RETRY_CONFIG, overrides) { const attempts = Math.max(1, Math.round(clampNumber(overrides?.attempts, defaults.attempts, 1))); const minDelayMs = Math.max(0, Math.round(clampNumber(overrides?.minDelayMs, defaults.minDelayMs, 0))); return { attempts, minDelayMs, maxDelayMs: Math.max(minDelayMs, Math.round(clampNumber(overrides?.maxDelayMs, defaults.maxDelayMs, 0))), jitter: clampNumber(overrides?.jitter, defaults.jitter, 0, 1) }; } function applyJitter(delayMs, jitter) { if (jitter <= 0) return delayMs; const offset = (Math.random() * 2 - 1) * jitter; return Math.max(0, Math.round(delayMs * (1 + offset))); } async function retryAsync(fn, attemptsOrOptions = 3, initialDelayMs = 300) { if (typeof attemptsOrOptions === "number") { const attempts = Math.max(1, Math.round(attemptsOrOptions)); let lastErr; for (let i = 0; i < attempts; i += 1) try { return await fn(); } catch (err) { lastErr = err; if (i === attempts - 1) break; await sleep(initialDelayMs * 2 ** i); } throw lastErr ?? /* @__PURE__ */ new Error("Retry failed"); } const options = attemptsOrOptions; const resolved = resolveRetryConfig(DEFAULT_RETRY_CONFIG, options); const maxAttempts = resolved.attempts; const minDelayMs = resolved.minDelayMs; const maxDelayMs = Number.isFinite(resolved.maxDelayMs) && resolved.maxDelayMs > 0 ? resolved.maxDelayMs : Number.POSITIVE_INFINITY; const jitter = resolved.jitter; const shouldRetry = options.shouldRetry ?? (() => true); let lastErr; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) try { return await fn(); } catch (err) { lastErr = err; if (attempt >= maxAttempts || !shouldRetry(err, attempt)) break; const retryAfterMs = options.retryAfterMs?.(err); const baseDelay = typeof retryAfterMs === "number" && Number.isFinite(retryAfterMs) ? Math.max(retryAfterMs, minDelayMs) : minDelayMs * 2 ** (attempt - 1); let delay = Math.min(baseDelay, maxDelayMs); delay = applyJitter(delay, jitter); delay = Math.min(Math.max(delay, minDelayMs), maxDelayMs); options.onRetry?.({ attempt, maxAttempts, delayMs: delay, err, label: options.label }); await sleep(delay); } throw lastErr ?? /* @__PURE__ */ new Error("Retry failed"); } //#endregion //#region src/memory/batch-openai.ts const OPENAI_BATCH_ENDPOINT = "/v1/embeddings"; const OPENAI_BATCH_COMPLETION_WINDOW = "24h"; const OPENAI_BATCH_MAX_REQUESTS = 5e4; function getOpenAiBaseUrl(openAi) { return openAi.baseUrl?.replace(/\/$/, "") ?? ""; } function getOpenAiHeaders(openAi, params) { const headers = openAi.headers ? { ...openAi.headers } : {}; if (params.json) { if (!headers["Content-Type"] && !headers["content-type"]) headers["Content-Type"] = "application/json"; } else { delete headers["Content-Type"]; delete headers["content-type"]; } return headers; } function splitOpenAiBatchRequests(requests) { if (requests.length <= OPENAI_BATCH_MAX_REQUESTS) return [requests]; const groups = []; for (let i = 0; i < requests.length; i += OPENAI_BATCH_MAX_REQUESTS) groups.push(requests.slice(i, i + OPENAI_BATCH_MAX_REQUESTS)); return groups; } async function submitOpenAiBatch(params) { const baseUrl = getOpenAiBaseUrl(params.openAi); const jsonl = params.requests.map((request) => JSON.stringify(request)).join("\n"); const form = new FormData(); form.append("purpose", "batch"); form.append("file", new Blob([jsonl], { type: "application/jsonl" }), `memory-embeddings.${hashText(String(Date.now()))}.jsonl`); const fileRes = await fetch(`${baseUrl}/files`, { method: "POST", headers: getOpenAiHeaders(params.openAi, { json: false }), body: form }); if (!fileRes.ok) { const text = await fileRes.text(); throw new Error(`openai batch file upload failed: ${fileRes.status} ${text}`); } const filePayload = await fileRes.json(); if (!filePayload.id) throw new Error("openai batch file upload failed: missing file id"); return await (await retryAsync(async () => { const res = await fetch(`${baseUrl}/batches`, { method: "POST", headers: getOpenAiHeaders(params.openAi, { json: true }), body: JSON.stringify({ input_file_id: filePayload.id, endpoint: OPENAI_BATCH_ENDPOINT, completion_window: OPENAI_BATCH_COMPLETION_WINDOW, metadata: { source: "openclaw-memory", agent: params.agentId } }) }); if (!res.ok) { const text = await res.text(); const err = /* @__PURE__ */ new Error(`openai batch create failed: ${res.status} ${text}`); err.status = res.status; throw err; } return res; }, { attempts: 3, minDelayMs: 300, maxDelayMs: 2e3, jitter: .2, shouldRetry: (err) => { const status = err.status; return status === 429 || typeof status === "number" && status >= 500; } })).json(); } async function fetchOpenAiBatchStatus(params) { const baseUrl = getOpenAiBaseUrl(params.openAi); const res = await fetch(`${baseUrl}/batches/${params.batchId}`, { headers: getOpenAiHeaders(params.openAi, { json: true }) }); if (!res.ok) { const text = await res.text(); throw new Error(`openai batch status failed: ${res.status} ${text}`); } return await res.json(); } async function fetchOpenAiFileContent(params) { const baseUrl = getOpenAiBaseUrl(params.openAi); const res = await fetch(`${baseUrl}/files/${params.fileId}/content`, { headers: getOpenAiHeaders(params.openAi, { json: true }) }); if (!res.ok) { const text = await res.text(); throw new Error(`openai batch file content failed: ${res.status} ${text}`); } return await res.text(); } function parseOpenAiBatchOutput(text) { if (!text.trim()) return []; return text.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line)); } async function readOpenAiBatchError(params) { try { const first = parseOpenAiBatchOutput(await fetchOpenAiFileContent({ openAi: params.openAi, fileId: params.errorFileId })).find((line) => line.error?.message || line.response?.body?.error); return first?.error?.message ?? (typeof first?.response?.body?.error?.message === "string" ? first?.response?.body?.error?.message : void 0); } catch (err) { const message = err instanceof Error ? err.message : String(err); return message ? `error file unavailable: ${message}` : void 0; } } async function waitForOpenAiBatch(params) { const start = Date.now(); let current = params.initial; while (true) { const status = current ?? await fetchOpenAiBatchStatus({ openAi: params.openAi, batchId: params.batchId }); const state = status.status ?? "unknown"; if (state === "completed") { if (!status.output_file_id) throw new Error(`openai batch ${params.batchId} completed without output file`); return { outputFileId: status.output_file_id, errorFileId: status.error_file_id ?? void 0 }; } if ([ "failed", "expired", "cancelled", "canceled" ].includes(state)) { const detail = status.error_file_id ? await readOpenAiBatchError({ openAi: params.openAi, errorFileId: status.error_file_id }) : void 0; const suffix = detail ? `: ${detail}` : ""; throw new Error(`openai batch ${params.batchId} ${state}${suffix}`); } if (!params.wait) throw new Error(`openai batch ${params.batchId} still ${state}; wait disabled`); if (Date.now() - start > params.timeoutMs) throw new Error(`openai batch ${params.batchId} timed out after ${params.timeoutMs}ms`); params.debug?.(`openai batch ${params.batchId} ${state}; waiting ${params.pollIntervalMs}ms`); await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs)); current = void 0; } } async function runWithConcurrency(tasks, limit) { if (tasks.length === 0) return []; const resolvedLimit = Math.max(1, Math.min(limit, tasks.length)); const results = Array.from({ length: tasks.length }); let next = 0; let firstError = null; const workers = Array.from({ length: resolvedLimit }, async () => { while (true) { if (firstError) return; const index = next; next += 1; if (index >= tasks.length) return; try { results[index] = await tasks[index](); } catch (err) { firstError = err; return; } } }); await Promise.allSettled(workers); if (firstError) throw firstError; return results; } async function runOpenAiEmbeddingBatches(params) { if (params.requests.length === 0) return /* @__PURE__ */ new Map(); const groups = splitOpenAiBatchRequests(params.requests); const byCustomId = /* @__PURE__ */ new Map(); const tasks = groups.map((group, groupIndex) => async () => { const batchInfo = await submitOpenAiBatch({ openAi: params.openAi, requests: group, agentId: params.agentId }); if (!batchInfo.id) throw new Error("openai batch create failed: missing batch id"); params.debug?.("memory embeddings: openai batch created", { batchId: batchInfo.id, status: batchInfo.status, group: groupIndex + 1, groups: groups.length, requests: group.length }); if (!params.wait && batchInfo.status !== "completed") throw new Error(`openai batch ${batchInfo.id} submitted; enable remote.batch.wait to await completion`); const completed = batchInfo.status === "completed" ? { outputFileId: batchInfo.output_file_id ?? "", errorFileId: batchInfo.error_file_id ?? void 0 } : await waitForOpenAiBatch({ openAi: params.openAi, batchId: batchInfo.id, wait: params.wait, pollIntervalMs: params.pollIntervalMs, timeoutMs: params.timeoutMs, debug: params.debug, initial: batchInfo }); if (!completed.outputFileId) throw new Error(`openai batch ${batchInfo.id} completed without output file`); const outputLines = parseOpenAiBatchOutput(await fetchOpenAiFileContent({ openAi: params.openAi, fileId: completed.outputFileId })); const errors = []; const remaining = new Set(group.map((request) => request.custom_id)); for (const line of outputLines) { const customId = line.custom_id; if (!customId) continue; remaining.delete(customId); if (line.error?.message) { errors.push(`${customId}: ${line.error.message}`); continue; } const response = line.response; if ((response?.status_code ?? 0) >= 400) { const message = response?.body?.error?.message ?? (typeof response?.body === "string" ? response.body : void 0) ?? "unknown error"; errors.push(`${customId}: ${message}`); continue; } const embedding = (response?.body?.data ?? [])[0]?.embedding ?? []; if (embedding.length === 0) { errors.push(`${customId}: empty embedding`); continue; } byCustomId.set(customId, embedding); } if (errors.length > 0) throw new Error(`openai batch ${batchInfo.id} failed: ${errors.join("; ")}`); if (remaining.size > 0) throw new Error(`openai batch ${batchInfo.id} missing ${remaining.size} embedding responses`); }); params.debug?.("memory embeddings: openai batch submit", { requests: params.requests.length, groups: groups.length, wait: params.wait, concurrency: params.concurrency, pollIntervalMs: params.pollIntervalMs, timeoutMs: params.timeoutMs }); await runWithConcurrency(tasks, params.concurrency); return byCustomId; } //#endregion //#region src/memory/embeddings-gemini.ts const DEFAULT_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"; const DEFAULT_GEMINI_EMBEDDING_MODEL = "gemini-embedding-001"; const debugEmbeddings = isTruthyEnvValue(process.env.OPENCLAW_DEBUG_MEMORY_EMBEDDINGS); const log$1 = createSubsystemLogger("memory/embeddings"); const debugLog = (message, meta) => { if (!debugEmbeddings) return; const suffix = meta ? ` ${JSON.stringify(meta)}` : ""; log$1.raw(`${message}${suffix}`); }; function resolveRemoteApiKey(remoteApiKey) { const trimmed = remoteApiKey?.trim(); if (!trimmed) return; if (trimmed === "GOOGLE_API_KEY" || trimmed === "GEMINI_API_KEY") return process.env[trimmed]?.trim(); return trimmed; } function normalizeGeminiModel(model) { const trimmed = model.trim(); if (!trimmed) return DEFAULT_GEMINI_EMBEDDING_MODEL; const withoutPrefix = trimmed.replace(/^models\//, ""); if (withoutPrefix.startsWith("gemini/")) return withoutPrefix.slice(7); if (withoutPrefix.startsWith("google/")) return withoutPrefix.slice(7); return withoutPrefix; } function normalizeGeminiBaseUrl(raw) { const trimmed = raw.replace(/\/+$/, ""); const openAiIndex = trimmed.indexOf("/openai"); if (openAiIndex > -1) return trimmed.slice(0, openAiIndex); return trimmed; } function buildGeminiModelPath(model) { return model.startsWith("models/") ? model : `models/${model}`; } async function createGeminiEmbeddingProvider(options) { const client = await resolveGeminiEmbeddingClient(options); const baseUrl = client.baseUrl.replace(/\/$/, ""); const embedUrl = `${baseUrl}/${client.modelPath}:embedContent`; const batchUrl = `${baseUrl}/${client.modelPath}:batchEmbedContents`; const embedQuery = async (text) => { if (!text.trim()) return []; const res = await fetch(embedUrl, { method: "POST", headers: client.headers, body: JSON.stringify({ content: { parts: [{ text }] }, taskType: "RETRIEVAL_QUERY" }) }); if (!res.ok) { const payload = await res.text(); throw new Error(`gemini embeddings failed: ${res.status} ${payload}`); } return (await res.json()).embedding?.values ?? []; }; const embedBatch = async (texts) => { if (texts.length === 0) return []; const requests = texts.map((text) => ({ model: client.modelPath, content: { parts: [{ text }] }, taskType: "RETRIEVAL_DOCUMENT" })); const res = await fetch(batchUrl, { method: "POST", headers: client.headers, body: JSON.stringify({ requests }) }); if (!res.ok) { const payload = await res.text(); throw new Error(`gemini embeddings failed: ${res.status} ${payload}`); } const payload = await res.json(); const embeddings = Array.isArray(payload.embeddings) ? payload.embeddings : []; return texts.map((_, index) => embeddings[index]?.values ?? []); }; return { provider: { id: "gemini", model: client.model, embedQuery, embedBatch }, client }; } async function resolveGeminiEmbeddingClient(options) { const remote = options.remote; const remoteApiKey = resolveRemoteApiKey(remote?.apiKey); const remoteBaseUrl = remote?.baseUrl?.trim(); const apiKey = remoteApiKey ? remoteApiKey : requireApiKey(await resolveApiKeyForProvider({ provider: "google", cfg: options.config, agentDir: options.agentDir }), "google"); const providerConfig = options.config.models?.providers?.google; const rawBaseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || DEFAULT_GEMINI_BASE_URL; const baseUrl = normalizeGeminiBaseUrl(rawBaseUrl); const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers); const headers = { "Content-Type": "application/json", "x-goog-api-key": apiKey, ...headerOverrides }; const model = normalizeGeminiModel(options.model); const modelPath = buildGeminiModelPath(model); debugLog("memory embeddings: gemini client", { rawBaseUrl, baseUrl, model, modelPath, embedEndpoint: `${baseUrl}/${modelPath}:embedContent`, batchEndpoint: `${baseUrl}/${modelPath}:batchEmbedContents` }); return { baseUrl, headers, model, modelPath }; } //#endregion //#region src/memory/embeddings-openai.ts const DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small"; const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1"; function normalizeOpenAiModel(model) { const trimmed = model.trim(); if (!trimmed) return DEFAULT_OPENAI_EMBEDDING_MODEL; if (trimmed.startsWith("openai/")) return trimmed.slice(7); return trimmed; } async function createOpenAiEmbeddingProvider(options) { const client = await resolveOpenAiEmbeddingClient(options); const url = `${client.baseUrl.replace(/\/$/, "")}/embeddings`; const embed = async (input) => { if (input.length === 0) return []; const res = await fetch(url, { method: "POST", headers: client.headers, body: JSON.stringify({ model: client.model, input }) }); if (!res.ok) { const text = await res.text(); throw new Error(`openai embeddings failed: ${res.status} ${text}`); } return ((await res.json()).data ?? []).map((entry) => entry.embedding ?? []); }; return { provider: { id: "openai", model: client.model, embedQuery: async (text) => { const [vec] = await embed([text]); return vec ?? []; }, embedBatch: embed }, client }; } async function resolveOpenAiEmbeddingClient(options) { const remote = options.remote; const remoteApiKey = remote?.apiKey?.trim(); const remoteBaseUrl = remote?.baseUrl?.trim(); const apiKey = remoteApiKey ? remoteApiKey : requireApiKey(await resolveApiKeyForProvider({ provider: "openai", cfg: options.config, agentDir: options.agentDir }), "openai"); const providerConfig = options.config.models?.providers?.openai; const baseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || DEFAULT_OPENAI_BASE_URL; const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers); return { baseUrl, headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, ...headerOverrides }, model: normalizeOpenAiModel(options.model) }; } //#endregion //#region src/memory/node-llama.ts async function importNodeLlamaCpp() { return import("node-llama-cpp"); } //#endregion //#region src/memory/embeddings.ts function sanitizeAndNormalizeEmbedding(vec) { const sanitized = vec.map((value) => Number.isFinite(value) ? value : 0); const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0)); if (magnitude < 1e-10) return sanitized; return sanitized.map((value) => value / magnitude); } const DEFAULT_LOCAL_MODEL = "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf"; function canAutoSelectLocal(options) { const modelPath = options.local?.modelPath?.trim(); if (!modelPath) return false; if (/^(hf:|https?:)/i.test(modelPath)) return false; const resolved = resolveUserPath(modelPath); try { return fs.statSync(resolved).isFile(); } catch { return false; } } function isMissingApiKeyError(err) { return formatError(err).includes("No API key found for provider"); } async function createLocalEmbeddingProvider(options) { const modelPath = options.local?.modelPath?.trim() || DEFAULT_LOCAL_MODEL; const modelCacheDir = options.local?.modelCacheDir?.trim(); const { getLlama, resolveModelFile, LlamaLogLevel } = await importNodeLlamaCpp(); let llama = null; let embeddingModel = null; let embeddingContext = null; const ensureContext = async () => { if (!llama) llama = await getLlama({ logLevel: LlamaLogLevel.error }); if (!embeddingModel) { const resolved = await resolveModelFile(modelPath, modelCacheDir || void 0); embeddingModel = await llama.loadModel({ modelPath: resolved }); } if (!embeddingContext) embeddingContext = await embeddingModel.createEmbeddingContext(); return embeddingContext; }; return { id: "local", model: modelPath, embedQuery: async (text) => { const embedding = await (await ensureContext()).getEmbeddingFor(text); return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector)); }, embedBatch: async (texts) => { const ctx = await ensureContext(); return await Promise.all(texts.map(async (text) => { const embedding = await ctx.getEmbeddingFor(text); return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector)); })); } }; } async function createEmbeddingProvider(options) { const requestedProvider = options.provider; const fallback = options.fallback; const createProvider = async (id) => { if (id === "local") return { provider: await createLocalEmbeddingProvider(options) }; if (id === "gemini") { const { provider, client } = await createGeminiEmbeddingProvider(options); return { provider, gemini: client }; } const { provider, client } = await createOpenAiEmbeddingProvider(options); return { provider, openAi: client }; }; const formatPrimaryError = (err, provider) => provider === "local" ? formatLocalSetupError(err) : formatError(err); if (requestedProvider === "auto") { const missingKeyErrors = []; let localError = null; if (canAutoSelectLocal(options)) try { return { ...await createProvider("local"), requestedProvider }; } catch (err) { localError = formatLocalSetupError(err); } for (const provider of ["openai", "gemini"]) try { return { ...await createProvider(provider), requestedProvider }; } catch (err) { const message = formatPrimaryError(err, provider); if (isMissingApiKeyError(err)) { missingKeyErrors.push(message); continue; } throw new Error(message, { cause: err }); } const details = [...missingKeyErrors, localError].filter(Boolean); if (details.length > 0) throw new Error(details.join("\n\n")); throw new Error("No embeddings provider available."); } try { return { ...await createProvider(requestedProvider), requestedProvider }; } catch (primaryErr) { const reason = formatPrimaryError(primaryErr, requestedProvider); if (fallback && fallback !== "none" && fallback !== requestedProvider) try { return { ...await createProvider(fallback), requestedProvider, fallbackFrom: requestedProvider, fallbackReason: reason }; } catch (fallbackErr) { throw new Error(`${reason}\n\nFallback to ${fallback} failed: ${formatError(fallbackErr)}`, { cause: fallbackErr }); } throw new Error(reason, { cause: primaryErr }); } } function formatError(err) { if (err instanceof Error) return err.message; return String(err); } function isNodeLlamaCppMissing(err) { if (!(err instanceof Error)) return false; if (err.code === "ERR_MODULE_NOT_FOUND") return err.message.includes("node-llama-cpp"); return false; } function formatLocalSetupError(err) { const detail = formatError(err); const missing = isNodeLlamaCppMissing(err); return [ "Local embeddings unavailable.", missing ? "Reason: optional dependency node-llama-cpp is missing (or failed to install)." : detail ? `Reason: ${detail}` : void 0, missing && detail ? `Detail: ${detail}` : null, "To enable local embeddings:", "1) Use Node 22 LTS (recommended for installs/updates)", missing ? "2) Reinstall OpenClaw (this should install node-llama-cpp): npm i -g openclaw@latest" : null, "3) If you use pnpm: pnpm approve-builds (select node-llama-cpp), then pnpm rebuild node-llama-cpp", "Or set agents.defaults.memorySearch.provider = \"openai\" (remote)." ].filter(Boolean).join("\n"); } //#endregion //#region src/memory/hybrid.ts function buildFtsQuery(raw) { const tokens = raw.match(/[A-Za-z0-9_]+/g)?.map((t) => t.trim()).filter(Boolean) ?? []; if (tokens.length === 0) return null; return tokens.map((t) => `"${t.replaceAll("\"", "")}"`).join(" AND "); } function bm25RankToScore(rank) { return 1 / (1 + (Number.isFinite(rank) ? Math.max(0, rank) : 999)); } function mergeHybridResults(params) { const byId = /* @__PURE__ */ new Map(); for (const r of params.vector) byId.set(r.id, { id: r.id, path: r.path, startLine: r.startLine, endLine: r.endLine, source: r.source, snippet: r.snippet, vectorScore: r.vectorScore, textScore: 0 }); for (const r of params.keyword) { const existing = byId.get(r.id); if (existing) { existing.textScore = r.textScore; if (r.snippet && r.snippet.length > 0) existing.snippet = r.snippet; } else byId.set(r.id, { id: r.id, path: r.path, startLine: r.startLine, endLine: r.endLine, source: r.source, snippet: r.snippet, vectorScore: 0, textScore: r.textScore }); } return Array.from(byId.values()).map((entry) => { const score = params.vectorWeight * entry.vectorScore + params.textWeight * entry.textScore; return { path: entry.path, startLine: entry.startLine, endLine: entry.endLine, score, snippet: entry.snippet, source: entry.source }; }).toSorted((a, b) => b.score - a.score); } //#endregion //#region src/memory/manager-search.ts const vectorToBlob$1 = (embedding) => Buffer.from(new Float32Array(embedding).buffer); async function searchVector(params) { if (params.queryVec.length === 0 || params.limit <= 0) return []; if (await params.ensureVectorReady(params.queryVec.length)) return params.db.prepare(`SELECT c.id, c.path, c.start_line, c.end_line, c.text, c.source, vec_distance_cosine(v.embedding, ?) AS dist FROM ${params.vectorTable} v\n JOIN chunks c ON c.id = v.id\n WHERE c.model = ?${params.sourceFilterVec.sql}\n ORDER BY dist ASC\n LIMIT ?`).all(vectorToBlob$1(params.queryVec), params.providerModel, ...params.sourceFilterVec.params, params.limit).map((row) => ({ id: row.id, path: row.path, startLine: row.start_line, endLine: row.end_line, score: 1 - row.dist, snippet: truncateUtf16Safe(row.text, params.snippetMaxChars), source: row.source })); return listChunks({ db: params.db, providerModel: params.providerModel, sourceFilter: params.sourceFilterChunks }).map((chunk) => ({ chunk, score: cosineSimilarity(params.queryVec, chunk.embedding) })).filter((entry) => Number.isFinite(entry.score)).toSorted((a, b) => b.score - a.score).slice(0, params.limit).map((entry) => ({ id: entry.chunk.id, path: entry.chunk.path, startLine: entry.chunk.startLine, endLine: entry.chunk.endLine, score: entry.score, snippet: truncateUtf16Safe(entry.chunk.text, params.snippetMaxChars), source: entry.chunk.source })); } function listChunks(params) { return params.db.prepare(`SELECT id, path, start_line, end_line, text, embedding, source FROM chunks WHERE model = ?${params.sourceFilter.sql}`).all(params.providerModel, ...params.sourceFilter.params).map((row) => ({ id: row.id, path: row.path, startLine: row.start_line, endLine: row.end_line, text: row.text, embedding: parseEmbedding(row.embedding), source: row.source })); } async function searchKeyword(params) { if (params.limit <= 0) return []; const ftsQuery = params.buildFtsQuery(params.query); if (!ftsQuery) return []; return params.db.prepare(`SELECT id, path, source, start_line, end_line, text,\n bm25(${params.ftsTable}) AS rank\n FROM ${params.ftsTable}\n WHERE ${params.ftsTable} MATCH ? AND model = ?${params.sourceFilter.sql}\n ORDER BY rank ASC\n LIMIT ?`).all(ftsQuery, params.providerModel, ...params.sourceFilter.params, params.limit).map((row) => { const textScore = params.bm25RankToScore(row.rank); return { id: row.id, path: row.path, startLine: row.start_line, endLine: row.end_line, score: textScore, textScore, snippet: truncateUtf16Safe(row.text, params.snippetMaxChars), source: row.source }; }); } //#endregion //#region src/memory/memory-schema.ts function ensureMemoryIndexSchema(params) { params.db.exec(` CREATE TABLE IF NOT EXISTS meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL ); `); params.db.exec(` CREATE TABLE IF NOT EXISTS files ( path TEXT PRIMARY KEY, source TEXT NOT NULL DEFAULT 'memory', hash TEXT NOT NULL, mtime INTEGER NOT NULL, size INTEGER NOT NULL ); `); params.db.exec(` CREATE TABLE IF NOT EXISTS chunks ( id TEXT PRIMARY KEY, path TEXT NOT NULL, source TEXT NOT NULL DEFAULT 'memory', start_line INTEGER NOT NULL, end_line INTEGER NOT NULL, hash TEXT NOT NULL, model TEXT NOT NULL, text TEXT NOT NULL, embedding TEXT NOT NULL, updated_at INTEGER NOT NULL ); `); params.db.exec(` CREATE TABLE IF NOT EXISTS ${params.embeddingCacheTable} ( provider TEXT NOT NULL, model TEXT NOT NULL, provider_key TEXT NOT NULL, hash TEXT NOT NULL, embedding TEXT NOT NULL, dims INTEGER, updated_at INTEGER NOT NULL, PRIMARY KEY (provider, model, provider_key, hash) ); `); params.db.exec(`CREATE INDEX IF NOT EXISTS idx_embedding_cache_updated_at ON ${params.embeddingCacheTable}(updated_at);`); let ftsAvailable = false; let ftsError; if (params.ftsEnabled) try { params.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${params.ftsTable} USING fts5(\n text,\n id UNINDEXED,\n path UNINDEXED,\n source UNINDEXED,\n model UNINDEXED,\n start_line UNINDEXED,\n end_line UNINDEXED\n);`); ftsAvailable = true; } catch (err) { const message = err instanceof Error ? err.message : String(err); ftsAvailable = false; ftsError = message; } ensureColumn(params.db, "files", "source", "TEXT NOT NULL DEFAULT 'memory'"); ensureColumn(params.db, "chunks", "source", "TEXT NOT NULL DEFAULT 'memory'"); params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);`); params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);`); return { ftsAvailable, ...ftsError ? { ftsError } : {} }; } function ensureColumn(db, table, column, definition) { if (db.prepare(`PRAGMA table_info(${table})`).all().some((row) => row.name === column)) return; db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`); } //#endregion //#region src/memory/sqlite-vec.ts async function loadSqliteVecExtension(params) { try { const sqliteVec = await import("sqlite-vec"); const resolvedPath = params.extensionPath?.trim() ? params.extensionPath.trim() : void 0; const extensionPath = resolvedPath ?? sqliteVec.getLoadablePath(); params.db.enableLoadExtension(true); if (resolvedPath) params.db.loadExtension(extensionPath); else sqliteVec.load(params.db); return { ok: true, extensionPath }; } catch (err) { return { ok: false, error: err instanceof Error ? err.message : String(err) }; } } //#endregion //#region src/memory/manager.ts const META_KEY = "memory_index_meta_v1"; const SNIPPET_MAX_CHARS = 700; const VECTOR_TABLE = "chunks_vec"; const FTS_TABLE = "chunks_fts"; const EMBEDDING_CACHE_TABLE = "embedding_cache"; const SESSION_DIRTY_DEBOUNCE_MS = 5e3; const EMBEDDING_BATCH_MAX_TOKENS = 8e3; const EMBEDDING_APPROX_CHARS_PER_TOKEN = 1; const EMBEDDING_INDEX_CONCURRENCY = 4; const EMBEDDING_RETRY_MAX_ATTEMPTS = 3; const EMBEDDING_RETRY_BASE_DELAY_MS = 500; const EMBEDDING_RETRY_MAX_DELAY_MS = 8e3; const BATCH_FAILURE_LIMIT = 2; const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024; const VECTOR_LOAD_TIMEOUT_MS = 3e4; const EMBEDDING_QUERY_TIMEOUT_REMOTE_MS = 6e4; const EMBEDDING_QUERY_TIMEOUT_LOCAL_MS = 5 * 6e4; const EMBEDDING_BATCH_TIMEOUT_REMOTE_MS = 2 * 6e4; const EMBEDDING_BATCH_TIMEOUT_LOCAL_MS = 10 * 6e4; const log = createSubsystemLogger("memory"); const INDEX_CACHE = /* @__PURE__ */ new Map(); const vectorToBlob = (embedding) => Buffer.from(new Float32Array(embedding).buffer); var MemoryIndexManager = class MemoryIndexManager { static async get(params) { co