mem0ai
Version:
The Memory Layer For Your AI Apps
1,605 lines (1,587 loc) • 241 kB
JavaScript
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
}) : x)(function(x) {
if (typeof require !== "undefined") return require.apply(this, arguments);
throw Error('Dynamic require of "' + x + '" is not supported');
});
// src/oss/src/memory/index.ts
import { v4 as uuidv43 } from "uuid";
import { createHash } from "crypto";
// src/oss/src/types/index.ts
import { z } from "zod";
var MemoryConfigSchema = z.object({
version: z.string().optional(),
embedder: z.object({
provider: z.string(),
config: z.object({
modelProperties: z.record(z.string(), z.any()).optional(),
apiKey: z.string().optional(),
model: z.union([z.string(), z.any()]).optional(),
baseURL: z.string().optional(),
embeddingDims: z.number().optional(),
url: z.string().optional()
})
}),
vectorStore: z.object({
provider: z.string(),
config: z.object({
collectionName: z.string().optional(),
dimension: z.number().optional(),
dbPath: z.string().optional(),
client: z.any().optional()
}).passthrough()
}),
llm: z.object({
provider: z.string(),
config: z.object({
apiKey: z.string().optional(),
model: z.union([z.string(), z.any()]).optional(),
modelProperties: z.record(z.string(), z.any()).optional(),
baseURL: z.string().optional(),
url: z.string().optional(),
timeout: z.number().optional()
})
}),
historyDbPath: z.string().optional(),
customInstructions: z.string().optional(),
historyStore: z.object({
provider: z.string(),
config: z.record(z.string(), z.any())
}).optional(),
disableHistory: z.boolean().optional()
});
// src/oss/src/embeddings/openai.ts
import OpenAI from "openai";
var OpenAIEmbedder = class {
constructor(config) {
this.openai = new OpenAI({
apiKey: config.apiKey,
baseURL: config.baseURL || config.url
});
this.model = config.model || "text-embedding-3-small";
this.embeddingDims = config.embeddingDims;
}
async embed(text) {
const response = await this.openai.embeddings.create({
model: this.model,
input: text,
...this.embeddingDims !== void 0 && {
dimensions: this.embeddingDims
}
});
return response.data[0].embedding;
}
async embedBatch(texts) {
const MAX_BATCH = 100;
const allEmbeddings = [];
for (let i = 0; i < texts.length; i += MAX_BATCH) {
const chunk = texts.slice(i, i + MAX_BATCH);
const response = await this.openai.embeddings.create({
model: this.model,
input: chunk,
...this.embeddingDims !== void 0 && {
dimensions: this.embeddingDims
}
});
allEmbeddings.push(
...response.data.sort((a, b) => a.index - b.index).map((item) => item.embedding)
);
}
return allEmbeddings;
}
};
// src/oss/src/embeddings/ollama.ts
import { Ollama } from "ollama";
// src/oss/src/utils/logger.ts
var logger = {
info: (message) => console.log(`[INFO] ${message}`),
error: (message) => console.error(`[ERROR] ${message}`),
debug: (message) => console.debug(`[DEBUG] ${message}`),
warn: (message) => console.warn(`[WARN] ${message}`)
};
// src/oss/src/embeddings/ollama.ts
var OllamaEmbedder = class _OllamaEmbedder {
constructor(config) {
// Using this variable to avoid calling the Ollama server multiple times
this.initialized = false;
this.ollama = new Ollama({
host: config.url || config.baseURL || "http://localhost:11434"
});
this.model = config.model || "nomic-embed-text:latest";
this.embeddingDims = config.embeddingDims || 768;
this.ensureModelExists().catch((err) => {
logger.error(`Error ensuring model exists: ${err}`);
});
}
async embed(text) {
try {
await this.ensureModelExists();
} catch (err) {
logger.error(`Error ensuring model exists: ${err}`);
}
const input = typeof text === "string" ? text : JSON.stringify(text);
const response = await this.ollama.embed({
model: this.model,
input
});
if (!response.embeddings || response.embeddings.length === 0) {
throw new Error(
`Ollama embed() returned no embeddings for model '${this.model}'`
);
}
return response.embeddings[0];
}
async embedBatch(texts) {
const response = await Promise.all(texts.map((text) => this.embed(text)));
return response;
}
static normalizeModelName(name) {
return name.includes(":") ? name : `${name}:latest`;
}
async ensureModelExists() {
if (this.initialized) {
return true;
}
const local_models = await this.ollama.list();
const target = _OllamaEmbedder.normalizeModelName(this.model);
if (!local_models.models.find(
(m) => _OllamaEmbedder.normalizeModelName(m.name) === target
)) {
logger.info(`Pulling model ${this.model}...`);
await this.ollama.pull({ model: this.model });
}
this.initialized = true;
return true;
}
};
// src/oss/src/embeddings/lmstudio.ts
import OpenAI2 from "openai";
var DEFAULT_BASE_URL = "http://localhost:1234/v1";
var DEFAULT_MODEL = "nomic-ai/nomic-embed-text-v1.5-GGUF/nomic-embed-text-v1.5.f16.gguf";
var DEFAULT_LMSTUDIO_API_KEY = "lm-studio";
var LMStudioEmbedder = class {
constructor(config) {
var _a2, _b;
const baseURL = (_b = (_a2 = config.baseURL) != null ? _a2 : config.url) != null ? _b : DEFAULT_BASE_URL;
const apiKey = config.apiKey || DEFAULT_LMSTUDIO_API_KEY;
this.openai = new OpenAI2({ apiKey, baseURL: String(baseURL) });
this.model = config.model || DEFAULT_MODEL;
}
async embed(text) {
const normalized = typeof text === "string" ? text.replace(/\n/g, " ") : String(text);
try {
const response = await this.openai.embeddings.create({
model: this.model,
input: normalized,
encoding_format: "float"
});
return response.data[0].embedding;
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
throw new Error(`LM Studio embedder failed: ${message}`);
}
}
async embedBatch(texts) {
const normalized = texts.map(
(t) => typeof t === "string" ? t.replace(/\n/g, " ") : String(t)
);
try {
const response = await this.openai.embeddings.create({
model: this.model,
input: normalized,
encoding_format: "float"
});
return response.data.map((item) => item.embedding);
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
throw new Error(`LM Studio embedder failed: ${message}`);
}
}
};
// src/oss/src/llms/openai.ts
import OpenAI3 from "openai";
var OpenAILLM = class {
constructor(config) {
this.openai = new OpenAI3({
apiKey: config.apiKey,
baseURL: config.baseURL,
...config.timeout != null && { timeout: config.timeout }
});
this.model = config.model || "gpt-5-mini";
}
async generateResponse(messages, responseFormat, tools) {
const completion = await this.openai.chat.completions.create({
messages: messages.map((msg) => {
const role = msg.role;
return {
role,
content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
};
}),
model: this.model,
response_format: responseFormat,
...tools && { tools, tool_choice: "auto" }
});
const response = completion.choices[0].message;
if (response.tool_calls) {
return {
content: response.content || "",
role: response.role,
toolCalls: response.tool_calls.map((call) => ({
name: call.function.name,
arguments: call.function.arguments
}))
};
}
return response.content || "";
}
async generateChat(messages) {
const completion = await this.openai.chat.completions.create({
messages: messages.map((msg) => {
const role = msg.role;
return {
role,
content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
};
}),
model: this.model
});
const response = completion.choices[0].message;
return {
content: response.content || "",
role: response.role
};
}
};
// src/oss/src/llms/openai_structured.ts
import OpenAI4 from "openai";
var OpenAIStructuredLLM = class {
constructor(config) {
this.openai = new OpenAI4({
apiKey: config.apiKey,
baseURL: config.baseURL,
...config.timeout != null && { timeout: config.timeout }
});
this.model = config.model || "gpt-5-mini";
}
async generateResponse(messages, responseFormat, tools) {
const completion = await this.openai.chat.completions.create({
messages: messages.map((msg) => ({
role: msg.role,
content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
})),
model: this.model,
...tools ? {
tools: tools.map((tool) => ({
type: "function",
function: {
name: tool.function.name,
description: tool.function.description,
parameters: tool.function.parameters
}
})),
tool_choice: "auto"
} : responseFormat ? {
response_format: {
type: responseFormat.type
}
} : {}
});
const response = completion.choices[0].message;
if (response.tool_calls) {
return {
content: response.content || "",
role: response.role,
toolCalls: response.tool_calls.map((call) => ({
name: call.function.name,
arguments: call.function.arguments
}))
};
}
return response.content || "";
}
async generateChat(messages) {
const completion = await this.openai.chat.completions.create({
messages: messages.map((msg) => ({
role: msg.role,
content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
})),
model: this.model
});
const response = completion.choices[0].message;
return {
content: response.content || "",
role: response.role
};
}
};
// src/oss/src/llms/anthropic.ts
import Anthropic from "@anthropic-ai/sdk";
var AnthropicLLM = class {
constructor(config) {
const apiKey = config.apiKey || process.env.ANTHROPIC_API_KEY;
if (!apiKey) {
throw new Error("Anthropic API key is required");
}
this.client = new Anthropic({ apiKey });
this.model = config.model || "claude-3-sonnet-20240229";
}
async generateResponse(messages, responseFormat) {
const systemMessage = messages.find((msg) => msg.role === "system");
const otherMessages = messages.filter((msg) => msg.role !== "system");
const response = await this.client.messages.create({
model: this.model,
messages: otherMessages.map((msg) => ({
role: msg.role,
content: typeof msg.content === "string" ? msg.content : msg.content.image_url.url
})),
system: typeof (systemMessage == null ? void 0 : systemMessage.content) === "string" ? systemMessage.content : void 0,
max_tokens: 4096
});
const firstBlock = response.content[0];
if (firstBlock.type === "text") {
return firstBlock.text;
} else {
throw new Error("Unexpected response type from Anthropic API");
}
}
async generateChat(messages) {
const response = await this.generateResponse(messages);
return {
content: response,
role: "assistant"
};
}
};
// src/oss/src/llms/groq.ts
import { Groq } from "groq-sdk";
var GroqLLM = class {
constructor(config) {
const apiKey = config.apiKey || process.env.GROQ_API_KEY;
if (!apiKey) {
throw new Error("Groq API key is required");
}
this.client = new Groq({ apiKey });
this.model = config.model || "llama3-70b-8192";
}
async generateResponse(messages, responseFormat) {
const response = await this.client.chat.completions.create({
model: this.model,
messages: messages.map((msg) => ({
role: msg.role,
content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
})),
response_format: responseFormat
});
return response.choices[0].message.content || "";
}
async generateChat(messages) {
const response = await this.client.chat.completions.create({
model: this.model,
messages: messages.map((msg) => ({
role: msg.role,
content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
}))
});
const message = response.choices[0].message;
return {
content: message.content || "",
role: message.role
};
}
};
// src/oss/src/llms/mistral.ts
import { Mistral } from "@mistralai/mistralai";
var MistralLLM = class {
constructor(config) {
if (!config.apiKey) {
throw new Error("Mistral API key is required");
}
this.client = new Mistral({
apiKey: config.apiKey
});
this.model = config.model || "mistral-tiny-latest";
}
// Helper function to convert content to string
contentToString(content) {
if (typeof content === "string") {
return content;
}
if (Array.isArray(content)) {
return content.map((chunk) => {
if (chunk.type === "text") {
return chunk.text;
} else {
return JSON.stringify(chunk);
}
}).join("");
}
return String(content || "");
}
async generateResponse(messages, responseFormat, tools) {
const response = await this.client.chat.complete({
model: this.model,
messages: messages.map((msg) => ({
role: msg.role,
content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
})),
...tools && { tools },
...responseFormat && { response_format: responseFormat }
});
if (!response || !response.choices || response.choices.length === 0) {
return "";
}
const message = response.choices[0].message;
if (!message) {
return "";
}
if (message.toolCalls && message.toolCalls.length > 0) {
return {
content: this.contentToString(message.content),
role: message.role || "assistant",
toolCalls: message.toolCalls.map((call) => ({
name: call.function.name,
arguments: typeof call.function.arguments === "string" ? call.function.arguments : JSON.stringify(call.function.arguments)
}))
};
}
return this.contentToString(message.content);
}
async generateChat(messages) {
const formattedMessages = messages.map((msg) => ({
role: msg.role,
content: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
}));
const response = await this.client.chat.complete({
model: this.model,
messages: formattedMessages
});
if (!response || !response.choices || response.choices.length === 0) {
return {
content: "",
role: "assistant"
};
}
const message = response.choices[0].message;
return {
content: this.contentToString(message.content),
role: message.role || "assistant"
};
}
};
// src/oss/src/vector_stores/memory.ts
import Database from "better-sqlite3";
import fs2 from "fs";
import path2 from "path";
// src/oss/src/utils/sqlite.ts
import fs from "fs";
import os from "os";
import path from "path";
function getDefaultVectorStoreDbPath() {
return path.join(os.homedir(), ".mem0", "vector_store.db");
}
function ensureSQLiteDirectory(dbPath) {
if (!dbPath || dbPath === ":memory:" || dbPath.startsWith("file:")) {
return;
}
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
}
// src/oss/src/vector_stores/memory.ts
var _MemoryVectorStore = class _MemoryVectorStore {
normalizePayload(payload) {
for (const [camel, snake] of Object.entries(
_MemoryVectorStore.CAMEL_TO_SNAKE
)) {
if (camel in payload && !(snake in payload)) {
payload[snake] = payload[camel];
delete payload[camel];
}
}
return payload;
}
constructor(config) {
this.dimension = config.dimension || 1536;
this.dbPath = config.dbPath || getDefaultVectorStoreDbPath();
if (!config.dbPath) {
const oldDefault = path2.join(process.cwd(), "vector_store.db");
if (fs2.existsSync(oldDefault) && oldDefault !== this.dbPath) {
console.warn(
`[mem0] Default vector_store.db location changed from ${oldDefault} to ${this.dbPath}. Move your existing file or set vectorStore.config.dbPath explicitly.`
);
}
}
ensureSQLiteDirectory(this.dbPath);
this.db = new Database(this.dbPath);
this.init();
}
init() {
this.db.exec(`
CREATE TABLE IF NOT EXISTS vectors (
id TEXT PRIMARY KEY,
vector BLOB NOT NULL,
payload TEXT NOT NULL
)
`);
this.db.exec(`
CREATE TABLE IF NOT EXISTS memory_migrations (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id TEXT NOT NULL UNIQUE
)
`);
}
cosineSimilarity(a, b) {
let dotProduct = 0;
let normA = 0;
let normB = 0;
for (let i = 0; i < a.length; i++) {
dotProduct += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
}
/**
* Check if a single field condition matches the payload.
* Supports comparison operators: eq, ne, gt, gte, lt, lte, in, nin, contains, icontains
*/
matchFieldCondition(payload, key, value) {
const payloadValue = payload[key];
if (typeof value !== "object" || value === null) {
if (value === "*") {
return true;
}
return payloadValue === value;
}
if (Array.isArray(value)) {
return value.includes(payloadValue);
}
if ("eq" in value) {
return payloadValue === value.eq;
}
if ("ne" in value) {
return payloadValue !== value.ne;
}
if ("gt" in value) {
return payloadValue > value.gt;
}
if ("gte" in value) {
return payloadValue >= value.gte;
}
if ("lt" in value) {
return payloadValue < value.lt;
}
if ("lte" in value) {
return payloadValue <= value.lte;
}
if ("in" in value) {
return Array.isArray(value.in) && value.in.includes(payloadValue);
}
if ("nin" in value) {
return !Array.isArray(value.nin) || !value.nin.includes(payloadValue);
}
if ("contains" in value) {
return typeof payloadValue === "string" && payloadValue.includes(value.contains);
}
if ("icontains" in value) {
return typeof payloadValue === "string" && payloadValue.toLowerCase().includes(value.icontains.toLowerCase());
}
return payloadValue === value;
}
/**
* Filter a vector by the given filters.
* Supports logical operators (AND, OR, NOT) and comparison operators.
*/
filterVector(vector, filters) {
if (!filters || Object.keys(filters).length === 0) return true;
const keyMap = {
$and: "AND",
$or: "OR",
$not: "NOT"
};
const normalized = {};
for (const [key, value] of Object.entries(filters)) {
const normKey = keyMap[key] || key;
if (!(normKey in normalized)) {
normalized[normKey] = value;
}
}
for (const [key, value] of Object.entries(normalized)) {
if (key === "AND") {
if (!Array.isArray(value)) {
throw new Error(
`AND filter value must be a list of filter dicts, got ${typeof value}`
);
}
const allMatch = value.every(
(sub) => this.filterVector(vector, sub)
);
if (!allMatch) return false;
} else if (key === "OR") {
if (!Array.isArray(value)) {
throw new Error(
`OR filter value must be a list of filter dicts, got ${typeof value}`
);
}
const anyMatch = value.some(
(sub) => this.filterVector(vector, sub)
);
if (!anyMatch) return false;
} else if (key === "NOT") {
if (!Array.isArray(value)) {
throw new Error(
`NOT filter value must be a list of filter dicts, got ${typeof value}`
);
}
const noneMatch = value.every(
(sub) => !this.filterVector(vector, sub)
);
if (!noneMatch) return false;
} else {
if (!this.matchFieldCondition(vector.payload, key, value)) {
return false;
}
}
}
return true;
}
async insert(vectors, ids, payloads) {
const stmt = this.db.prepare(
`INSERT OR REPLACE INTO vectors (id, vector, payload) VALUES (?, ?, ?)`
);
const insertMany = this.db.transaction(
(vecs, vIds, vPayloads) => {
for (let i = 0; i < vecs.length; i++) {
if (vecs[i].length !== this.dimension) {
throw new Error(
`Vector dimension mismatch. Expected ${this.dimension}, got ${vecs[i].length}`
);
}
const vectorBuffer = Buffer.from(new Float32Array(vecs[i]).buffer);
stmt.run(vIds[i], vectorBuffer, JSON.stringify(vPayloads[i]));
}
}
);
insertMany(vectors, ids, payloads);
}
tokenize(text) {
return text.toLowerCase().split(/\s+/).filter(Boolean);
}
async keywordSearch(query, topK = 10, filters) {
try {
const rows = this.db.prepare(`SELECT * FROM vectors`).all();
const candidates = [];
for (const row of rows) {
const payload = this.normalizePayload(JSON.parse(row.payload));
const memoryVector = {
id: row.id,
vector: Array.from(
new Float32Array(
row.vector.buffer,
row.vector.byteOffset,
row.vector.byteLength / 4
)
),
payload
};
if (this.filterVector(memoryVector, filters)) {
const text = payload.textLemmatized || payload.data || "";
candidates.push({ id: row.id, payload, tokens: this.tokenize(text) });
}
}
if (candidates.length === 0) {
return [];
}
const tokenizedQuery = this.tokenize(query);
if (tokenizedQuery.length === 0) {
return [];
}
const k1 = 1.5;
const b = 0.75;
const N = candidates.length;
const avgDocLength = candidates.reduce((sum, c) => sum + c.tokens.length, 0) / N;
const docFreq = /* @__PURE__ */ new Map();
for (const term of tokenizedQuery) {
if (!docFreq.has(term)) {
let count = 0;
for (const c of candidates) {
if (c.tokens.includes(term)) count++;
}
docFreq.set(term, count);
}
}
const idf = /* @__PURE__ */ new Map();
for (const [term, freq] of docFreq) {
idf.set(term, Math.log((N - freq + 0.5) / (freq + 0.5) + 1));
}
const scored = candidates.map((candidate) => {
let score = 0;
const docLength = candidate.tokens.length;
for (const term of tokenizedQuery) {
const tf = candidate.tokens.filter((t) => t === term).length;
const termIdf = idf.get(term) || 0;
score += termIdf * tf * (k1 + 1) / (tf + k1 * (1 - b + b * docLength / avgDocLength));
}
return { ...candidate, score };
});
const results = scored.filter((s) => s.score > 0).sort((a, b2) => b2.score - a.score).slice(0, topK).map((s) => ({
id: s.id,
payload: s.payload,
score: s.score
}));
return results;
} catch (error) {
console.error("Error during keyword search:", error);
return null;
}
}
async search(query, topK = 10, filters) {
if (query.length !== this.dimension) {
throw new Error(
`Query dimension mismatch. Expected ${this.dimension}, got ${query.length}`
);
}
const rows = this.db.prepare(`SELECT * FROM vectors`).all();
const results = [];
for (const row of rows) {
const vector = new Float32Array(
row.vector.buffer,
row.vector.byteOffset,
row.vector.byteLength / 4
);
const payload = this.normalizePayload(JSON.parse(row.payload));
const memoryVector = {
id: row.id,
vector: Array.from(vector),
payload
};
if (this.filterVector(memoryVector, filters)) {
const score = this.cosineSimilarity(query, Array.from(vector));
results.push({
id: memoryVector.id,
payload: memoryVector.payload,
score
});
}
}
results.sort((a, b) => (b.score || 0) - (a.score || 0));
return results.slice(0, topK);
}
async get(vectorId) {
const row = this.db.prepare(`SELECT * FROM vectors WHERE id = ?`).get(vectorId);
if (!row) return null;
const payload = this.normalizePayload(JSON.parse(row.payload));
return {
id: row.id,
payload
};
}
async update(vectorId, vector, payload) {
if (vector.length !== this.dimension) {
throw new Error(
`Vector dimension mismatch. Expected ${this.dimension}, got ${vector.length}`
);
}
const vectorBuffer = Buffer.from(new Float32Array(vector).buffer);
this.db.prepare(`UPDATE vectors SET vector = ?, payload = ? WHERE id = ?`).run(vectorBuffer, JSON.stringify(payload), vectorId);
}
async delete(vectorId) {
this.db.prepare(`DELETE FROM vectors WHERE id = ?`).run(vectorId);
}
async deleteCol() {
this.db.exec(`DROP TABLE IF EXISTS vectors`);
this.init();
}
async list(filters, topK = 100) {
const rows = this.db.prepare(`SELECT * FROM vectors`).all();
const results = [];
for (const row of rows) {
const payload = this.normalizePayload(JSON.parse(row.payload));
const memoryVector = {
id: row.id,
vector: Array.from(
new Float32Array(
row.vector.buffer,
row.vector.byteOffset,
row.vector.byteLength / 4
)
),
payload
};
if (this.filterVector(memoryVector, filters)) {
results.push({
id: memoryVector.id,
payload: memoryVector.payload
});
}
}
return [results.slice(0, topK), results.length];
}
async getUserId() {
const row = this.db.prepare(`SELECT user_id FROM memory_migrations LIMIT 1`).get();
if (row) {
return row.user_id;
}
const randomUserId2 = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
this.db.prepare(`INSERT INTO memory_migrations (user_id) VALUES (?)`).run(randomUserId2);
return randomUserId2;
}
async setUserId(userId) {
this.db.prepare(`DELETE FROM memory_migrations`).run();
this.db.prepare(`INSERT INTO memory_migrations (user_id) VALUES (?)`).run(userId);
}
async initialize() {
this.init();
}
};
_MemoryVectorStore.CAMEL_TO_SNAKE = {
userId: "user_id",
agentId: "agent_id",
runId: "run_id"
};
var MemoryVectorStore = _MemoryVectorStore;
// src/oss/src/vector_stores/qdrant.ts
import { QdrantClient } from "@qdrant/js-client-rest";
import * as fs3 from "fs";
var KEY_MAP = {
$and: "AND",
$or: "OR",
$not: "NOT"
};
var Qdrant = class {
constructor(config) {
if (config.client) {
this.client = config.client;
} else {
const params = {};
if (config.apiKey) {
params.apiKey = config.apiKey;
}
if (config.url) {
params.url = config.url;
try {
const parsedUrl = new URL(config.url);
params.port = parsedUrl.port ? parseInt(parsedUrl.port, 10) : 6333;
} catch (_) {
params.port = 6333;
}
}
if (config.host && config.port) {
params.host = config.host;
params.port = config.port;
}
if (!Object.keys(params).length) {
params.path = config.path;
if (!config.onDisk && config.path) {
if (fs3.existsSync(config.path) && fs3.statSync(config.path).isDirectory()) {
fs3.rmSync(config.path, { recursive: true });
}
}
}
this.client = new QdrantClient(params);
}
this.collectionName = config.collectionName;
this.dimension = config.dimension || 1536;
this.initialize().catch(console.error);
}
/**
* Build a single field condition from a key-value filter pair.
* Supports enhanced filter syntax with comparison operators.
*/
buildFieldCondition(key, value) {
if (typeof value !== "object" || value === null) {
if (value === "*") {
return null;
}
return { key, match: { value } };
}
if (Array.isArray(value)) {
return { key, match: { any: value } };
}
const ops = Object.keys(value);
const rangeOps = ["gt", "gte", "lt", "lte"];
const hasRangeOps = ops.some((op) => rangeOps.includes(op));
const nonRangeOps = ops.filter((op) => !rangeOps.includes(op));
if (hasRangeOps) {
if (nonRangeOps.length > 0) {
throw new Error(
`Cannot mix range operators (${ops.filter((o) => rangeOps.includes(o)).join(", ")}) with non-range operators (${nonRangeOps.join(", ")}) for field '${key}'. Use AND to combine them as separate conditions.`
);
}
const range = {};
for (const op of rangeOps) {
if (op in value) {
range[op] = value[op];
}
}
return { key, range };
}
if ("eq" in value) {
return { key, match: { value: value.eq } };
}
if ("ne" in value) {
return { key, match: { except: [value.ne] } };
}
if ("in" in value) {
return { key, match: { any: value.in } };
}
if ("nin" in value) {
return { key, match: { except: value.nin } };
}
if ("contains" in value || "icontains" in value) {
const text = value.contains || value.icontains;
return { key, match: { text } };
}
const supportedOps = [
"eq",
"ne",
"gt",
"gte",
"lt",
"lte",
"in",
"nin",
"contains",
"icontains"
];
throw new Error(
`Unsupported filter operator(s) for field '${key}': ${ops.join(", ")}. Supported operators: ${supportedOps.join(", ")}`
);
}
/**
* Create a Filter object from the provided filters.
* Supports logical operators (AND, OR, NOT) and comparison operators.
*/
createFilter(filters) {
if (!filters || Object.keys(filters).length === 0) return void 0;
const normalized = {};
for (const [key, value] of Object.entries(filters)) {
const normKey = KEY_MAP[key] || key;
if (!(normKey in normalized)) {
normalized[normKey] = value;
}
}
const must = [];
const should = [];
const mustNot = [];
for (const [key, value] of Object.entries(normalized)) {
if (key === "AND" || key === "OR" || key === "NOT") {
if (!Array.isArray(value)) {
throw new Error(
`${key} filter value must be a list of filter dicts, got ${typeof value}`
);
}
for (let i = 0; i < value.length; i++) {
const item = value[i];
if (typeof item !== "object" || item === null || Array.isArray(item)) {
throw new Error(
`${key} filter list item at index ${i} must be a dict, got ${typeof item}`
);
}
}
if (key === "AND") {
for (const sub of value) {
const built = this.createFilter(sub);
if (built) {
must.push(built);
}
}
} else if (key === "OR") {
for (const sub of value) {
const built = this.createFilter(sub);
if (built) {
should.push(built);
}
}
} else if (key === "NOT") {
for (const sub of value) {
const built = this.createFilter(sub);
if (built) {
mustNot.push(built);
}
}
}
} else {
const condition = this.buildFieldCondition(key, value);
if (condition !== null) {
must.push(condition);
}
}
}
if (must.length === 0 && should.length === 0 && mustNot.length === 0) {
return void 0;
}
return {
must: must.length > 0 ? must : void 0,
should: should.length > 0 ? should : void 0,
must_not: mustNot.length > 0 ? mustNot : void 0
};
}
async insert(vectors, ids, payloads) {
const points = vectors.map((vector, idx) => ({
id: ids[idx],
vector,
payload: payloads[idx] || {}
}));
await this.client.upsert(this.collectionName, {
points
});
}
async keywordSearch() {
return null;
}
async search(query, topK = 5, filters) {
const queryFilter = this.createFilter(filters);
const results = await this.client.search(this.collectionName, {
vector: query,
filter: queryFilter,
limit: topK
});
return results.map((hit) => ({
id: String(hit.id),
payload: hit.payload || {},
score: hit.score
}));
}
async get(vectorId) {
const results = await this.client.retrieve(this.collectionName, {
ids: [vectorId],
with_payload: true
});
if (!results.length) return null;
return {
id: vectorId,
payload: results[0].payload || {}
};
}
async update(vectorId, vector, payload) {
const point = {
id: vectorId,
vector,
payload
};
await this.client.upsert(this.collectionName, {
points: [point]
});
}
async delete(vectorId) {
await this.client.delete(this.collectionName, {
points: [vectorId]
});
}
async deleteCol() {
await this.client.deleteCollection(this.collectionName);
}
async list(filters, topK = 100) {
const scrollRequest = {
limit: topK,
filter: this.createFilter(filters),
with_payload: true,
with_vectors: false
};
const response = await this.client.scroll(
this.collectionName,
scrollRequest
);
const results = response.points.map((point) => ({
id: String(point.id),
payload: point.payload || {}
}));
return [results, response.points.length];
}
generateUUID() {
return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(
/[xy]/g,
function(c) {
const r = Math.random() * 16 | 0;
const v = c === "x" ? r : r & 3 | 8;
return v.toString(16);
}
);
}
async getUserId() {
var _a2;
try {
await this.ensureCollection("memory_migrations", 1);
const result = await this.client.scroll("memory_migrations", {
limit: 1,
with_payload: true
});
if (result.points.length > 0) {
return (_a2 = result.points[0].payload) == null ? void 0 : _a2.user_id;
}
const randomUserId2 = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
await this.client.upsert("memory_migrations", {
points: [
{
id: this.generateUUID(),
vector: [0],
payload: { user_id: randomUserId2 }
}
]
});
return randomUserId2;
} catch (error) {
console.error("Error getting user ID:", error);
throw error;
}
}
async setUserId(userId) {
try {
const result = await this.client.scroll("memory_migrations", {
limit: 1,
with_payload: true
});
const pointId = result.points.length > 0 ? result.points[0].id : this.generateUUID();
await this.client.upsert("memory_migrations", {
points: [
{
id: pointId,
vector: [0],
payload: { user_id: userId }
}
]
});
} catch (error) {
console.error("Error setting user ID:", error);
throw error;
}
}
async ensureCollection(name, size) {
var _a2, _b, _c;
try {
await this.client.createCollection(name, {
vectors: {
size,
distance: "Cosine"
}
});
} catch (error) {
if ((error == null ? void 0 : error.status) === 409 || (error == null ? void 0 : error.status) === 401 || (error == null ? void 0 : error.status) === 403) {
if (name === this.collectionName) {
try {
const collectionInfo = await this.client.getCollection(name);
const vectorConfig = (_b = (_a2 = collectionInfo.config) == null ? void 0 : _a2.params) == null ? void 0 : _b.vectors;
if (vectorConfig && vectorConfig.size !== size) {
throw new Error(
`Collection ${name} exists but has wrong vector size. Expected: ${size}, got: ${vectorConfig.size}`
);
}
} catch (verifyError) {
if ((_c = verifyError == null ? void 0 : verifyError.message) == null ? void 0 : _c.includes("wrong vector size")) {
throw verifyError;
}
console.warn(
`Collection '${name}' exists (409) but dimension verification failed: ${(verifyError == null ? void 0 : verifyError.message) || verifyError}. Proceeding anyway.`
);
}
}
} else {
throw error;
}
}
}
async initialize() {
if (!this._initPromise) {
this._initPromise = this._doInitialize();
}
return this._initPromise;
}
async _doInitialize() {
try {
await this.ensureCollection(this.collectionName, this.dimension);
await this.ensureCollection("memory_migrations", 1);
} catch (error) {
console.error("Error initializing Qdrant:", error);
throw error;
}
}
};
// src/oss/src/vector_stores/vectorize.ts
import Cloudflare from "cloudflare";
var VectorizeDB = class {
constructor(config) {
this.client = null;
this.client = new Cloudflare({ apiToken: config.apiKey });
this.dimensions = config.dimension || 1536;
this.indexName = config.indexName;
this.accountId = config.accountId;
this.initialize().catch(console.error);
}
async insert(vectors, ids, payloads) {
var _a2;
try {
const vectorObjects = vectors.map(
(vector, index) => ({
id: ids[index],
values: vector,
metadata: payloads[index] || {}
})
);
const ndjsonPayload = vectorObjects.map((v) => JSON.stringify(v)).join("\n");
const response = await fetch(
`https://api.cloudflare.com/client/v4/accounts/${this.accountId}/vectorize/v2/indexes/${this.indexName}/insert`,
{
method: "POST",
headers: {
"Content-Type": "application/x-ndjson",
Authorization: `Bearer ${(_a2 = this.client) == null ? void 0 : _a2.apiToken}`
},
body: ndjsonPayload
}
);
if (!response.ok) {
const errorText = await response.text();
throw new Error(
`Failed to insert vectors: ${response.status} ${errorText}`
);
}
} catch (error) {
console.error("Error inserting vectors:", error);
throw new Error(
`Failed to insert vectors: ${error instanceof Error ? error.message : String(error)}`
);
}
}
async keywordSearch() {
return null;
}
async search(query, topK = 5, filters) {
var _a2, _b;
try {
const result = await ((_a2 = this.client) == null ? void 0 : _a2.vectorize.indexes.query(
this.indexName,
{
account_id: this.accountId,
vector: query,
filter: filters,
returnMetadata: "all",
topK
}
));
return ((_b = result == null ? void 0 : result.matches) == null ? void 0 : _b.map((match) => ({
id: match.id,
payload: match.metadata,
score: match.score
}))) || [];
} catch (error) {
console.error("Error searching vectors:", error);
throw new Error(
`Failed to search vectors: ${error instanceof Error ? error.message : String(error)}`
);
}
}
async get(vectorId) {
var _a2;
try {
const result = await ((_a2 = this.client) == null ? void 0 : _a2.vectorize.indexes.getByIds(
this.indexName,
{
account_id: this.accountId,
ids: [vectorId]
}
));
if (!(result == null ? void 0 : result.length)) return null;
return {
id: vectorId,
payload: result[0].metadata
};
} catch (error) {
console.error("Error getting vector:", error);
throw new Error(
`Failed to get vector: ${error instanceof Error ? error.message : String(error)}`
);
}
}
async update(vectorId, vector, payload) {
var _a2;
try {
const data = {
id: vectorId,
values: vector,
metadata: payload
};
const response = await fetch(
`https://api.cloudflare.com/client/v4/accounts/${this.accountId}/vectorize/v2/indexes/${this.indexName}/upsert`,
{
method: "POST",
headers: {
"Content-Type": "application/x-ndjson",
Authorization: `Bearer ${(_a2 = this.client) == null ? void 0 : _a2.apiToken}`
},
body: JSON.stringify(data) + "\n"
// ndjson format
}
);
if (!response.ok) {
const errorText = await response.text();
throw new Error(
`Failed to update vector: ${response.status} ${errorText}`
);
}
} catch (error) {
console.error("Error updating vector:", error);
throw new Error(
`Failed to update vector: ${error instanceof Error ? error.message : String(error)}`
);
}
}
async delete(vectorId) {
var _a2;
try {
await ((_a2 = this.client) == null ? void 0 : _a2.vectorize.indexes.deleteByIds(this.indexName, {
account_id: this.accountId,
ids: [vectorId]
}));
} catch (error) {
console.error("Error deleting vector:", error);
throw new Error(
`Failed to delete vector: ${error instanceof Error ? error.message : String(error)}`
);
}
}
async deleteCol() {
var _a2;
try {
await ((_a2 = this.client) == null ? void 0 : _a2.vectorize.indexes.delete(this.indexName, {
account_id: this.accountId
}));
} catch (error) {
console.error("Error deleting collection:", error);
throw new Error(
`Failed to delete collection: ${error instanceof Error ? error.message : String(error)}`
);
}
}
async list(filters, topK = 20) {
var _a2, _b;
try {
const result = await ((_a2 = this.client) == null ? void 0 : _a2.vectorize.indexes.query(
this.indexName,
{
account_id: this.accountId,
vector: Array(this.dimensions).fill(0),
// Dummy vector for listing
filter: filters,
topK,
returnMetadata: "all"
}
));
const matches = ((_b = result == null ? void 0 : result.matches) == null ? void 0 : _b.map((match) => ({
id: match.id,
payload: match.metadata,
score: match.score
}))) || [];
return [matches, matches.length];
} catch (error) {
console.error("Error listing vectors:", error);
throw new Error(
`Failed to list vectors: ${error instanceof Error ? error.message : String(error)}`
);
}
}
generateUUID() {
return "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx".replace(
/[xy]/g,
function(c) {
const r = Math.random() * 16 | 0;
const v = c === "x" ? r : r & 3 | 8;
return v.toString(16);
}
);
}
async getUserId() {
var _a2, _b, _c;
try {
let found = false;
for await (const index of this.client.vectorize.indexes.list({
account_id: this.accountId
})) {
if (index.name === "memory_migrations") {
found = true;
}
}
if (!found) {
await ((_a2 = this.client) == null ? void 0 : _a2.vectorize.indexes.create({
account_id: this.accountId,
name: "memory_migrations",
config: {
dimensions: 1,
metric: "cosine"
}
}));
}
const result = await ((_b = this.client) == null ? void 0 : _b.vectorize.indexes.query(
"memory_migrations",
{
account_id: this.accountId,
vector: [0],
topK: 1,
returnMetadata: "all"
}
));
if (result.matches.length > 0) {
return result.matches[0].metadata.userId;
}
const randomUserId2 = Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
const data = {
id: this.generateUUID(),
values: [0],
metadata: { userId: randomUserId2 }
};
await fetch(
`https://api.cloudflare.com/client/v4/accounts/${this.accountId}/vectorize/v2/indexes/memory_migrations/upsert`,
{
method: "POST",
headers: {
"Content-Type": "application/x-ndjson",
Authorization: `Bearer ${(_c = this.client) == null ? void 0 : _c.apiToken}`
},
body: JSON.stringify(data) + "\n"
// ndjson format
}
);
return randomUserId2;
} catch (error) {
console.error("Error getting user ID:", error);
throw new Error(
`Failed to get user ID: ${error instanceof Error ? error.message : String(error)}`
);
}
}
async setUserId(userId) {
var _a2, _b;
try {
const result = await ((_a2 = this.client) == null ? void 0 : _a2.vectorize.indexes.query(
"memory_migrations",
{
account_id: this.accountId,
vector: [0],
topK: 1,
returnMetadata: "all"
}
));
const pointId = result.matches.length > 0 ? result.matches[0].id : this.generateUUID();
const data = {
id: pointId,
values: [0],
metadata: { userId }
};
await fetch(
`https://api.cloudflare.com/client/v4/accounts/${this.accountId}/vectorize/v2/indexes/memory_migrations/upsert`,
{
method: "POST",
headers: {
"Content-Type": "application/x-ndjson",
Authorization: `Bearer ${(_b = this.client) == null ? void 0 : _b.apiToken}`
},
body: JSON.stringify(data) + "\n"
// ndjson format
}
);
} catch (error) {
console.error("Error setting user ID:", error);
throw new Error(
`Failed to set user ID: ${error instanceof Error ? error.message : String(error)}`
);
}
}
async initialize() {
if (!this._initPromise) {
this._initPromise = this._doInitialize();
}
return this._initPromise;
}
async _doInitialize() {
var _a2, _b, _c, _d, _e;
try {
let indexFound = false;
for await (const idx of this.client.vectorize.indexes.list({
account_id: this.accountId
})) {
if (idx.name === this.indexName) {
indexFound = true;
break;
}
}
if (!indexFound) {
try {
await ((_a2 = this.client) == null ? void 0 : _a2.vectorize.indexes.create({
account_id: this.accountId,
name: this.indexName,
config: {
dimensions: this.dimensions,
metric: "cosine"
}
}));
const properties2 = ["userId", "agentId", "runId"];
for (const propertyName of properties2) {
await ((_b = this.client) == null ? void 0 : _b.vectorize.indexes.metadataIndex.create(
this.indexName,
{
account_id: this.accountId,
indexType: "string",
propertyName
}
));
}
} catch (err) {
throw new Error(err);
}
}
const metadataIndexes = await ((_c = this.client) == null ? void 0 : _c.vectorize.indexes.metadataIndex.list(
this.indexName,
{
account_id: this.accountId
}
));
const existingMetadataIndexes = /* @__PURE__ */ new Set();
for (const metadataIndex of (metadataIndexes == null ? void 0 : metadataIndexes.metadataIndexes) || []) {
existingMetadataIndexes.add(metadataIndex.propertyName);
}
const properties = ["userId", "agentId", "runId"];
for (const propertyName of properties) {
if (!existingMetadataIndexes.has(propertyName)) {
await ((_d = this.client) == null ? void 0 : _d.vectorize.indexes.metadataIndex.create(
this.indexName,
{
account_id: this.accountId,
indexType: "string",
propertyName
}
));
}
}
let found = false;
for await (const index of this.client.vectorize.indexes.list({
account_id: this.accountId
})) {
if (index.name === "memory_migrations") {
found = true;
break;
}
}
if (!found) {
await ((_e = this.client) == null ? void 0 : _e.vectorize.indexes.create({
account_id: this.accountId,
name: "memory_migrations",
config: {
dimensions: 1,
metric: "cosine"
}
}));
}
} catch (err) {
throw new Error(err);
}
}
};
// src/oss/src/vector_stores/redis.ts
import { createClient } from "redis";
function escapeRedisTagValue(value) {
return String(value).replace(
/([,.<>{}\[\]"':;!@#$%^&*()\-+=~|/\\\s])/g,
"\\$1"
);
}
var DEFAULT_FIELDS = [
{ name: