@codai/memorai-core
Version:
Simplified advanced memory engine - no tiers, just powerful semantic search with persistence
411 lines (410 loc) • 14.5 kB
JavaScript
import { QdrantClient } from '@qdrant/js-client-rest';
import { VectorStoreError } from '../types/index.js';
export class QdrantVectorStore {
constructor(url, collection, dimension, apiKey) {
const clientConfig = {
url,
checkCompatibility: false, // Disable version compatibility check
};
if (apiKey) {
clientConfig.apiKey = apiKey;
}
this.client = new QdrantClient(clientConfig);
this.collection = collection;
this.dimension = dimension;
}
async initialize() {
try {
// Check if collection exists
const collections = await this.client.getCollections();
const exists = collections.collections.some((c) => c.name === this.collection);
if (!exists) {
await this.client.createCollection(this.collection, {
vectors: {
size: this.dimension,
distance: 'Cosine',
},
optimizers_config: {
default_segment_number: 2,
max_segment_size: 20000,
memmap_threshold: 50000,
indexing_threshold: 20000,
flush_interval_sec: 5,
},
hnsw_config: {
m: 16,
ef_construct: 100,
full_scan_threshold: 10000,
},
});
// Create indexes for filtering
await this.client.createPayloadIndex(this.collection, {
field_name: 'tenant_id',
field_schema: 'keyword',
});
await this.client.createPayloadIndex(this.collection, {
field_name: 'type',
field_schema: 'keyword',
});
await this.client.createPayloadIndex(this.collection, {
field_name: 'created_at',
field_schema: 'keyword', // Changed from datetime to keyword as v1.7.0 doesn't support datetime
});
}
}
catch (error) {
if (error instanceof Error) {
throw new VectorStoreError(`Failed to initialize Qdrant collection: ${error.message}`);
}
throw new VectorStoreError('Unknown initialization error');
}
}
convertToUuidFormat(id) {
// If already UUID format (contains dashes in right positions), use as-is
if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)) {
return id;
}
// Convert string to a UUID-like format by padding/hashing
// Use a simple deterministic approach to ensure consistency
const hash = id.split('').reduce((a, b) => {
a = (a << 5) - a + b.charCodeAt(0);
return a & a;
}, 0);
const hex = Math.abs(hash).toString(16).padStart(8, '0');
const uuid = `${hex.slice(0, 8)}-${hex.slice(0, 4)}-4${hex.slice(1, 4)}-a${hex.slice(1, 4)}-${hex.padEnd(12, '0')}`;
return uuid;
}
async upsert(points) {
if (points.length === 0) {
return;
}
try {
const qdrantPoints = points.map(point => ({
id: this.convertToUuidFormat(point.id),
vector: point.vector,
payload: {
...point.payload,
original_id: point.id, // Store original ID in payload for reverse lookup
},
}));
// Minimal logging without vector data to prevent performance issues
if (process.env.NODE_ENV === 'development' && qdrantPoints.length > 0) {
console.log(`Upserting ${qdrantPoints.length} points to Qdrant collection: ${this.collection}`);
}
await this.client.upsert(this.collection, {
wait: true,
points: qdrantPoints,
});
}
catch (error) {
if (error instanceof Error) {
throw new VectorStoreError(`Failed to upsert points: ${error.message}`, {
point_count: points.length,
});
}
throw new VectorStoreError('Unknown upsert error');
}
}
async search(vector, query) {
try {
const filter = {
must: [
{
key: 'tenant_id',
match: { value: query.tenant_id },
},
],
};
// Add optional filters
if (query.type) {
filter.must.push({
key: 'type',
match: { value: query.type },
});
}
if (query.agent_id) {
filter.must.push({
key: 'agent_id',
match: { value: query.agent_id },
});
}
const searchResult = await this.client.search(this.collection, {
vector,
filter,
limit: query.limit,
score_threshold: query.threshold,
with_payload: true,
}); // eslint-disable-next-line @typescript-eslint/no-explicit-any
return searchResult.map((point) => ({
id: point.payload?.original_id || point.id, // Use original_id if available
score: point.score,
payload: point.payload ?? {},
}));
}
catch (error) {
if (error instanceof Error) {
throw new VectorStoreError(`Search failed: ${error.message}`, {
query: query.query.substring(0, 100),
tenant_id: query.tenant_id,
});
}
throw new VectorStoreError('Unknown search error');
}
}
async delete(ids) {
if (ids.length === 0) {
return;
}
try {
await this.client.delete(this.collection, {
wait: true,
points: ids,
});
}
catch (error) {
if (error instanceof Error) {
throw new VectorStoreError(`Failed to delete points: ${error.message}`, {
id_count: ids.length,
});
}
throw new VectorStoreError('Unknown delete error');
}
}
async count(tenantId) {
try {
const result = await this.client.count(this.collection, {
filter: {
must: [
{
key: 'tenant_id',
match: { value: tenantId },
},
],
},
});
return result.count;
}
catch (error) {
if (error instanceof Error) {
throw new VectorStoreError(`Failed to count points: ${error.message}`, {
tenant_id: tenantId,
});
}
throw new VectorStoreError('Unknown count error');
}
}
async healthCheck() {
try {
const collections = await this.client.getCollections(); // eslint-disable-next-line @typescript-eslint/no-explicit-any
return collections.collections.some((c) => c.name === this.collection);
}
catch {
return false;
}
}
}
export class MemoryVectorStore {
constructor(store) {
this.isInitialized = false;
this.store = store;
}
async initialize() {
if (this.isInitialized) {
return;
}
await this.store.initialize();
this.isInitialized = true;
}
async storeMemory(memory, embedding) {
await this.ensureInitialized();
// Create schema-compliant payload for Qdrant collection
// Collection expects only: created_at (datetime), type (keyword), tenant_id (keyword)
const point = {
id: memory.id,
vector: embedding,
payload: {
created_at: memory.createdAt.toISOString(),
type: memory.type || 'default',
tenant_id: memory.tenant_id || 'default',
},
};
await this.store.upsert([point]);
}
async storeMemories(memories, embeddings) {
await this.ensureInitialized();
if (memories.length !== embeddings.length) {
throw new VectorStoreError('Memories and embeddings count mismatch');
}
const points = memories.map((memory, index) => {
// Create schema-compliant payload for Qdrant collection
// Collection expects only: created_at (datetime), type (keyword), tenant_id (keyword)
return {
id: memory.id,
vector: embeddings[index],
payload: {
created_at: memory.createdAt.toISOString(),
type: memory.type || 'default',
tenant_id: memory.tenant_id || 'default',
},
};
});
await this.store.upsert(points);
}
async searchMemories(queryEmbedding, query) {
await this.ensureInitialized();
const results = await this.store.search(queryEmbedding, query);
return results.map(result => {
const payload = result.payload;
return {
memory: {
id: payload.id,
type: payload.type,
content: payload.content,
confidence: payload.confidence,
createdAt: new Date(payload.created_at),
updatedAt: new Date(payload.updated_at),
lastAccessedAt: new Date(payload.last_accessed_at),
accessCount: payload.accessCount,
importance: payload.importance,
emotional_weight: payload.emotional_weight,
tags: payload.tags,
context: payload.context,
tenant_id: payload.tenant_id,
agent_id: payload.agent_id,
ttl: payload.ttl ? new Date(payload.ttl) : undefined,
},
score: result.score,
relevance_reason: this.generateRelevanceReason(result.score, query.query),
};
});
}
async deleteMemories(ids) {
await this.ensureInitialized();
await this.store.delete(ids);
}
async getMemoryCount(tenantId) {
await this.ensureInitialized();
return this.store.count(tenantId);
}
async healthCheck() {
if (!this.isInitialized) {
return false;
}
return this.store.healthCheck();
}
async close() {
if (this.store.close) {
await this.store.close();
}
this.isInitialized = false;
}
async getHealth() {
try {
const isHealthy = await this.healthCheck();
return { status: isHealthy ? 'healthy' : 'unhealthy' };
}
catch (error) {
return {
status: 'unhealthy',
error: error instanceof Error ? error.message : 'Unknown error',
};
}
}
async ensureInitialized() {
if (!this.isInitialized) {
await this.initialize();
}
}
generateRelevanceReason(score, query) {
if (score >= 0.9) {
return `Highly relevant to "${query}" with excellent semantic match`;
}
else if (score >= 0.8) {
return `Strong relevance to "${query}" with good semantic similarity`;
}
else if (score >= 0.7) {
return `Moderately relevant to "${query}" with decent semantic overlap`;
}
else {
return `Some relevance to "${query}" but weaker semantic connection`;
}
}
}
/**
* Simple in-memory vector store for BASIC tier - no external dependencies
*/
export class InMemoryVectorStore {
constructor() {
this.vectors = new Map();
}
async initialize() {
// No initialization needed for in-memory store
return Promise.resolve();
}
async upsert(points) {
for (const point of points) {
this.vectors.set(point.id, point);
}
}
async search(vector, query) {
const results = [];
for (const [id, point] of this.vectors.entries()) {
// Simple cosine similarity calculation
const score = this.cosineSimilarity(vector, point.vector);
// Apply tenant filtering if specified
if (query.tenant_id && point.payload.tenant_id !== query.tenant_id) {
continue;
}
// Apply type filtering if specified
if (query.type && point.payload.type !== query.type) {
continue;
}
results.push({
id,
score,
payload: point.payload,
});
}
// Sort by score (highest first) and limit results
return results
.sort((a, b) => b.score - a.score)
.slice(0, query.limit || 10);
}
async delete(ids) {
for (const id of ids) {
this.vectors.delete(id);
}
}
async count(tenantId) {
let count = 0;
for (const point of this.vectors.values()) {
if (!tenantId || point.payload.tenant_id === tenantId) {
count++;
}
}
return count;
}
async healthCheck() {
return true; // In-memory store is always healthy
}
async close() {
this.vectors.clear();
}
cosineSimilarity(a, b) {
if (a.length !== b.length) {
return 0;
}
let dotProduct = 0;
let normA = 0;
let normB = 0;
for (let i = 0; i < a.length; i++) {
dotProduct += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
if (normA === 0 || normB === 0) {
return 0;
}
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
}
}