@masuidrive/bloom-local-rag/dist/lib/vectorStore.js

RAG (Retrieval-Augmented Generation) system for local directories - Index and search your documents with AI-powered answers

import { join } from 'path';
import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
import { LanceDB } from '@langchain/community/vectorstores/lancedb';
import { connect } from '@lancedb/lancedb';
import { Document } from '@langchain/core/documents';
import { BLOOM_DIR, DB_DIR } from './constants.js';
import { createEmbeddings } from './embeddings.js';
export class VectorStoreManager {
    vectorStore = null;
    embeddings;
    config;
    splitter;
    dbPath = null;
    constructor(config) {
        this.config = config;
        this.embeddings = createEmbeddings(config);
        this.splitter = new RecursiveCharacterTextSplitter({
            chunkSize: config.embedding.chunkSize,
            chunkOverlap: config.embedding.chunkOverlap,
        });
    }
    async initialize(directory) {
        this.dbPath = join(directory, BLOOM_DIR, DB_DIR);
        // Initialize LanceDB vector store
        this.vectorStore = new LanceDB(this.embeddings, {
            uri: this.dbPath,
            tableName: 'documents',
            mode: 'overwrite',
        });
    }
    async indexFiles(files) {
        if (!this.vectorStore) {
            throw new Error('Vector store not initialized');
        }
        let totalChunks = 0;
        const documents = [];
        for (const file of files) {
            // Skip empty files
            if (!file.content || file.content.trim().length === 0) {
                continue;
            }
            const chunks = await this.splitter.splitText(file.content);
            for (let i = 0; i < chunks.length; i++) {
                // Skip empty chunks
                if (!chunks[i] || chunks[i].trim().length === 0) {
                    continue;
                }
                // Filter out complex metadata that could cause schema issues
                const simpleMetadata = {};
                for (const [key, value] of Object.entries(file.metadata || {})) {
                    if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
                        simpleMetadata[key] = value;
                    }
                }
                documents.push(new Document({
                    pageContent: chunks[i],
                    metadata: {
                        ...simpleMetadata,
                        path: file.path,
                        chunkId: `${file.path}#${i}`,
                        chunkIndex: i,
                        totalChunks: chunks.length,
                    },
                }));
                totalChunks++;
            }
        }
        if (documents.length > 0) {
            await this.vectorStore.addDocuments(documents);
        }
        return totalChunks;
    }
    async search(query, limit = 5) {
        if (!this.dbPath) {
            throw new Error('Vector store not initialized');
        }
        // Connect to existing LanceDB and open the table
        const db = await connect(this.dbPath);
        const table = await db.openTable('documents');
        // Create a new LanceDB instance with the existing table
        const searchStore = new LanceDB(this.embeddings, {
            table,
        });
        const results = await searchStore.similaritySearchWithScore(query, limit);
        return results.map(([doc, score]) => ({
            content: doc.pageContent,
            metadata: {
                path: doc.metadata.path,
                chunkId: doc.metadata.chunkId,
                score,
            },
        }));
    }
    async deleteByPaths(paths) {
        if (!this.dbPath) {
            throw new Error('Vector store not initialized');
        }
        if (paths.length === 0) {
            return;
        }
        try {
            // Connect to LanceDB and open the table
            const db = await connect(this.dbPath);
            const table = await db.openTable('documents');
            // Escape single quotes in paths for SQL safety
            const escapedPaths = paths.map(path => path.replace(/'/g, "''"));
            if (escapedPaths.length === 1) {
                // Delete single path
                await table.delete(`path = '${escapedPaths[0]}'`);
            }
            else {
                // Delete multiple paths using IN clause
                const pathList = escapedPaths.map(p => `'${p}'`).join(', ');
                await table.delete(`path IN (${pathList})`);
            }
            // Log deletion only if BLOOM_VERBOSE is set (for debugging)
            if (process.env.BLOOM_VERBOSE) {
                console.log(`Deleted vectors for ${paths.length} file(s)`);
            }
        }
        catch (error) {
            console.error('Error deleting vectors:', error);
            throw new Error(`Failed to delete vectors: ${error instanceof Error ? error.message : 'Unknown error'}`);
        }
    }
}
//# sourceMappingURL=vectorStore.js.map