@masuidrive/bloom-local-rag/dist/lib/indexer.js

RAG (Retrieval-Augmented Generation) system for local directories - Index and search your documents with AI-powered answers

import { readFile, stat } from 'fs/promises';
import { join } from 'path';
import pLimit from 'p-limit';
import { scanFiles, hashFile } from './fileScanner.js';
import { loadCache, saveCache } from './config.js';
export class Indexer {
    config;
    vectorStore;
    limit = pLimit(5); // Process 5 files concurrently
    constructor(config, vectorStore) {
        this.config = config;
        this.vectorStore = vectorStore;
    }
    async index(directory, force = false) {
        const cache = await loadCache(directory);
        const files = await scanFiles(directory, this.config.extensions, this.config.exclude);
        const result = {
            added: [],
            updated: [],
            deleted: [],
            totalChunks: 0,
        };
        // Create a map of current files
        const currentFiles = new Map();
        for (const file of files) {
            currentFiles.set(file.path, file);
        }
        // Check for deleted files
        for (const [path, entry] of Object.entries(cache.entries)) {
            if (!currentFiles.has(path)) {
                result.deleted.push(path);
                delete cache.entries[path];
            }
        }
        // Process files for addition or update
        const filesToIndex = [];
        await Promise.all(Array.from(currentFiles.entries()).map(([path, fileInfo]) => this.limit(async () => {
            const filePath = join(directory, path);
            const stats = await stat(filePath);
            const content = await readFile(filePath, 'utf-8');
            const hash = await hashFile(content);
            const cacheEntry = cache.entries[path];
            if (force || !cacheEntry || cacheEntry.hash !== hash) {
                filesToIndex.push(fileInfo);
                if (cacheEntry) {
                    result.updated.push(path);
                }
                else {
                    result.added.push(path);
                }
            }
        })));
        // Index the files that need indexing
        if (filesToIndex.length > 0) {
            result.totalChunks = await this.vectorStore.indexFiles(filesToIndex);
            // Update cache
            for (const file of filesToIndex) {
                const filePath = join(directory, file.path);
                const stats = await stat(filePath);
                const content = await readFile(filePath, 'utf-8');
                const hash = await hashFile(content);
                cache.entries[file.path] = {
                    path: file.path,
                    hash,
                    modifiedAt: stats.mtime.getTime(),
                    chunks: 0, // We don't track individual chunk counts in this implementation
                };
            }
        }
        // Delete removed files from vector store
        if (result.deleted.length > 0) {
            await this.vectorStore.deleteByPaths(result.deleted);
        }
        // Save updated cache
        await saveCache(directory, cache);
        return result;
    }
}
//# sourceMappingURL=indexer.js.map