@masuidrive/bloom-local-rag/dist/lib/fileScanner.js

RAG (Retrieval-Augmented Generation) system for local directories - Index and search your documents with AI-powered answers

import { readFile, stat } from 'fs/promises';
import { join, relative, extname } from 'path';
import { glob } from 'glob';
import ignore from 'ignore';
import matter from 'gray-matter';
import yaml from 'js-yaml';
import createXXHash from 'xxhash-wasm';
import { DEFAULT_EXCLUDE_PATTERNS } from './constants.js';
import { createGitignoreChecker } from './gitignore.js';
let xxhash = null;
async function getHasher() {
    if (!xxhash) {
        xxhash = await createXXHash();
    }
    return xxhash;
}
export async function hashFile(content) {
    const hasher = await getHasher();
    return hasher.h64(content).toString(16);
}
// Re-export from gitignore.ts for backward compatibility
export { loadGitignore } from './gitignore.js';
export async function scanFiles(directory, extensions, excludePatterns = []) {
    const ig = ignore();
    ig.add(DEFAULT_EXCLUDE_PATTERNS);
    ig.add(excludePatterns);
    // Create gitignore checker that respects hierarchy
    const isGitignored = await createGitignoreChecker(directory);
    const patterns = extensions.map(ext => `**/*${ext}`);
    const files = await glob(patterns, {
        cwd: directory,
        absolute: false,
        ignore: DEFAULT_EXCLUDE_PATTERNS.concat(excludePatterns),
    });
    const fileInfos = [];
    for (const file of files) {
        const relativePath = relative(directory, join(directory, file));
        if (ig.ignores(relativePath) || isGitignored(relativePath)) {
            continue;
        }
        try {
            const filePath = join(directory, file);
            const content = await readFile(filePath, 'utf-8');
            const stats = await stat(filePath);
            const ext = extname(file);
            let metadata = {
                path: relativePath,
                modifiedAt: stats.mtime.getTime(),
            };
            if (ext === '.md' || ext === '.mdx') {
                const { data, content: body } = matter(content);
                metadata = { ...metadata, ...data };
                fileInfos.push({
                    path: relativePath,
                    content: body,
                    metadata,
                });
            }
            else if (ext === '.yaml' || ext === '.yml') {
                try {
                    const data = yaml.load(content);
                    metadata = { ...metadata, ...data };
                }
                catch {
                    // If YAML parsing fails, treat as plain text
                }
                fileInfos.push({
                    path: relativePath,
                    content,
                    metadata,
                });
            }
            else {
                fileInfos.push({
                    path: relativePath,
                    content,
                    metadata,
                });
            }
        }
        catch (error) {
            console.error(`Error reading file ${file}:`, error);
        }
    }
    return fileInfos;
}
//# sourceMappingURL=fileScanner.js.map