@hpbyte/h-codex-core/dist/src/ingestion/explorer/index.js

Core indexing and search functionality for h-codex

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.fileExplorer = exports.FileExplorer = void 0;
const fast_glob_1 = __importDefault(require("fast-glob"));
const fs_1 = require("fs");
const path_1 = require("path");
const utils_1 = require("../../utils");
class FileExplorer {
    constructor(config = utils_1.DEFAULT_CONFIG) {
        this.config = config;
    }
    async discover(folderPath) {
        const absolutePath = (0, path_1.resolve)(folderPath);
        try {
            const stats = await fs_1.promises.stat(absolutePath);
            if (!stats.isDirectory()) {
                throw new Error(`Path is not a directory: ${absolutePath}`);
            }
        }
        catch {
            throw new Error(`Cannot access folder: ${absolutePath}`);
        }
        const patterns = this.config.supportedExtensions.map(ext => `**/*${ext}`);
        const ignorePatterns = await this.loadIgnorePatterns(absolutePath);
        const files = await (0, fast_glob_1.default)(patterns, {
            cwd: absolutePath,
            absolute: true,
            ignore: ignorePatterns,
            onlyFiles: true,
            followSymbolicLinks: false,
            suppressErrors: true,
        });
        return files.sort();
    }
    async loadIgnorePatterns(rootPath) {
        const allPatterns = [...this.config.defaultIgnorePatterns];
        for (const ignoreFile of this.config.ignoreFiles) {
            const filePath = (0, path_1.join)(rootPath, ignoreFile);
            try {
                const content = await fs_1.promises.readFile(filePath, 'utf-8');
                const patterns = this.parseIgnoreFile(content);
                allPatterns.push(...patterns);
            }
            catch {
                // Ignore file doesn't exist or can't be read, skip it
            }
        }
        return allPatterns;
    }
    parseIgnoreFile(content) {
        return content
            .split('\n')
            .map(line => line.trim())
            .filter(line => line && !line.startsWith('#'))
            .map(line => {
            // Convert gitignore patterns to fast-glob patterns
            if (line.endsWith('/')) {
                return `${line}**`;
            }
            if (!line.includes('/') && !line.includes('*')) {
                return `**/${line}`;
            }
            return line;
        });
    }
    async validateFile(filePath) {
        try {
            const stats = await fs_1.promises.stat(filePath);
            return stats.isFile() && stats.size > 0;
        }
        catch {
            return false;
        }
    }
    createBatches(items, batchSize) {
        const batches = [];
        for (let i = 0; i < items.length; i += batchSize) {
            batches.push(items.slice(i, i + batchSize));
        }
        return batches;
    }
    async processInBatches(items, processor, options = { batchSize: 10, maxConcurrency: 5 }) {
        const batches = this.createBatches(items, options.batchSize);
        const results = [];
        const errors = [];
        for (let i = 0; i < batches.length; i += options.maxConcurrency) {
            const currentBatches = batches.slice(i, i + options.maxConcurrency);
            const batchPromises = currentBatches.map(async (batch) => {
                const batchResults = await Promise.allSettled(batch.map(item => processor(item)));
                batchResults.forEach((result, index) => {
                    if (result.status === 'fulfilled') {
                        results.push(result.value);
                    }
                    else {
                        errors.push({
                            item: batch[index],
                            error: result.reason,
                        });
                    }
                });
            });
            await Promise.all(batchPromises);
        }
        return { results, errors };
    }
    getFileExtension(filePath) {
        const lastDotIndex = filePath.lastIndexOf('.');
        return lastDotIndex === -1 ? '' : filePath.substring(lastDotIndex);
    }
    isFileSupported(filePath) {
        const extension = this.getFileExtension(filePath);
        return this.config.supportedExtensions.includes(extension);
    }
    updateConfig(newConfig) {
        this.config = { ...this.config, ...newConfig };
    }
}
exports.FileExplorer = FileExplorer;
exports.fileExplorer = new FileExplorer();
//# sourceMappingURL=index.js.map