ai-knowledge/src/loaders/directoryLoader.js

ai-knowledge

const path = require('path');
const { getAllFiles, isSupportedFile } = require('../utils/fileUtils');
const { loadFile } = require('./fileLoader');
const { v4: uuidv4 } = require('uuid');

/**
 * 加载目录中的所有文件到知识库
 */
async function loadDirectory(ragApplication, directoryPath, options = {}) {
    try {
        console.log(`Processing directory: ${directoryPath}`);
        const {
            forceReload = false,
            filter = isSupportedFile
        } = options;
        // 获取所有文件
        const allFiles = getAllFiles(directoryPath);
        // 过滤支持的文件
        const filesToProcess = allFiles.filter(filter);
        const taskIds = [];
        // 将每个文件添加到队列中
        for (const file of filesToProcess) {
            // loadFile 函数内部会将任务添加到队列
            const result = await loadFile(ragApplication, file, forceReload);
            taskIds.push(result.taskId);
        }
        return {
            success: true,
            source: directoryPath,
            type: 'directory',
            status: 'queued',
            taskIds: taskIds
        };
    } catch (error) {
        console.error(`Error loading directory ${directoryPath}:`, error);
        return {
            success: false,
            source: directoryPath,
            type: 'directory',
            error: error.message
        };
    }
}

module.exports = { loadDirectory };