ai-knowledge/src/loaders/urlLoader.js

ai-knowledge

const { WebLoader } = require('@llm-tools/embedjs-loader-web');
const config = require('../config');
const loaderQueue = require('./queueManager');
const { v4: uuidv4 } = require('uuid');
/**
 * 加载URL内容到知识库
 */
function loadUrl(ragApplication, url, forceReload = false) {
    const taskId = uuidv4();
    const loader = new WebLoader({
        urlOrContent: url,
        chunkSize: config.chunking.chunkSize,
        chunkOverlap: config.chunking.chunkOverlap,
        metadata: {
            source: url,
            type: 'url',
            timestamp: new Date().toISOString()
        }
    });

    loaderQueue.add({
        taskId,
        ragApplication,
        loader,
        source: url,
        type: 'url',
        forceReload
    });

    return {
        success: true,
        source: url,
        type: 'url',
        status: 'queued',
        taskId
    };
}

module.exports = { loadUrl };