ai-knowledge/src/loaders/textLoader.js

ai-knowledge

const { TextLoader } = require('@llm-tools/embedjs');
const config = require('../config');
const loaderQueue = require('./queueManager');
const { v4: uuidv4 } = require('uuid');

/**
 * 加载文本内容到知识库
 * @param {object} ragApplication RAG应用实例
 * @param {string} text 文本内容
 * @param {boolean} forceReload 是否强制重新加载
 * @param {object} metadata 可选的元数据
 */
function loadText(ragApplication, text, forceReload = false, metadata = {}) {
    const taskId = uuidv4();
    
    const loader = new TextLoader({
        text,
        chunkSize: config.chunking.chunkSize,
        chunkOverlap: config.chunking.chunkOverlap,
        metadata: {
            type: 'text',
            timestamp: new Date().toISOString(),
            ...metadata
        }
    });

    loaderQueue.add({
        taskId,
        ragApplication,
        loader,
        source: metadata.source || '直接输入文本',
        type: 'text',
        forceReload
    });

    return {
        success: true,
        source: metadata.source || '直接输入文本',
        type: 'text',
        status: 'queued',
        taskId
    };
}

module.exports = { loadText };