better-qdrant-mcp-server/build/services/text-processing.js

MCP server for enhanced Qdrant vector database functionality

import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
export class TextProcessor {
    constructor(chunkSize = 1000, chunkOverlap = 200) {
        this.chunkSize = chunkSize;
        this.chunkOverlap = chunkOverlap;
        this.splitter = new RecursiveCharacterTextSplitter({
            chunkSize: this.chunkSize,
            chunkOverlap: this.chunkOverlap,
        });
    }
    async processText(text, source) {
        const documents = await this.splitter.createDocuments([text]);
        return documents.map((doc, index) => ({
            text: doc.pageContent,
            metadata: {
                index,
                source,
                start: doc.metadata?.start,
                end: doc.metadata?.end,
            },
        }));
    }
    async processFile(content, filename) {
        return this.processText(content, filename);
    }
    setChunkSize(size) {
        this.chunkSize = size;
        this.splitter = new RecursiveCharacterTextSplitter({
            chunkSize: this.chunkSize,
            chunkOverlap: this.chunkOverlap,
        });
    }
    setChunkOverlap(overlap) {
        this.chunkOverlap = overlap;
        this.splitter = new RecursiveCharacterTextSplitter({
            chunkSize: this.chunkSize,
            chunkOverlap: this.chunkOverlap,
        });
    }
}
//# sourceMappingURL=text-processing.js.map