@juspay/neurolink/dist/rag/chunkers/CharacterChunker.js

The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002

/**
 * Character Chunker
 *
 * Splits text into fixed-size character chunks with optional overlap.
 * The simplest chunking strategy for language-agnostic processing.
 */
import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
/**
 * Character Chunker
 *
 * Splits content into fixed-size character chunks.
 */
export class CharacterChunker extends BaseChunker {
    strategy = "character";
    getDefaultConfig() {
        return {
            ...DEFAULT_CHUNKER_CONFIG,
            maxSize: 1000,
            overlap: 100,
        };
    }
    async doChunk(content, config) {
        const maxSize = config.maxSize ?? 1000;
        const overlap = config.overlap ?? 100;
        const segments = this.splitBySizeWithOverlap(content, maxSize, overlap);
        return segments.map((segment, index) => this.createChunk(segment.text, index, segment.start, segment.end));
    }
}