@juspay/neurolink/dist/rag/chunkers/TokenChunker.d.ts

Universal AI Development Platform with working MCP integration, multi-provider support, voice (TTS/STT/realtime), and professional CLI. 58+ external MCP servers discoverable, multimodal file processing, RAG pipelines. Build, test, and deploy AI applicatio

/**
 * Token Chunker
 *
 * Splits text by token count using a tokenizer.
 * Useful for precise token budget management.
 */
import type { Chunk, ChunkerConfig, ChunkingStrategy } from "../../types/index.js";
import { BaseChunker } from "./BaseChunker.js";
/**
 * Token Chunker
 *
 * Approximates token-based splitting using word count.
 * For production, integrate with a proper tokenizer (tiktoken, etc.)
 */
export declare class TokenChunker extends BaseChunker {
    readonly strategy: ChunkingStrategy;
    getDefaultConfig(): ChunkerConfig;
    protected doChunk(content: string, config: ChunkerConfig): Promise<Chunk[]>;
}