@juspay/neurolink/dist/rag/chunkers/HTMLChunker.js

The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002

/**
 * HTML Chunker
 *
 * Splits HTML content by semantic tags.
 */
import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
/**
 * HTML Chunker
 */
export class HTMLChunker extends BaseChunker {
    strategy = "html";
    getDefaultConfig() {
        return {
            ...DEFAULT_CHUNKER_CONFIG,
            maxSize: 1000,
            overlap: 0,
        };
    }
    async doChunk(content, config) {
        const maxSize = config.maxSize ?? 1000;
        // Strip HTML tags for text content
        const textContent = this.stripHtml(content);
        // Use simple character-based splitting for now
        const segments = this.splitBySizeWithOverlap(textContent, maxSize, 0);
        return segments.map((segment, index) => this.createChunk(segment.text, index, segment.start, segment.end));
    }
    /**
     * Strip HTML tags from content
     */
    stripHtml(html) {
        return html
            .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
            .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
            .replace(/<[^>]+>/g, " ")
            .replace(/\s+/g, " ")
            .trim();
    }
}