@juspay/neurolink/dist/rag/chunkers/HTMLChunker.js

The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002

/**
 * HTML Chunker
 *
 * Splits HTML content by semantic tags.
 */
import { removeHtmlMarkup } from "../../utils/htmlText.js";
import { BaseChunker, DEFAULT_CHUNKER_CONFIG } from "./BaseChunker.js";
/**
 * HTML Chunker
 */
export class HTMLChunker extends BaseChunker {
    strategy = "html";
    getDefaultConfig() {
        return {
            ...DEFAULT_CHUNKER_CONFIG,
            maxSize: 1000,
            overlap: 0,
        };
    }
    async doChunk(content, config) {
        const maxSize = config.maxSize ?? 1000;
        // Strip HTML tags for text content
        const textContent = this.stripHtml(content);
        // Use simple character-based splitting for now
        const segments = this.splitBySizeWithOverlap(textContent, maxSize, 0);
        return segments.map((segment, index) => this.createChunk(segment.text, index, segment.start, segment.end));
    }
    /**
     * Strip HTML tags from content
     */
    stripHtml(html) {
        return removeHtmlMarkup(html, {
            tagReplacement: " ",
            commentReplacement: " ",
            lineBreakEndTags: [],
            lineBreakVoidTags: [],
        })
            .replace(/\s+/g, " ")
            .trim();
    }
}