agentfootprint/dist/esm/rag/loaders/html.d.ts

The explainable agent framework — backtrack a wrong answer to the exact context that caused it (evidence, not guesses). Built on footprintjs.

/**
 * htmlLoader — HTML with the tags taken out, without a parser.
 *
 * Pattern: Adapter behind `DocumentLoader`.
 * Role:    rag/ layer. Zero dependency.
 * Emits:   N/A.
 *
 * ── What this is, and what it is not ────────────────────────────────────────
 * This is a **tag stripper**, not an HTML parser, and the distinction is worth
 * being blunt about because the failure mode is silent. It removes `<script>`
 * and `<style>` bodies, replaces tags with whitespace, and decodes the handful
 * of entities that appear in real prose. That is enough for documentation
 * pages, exported articles, and saved knowledge-base entries — the things
 * people actually put in a corpus.
 *
 * It is NOT enough for a modern application page. A single-page app's HTML is
 * mostly scaffolding, and what you get back will be navigation labels and
 * button text. If your corpus is that, run a real extractor (Readability,
 * Mercury, your CMS's own export) and feed the result in as text — the
 * `{ text, uri }` arm of `DocumentSource` exists for exactly that.
 *
 * A parser was considered and rejected for v1: every option is a dependency
 * measured in megabytes, and the difference it buys on the documents people
 * index is small. If that stops being true, the fix is another
 * `DocumentLoader` passed ahead of this one — no change here.
 *
 * ── Offsets ─────────────────────────────────────────────────────────────────
 * Tags are replaced with whitespace of the SAME LENGTH rather than deleted, so
 * offsets into the extracted text line up with offsets into the original file.
 * A chunk's `charStart` therefore points at the same place in both, and the
 * text that gets stored is the extracted text, which is what the offsets index.
 */
import type { DocumentLoader } from '../types.js';
/**
 * Every `open … close` region blanked, leftmost first, each ending at the
 * first `close` after its `open` — `/<!--[\s\S]*?-->/g`, and `/<[^>]*>/g` for
 * `<` … `>` (`[^>]*` stops at the first `>` too).
 */
export declare function blankDelimited(text: string, open: string, close: string): string;
/**
 * Every `<name …>` … `</name …>` element blanked, body included —
 * `/<name\b[^>]*>[\s\S]*?<\/name(?=[\t\n\f\r />])[^>]*>/gi`. The end tag is the
 * one an HTML tokenizer ends the element at: `</script`, then tab, LF, FF, CR,
 * space, `/` or `>`, then anything up to the next `>` — so `</script foo>` and
 * `</script/>` close the body too, where the old `</script\s*>` left that body
 * in the text. Like a browser, other blanks (`</script` + U+00A0, U+2028 …)
 * no longer end the tag name; the old `\s` did.
 */
export declare function blankRawText(text: string, name: 'script' | 'style'): string;
/**
 * Strip markup to text.
 *
 * Exported because the splitter tests and the security tests both need to
 * assert on it directly, and because a consumer writing their own loader for a
 * markup-ish format may reasonably want the same pass.
 */
export declare function stripTags(html: string): string;
export declare function htmlLoader(): DocumentLoader;