@humanspeak/svelte-markdown
Version:
Markdown and HTML renderer for Svelte 5 — built for rendering streaming AI agent output from Claude Code, ChatGPT, and agentic workflows. XSS-safe defaults, streaming-aware sanitization, token caching, TypeScript types, and Svelte 5 runes.
296 lines (295 loc) • 13.5 kB
TypeScript
/**
* Incremental Markdown Parser for Streaming
*
* Optimizes streaming scenarios (LLM token-by-token updates) by performing
* a full re-parse but diffing the result against the previous token array.
* Only changed/appended tokens are returned as updates, allowing Svelte to
* skip re-rendering unchanged components.
*
* @module incremental-parser
*/
import type { SvelteMarkdownOptions } from '../types.js';
import type { Token } from './markdown-parser.js';
/**
* Result of an incremental parse update.
*/
export interface IncrementalUpdateResult {
/** The full new token array */
tokens: Token[];
/** Index of the first token that differs from the previous parse */
divergeAt: number;
/** Source offset where the divergent token begins, when known without scanning the stable prefix */
divergeOffset?: number;
/** Whether consumers can safely reuse stable token objects from the previous parse */
canReuse: boolean;
/** Whether this update re-lexed only the appended tail (vs the whole source) */
usedTailWindow: boolean;
}
/**
* Streaming-optimized parser that performs full re-parses but diffs results
* against the previous token array to minimize DOM updates.
*
* For append-only streaming (typical LLM use case), most tokens are identical
* between updates. By comparing `raw` strings, we identify which tokens changed
* so Svelte can skip re-rendering unchanged components.
*
* @example
* ```typescript
* const parser = new IncrementalParser({ gfm: true })
*
* // First update — all tokens are "new"
* const r1 = parser.update('# Hello')
* // r1.divergeAt === 0
*
* // Second update — heading unchanged, paragraph appended
* const r2 = parser.update('# Hello\n\nWorld')
* // r2.divergeAt === 1 (heading at index 0 unchanged)
* ```
*/
export declare class IncrementalParser {
/** Previous parse result for diffing */
private prevTokens;
/** Previous full source string for append-only tail reparsing */
private prevSource;
/** Parser options passed to the Marked lexer */
private options;
/** Whether caller-supplied parser hooks make tail-window reparsing unsafe */
private tailWindowDisabled;
/** True iff any token in `prevTokens` is an HTML opening whose actual
* source span is unknown. Closed HTML spans record `sourceLength`
* during cleanup and remain safe for tail-window offset arithmetic;
* unclosed spans do not. Cached to keep `getTailWindowBoundary` O(1)
* on the hot path. */
private prevHasHtmlSpanMismatch;
/** Cached boundary for the next append-only update. Computed when
* parser state is committed so `getTailWindowBoundary` stays O(1). */
private prevTailWindowBoundary;
/** Cached reference-syntax facts for `prevSource`. These avoid scanning
* the accumulated stream on every append. */
private prevHasPotentialReferenceUse;
private prevHasReferenceDefinition;
/**
* Creates a new incremental parser instance.
*
* @param options - Svelte markdown parser options forwarded to Marked's Lexer
*/
constructor(options: SvelteMarkdownOptions);
private getTailWindowBoundary;
/**
* True for an HTML opening tag whose actual source span is unknown.
* After token cleanup, closed HTML tokens keep children on `.tokens`
* and record their full source span as `sourceLength`. Unclosed HTML
* openings have neither a full span nor a closing tag yet, so serving
* them as a stable tail-window prefix would corrupt the offset math.
*/
private hasHtmlSpanMismatch;
/**
* Source characters this token consumed. Closed HTML tokens record
* their full span as `sourceLength` during cleanup; every other token's
* `.raw` already equals its source span, so we fall back to `raw.length`.
*/
private getTokenSourceLength;
private isStableAtSourceEnd;
/**
* True when `source` contains reference-style link syntax that could
* resolve against a definition — either a full reference (`[text][id]`)
* or a shortcut (`[text]`). It says nothing about whether a matching
* definition exists; pair it with {@link hasReferenceDefinition} for that.
*
* @param source - Markdown source to scan
* @returns `true` if a full or shortcut reference use is present
* @example
* ```typescript
* this.hasPotentialReferenceUse('see [docs]') // true (shortcut)
* this.hasPotentialReferenceUse('see [a][b]') // true (full ref)
* this.hasPotentialReferenceUse('see [x](/y)') // false (inline link)
* ```
*/
private hasPotentialReferenceUse;
/**
* True when `source` contains reference-style link syntax outside
* reference definition lines. Definition labels (`[docs]: /docs`) look
* like shortcut references to `SHORTCUT_REFERENCE_RE`, but they are not
* renderable uses and should not keep a stream reference-sensitive after
* the definition has already been handled.
*
* @param source - Markdown source or source slice to scan
* @returns `true` if a full or shortcut reference use appears on a
* non-definition line
* @example
* ```typescript
* this.hasPotentialReferenceUseOutsideDefinitions('[docs]: /docs') // false
* this.hasPotentialReferenceUseOutsideDefinitions('see [docs]') // true
* ```
*/
private hasPotentialReferenceUseOutsideDefinitions;
/**
* True when `source` contains a link reference definition line
* (`[label]: url`). A definition can retroactively change how reference
* uses elsewhere in the document render, which is what makes it relevant
* to tail-window safety.
*
* @param source - Markdown source to scan
* @returns `true` if a reference definition line is present
* @example
* ```typescript
* this.hasReferenceDefinition('[docs]: /docs') // true
* this.hasReferenceDefinition('see [docs]') // false
* ```
*/
private hasReferenceDefinition;
/**
* True when an append-only update newly introduces text accepted by
* `matches`. Reference uses and definitions cannot span newlines, so a
* token split across the append boundary can only complete on the line
* that straddles it; checking the appended slice plus that single boundary
* line catches every case without rescanning the accumulated source.
* Assumes `source` starts with `prevSource` — callers guard the non-append
* case. (The one unbounded input is a document streamed as a single
* newline-free line, where the boundary line grows with the document.)
*
* @param source - Full source string for an append-only update
* @param matches - Predicate identifying the reference syntax of interest
* @returns `true` if the append introduces a match not already present
* @example
* ```typescript
* // prevSource === 'see [do'
* this.appendIntroducesMatch('see [docs]', this.hasPotentialReferenceUseOutsideDefinitions) // true
* ```
*/
private appendIntroducesMatch;
/**
* True when `source` is a pure append onto the previously parsed source —
* i.e. this is not the first update and `source` begins with `prevSource`.
* Computed once per `update` and threaded into `canUseTailWindow` /
* `parseSource` so the full-length `startsWith` scan runs a single time.
*
* @param source - The full new source string for this update
* @returns `true` if this update only appends to `prevSource`
* @example
* ```typescript
* // prevSource === '# A\n\nB'
* this.isAppendOnlyUpdate('# A\n\nB\n\nC') // true
* this.isAppendOnlyUpdate('# A\n\nX') // false (diverges from prevSource)
* ```
*/
private isAppendOnlyUpdate;
/**
* True when appending to `prevSource` introduces a reference definition
* that was not already present. Definitions can arrive wholly in the
* appended slice or be completed across the append boundary, such as
* `[do` followed by `cs]: /docs`. Returns false for any update that is not
* a pure append of `prevSource`.
*
* @param source - The full new source, expected to start with `prevSource`
* @returns `true` if the appended update adds a reference definition
* @example
* ```typescript
* // prevSource === '[do'
* this.hasNewReferenceDefinition('[docs]: /d') // true
* ```
*/
private hasNewReferenceDefinition;
/**
* True when a reference definition arriving in the appended tail can
* retroactively change how existing reference-style uses in the stable
* prefix render — the one case where an append-only stream is not safe
* to serve incrementally. This is the standalone form used as
* `canUseTailWindow`'s default argument; the hot path in `update` computes
* the same value inline (reusing `appendAddsDefinition`) so the boundary
* scan runs a single time per update.
*
* @param source - The full new source string for this update
* @returns `true` if a newly appended definition can change how the reused
* prefix renders, meaning the tail window must be bypassed
* @example
* ```typescript
* // prevSource === 'see [docs]\n\n' (a shortcut use already rendered)
* this.appendedDefinitionInvalidatesTail('see [docs]\n\n[docs]: /d') // true
* ```
*/
private appendedDefinitionInvalidatesTail;
/**
* Decides whether an update may reuse the stable token prefix and re-lex
* only the appended tail (`boundary.reparseOffset` onward) instead of the
* whole document. Returns false whenever that shortcut could diverge from a
* full parse: caller parser hooks are active, the update is not append-only,
* the boundary is empty, or reference syntax straddles the prefix/tail split
* in a way a tail-only re-lex cannot resolve.
*
* @param source - The full new source string for this update
* @param boundary - The stable-prefix boundary from `getTailWindowBoundary`
* @param isAppendOnly - Precomputed append-only fact from `update`; direct
* helper callers may omit it to compute the same fact locally
* @param referenceInvalidatesTail - Precomputed reference-safety flag;
* defaults to `appendedDefinitionInvalidatesTail(source)` for standalone
* callers that have not computed it already
* @returns `true` if the appended tail can be re-lexed in isolation
* @example
* ```typescript
* const boundary = this.getTailWindowBoundary()
* if (this.canUseTailWindow(source, boundary)) {
* // safe to re-lex only source.slice(boundary.reparseOffset)
* }
* ```
*/
private canUseTailWindow;
private parseSource;
/**
* True when any token in `tokens` has an unknown HTML source span. Tail
* reparsing can reuse a prefix only when prefix token lengths still map to
* source offsets; unclosed HTML openings break that invariant.
*
* @param tokens - Tokens to inspect for unknown HTML spans
* @returns `true` if at least one token makes tail-window offsets unsafe
* @example
* ```typescript
* this.hasAnyHtmlSpanMismatch(tailTokens) // scan only the reparsed tail
* ```
*/
private hasAnyHtmlSpanMismatch;
/**
* Computes and caches the next stable-prefix boundary once per committed
* parser state. The hot-path `getTailWindowBoundary` then returns this
* object without summing every prefix token on every append.
*
* @param tokens - Latest token array after parsing the current source
* @param sourceLength - Character length of the current source
* @param hasHtmlSpanMismatch - Whether any current token has an unknown
* source span
* @returns The prefix token count and source offset to reuse on the next
* append-only update
* @example
* ```typescript
* this.prevTailWindowBoundary = this.getNextTailWindowBoundary(tokens, source.length, false)
* ```
*/
private getNextTailWindowBoundary;
/**
* Commits parser state and refreshes cached bookkeeping facts for the next
* update. When the current parse reused a stable prefix, only the reparsed
* tail is inspected for HTML span mismatches so append-only streaming stays
* proportional to the newly parsed region.
*
* @param source - Full source string for the just-completed update
* @param parseResult - Parsed tokens plus metadata describing whether the
* tail-window shortcut was used
* @param isAppendOnly - Whether `source` appended to the previous source
* @param appendAddsDefinition - Whether the append introduced a reference
* definition, already computed by `update` so the boundary scan is not
* repeated here
* @returns Nothing; updates `prevSource`, `prevTokens`, and cached flags
* @example
* ```typescript
* this.updateCachedState(source, parseResult, isAppendOnly, appendAddsDefinition)
* ```
*/
private updateCachedState;
/**
* Parses the full source and diffs against the previous result.
*
* @param source - The full accumulated markdown source string
* @returns The new tokens and the index where they diverge from the previous parse
*/
update: (source: string) => IncrementalUpdateResult;
}