@humanspeak/svelte-markdown/dist/utils/parse-and-cache.js

Markdown and HTML renderer for Svelte 5 — built for rendering streaming AI agent output from Claude Code, ChatGPT, and agentic workflows. XSS-safe defaults, streaming-aware sanitization, token caching, TypeScript types, and Svelte 5 runes.

/**
 * Parse and Cache Utility
 *
 * Handles markdown parsing with intelligent caching.
 * Separates parsing logic from component code for better testability.
 *
 * @module parse-and-cache
 */
import { tokenCache } from './token-cache.js';
import { shrinkHtmlTokens } from './token-cleanup.js';
import { Lexer, Marked } from 'marked';
/**
 * Lexes markdown source and cleans the resulting tokens. Shared by sync and async paths.
 *
 * @param source - Raw markdown string to lex
 * @param options - Parser options forwarded to the Marked lexer
 * @param isInline - When true, uses inline tokenization (no block elements)
 * @returns Cleaned token array with HTML tokens properly nested
 *
 * @example
 * ```typescript
 * import { lexAndClean } from './parse-and-cache.js'
 *
 * const tokens = lexAndClean('# Hello **world**', { gfm: true }, false)
 * ```
 *
 * @internal
 */
export const lexAndClean = (source, options, isInline) => {
    const lexer = new Lexer(options);
    const parsedTokens = isInline ? lexer.inlineTokens(source) : lexer.lex(source);
    return shrinkHtmlTokens(parsedTokens);
};
/**
 * Parses markdown source with caching (synchronous path).
 * Checks cache first, parses on miss, stores result, and returns tokens.
 *
 * @param source - Raw markdown string to parse
 * @param options - Svelte markdown parser options
 * @param isInline - Whether to parse as inline markdown (no block elements)
 * @returns Cleaned and cached token array
 *
 * @example
 * ```typescript
 * import { parseAndCacheTokens } from './parse-and-cache.js'
 *
 * // Parse markdown with caching
 * const tokens = parseAndCacheTokens('# Hello World', { gfm: true }, false)
 *
 * // Second call with same input returns cached result (<1ms)
 * const cachedTokens = parseAndCacheTokens('# Hello World', { gfm: true }, false)
 * ```
 */
export const parseAndCacheTokens = (source, options, isInline) => {
    // Check cache first - avoids expensive parsing
    const cached = tokenCache.getTokens(source, options);
    if (cached) {
        return cached;
    }
    // Cache miss - parse and store
    const cleanedTokens = lexAndClean(source, options, isInline);
    if (typeof options.walkTokens === 'function') {
        cleanedTokens.forEach(options.walkTokens);
    }
    // Cache the cleaned tokens for next time
    tokenCache.setTokens(source, options, cleanedTokens);
    return cleanedTokens;
};
/**
 * Parses markdown source with caching (async path).
 * Uses Marked's recursive walkTokens with Promise.all to properly
 * handle async walkTokens callbacks (e.g. marked-code-format).
 *
 * @param source - Raw markdown string to parse
 * @param options - Svelte markdown parser options
 * @param isInline - Whether to parse as inline markdown (no block elements)
 * @returns Promise resolving to cleaned and cached token array
 *
 * @example
 * ```typescript
 * import { parseAndCacheTokensAsync } from './parse-and-cache.js'
 *
 * const tokens = await parseAndCacheTokensAsync('# Hello', opts, false)
 * ```
 */
export const parseAndCacheTokensAsync = async (source, options, isInline) => {
    // Check cache first - avoids expensive parsing
    const cached = tokenCache.getTokens(source, options);
    if (cached) {
        return cached;
    }
    // Cache miss - parse and store
    const cleanedTokens = lexAndClean(source, options, isInline);
    if (typeof options.walkTokens === 'function') {
        // Use Marked's recursive walkTokens which handles tables, lists,
        // nested tokens, and extension childTokens. Await all returned
        // promises so async walkTokens callbacks complete before caching.
        const marked = new Marked();
        marked.defaults = { ...marked.defaults, ...options };
        const results = marked.walkTokens(cleanedTokens, options.walkTokens);
        await Promise.all(results);
    }
    // Cache the cleaned tokens for next time
    tokenCache.setTokens(source, options, cleanedTokens);
    return cleanedTokens;
};