@lobehub/ui/es/hooks/useStreamHighlight.mjs.map

Lobe UI is an open-source UI component library for building AIGC web apps

{"version":3,"file":"useStreamHighlight.mjs","names":["lobeTheme"],"sources":["../../src/hooks/useStreamHighlight.ts"],"sourcesContent":["'use client';\n\nimport { ShikiStreamTokenizer } from '@shikijs/stream';\nimport { type CSSProperties, useCallback, useEffect, useMemo, useRef, useState } from 'react';\nimport { type BuiltinTheme, type ThemedToken } from 'shiki';\n\nimport { getCodeLanguageByInput } from '@/Highlighter/const';\nimport lobeTheme from '@/Highlighter/theme/lobe-theme';\n\nimport { shikiModulePromise, type StreamingHighlightResult } from './useHighlight';\n\ntype StreamingOptions = {\n  customThemes?: Record<string, any>;\n  enabled?: boolean;\n  language: string;\n  theme: string;\n};\n\n// Optimized version: reduce array allocations and object spreading\nconst tokensToLineTokens = (tokens: ThemedToken[]): ThemedToken[][] => {\n  if (!tokens.length) return [[]];\n\n  const lines: ThemedToken[][] = [];\n  let currentLine: ThemedToken[] = [];\n\n  for (const token of tokens) {\n    const content = token.content ?? '';\n\n    if (content === '\\n') {\n      lines.push(currentLine);\n      currentLine = [];\n      continue;\n    }\n\n    const newlineIndex = content.indexOf('\\n');\n    if (newlineIndex === -1) {\n      // No newline, add token directly\n      currentLine.push(token);\n    } else {\n      // Split on newlines\n      const segments = content.split('\\n');\n      for (const [j, segment] of segments.entries()) {\n        if (segment) {\n          // Only create new object if we need to modify content\n          currentLine.push(j === 0 && segment === content ? token : { ...token, content: segment });\n        }\n        if (j < segments.length - 1) {\n          lines.push(currentLine);\n          currentLine = [];\n        }\n      }\n    }\n  }\n\n  // Don't forget the last line\n  if (currentLine.length > 0 || lines.length === 0) {\n    lines.push(currentLine);\n  }\n\n  return lines.length > 0 ? lines : [[]];\n};\n\nconst createPreStyle = (bg?: string, fg?: string): CSSProperties | undefined => {\n  if (!bg && !fg) return undefined;\n  return {\n    backgroundColor: bg,\n    color: fg,\n  };\n};\n\nconst useStreamingHighlighter = (\n  text: string,\n  options: StreamingOptions,\n): StreamingHighlightResult | undefined => {\n  const { customThemes, enabled, language, theme } = options;\n  const [result, setResult] = useState<StreamingHighlightResult>();\n  const tokenizerRef = useRef<ShikiStreamTokenizer | null>(null);\n  const previousTextRef = useRef('');\n  const safeText = text ?? '';\n  const latestTextRef = useRef(safeText);\n  const preStyleRef = useRef<CSSProperties | undefined>(undefined);\n  const linesRef = useRef<ThemedToken[][]>([[]]);\n\n  useEffect(() => {\n    latestTextRef.current = safeText;\n  }, [safeText]);\n\n  // Use ref to store callback to avoid recreating it\n  const setStreamingResultRef = useRef((rawLines: ThemedToken[][]) => {\n    const previousLines = linesRef.current;\n    const newLinesLength = rawLines.length;\n    const prevLinesLength = previousLines.length;\n\n    // Fast path: if lengths differ or it's a complete reset, use new lines directly\n    if (newLinesLength !== prevLinesLength || newLinesLength === 0) {\n      linesRef.current = rawLines;\n      setResult({\n        lines: rawLines,\n        preStyle: preStyleRef.current,\n      });\n      return;\n    }\n\n    // Optimized comparison: only check changed lines\n    let hasChanges = false;\n    const mergedLines: ThemedToken[][] = [];\n\n    for (let i = 0; i < newLinesLength; i++) {\n      const newLine = rawLines[i];\n      const prevLine = previousLines[i];\n\n      // Quick reference equality check first\n      if (prevLine === newLine) {\n        mergedLines[i] = prevLine;\n        continue;\n      }\n\n      // Length check\n      if (!prevLine || prevLine.length !== newLine.length) {\n        mergedLines[i] = newLine;\n        hasChanges = true;\n        continue;\n      }\n\n      // Deep comparison only for lines that might have changed\n      let lineChanged = false;\n      for (const [j, newToken] of newLine.entries()) {\n        if (prevLine[j] !== newToken) {\n          lineChanged = true;\n          break;\n        }\n      }\n\n      if (lineChanged) {\n        mergedLines[i] = newLine;\n        hasChanges = true;\n      } else {\n        mergedLines[i] = prevLine;\n      }\n    }\n\n    // Only update state if there are actual changes\n    if (hasChanges) {\n      linesRef.current = mergedLines;\n      setResult({\n        lines: mergedLines,\n        preStyle: preStyleRef.current,\n      });\n    }\n  });\n\n  const updateTokens = useCallback(async (nextText: string, forceReset = false) => {\n    const tokenizer = tokenizerRef.current;\n    if (!tokenizer) return;\n\n    if (forceReset) {\n      tokenizer.clear();\n      previousTextRef.current = '';\n    }\n\n    const previousText = previousTextRef.current;\n    let chunk = nextText;\n    const canAppend = !forceReset && nextText.startsWith(previousText);\n\n    if (canAppend) {\n      chunk = nextText.slice(previousText.length);\n    } else if (!forceReset) {\n      tokenizer.clear();\n    }\n\n    previousTextRef.current = nextText;\n\n    if (!chunk) {\n      // Optimize: avoid array spread if possible\n      const stableTokens = tokenizer.tokensStable;\n      const unstableTokens = tokenizer.tokensUnstable;\n      const totalLength = stableTokens.length + unstableTokens.length;\n\n      if (totalLength === 0) {\n        setStreamingResultRef.current([[]]);\n        return;\n      }\n\n      // Only create merged array if we have both stable and unstable tokens\n      const mergedTokens =\n        stableTokens.length === 0\n          ? unstableTokens\n          : unstableTokens.length === 0\n            ? stableTokens\n            : [...stableTokens, ...unstableTokens];\n\n      setStreamingResultRef.current(tokensToLineTokens(mergedTokens));\n      return;\n    }\n\n    try {\n      await tokenizer.enqueue(chunk);\n      // Optimize: avoid array spread if possible\n      const stableTokens = tokenizer.tokensStable;\n      const unstableTokens = tokenizer.tokensUnstable;\n      const mergedTokens =\n        stableTokens.length === 0\n          ? unstableTokens\n          : unstableTokens.length === 0\n            ? stableTokens\n            : [...stableTokens, ...unstableTokens];\n      setStreamingResultRef.current(tokensToLineTokens(mergedTokens));\n    } catch (error) {\n      console.error('Streaming highlighting failed:', error);\n    }\n  }, []);\n\n  // Cache highlighter key to avoid unnecessary recreations\n  const highlighterKeyRef = useRef<string>('');\n\n  useEffect(() => {\n    if (!enabled) {\n      tokenizerRef.current?.clear();\n      tokenizerRef.current = null;\n      previousTextRef.current = '';\n      preStyleRef.current = undefined;\n      linesRef.current = [[]];\n      setResult(undefined);\n      highlighterKeyRef.current = '';\n      return;\n    }\n\n    // Skip if language/theme combination hasn't changed\n    const currentKey = `${language}-${theme}`;\n    if (highlighterKeyRef.current === currentKey && tokenizerRef.current) {\n      return;\n    }\n\n    let cancelled = false;\n\n    (async () => {\n      const mod = await shikiModulePromise;\n      if (!mod || cancelled) return;\n\n      try {\n        // Load custom theme if using slack-dark or slack-ochin\n        let themesToLoad: any[] = [theme];\n        if (customThemes && theme === 'lobe-theme') {\n          const customTheme = customThemes[theme];\n          if (customTheme) {\n            themesToLoad = [customTheme as any];\n          }\n        }\n\n        // Only load the specific language and theme needed\n        // getSingletonHighlighter will cache the instance internally\n        const highlighter = await mod.getSingletonHighlighter({\n          langs: language ? [language] : ['plaintext'],\n          themes: themesToLoad,\n        });\n\n        if (!highlighter || cancelled) return;\n\n        // Only create new tokenizer if key changed\n        if (highlighterKeyRef.current !== currentKey) {\n          // Clear old tokenizer\n          tokenizerRef.current?.clear();\n\n          const tokenizer = new ShikiStreamTokenizer({\n            highlighter,\n            lang: language,\n            theme,\n          });\n\n          tokenizerRef.current = tokenizer;\n          highlighterKeyRef.current = currentKey;\n          previousTextRef.current = '';\n          linesRef.current = [[]];\n\n          const themeInfo = highlighter.getTheme(theme);\n          preStyleRef.current = createPreStyle(themeInfo?.bg, themeInfo?.fg);\n        }\n\n        const currentText = latestTextRef.current;\n        if (currentText) {\n          await updateTokens(currentText, true);\n        } else {\n          setStreamingResultRef.current([[]]);\n        }\n      } catch (error) {\n        console.error('Streaming highlighter initialization failed:', error);\n        // Reset on error\n        highlighterKeyRef.current = '';\n      }\n    })();\n\n    return () => {\n      cancelled = true;\n      // Cleanup only if this effect was cancelled before completion\n      // The next effect will handle cleanup if key changed\n    };\n  }, [enabled, language, theme, updateTokens, customThemes]);\n\n  // Separate effect for text updates to avoid unnecessary tokenizer recreation\n  useEffect(() => {\n    if (!enabled) return;\n    if (!tokenizerRef.current) return;\n    // Use ref to get latest text to avoid stale closures\n    const currentText = latestTextRef.current;\n    updateTokens(currentText);\n  }, [enabled, safeText, updateTokens]);\n\n  return result;\n};\n\nexport const useStreamHighlight = (\n  text: string,\n  {\n    language,\n    theme: builtinTheme,\n    streaming,\n  }: { enableTransformer?: boolean; language: string; streaming?: boolean; theme?: BuiltinTheme },\n) => {\n  // Safely handle language and text with boundary checks\n  const safeText = text ?? '';\n  const lang = (language ?? 'plaintext').toLowerCase();\n\n  // Match supported languages\n  const matchedLanguage = useMemo(() => getCodeLanguageByInput(lang), [lang]);\n\n  const effectiveTheme = builtinTheme || 'lobe-theme';\n\n  return useStreamingHighlighter(safeText, {\n    customThemes: {\n      'lobe-theme': lobeTheme,\n    },\n    enabled: streaming,\n    language: matchedLanguage,\n    theme: effectiveTheme,\n  });\n};\n"],"mappings":";;;;;;;AAmBA,MAAM,sBAAsB,WAA2C;CACrE,IAAI,CAAC,OAAO,QAAQ,OAAO,CAAC,CAAC,CAAC;CAE9B,MAAM,QAAyB,CAAC;CAChC,IAAI,cAA6B,CAAC;CAElC,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,UAAU,MAAM,WAAW;EAEjC,IAAI,YAAY,MAAM;GACpB,MAAM,KAAK,WAAW;GACtB,cAAc,CAAC;GACf;EACF;EAGA,IADqB,QAAQ,QAAQ,IACtB,MAAM,IAEnB,YAAY,KAAK,KAAK;OACjB;GAEL,MAAM,WAAW,QAAQ,MAAM,IAAI;GACnC,KAAK,MAAM,CAAC,GAAG,YAAY,SAAS,QAAQ,GAAG;IAC7C,IAAI,SAEF,YAAY,KAAK,MAAM,KAAK,YAAY,UAAU,QAAQ;KAAE,GAAG;KAAO,SAAS;IAAQ,CAAC;IAE1F,IAAI,IAAI,SAAS,SAAS,GAAG;KAC3B,MAAM,KAAK,WAAW;KACtB,cAAc,CAAC;IACjB;GACF;EACF;CACF;CAGA,IAAI,YAAY,SAAS,KAAK,MAAM,WAAW,GAC7C,MAAM,KAAK,WAAW;CAGxB,OAAO,MAAM,SAAS,IAAI,QAAQ,CAAC,CAAC,CAAC;AACvC;AAEA,MAAM,kBAAkB,IAAa,OAA2C;CAC9E,IAAI,CAAC,MAAM,CAAC,IAAI,OAAO,KAAA;CACvB,OAAO;EACL,iBAAiB;EACjB,OAAO;CACT;AACF;AAEA,MAAM,2BACJ,MACA,YACyC;CACzC,MAAM,EAAE,cAAc,SAAS,UAAU,UAAU;CACnD,MAAM,CAAC,QAAQ,aAAa,SAAmC;CAC/D,MAAM,eAAe,OAAoC,IAAI;CAC7D,MAAM,kBAAkB,OAAO,EAAE;CACjC,MAAM,WAAW,QAAQ;CACzB,MAAM,gBAAgB,OAAO,QAAQ;CACrC,MAAM,cAAc,OAAkC,KAAA,CAAS;CAC/D,MAAM,WAAW,OAAwB,CAAC,CAAC,CAAC,CAAC;CAE7C,gBAAgB;EACd,cAAc,UAAU;CAC1B,GAAG,CAAC,QAAQ,CAAC;CAGb,MAAM,wBAAwB,QAAQ,aAA8B;EAClE,MAAM,gBAAgB,SAAS;EAC/B,MAAM,iBAAiB,SAAS;EAIhC,IAAI,mBAHoB,cAAc,UAGI,mBAAmB,GAAG;GAC9D,SAAS,UAAU;GACnB,UAAU;IACR,OAAO;IACP,UAAU,YAAY;GACxB,CAAC;GACD;EACF;EAGA,IAAI,aAAa;EACjB,MAAM,cAA+B,CAAC;EAEtC,KAAK,IAAI,IAAI,GAAG,IAAI,gBAAgB,KAAK;GACvC,MAAM,UAAU,SAAS;GACzB,MAAM,WAAW,cAAc;GAG/B,IAAI,aAAa,SAAS;IACxB,YAAY,KAAK;IACjB;GACF;GAGA,IAAI,CAAC,YAAY,SAAS,WAAW,QAAQ,QAAQ;IACnD,YAAY,KAAK;IACjB,aAAa;IACb;GACF;GAGA,IAAI,cAAc;GAClB,KAAK,MAAM,CAAC,GAAG,aAAa,QAAQ,QAAQ,GAC1C,IAAI,SAAS,OAAO,UAAU;IAC5B,cAAc;IACd;GACF;GAGF,IAAI,aAAa;IACf,YAAY,KAAK;IACjB,aAAa;GACf,OACE,YAAY,KAAK;EAErB;EAGA,IAAI,YAAY;GACd,SAAS,UAAU;GACnB,UAAU;IACR,OAAO;IACP,UAAU,YAAY;GACxB,CAAC;EACH;CACF,CAAC;CAED,MAAM,eAAe,YAAY,OAAO,UAAkB,aAAa,UAAU;EAC/E,MAAM,YAAY,aAAa;EAC/B,IAAI,CAAC,WAAW;EAEhB,IAAI,YAAY;GACd,UAAU,MAAM;GAChB,gBAAgB,UAAU;EAC5B;EAEA,MAAM,eAAe,gBAAgB;EACrC,IAAI,QAAQ;EAGZ,IAFkB,CAAC,cAAc,SAAS,WAAW,YAAY,GAG/D,QAAQ,SAAS,MAAM,aAAa,MAAM;OACrC,IAAI,CAAC,YACV,UAAU,MAAM;EAGlB,gBAAgB,UAAU;EAE1B,IAAI,CAAC,OAAO;GAEV,MAAM,eAAe,UAAU;GAC/B,MAAM,iBAAiB,UAAU;GAGjC,IAFoB,aAAa,SAAS,eAAe,WAErC,GAAG;IACrB,sBAAsB,QAAQ,CAAC,CAAC,CAAC,CAAC;IAClC;GACF;GAGA,MAAM,eACJ,aAAa,WAAW,IACpB,iBACA,eAAe,WAAW,IACxB,eACA,CAAC,GAAG,cAAc,GAAG,cAAc;GAE3C,sBAAsB,QAAQ,mBAAmB,YAAY,CAAC;GAC9D;EACF;EAEA,IAAI;GACF,MAAM,UAAU,QAAQ,KAAK;GAE7B,MAAM,eAAe,UAAU;GAC/B,MAAM,iBAAiB,UAAU;GACjC,MAAM,eACJ,aAAa,WAAW,IACpB,iBACA,eAAe,WAAW,IACxB,eACA,CAAC,GAAG,cAAc,GAAG,cAAc;GAC3C,sBAAsB,QAAQ,mBAAmB,YAAY,CAAC;EAChE,SAAS,OAAO;GACd,QAAQ,MAAM,kCAAkC,KAAK;EACvD;CACF,GAAG,CAAC,CAAC;CAGL,MAAM,oBAAoB,OAAe,EAAE;CAE3C,gBAAgB;EACd,IAAI,CAAC,SAAS;GACZ,aAAa,SAAS,MAAM;GAC5B,aAAa,UAAU;GACvB,gBAAgB,UAAU;GAC1B,YAAY,UAAU,KAAA;GACtB,SAAS,UAAU,CAAC,CAAC,CAAC;GACtB,UAAU,KAAA,CAAS;GACnB,kBAAkB,UAAU;GAC5B;EACF;EAGA,MAAM,aAAa,GAAG,SAAS,GAAG;EAClC,IAAI,kBAAkB,YAAY,cAAc,aAAa,SAC3D;EAGF,IAAI,YAAY;EAEhB,CAAC,YAAY;GACX,MAAM,MAAM,MAAM;GAClB,IAAI,CAAC,OAAO,WAAW;GAEvB,IAAI;IAEF,IAAI,eAAsB,CAAC,KAAK;IAChC,IAAI,gBAAgB,UAAU,cAAc;KAC1C,MAAM,cAAc,aAAa;KACjC,IAAI,aACF,eAAe,CAAC,WAAkB;IAEtC;IAIA,MAAM,cAAc,MAAM,IAAI,wBAAwB;KACpD,OAAO,WAAW,CAAC,QAAQ,IAAI,CAAC,WAAW;KAC3C,QAAQ;IACV,CAAC;IAED,IAAI,CAAC,eAAe,WAAW;IAG/B,IAAI,kBAAkB,YAAY,YAAY;KAE5C,aAAa,SAAS,MAAM;KAE5B,MAAM,YAAY,IAAI,qBAAqB;MACzC;MACA,MAAM;MACN;KACF,CAAC;KAED,aAAa,UAAU;KACvB,kBAAkB,UAAU;KAC5B,gBAAgB,UAAU;KAC1B,SAAS,UAAU,CAAC,CAAC,CAAC;KAEtB,MAAM,YAAY,YAAY,SAAS,KAAK;KAC5C,YAAY,UAAU,eAAe,WAAW,IAAI,WAAW,EAAE;IACnE;IAEA,MAAM,cAAc,cAAc;IAClC,IAAI,aACF,MAAM,aAAa,aAAa,IAAI;SAEpC,sBAAsB,QAAQ,CAAC,CAAC,CAAC,CAAC;GAEtC,SAAS,OAAO;IACd,QAAQ,MAAM,gDAAgD,KAAK;IAEnE,kBAAkB,UAAU;GAC9B;EACF,EAAA,CAAG;EAEH,aAAa;GACX,YAAY;EAGd;CACF,GAAG;EAAC;EAAS;EAAU;EAAO;EAAc;CAAY,CAAC;CAGzD,gBAAgB;EACd,IAAI,CAAC,SAAS;EACd,IAAI,CAAC,aAAa,SAAS;EAE3B,MAAM,cAAc,cAAc;EAClC,aAAa,WAAW;CAC1B,GAAG;EAAC;EAAS;EAAU;CAAY,CAAC;CAEpC,OAAO;AACT;AAEA,MAAa,sBACX,MACA,EACE,UACA,OAAO,cACP,gBAEC;CAEH,MAAM,WAAW,QAAQ;CACzB,MAAM,QAAQ,YAAY,YAAA,CAAa,YAAY;CAGnD,MAAM,kBAAkB,cAAc,uBAAuB,IAAI,GAAG,CAAC,IAAI,CAAC;CAI1E,OAAO,wBAAwB,UAAU;EACvC,cAAc,EACZ,cAAcA,mBAChB;EACA,SAAS;EACT,UAAU;EACV,OARqB,gBAAgB;CASvC,CAAC;AACH"}