@lobehub/ui/es/hooks/useStreamHighlight.mjs.map

Lobe UI is an open-source UI component library for building AIGC web apps

{"version":3,"file":"useStreamHighlight.mjs","names":["lines: ThemedToken[][]","currentLine: ThemedToken[]","mergedLines: ThemedToken[][]","themesToLoad: any[]","lobeTheme"],"sources":["../../src/hooks/useStreamHighlight.ts"],"sourcesContent":["'use client';\n\nimport { CSSProperties, useCallback, useEffect, useMemo, useRef, useState } from 'react';\nimport type { BuiltinTheme, ThemedToken } from 'shiki';\nimport { ShikiStreamTokenizer } from 'shiki-stream';\n\nimport { getCodeLanguageByInput } from '@/Highlighter/const';\nimport lobeTheme from '@/Highlighter/theme/lobe-theme';\n\nimport { StreamingHighlightResult, shikiModulePromise } from './useHighlight';\n\ntype StreamingOptions = {\n  customThemes?: Record<string, any>;\n  enabled?: boolean;\n  language: string;\n  theme: string;\n};\n\n// Optimized version: reduce array allocations and object spreading\nconst tokensToLineTokens = (tokens: ThemedToken[]): ThemedToken[][] => {\n  if (!tokens.length) return [[]];\n\n  const lines: ThemedToken[][] = [];\n  let currentLine: ThemedToken[] = [];\n\n  for (const token of tokens) {\n    const content = token.content ?? '';\n\n    if (content === '\\n') {\n      lines.push(currentLine);\n      currentLine = [];\n      continue;\n    }\n\n    const newlineIndex = content.indexOf('\\n');\n    if (newlineIndex === -1) {\n      // No newline, add token directly\n      currentLine.push(token);\n    } else {\n      // Split on newlines\n      const segments = content.split('\\n');\n      for (const [j, segment] of segments.entries()) {\n        if (segment) {\n          // Only create new object if we need to modify content\n          currentLine.push(j === 0 && segment === content ? token : { ...token, content: segment });\n        }\n        if (j < segments.length - 1) {\n          lines.push(currentLine);\n          currentLine = [];\n        }\n      }\n    }\n  }\n\n  // Don't forget the last line\n  if (currentLine.length > 0 || lines.length === 0) {\n    lines.push(currentLine);\n  }\n\n  return lines.length > 0 ? lines : [[]];\n};\n\nconst createPreStyle = (bg?: string, fg?: string): CSSProperties | undefined => {\n  if (!bg && !fg) return undefined;\n  return {\n    backgroundColor: bg,\n    color: fg,\n  };\n};\n\nconst useStreamingHighlighter = (\n  text: string,\n  options: StreamingOptions,\n): StreamingHighlightResult | undefined => {\n  const { customThemes, enabled, language, theme } = options;\n  const [result, setResult] = useState<StreamingHighlightResult>();\n  const tokenizerRef = useRef<ShikiStreamTokenizer | null>(null);\n  const previousTextRef = useRef('');\n  const safeText = text ?? '';\n  const latestTextRef = useRef(safeText);\n  const preStyleRef = useRef<CSSProperties | undefined>(undefined);\n  const linesRef = useRef<ThemedToken[][]>([[]]);\n\n  useEffect(() => {\n    latestTextRef.current = safeText;\n  }, [safeText]);\n\n  // Use ref to store callback to avoid recreating it\n  const setStreamingResultRef = useRef((rawLines: ThemedToken[][]) => {\n    const previousLines = linesRef.current;\n    const newLinesLength = rawLines.length;\n    const prevLinesLength = previousLines.length;\n\n    // Fast path: if lengths differ or it's a complete reset, use new lines directly\n    if (newLinesLength !== prevLinesLength || newLinesLength === 0) {\n      linesRef.current = rawLines;\n      setResult({\n        lines: rawLines,\n        preStyle: preStyleRef.current,\n      });\n      return;\n    }\n\n    // Optimized comparison: only check changed lines\n    let hasChanges = false;\n    const mergedLines: ThemedToken[][] = [];\n\n    for (let i = 0; i < newLinesLength; i++) {\n      const newLine = rawLines[i];\n      const prevLine = previousLines[i];\n\n      // Quick reference equality check first\n      if (prevLine === newLine) {\n        mergedLines[i] = prevLine;\n        continue;\n      }\n\n      // Length check\n      if (!prevLine || prevLine.length !== newLine.length) {\n        mergedLines[i] = newLine;\n        hasChanges = true;\n        continue;\n      }\n\n      // Deep comparison only for lines that might have changed\n      let lineChanged = false;\n      for (const [j, newToken] of newLine.entries()) {\n        if (prevLine[j] !== newToken) {\n          lineChanged = true;\n          break;\n        }\n      }\n\n      if (lineChanged) {\n        mergedLines[i] = newLine;\n        hasChanges = true;\n      } else {\n        mergedLines[i] = prevLine;\n      }\n    }\n\n    // Only update state if there are actual changes\n    if (hasChanges) {\n      linesRef.current = mergedLines;\n      setResult({\n        lines: mergedLines,\n        preStyle: preStyleRef.current,\n      });\n    }\n  });\n\n  const updateTokens = useCallback(async (nextText: string, forceReset = false) => {\n    const tokenizer = tokenizerRef.current;\n    if (!tokenizer) return;\n\n    if (forceReset) {\n      tokenizer.clear();\n      previousTextRef.current = '';\n    }\n\n    const previousText = previousTextRef.current;\n    let chunk = nextText;\n    const canAppend = !forceReset && nextText.startsWith(previousText);\n\n    if (canAppend) {\n      chunk = nextText.slice(previousText.length);\n    } else if (!forceReset) {\n      tokenizer.clear();\n    }\n\n    previousTextRef.current = nextText;\n\n    if (!chunk) {\n      // Optimize: avoid array spread if possible\n      const stableTokens = tokenizer.tokensStable;\n      const unstableTokens = tokenizer.tokensUnstable;\n      const totalLength = stableTokens.length + unstableTokens.length;\n\n      if (totalLength === 0) {\n        setStreamingResultRef.current([[]]);\n        return;\n      }\n\n      // Only create merged array if we have both stable and unstable tokens\n      const mergedTokens =\n        stableTokens.length === 0\n          ? unstableTokens\n          : unstableTokens.length === 0\n            ? stableTokens\n            : [...stableTokens, ...unstableTokens];\n\n      setStreamingResultRef.current(tokensToLineTokens(mergedTokens));\n      return;\n    }\n\n    try {\n      await tokenizer.enqueue(chunk);\n      // Optimize: avoid array spread if possible\n      const stableTokens = tokenizer.tokensStable;\n      const unstableTokens = tokenizer.tokensUnstable;\n      const mergedTokens =\n        stableTokens.length === 0\n          ? unstableTokens\n          : unstableTokens.length === 0\n            ? stableTokens\n            : [...stableTokens, ...unstableTokens];\n      setStreamingResultRef.current(tokensToLineTokens(mergedTokens));\n    } catch (error) {\n      console.error('Streaming highlighting failed:', error);\n    }\n  }, []);\n\n  // Cache highlighter key to avoid unnecessary recreations\n  const highlighterKeyRef = useRef<string>('');\n\n  useEffect(() => {\n    if (!enabled) {\n      tokenizerRef.current?.clear();\n      tokenizerRef.current = null;\n      previousTextRef.current = '';\n      preStyleRef.current = undefined;\n      linesRef.current = [[]];\n      setResult(undefined);\n      highlighterKeyRef.current = '';\n      return;\n    }\n\n    // Skip if language/theme combination hasn't changed\n    const currentKey = `${language}-${theme}`;\n    if (highlighterKeyRef.current === currentKey && tokenizerRef.current) {\n      return;\n    }\n\n    let cancelled = false;\n\n    (async () => {\n      const mod = await shikiModulePromise;\n      if (!mod || cancelled) return;\n\n      try {\n        // Load custom theme if using slack-dark or slack-ochin\n        let themesToLoad: any[] = [theme];\n        if (customThemes && theme === 'lobe-theme') {\n          const customTheme = customThemes[theme];\n          if (customTheme) {\n            themesToLoad = [customTheme as any];\n          }\n        }\n\n        // Only load the specific language and theme needed\n        // getSingletonHighlighter will cache the instance internally\n        const highlighter = await mod.getSingletonHighlighter({\n          langs: language ? [language] : ['plaintext'],\n          themes: themesToLoad,\n        });\n\n        if (!highlighter || cancelled) return;\n\n        // Only create new tokenizer if key changed\n        if (highlighterKeyRef.current !== currentKey) {\n          // Clear old tokenizer\n          tokenizerRef.current?.clear();\n\n          const tokenizer = new ShikiStreamTokenizer({\n            highlighter,\n            lang: language,\n            theme,\n          });\n\n          tokenizerRef.current = tokenizer;\n          highlighterKeyRef.current = currentKey;\n          previousTextRef.current = '';\n          linesRef.current = [[]];\n\n          const themeInfo = highlighter.getTheme(theme);\n          preStyleRef.current = createPreStyle(themeInfo?.bg, themeInfo?.fg);\n        }\n\n        const currentText = latestTextRef.current;\n        if (currentText) {\n          await updateTokens(currentText, true);\n        } else {\n          setStreamingResultRef.current([[]]);\n        }\n      } catch (error) {\n        console.error('Streaming highlighter initialization failed:', error);\n        // Reset on error\n        highlighterKeyRef.current = '';\n      }\n    })();\n\n    return () => {\n      cancelled = true;\n      // Cleanup only if this effect was cancelled before completion\n      // The next effect will handle cleanup if key changed\n    };\n  }, [enabled, language, theme, updateTokens, customThemes]);\n\n  // Separate effect for text updates to avoid unnecessary tokenizer recreation\n  useEffect(() => {\n    if (!enabled) return;\n    if (!tokenizerRef.current) return;\n    // Use ref to get latest text to avoid stale closures\n    const currentText = latestTextRef.current;\n    updateTokens(currentText);\n  }, [enabled, safeText, updateTokens]);\n\n  return result;\n};\n\nexport const useStreamHighlight = (\n  text: string,\n  {\n    language,\n    theme: builtinTheme,\n    streaming,\n  }: { enableTransformer?: boolean; language: string; streaming?: boolean; theme?: BuiltinTheme },\n) => {\n  // Safely handle language and text with boundary checks\n  const safeText = text ?? '';\n  const lang = (language ?? 'plaintext').toLowerCase();\n\n  // Match supported languages\n  const matchedLanguage = useMemo(() => getCodeLanguageByInput(lang), [lang]);\n\n  const effectiveTheme = builtinTheme || 'lobe-theme';\n\n  return useStreamingHighlighter(safeText, {\n    customThemes: {\n      'lobe-theme': lobeTheme,\n    },\n    enabled: streaming,\n    language: matchedLanguage,\n    theme: effectiveTheme,\n  });\n};\n"],"mappings":";;;;;;;;;AAmBA,MAAM,sBAAsB,WAA2C;AACrE,KAAI,CAAC,OAAO,OAAQ,QAAO,CAAC,EAAE,CAAC;CAE/B,MAAMA,QAAyB,EAAE;CACjC,IAAIC,cAA6B,EAAE;AAEnC,MAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,UAAU,MAAM,WAAW;AAEjC,MAAI,YAAY,MAAM;AACpB,SAAM,KAAK,YAAY;AACvB,iBAAc,EAAE;AAChB;;AAIF,MADqB,QAAQ,QAAQ,KAAK,KACrB,GAEnB,aAAY,KAAK,MAAM;OAClB;GAEL,MAAM,WAAW,QAAQ,MAAM,KAAK;AACpC,QAAK,MAAM,CAAC,GAAG,YAAY,SAAS,SAAS,EAAE;AAC7C,QAAI,QAEF,aAAY,KAAK,MAAM,KAAK,YAAY,UAAU,QAAQ;KAAE,GAAG;KAAO,SAAS;KAAS,CAAC;AAE3F,QAAI,IAAI,SAAS,SAAS,GAAG;AAC3B,WAAM,KAAK,YAAY;AACvB,mBAAc,EAAE;;;;;AAOxB,KAAI,YAAY,SAAS,KAAK,MAAM,WAAW,EAC7C,OAAM,KAAK,YAAY;AAGzB,QAAO,MAAM,SAAS,IAAI,QAAQ,CAAC,EAAE,CAAC;;AAGxC,MAAM,kBAAkB,IAAa,OAA2C;AAC9E,KAAI,CAAC,MAAM,CAAC,GAAI,QAAO;AACvB,QAAO;EACL,iBAAiB;EACjB,OAAO;EACR;;AAGH,MAAM,2BACJ,MACA,YACyC;CACzC,MAAM,EAAE,cAAc,SAAS,UAAU,UAAU;CACnD,MAAM,CAAC,QAAQ,aAAa,UAAoC;CAChE,MAAM,eAAe,OAAoC,KAAK;CAC9D,MAAM,kBAAkB,OAAO,GAAG;CAClC,MAAM,WAAW,QAAQ;CACzB,MAAM,gBAAgB,OAAO,SAAS;CACtC,MAAM,cAAc,OAAkC,OAAU;CAChE,MAAM,WAAW,OAAwB,CAAC,EAAE,CAAC,CAAC;AAE9C,iBAAgB;AACd,gBAAc,UAAU;IACvB,CAAC,SAAS,CAAC;CAGd,MAAM,wBAAwB,QAAQ,aAA8B;EAClE,MAAM,gBAAgB,SAAS;EAC/B,MAAM,iBAAiB,SAAS;AAIhC,MAAI,mBAHoB,cAAc,UAGI,mBAAmB,GAAG;AAC9D,YAAS,UAAU;AACnB,aAAU;IACR,OAAO;IACP,UAAU,YAAY;IACvB,CAAC;AACF;;EAIF,IAAI,aAAa;EACjB,MAAMC,cAA+B,EAAE;AAEvC,OAAK,IAAI,IAAI,GAAG,IAAI,gBAAgB,KAAK;GACvC,MAAM,UAAU,SAAS;GACzB,MAAM,WAAW,cAAc;AAG/B,OAAI,aAAa,SAAS;AACxB,gBAAY,KAAK;AACjB;;AAIF,OAAI,CAAC,YAAY,SAAS,WAAW,QAAQ,QAAQ;AACnD,gBAAY,KAAK;AACjB,iBAAa;AACb;;GAIF,IAAI,cAAc;AAClB,QAAK,MAAM,CAAC,GAAG,aAAa,QAAQ,SAAS,CAC3C,KAAI,SAAS,OAAO,UAAU;AAC5B,kBAAc;AACd;;AAIJ,OAAI,aAAa;AACf,gBAAY,KAAK;AACjB,iBAAa;SAEb,aAAY,KAAK;;AAKrB,MAAI,YAAY;AACd,YAAS,UAAU;AACnB,aAAU;IACR,OAAO;IACP,UAAU,YAAY;IACvB,CAAC;;GAEJ;CAEF,MAAM,eAAe,YAAY,OAAO,UAAkB,aAAa,UAAU;EAC/E,MAAM,YAAY,aAAa;AAC/B,MAAI,CAAC,UAAW;AAEhB,MAAI,YAAY;AACd,aAAU,OAAO;AACjB,mBAAgB,UAAU;;EAG5B,MAAM,eAAe,gBAAgB;EACrC,IAAI,QAAQ;AAGZ,MAFkB,CAAC,cAAc,SAAS,WAAW,aAAa,CAGhE,SAAQ,SAAS,MAAM,aAAa,OAAO;WAClC,CAAC,WACV,WAAU,OAAO;AAGnB,kBAAgB,UAAU;AAE1B,MAAI,CAAC,OAAO;GAEV,MAAM,eAAe,UAAU;GAC/B,MAAM,iBAAiB,UAAU;AAGjC,OAFoB,aAAa,SAAS,eAAe,WAErC,GAAG;AACrB,0BAAsB,QAAQ,CAAC,EAAE,CAAC,CAAC;AACnC;;GAIF,MAAM,eACJ,aAAa,WAAW,IACpB,iBACA,eAAe,WAAW,IACxB,eACA,CAAC,GAAG,cAAc,GAAG,eAAe;AAE5C,yBAAsB,QAAQ,mBAAmB,aAAa,CAAC;AAC/D;;AAGF,MAAI;AACF,SAAM,UAAU,QAAQ,MAAM;GAE9B,MAAM,eAAe,UAAU;GAC/B,MAAM,iBAAiB,UAAU;GACjC,MAAM,eACJ,aAAa,WAAW,IACpB,iBACA,eAAe,WAAW,IACxB,eACA,CAAC,GAAG,cAAc,GAAG,eAAe;AAC5C,yBAAsB,QAAQ,mBAAmB,aAAa,CAAC;WACxD,OAAO;AACd,WAAQ,MAAM,kCAAkC,MAAM;;IAEvD,EAAE,CAAC;CAGN,MAAM,oBAAoB,OAAe,GAAG;AAE5C,iBAAgB;AACd,MAAI,CAAC,SAAS;AACZ,gBAAa,SAAS,OAAO;AAC7B,gBAAa,UAAU;AACvB,mBAAgB,UAAU;AAC1B,eAAY,UAAU;AACtB,YAAS,UAAU,CAAC,EAAE,CAAC;AACvB,aAAU,OAAU;AACpB,qBAAkB,UAAU;AAC5B;;EAIF,MAAM,aAAa,GAAG,SAAS,GAAG;AAClC,MAAI,kBAAkB,YAAY,cAAc,aAAa,QAC3D;EAGF,IAAI,YAAY;AAEhB,GAAC,YAAY;GACX,MAAM,MAAM,MAAM;AAClB,OAAI,CAAC,OAAO,UAAW;AAEvB,OAAI;IAEF,IAAIC,eAAsB,CAAC,MAAM;AACjC,QAAI,gBAAgB,UAAU,cAAc;KAC1C,MAAM,cAAc,aAAa;AACjC,SAAI,YACF,gBAAe,CAAC,YAAmB;;IAMvC,MAAM,cAAc,MAAM,IAAI,wBAAwB;KACpD,OAAO,WAAW,CAAC,SAAS,GAAG,CAAC,YAAY;KAC5C,QAAQ;KACT,CAAC;AAEF,QAAI,CAAC,eAAe,UAAW;AAG/B,QAAI,kBAAkB,YAAY,YAAY;AAE5C,kBAAa,SAAS,OAAO;AAQ7B,kBAAa,UANK,IAAI,qBAAqB;MACzC;MACA,MAAM;MACN;MACD,CAAC;AAGF,uBAAkB,UAAU;AAC5B,qBAAgB,UAAU;AAC1B,cAAS,UAAU,CAAC,EAAE,CAAC;KAEvB,MAAM,YAAY,YAAY,SAAS,MAAM;AAC7C,iBAAY,UAAU,eAAe,WAAW,IAAI,WAAW,GAAG;;IAGpE,MAAM,cAAc,cAAc;AAClC,QAAI,YACF,OAAM,aAAa,aAAa,KAAK;QAErC,uBAAsB,QAAQ,CAAC,EAAE,CAAC,CAAC;YAE9B,OAAO;AACd,YAAQ,MAAM,gDAAgD,MAAM;AAEpE,sBAAkB,UAAU;;MAE5B;AAEJ,eAAa;AACX,eAAY;;IAIb;EAAC;EAAS;EAAU;EAAO;EAAc;EAAa,CAAC;AAG1D,iBAAgB;AACd,MAAI,CAAC,QAAS;AACd,MAAI,CAAC,aAAa,QAAS;EAE3B,MAAM,cAAc,cAAc;AAClC,eAAa,YAAY;IACxB;EAAC;EAAS;EAAU;EAAa,CAAC;AAErC,QAAO;;AAGT,MAAa,sBACX,MACA,EACE,UACA,OAAO,cACP,gBAEC;CAEH,MAAM,WAAW,QAAQ;CACzB,MAAM,QAAQ,YAAY,aAAa,aAAa;CAGpD,MAAM,kBAAkB,cAAc,uBAAuB,KAAK,EAAE,CAAC,KAAK,CAAC;CAE3E,MAAM,iBAAiB,gBAAgB;AAEvC,QAAO,wBAAwB,UAAU;EACvC,cAAc,EACZ,cAAcC,oBACf;EACD,SAAS;EACT,UAAU;EACV,OAAO;EACR,CAAC"}