@shikijs/core
Version:
Core of Shiki
1,407 lines (1,406 loc) • 49.4 kB
JavaScript
import { ShikiError as ShikiError$1 } from "@shikijs/types";
import { ShikiError, applyColorReplacements, codeToTokensBase as codeToTokensBase$1, codeToTokensWithThemes, codeToTokensWithThemes as codeToTokensWithThemes$1, createShikiInternal, createShikiInternalSync, createShikiPrimitive, createShikiPrimitive as createShikiPrimitive$1, createShikiPrimitiveAsync, createShikiPrimitiveAsync as createShikiPrimitiveAsync$1, getLastGrammarState, getLastGrammarStateFromMap, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, normalizeGetter, normalizeTheme, resolveColorReplacements, setLastGrammarStateToMap, splitLines, splitLines as splitLines$1, toArray, tokenizeWithTheme } from "@shikijs/primitive";
import { FontStyle } from "@shikijs/vscode-textmate";
import { toHtml } from "hast-util-to-html";
export * from "@shikijs/types";
//#region src/utils/hast.ts
const RE_WHITESPACE = /\s+/g;
/**
* Utility to append class to a hast node
*
* If the `property.class` is a string, it will be splitted by space and converted to an array.
*/
function addClassToHast(node, className) {
if (!className) return node;
node.properties ||= {};
node.properties.class ||= [];
if (typeof node.properties.class === "string") node.properties.class = node.properties.class.split(RE_WHITESPACE);
if (!Array.isArray(node.properties.class)) node.properties.class = [];
const targets = Array.isArray(className) ? className : className.split(RE_WHITESPACE);
for (const c of targets) if (c && !node.properties.class.includes(c)) node.properties.class.push(c);
return node;
}
//#endregion
//#region src/utils/strings.ts
const RE_LANG_ATTR = /:?lang=["']([^"']+)["']/g;
const RE_CODE_FENCE = /(?:```|~~~)([\w-]+)/g;
const RE_LATEX_BEGIN = /\\begin\{([\w-]+)\}/g;
const RE_SCRIPT_LANG = /<script\s+(?:type|lang)=["']([^"']+)["']/gi;
/**
* Creates a converter between index and position in a code block.
*
* Overflow/underflow are unchecked.
*/
function createPositionConverter(code) {
const lines = splitLines$1(code, true).map(([line]) => line);
function indexToPos(index) {
if (index === code.length) return {
line: lines.length - 1,
character: lines.at(-1).length
};
let character = index;
let line = 0;
for (const lineText of lines) {
if (character < lineText.length) break;
character -= lineText.length;
line++;
}
return {
line,
character
};
}
function posToIndex(line, character) {
let index = 0;
for (let i = 0; i < line; i++) index += lines[i].length;
index += character;
return index;
}
return {
lines,
indexToPos,
posToIndex
};
}
/**
* Guess embedded languages from given code and highlighter.
*
* When highlighter is provided, only bundled languages will be included.
*
* @param code - The code string to analyze
* @param _lang - The primary language of the code (currently unused)
* @param highlighter - Optional highlighter instance to validate languages
* @returns Array of detected language identifiers
*
* @example
* ```ts
* // Detects 'javascript' from Vue SFC
* guessEmbeddedLanguages('<script lang="javascript">')
*
* // Detects 'python' from markdown code block
* guessEmbeddedLanguages('```python\nprint("hi")\n```')
* ```
*/
function guessEmbeddedLanguages(code, _lang, highlighter) {
const langs = /* @__PURE__ */ new Set();
for (const match of code.matchAll(RE_LANG_ATTR)) {
const lang = match[1].toLowerCase().trim();
if (lang) langs.add(lang);
}
for (const match of code.matchAll(RE_CODE_FENCE)) {
const lang = match[1].toLowerCase().trim();
if (lang) langs.add(lang);
}
for (const match of code.matchAll(RE_LATEX_BEGIN)) {
const lang = match[1].toLowerCase().trim();
if (lang) langs.add(lang);
}
for (const match of code.matchAll(RE_SCRIPT_LANG)) {
const fullType = match[1].toLowerCase().trim();
const lang = fullType.includes("/") ? fullType.split("/").pop() : fullType;
if (lang) langs.add(lang);
}
if (!highlighter) return [...langs];
const bundle = highlighter.getBundledLanguages();
return [...langs].filter((l) => l && bundle[l]);
}
const COLOR_KEYS = ["color", "background-color"];
//#endregion
//#region src/utils/tokens.ts
/**
* Split a token into multiple tokens by given offsets.
*
* The offsets are relative to the token, and should be sorted.
*/
function splitToken(token, offsets) {
let lastOffset = 0;
const tokens = [];
for (const offset of offsets) {
if (offset > lastOffset) tokens.push({
...token,
content: token.content.slice(lastOffset, offset),
offset: token.offset + lastOffset
});
lastOffset = offset;
}
if (lastOffset < token.content.length) tokens.push({
...token,
content: token.content.slice(lastOffset),
offset: token.offset + lastOffset
});
return tokens;
}
/**
* Split 2D tokens array by given breakpoints.
*/
function splitTokens(tokens, breakpoints) {
const sorted = [...breakpoints instanceof Set ? breakpoints : new Set(breakpoints)].sort((a, b) => a - b);
if (!sorted.length) return tokens;
return tokens.map((line) => {
return line.flatMap((token) => {
const breakpointsInToken = sorted.filter((i) => token.offset < i && i < token.offset + token.content.length).map((i) => i - token.offset).sort((a, b) => a - b);
if (!breakpointsInToken.length) return token;
return splitToken(token, breakpointsInToken);
});
});
}
function flatTokenVariants(merged, variantsOrder, cssVariablePrefix, defaultColor, colorsRendering = "css-vars") {
const token = {
content: merged.content,
explanation: merged.explanation,
offset: merged.offset
};
const styles = variantsOrder.map((t) => getTokenStyleObject(merged.variants[t]));
const styleKeys = new Set(styles.flatMap((t) => Object.keys(t)));
const mergedStyles = {};
const varKey = (idx, key) => {
const keyName = key === "color" ? "" : key === "background-color" ? "-bg" : `-${key}`;
return cssVariablePrefix + variantsOrder[idx] + (key === "color" ? "" : keyName);
};
styles.forEach((cur, idx) => {
for (const key of styleKeys) {
const value = cur[key] || "inherit";
if (idx === 0 && defaultColor && COLOR_KEYS.includes(key)) if (defaultColor === "light-dark()" && styles.length > 1) {
const lightIndex = variantsOrder.findIndex((t) => t === "light");
const darkIndex = variantsOrder.findIndex((t) => t === "dark");
if (lightIndex === -1 || darkIndex === -1) throw new ShikiError$1("When using `defaultColor: \"light-dark()\"`, you must provide both `light` and `dark` themes");
mergedStyles[key] = `light-dark(${styles[lightIndex][key] || "inherit"}, ${styles[darkIndex][key] || "inherit"})`;
if (colorsRendering === "css-vars") mergedStyles[varKey(idx, key)] = value;
} else mergedStyles[key] = value;
else if (colorsRendering === "css-vars") mergedStyles[varKey(idx, key)] = value;
}
});
token.htmlStyle = mergedStyles;
return token;
}
function getTokenStyleObject(token) {
const styles = {};
if (token.color) styles.color = token.color;
if (token.bgColor) styles["background-color"] = token.bgColor;
if (token.fontStyle) {
if (token.fontStyle & FontStyle.Italic) styles["font-style"] = "italic";
if (token.fontStyle & FontStyle.Bold) styles["font-weight"] = "bold";
const decorations = [];
if (token.fontStyle & FontStyle.Underline) decorations.push("underline");
if (token.fontStyle & FontStyle.Strikethrough) decorations.push("line-through");
if (decorations.length) styles["text-decoration"] = decorations.join(" ");
}
return styles;
}
function stringifyTokenStyle(token) {
if (typeof token === "string") return token;
return Object.entries(token).map(([key, value]) => `${key}:${value}`).join(";");
}
//#endregion
//#region src/transformer-decorations.ts
/**
* A built-in transformer to add decorations to the highlighted code.
*/
function transformerDecorations() {
const map = /* @__PURE__ */ new WeakMap();
function getContext(shiki) {
if (!map.has(shiki.meta)) {
const converter = createPositionConverter(shiki.source);
function normalizePosition(p) {
if (typeof p === "number") {
if (p < 0 || p > shiki.source.length) throw new ShikiError$1(`Invalid decoration offset: ${p}. Code length: ${shiki.source.length}`);
return {
...converter.indexToPos(p),
offset: p
};
} else {
const line = converter.lines[p.line];
if (line === void 0) throw new ShikiError$1(`Invalid decoration position ${JSON.stringify(p)}. Lines length: ${converter.lines.length}`);
let character = p.character;
if (character < 0) character = line.length + character;
if (character < 0 || character > line.length) throw new ShikiError$1(`Invalid decoration position ${JSON.stringify(p)}. Line ${p.line} length: ${line.length}`);
return {
...p,
character,
offset: converter.posToIndex(p.line, character)
};
}
}
const decorations = (shiki.options.decorations || []).map((d) => ({
...d,
start: normalizePosition(d.start),
end: normalizePosition(d.end)
}));
verifyIntersections(decorations);
map.set(shiki.meta, {
decorations,
converter,
source: shiki.source
});
}
return map.get(shiki.meta);
}
return {
name: "shiki:decorations",
tokens(tokens) {
if (!this.options.decorations?.length) return;
return splitTokens(tokens, getContext(this).decorations.flatMap((d) => [d.start.offset, d.end.offset]));
},
code(codeEl) {
if (!this.options.decorations?.length) return;
const ctx = getContext(this);
const lines = [...codeEl.children].filter((i) => i.type === "element" && i.tagName === "span");
if (lines.length !== ctx.converter.lines.length) throw new ShikiError$1(`Number of lines in code element (${lines.length}) does not match the number of lines in the source (${ctx.converter.lines.length}). Failed to apply decorations.`);
function applyLineSection(line, start, end, decoration) {
const lineEl = lines[line];
let text = "";
let startIndex = -1;
let endIndex = -1;
if (start === 0) startIndex = 0;
if (end === 0) endIndex = 0;
if (end === Number.POSITIVE_INFINITY) endIndex = lineEl.children.length;
if (startIndex === -1 || endIndex === -1) for (let i = 0; i < lineEl.children.length; i++) {
text += stringify(lineEl.children[i]);
if (startIndex === -1 && text.length === start) startIndex = i + 1;
if (endIndex === -1 && text.length === end) endIndex = i + 1;
}
if (startIndex === -1) throw new ShikiError$1(`Failed to find start index for decoration ${JSON.stringify(decoration.start)}`);
if (endIndex === -1) throw new ShikiError$1(`Failed to find end index for decoration ${JSON.stringify(decoration.end)}`);
const children = lineEl.children.slice(startIndex, endIndex);
if (!decoration.alwaysWrap && children.length === lineEl.children.length) applyDecoration(lineEl, decoration, "line");
else if (!decoration.alwaysWrap && children.length === 1 && children[0].type === "element") applyDecoration(children[0], decoration, "token");
else {
const wrapper = {
type: "element",
tagName: "span",
properties: {},
children
};
applyDecoration(wrapper, decoration, "wrapper");
lineEl.children.splice(startIndex, children.length, wrapper);
}
}
function applyLine(line, decoration) {
lines[line] = applyDecoration(lines[line], decoration, "line");
}
function applyDecoration(el, decoration, type) {
const properties = decoration.properties || {};
const transform = decoration.transform || ((i) => i);
el.tagName = decoration.tagName || "span";
el.properties = {
...el.properties,
...properties,
class: el.properties.class
};
if (decoration.properties?.class) addClassToHast(el, decoration.properties.class);
el = transform(el, type) || el;
return el;
}
const lineApplies = [];
const sorted = ctx.decorations.sort((a, b) => b.start.offset - a.start.offset || a.end.offset - b.end.offset);
for (const decoration of sorted) {
const { start, end } = decoration;
if (start.line === end.line) applyLineSection(start.line, start.character, end.character, decoration);
else if (start.line < end.line) {
applyLineSection(start.line, start.character, Number.POSITIVE_INFINITY, decoration);
for (let i = start.line + 1; i < end.line; i++) lineApplies.unshift(() => applyLine(i, decoration));
applyLineSection(end.line, 0, end.character, decoration);
}
}
lineApplies.forEach((i) => i());
}
};
}
function verifyIntersections(items) {
for (let i = 0; i < items.length; i++) {
const foo = items[i];
if (foo.start.offset > foo.end.offset) throw new ShikiError$1(`Invalid decoration range: ${JSON.stringify(foo.start)} - ${JSON.stringify(foo.end)}`);
for (let j = i + 1; j < items.length; j++) {
const bar = items[j];
const isFooHasBarStart = foo.start.offset <= bar.start.offset && bar.start.offset < foo.end.offset;
const isFooHasBarEnd = foo.start.offset < bar.end.offset && bar.end.offset <= foo.end.offset;
const isBarHasFooStart = bar.start.offset <= foo.start.offset && foo.start.offset < bar.end.offset;
const isBarHasFooEnd = bar.start.offset < foo.end.offset && foo.end.offset <= bar.end.offset;
if (isFooHasBarStart || isFooHasBarEnd || isBarHasFooStart || isBarHasFooEnd) {
if (isFooHasBarStart && isFooHasBarEnd) continue;
if (isBarHasFooStart && isBarHasFooEnd) continue;
if (isBarHasFooStart && foo.start.offset === foo.end.offset) continue;
if (isFooHasBarEnd && bar.start.offset === bar.end.offset) continue;
throw new ShikiError$1(`Decorations ${JSON.stringify(foo.start)} and ${JSON.stringify(bar.start)} intersect.`);
}
}
}
}
function stringify(el) {
if (el.type === "text") return el.value;
if (el.type === "element") return el.children.map(stringify).join("");
return "";
}
//#endregion
//#region src/highlight/_get-transformers.ts
const builtInTransformers = [/* @__PURE__ */ transformerDecorations()];
function getTransformers(options) {
const transformers = sortTransformersByEnforcement(options.transformers || []);
return [
...transformers.pre,
...transformers.normal,
...transformers.post,
...builtInTransformers
];
}
function sortTransformersByEnforcement(transformers) {
const pre = [];
const post = [];
const normal = [];
for (const transformer of transformers) switch (transformer.enforce) {
case "pre":
pre.push(transformer);
break;
case "post":
post.push(transformer);
break;
default: normal.push(transformer);
}
return {
pre,
post,
normal
};
}
//#endregion
//#region ../../node_modules/.pnpm/ansi-sequence-parser@1.1.3/node_modules/ansi-sequence-parser/dist/index.js
var namedColors = [
"black",
"red",
"green",
"yellow",
"blue",
"magenta",
"cyan",
"white",
"brightBlack",
"brightRed",
"brightGreen",
"brightYellow",
"brightBlue",
"brightMagenta",
"brightCyan",
"brightWhite"
];
var decorations = {
1: "bold",
2: "dim",
3: "italic",
4: "underline",
7: "reverse",
8: "hidden",
9: "strikethrough"
};
function findSequence(value, position) {
const nextEscape = value.indexOf("\x1B", position);
if (nextEscape !== -1) {
if (value[nextEscape + 1] === "[") {
const nextClose = value.indexOf("m", nextEscape);
if (nextClose !== -1) return {
sequence: value.substring(nextEscape + 2, nextClose).split(";"),
startPosition: nextEscape,
position: nextClose + 1
};
}
}
return { position: value.length };
}
function parseColor(sequence) {
const colorMode = sequence.shift();
if (colorMode === "2") {
const rgb = sequence.splice(0, 3).map((x) => Number.parseInt(x));
if (rgb.length !== 3 || rgb.some((x) => Number.isNaN(x))) return;
return {
type: "rgb",
rgb
};
} else if (colorMode === "5") {
const index = sequence.shift();
if (index) return {
type: "table",
index: Number(index)
};
}
}
function parseSequence(sequence) {
const commands = [];
while (sequence.length > 0) {
const code = sequence.shift();
if (!code) continue;
const codeInt = Number.parseInt(code);
if (Number.isNaN(codeInt)) continue;
if (codeInt === 0) commands.push({ type: "resetAll" });
else if (codeInt <= 9) {
if (decorations[codeInt]) commands.push({
type: "setDecoration",
value: decorations[codeInt]
});
} else if (codeInt <= 29) {
const decoration = decorations[codeInt - 20];
if (decoration) {
commands.push({
type: "resetDecoration",
value: decoration
});
if (decoration === "dim") commands.push({
type: "resetDecoration",
value: "bold"
});
}
} else if (codeInt <= 37) commands.push({
type: "setForegroundColor",
value: {
type: "named",
name: namedColors[codeInt - 30]
}
});
else if (codeInt === 38) {
const color = parseColor(sequence);
if (color) commands.push({
type: "setForegroundColor",
value: color
});
} else if (codeInt === 39) commands.push({ type: "resetForegroundColor" });
else if (codeInt <= 47) commands.push({
type: "setBackgroundColor",
value: {
type: "named",
name: namedColors[codeInt - 40]
}
});
else if (codeInt === 48) {
const color = parseColor(sequence);
if (color) commands.push({
type: "setBackgroundColor",
value: color
});
} else if (codeInt === 49) commands.push({ type: "resetBackgroundColor" });
else if (codeInt === 53) commands.push({
type: "setDecoration",
value: "overline"
});
else if (codeInt === 55) commands.push({
type: "resetDecoration",
value: "overline"
});
else if (codeInt >= 90 && codeInt <= 97) commands.push({
type: "setForegroundColor",
value: {
type: "named",
name: namedColors[codeInt - 90 + 8]
}
});
else if (codeInt >= 100 && codeInt <= 107) commands.push({
type: "setBackgroundColor",
value: {
type: "named",
name: namedColors[codeInt - 100 + 8]
}
});
}
return commands;
}
function createAnsiSequenceParser() {
let foreground = null;
let background = null;
let decorations2 = /* @__PURE__ */ new Set();
return { parse(value) {
const tokens = [];
let position = 0;
do {
const findResult = findSequence(value, position);
const text = findResult.sequence ? value.substring(position, findResult.startPosition) : value.substring(position);
if (text.length > 0) tokens.push({
value: text,
foreground,
background,
decorations: new Set(decorations2)
});
if (findResult.sequence) {
const commands = parseSequence(findResult.sequence);
for (const styleToken of commands) if (styleToken.type === "resetAll") {
foreground = null;
background = null;
decorations2.clear();
} else if (styleToken.type === "resetForegroundColor") foreground = null;
else if (styleToken.type === "resetBackgroundColor") background = null;
else if (styleToken.type === "resetDecoration") decorations2.delete(styleToken.value);
for (const styleToken of commands) if (styleToken.type === "setForegroundColor") foreground = styleToken.value;
else if (styleToken.type === "setBackgroundColor") background = styleToken.value;
else if (styleToken.type === "setDecoration") decorations2.add(styleToken.value);
}
position = findResult.position;
} while (position < value.length);
return tokens;
} };
}
var defaultNamedColorsMap = {
black: "#000000",
red: "#bb0000",
green: "#00bb00",
yellow: "#bbbb00",
blue: "#0000bb",
magenta: "#ff00ff",
cyan: "#00bbbb",
white: "#eeeeee",
brightBlack: "#555555",
brightRed: "#ff5555",
brightGreen: "#00ff00",
brightYellow: "#ffff55",
brightBlue: "#5555ff",
brightMagenta: "#ff55ff",
brightCyan: "#55ffff",
brightWhite: "#ffffff"
};
function createColorPalette(namedColorsMap = defaultNamedColorsMap) {
function namedColor(name) {
return namedColorsMap[name];
}
function rgbColor(rgb) {
return `#${rgb.map((x) => Math.max(0, Math.min(x, 255)).toString(16).padStart(2, "0")).join("")}`;
}
let colorTable;
function getColorTable() {
if (colorTable) return colorTable;
colorTable = [];
for (let i = 0; i < namedColors.length; i++) colorTable.push(namedColor(namedColors[i]));
let levels = [
0,
95,
135,
175,
215,
255
];
for (let r = 0; r < 6; r++) for (let g = 0; g < 6; g++) for (let b = 0; b < 6; b++) colorTable.push(rgbColor([
levels[r],
levels[g],
levels[b]
]));
let level = 8;
for (let i = 0; i < 24; i++, level += 10) colorTable.push(rgbColor([
level,
level,
level
]));
return colorTable;
}
function tableColor(index) {
return getColorTable()[index];
}
function value(color) {
switch (color.type) {
case "named": return namedColor(color.name);
case "rgb": return rgbColor(color.rgb);
case "table": return tableColor(color.index);
}
}
return { value };
}
//#endregion
//#region src/highlight/code-to-tokens-ansi.ts
const RE_HEX_COLOR = /#([0-9a-f]{3,8})/i;
const RE_CSS_VAR_ANSI = /var\((--[\w-]+-ansi-[\w-]+)\)/;
/**
* Default ANSI palette (VSCode compatible fallbacks)
* Used when the theme does not define terminal.ansi* colors.
*/
const defaultAnsiColors = {
black: "#000000",
red: "#cd3131",
green: "#0DBC79",
yellow: "#E5E510",
blue: "#2472C8",
magenta: "#BC3FBC",
cyan: "#11A8CD",
white: "#E5E5E5",
brightBlack: "#666666",
brightRed: "#F14C4C",
brightGreen: "#23D18B",
brightYellow: "#F5F543",
brightBlue: "#3B8EEA",
brightMagenta: "#D670D6",
brightCyan: "#29B8DB",
brightWhite: "#FFFFFF"
};
function tokenizeAnsiWithTheme(theme, fileContents, options) {
const colorReplacements = resolveColorReplacements(theme, options);
const lines = splitLines(fileContents);
const colorPalette = createColorPalette(Object.fromEntries(namedColors.map((name) => {
const key = `terminal.ansi${name[0].toUpperCase()}${name.substring(1)}`;
return [name, theme.colors?.[key] || defaultAnsiColors[name]];
})));
const parser = createAnsiSequenceParser();
return lines.map((line) => parser.parse(line[0]).map((token) => {
let color;
let bgColor;
if (token.decorations.has("reverse")) {
color = token.background ? colorPalette.value(token.background) : theme.bg;
bgColor = token.foreground ? colorPalette.value(token.foreground) : theme.fg;
} else {
color = token.foreground ? colorPalette.value(token.foreground) : theme.fg;
bgColor = token.background ? colorPalette.value(token.background) : void 0;
}
color = applyColorReplacements(color, colorReplacements);
bgColor = applyColorReplacements(bgColor, colorReplacements);
if (token.decorations.has("dim")) color = dimColor(color);
let fontStyle = FontStyle.None;
if (token.decorations.has("bold")) fontStyle |= FontStyle.Bold;
if (token.decorations.has("italic")) fontStyle |= FontStyle.Italic;
if (token.decorations.has("underline")) fontStyle |= FontStyle.Underline;
if (token.decorations.has("strikethrough")) fontStyle |= FontStyle.Strikethrough;
return {
content: token.value,
offset: line[1],
color,
bgColor,
fontStyle
};
}));
}
/**
* Adds 50% alpha to a hex color string or the "-dim" postfix to a CSS variable
*/
function dimColor(color) {
const hexMatch = color.match(RE_HEX_COLOR);
if (hexMatch) {
const hex = hexMatch[1];
if (hex.length === 8) {
const alpha = Math.round(Number.parseInt(hex.slice(6, 8), 16) / 2).toString(16).padStart(2, "0");
return `#${hex.slice(0, 6)}${alpha}`;
} else if (hex.length === 6) return `#${hex}80`;
else if (hex.length === 4) {
const r = hex[0];
const g = hex[1];
const b = hex[2];
const a = hex[3];
return `#${r}${r}${g}${g}${b}${b}${Math.round(Number.parseInt(`${a}${a}`, 16) / 2).toString(16).padStart(2, "0")}`;
} else if (hex.length === 3) {
const r = hex[0];
const g = hex[1];
const b = hex[2];
return `#${r}${r}${g}${g}${b}${b}80`;
}
}
const cssVarMatch = color.match(RE_CSS_VAR_ANSI);
if (cssVarMatch) return `var(${cssVarMatch[1]}-dim)`;
return color;
}
//#endregion
//#region src/highlight/code-to-tokens-base.ts
/**
* Code to tokens, with a simple theme.
* This wraps the tokenizer's implementation to add ANSI support.
*/
function codeToTokensBase(primitive, code, options = {}) {
const lang = primitive.resolveLangAlias(options.lang || "text");
const { theme: themeName = primitive.getLoadedThemes()[0] } = options;
if (!isPlainLang(lang) && !isNoneTheme(themeName) && lang === "ansi") {
const { theme } = primitive.setTheme(themeName);
return tokenizeAnsiWithTheme(theme, code, options);
}
return codeToTokensBase$1(primitive, code, options);
}
//#endregion
//#region src/highlight/code-to-tokens.ts
/**
* High-level code-to-tokens API.
*
* It will use `codeToTokensWithThemes` or `codeToTokensBase` based on the options.
*/
function codeToTokens(primitive, code, options) {
let bg;
let fg;
let tokens;
let themeName;
let rootStyle;
let grammarState;
if ("themes" in options) {
const { defaultColor = "light", cssVariablePrefix = "--shiki-", colorsRendering = "css-vars" } = options;
const themes = Object.entries(options.themes).filter((i) => i[1]).map((i) => ({
color: i[0],
theme: i[1]
})).sort((a, b) => a.color === defaultColor ? -1 : b.color === defaultColor ? 1 : 0);
if (themes.length === 0) throw new ShikiError$1("`themes` option must not be empty");
const themeTokens = codeToTokensWithThemes$1(primitive, code, options, codeToTokensBase);
grammarState = getLastGrammarStateFromMap(themeTokens);
if (defaultColor && "light-dark()" !== defaultColor && !themes.some((t) => t.color === defaultColor)) throw new ShikiError$1(`\`themes\` option must contain the defaultColor key \`${defaultColor}\``);
const themeRegs = themes.map((t) => primitive.getTheme(t.theme));
const themesOrder = themes.map((t) => t.color);
tokens = themeTokens.map((line) => line.map((token) => flatTokenVariants(token, themesOrder, cssVariablePrefix, defaultColor, colorsRendering)));
if (grammarState) setLastGrammarStateToMap(tokens, grammarState);
const themeColorReplacements = themes.map((t) => resolveColorReplacements(t.theme, options));
fg = mapThemeColors(themes, themeRegs, themeColorReplacements, cssVariablePrefix, defaultColor, "fg", colorsRendering);
bg = mapThemeColors(themes, themeRegs, themeColorReplacements, cssVariablePrefix, defaultColor, "bg", colorsRendering);
themeName = `shiki-themes ${themeRegs.map((t) => t.name).join(" ")}`;
rootStyle = defaultColor ? void 0 : [fg, bg].join(";");
} else if ("theme" in options) {
const colorReplacements = resolveColorReplacements(options.theme, options);
tokens = codeToTokensBase(primitive, code, options);
const _theme = primitive.getTheme(options.theme);
bg = applyColorReplacements(_theme.bg, colorReplacements);
fg = applyColorReplacements(_theme.fg, colorReplacements);
themeName = _theme.name;
grammarState = getLastGrammarStateFromMap(tokens);
} else throw new ShikiError$1("Invalid options, either `theme` or `themes` must be provided");
return {
tokens,
fg,
bg,
themeName,
rootStyle,
grammarState
};
}
function mapThemeColors(themes, themeRegs, themeColorReplacements, cssVariablePrefix, defaultColor, property, colorsRendering) {
return themes.map((t, idx) => {
const value = applyColorReplacements(themeRegs[idx][property], themeColorReplacements[idx]) || "inherit";
const cssVar = `${cssVariablePrefix + t.color}${property === "bg" ? "-bg" : ""}:${value}`;
if (idx === 0 && defaultColor) {
if (defaultColor === "light-dark()" && themes.length > 1) {
const lightIndex = themes.findIndex((t) => t.color === "light");
const darkIndex = themes.findIndex((t) => t.color === "dark");
if (lightIndex === -1 || darkIndex === -1) throw new ShikiError$1("When using `defaultColor: \"light-dark()\"`, you must provide both `light` and `dark` themes");
return `light-dark(${applyColorReplacements(themeRegs[lightIndex][property], themeColorReplacements[lightIndex]) || "inherit"}, ${applyColorReplacements(themeRegs[darkIndex][property], themeColorReplacements[darkIndex]) || "inherit"});${cssVar}`;
}
return value;
}
if (colorsRendering === "css-vars") return cssVar;
return null;
}).filter((i) => !!i).join(";");
}
//#endregion
//#region src/highlight/code-to-hast.ts
const RE_WHITESPACE_ONLY = /^\s+$/;
const RE_LEADING_TRAILING_WHITESPACE = /^(\s*)(.*?)(\s*)$/;
function codeToHast(primitive, code, options, transformerContext = {
meta: {},
options,
codeToHast: (_code, _options) => codeToHast(primitive, _code, _options),
codeToTokens: (_code, _options) => codeToTokens(primitive, _code, _options)
}) {
let input = code;
for (const transformer of getTransformers(options)) input = transformer.preprocess?.call(transformerContext, input, options) || input;
let { tokens, fg, bg, themeName, rootStyle, grammarState } = codeToTokens(primitive, input, options);
const { mergeWhitespaces = true, mergeSameStyleTokens = false } = options;
if (mergeWhitespaces === true) tokens = mergeWhitespaceTokens(tokens);
else if (mergeWhitespaces === "never") tokens = splitWhitespaceTokens(tokens);
if (mergeSameStyleTokens) tokens = mergeAdjacentStyledTokens(tokens);
const contextSource = {
...transformerContext,
get source() {
return input;
}
};
for (const transformer of getTransformers(options)) tokens = transformer.tokens?.call(contextSource, tokens) || tokens;
return tokensToHast(tokens, {
...options,
fg,
bg,
themeName,
rootStyle: options.rootStyle === false ? false : options.rootStyle ?? rootStyle
}, contextSource, grammarState);
}
function tokensToHast(tokens, options, transformerContext, grammarState = getLastGrammarStateFromMap(tokens)) {
const transformers = getTransformers(options);
const lines = [];
const root = {
type: "root",
children: []
};
const { structure = "classic", tabindex = "0" } = options;
const properties = { class: `shiki ${options.themeName || ""}` };
if (options.rootStyle !== false) if (options.rootStyle != null) properties.style = options.rootStyle;
else properties.style = `background-color:${options.bg};color:${options.fg}`;
if (tabindex !== false && tabindex != null) properties.tabindex = tabindex.toString();
for (const [key, value] of Object.entries(options.meta || {})) if (!key.startsWith("_")) properties[key] = value;
let preNode = {
type: "element",
tagName: "pre",
properties,
children: [],
data: options.data
};
let codeNode = {
type: "element",
tagName: "code",
properties: {},
children: lines
};
const lineNodes = [];
const context = {
...transformerContext,
structure,
addClassToHast,
get source() {
return transformerContext.source;
},
get tokens() {
return tokens;
},
get options() {
return options;
},
get root() {
return root;
},
get pre() {
return preNode;
},
get code() {
return codeNode;
},
get lines() {
return lineNodes;
}
};
tokens.forEach((line, idx) => {
if (idx) {
if (structure === "inline") root.children.push({
type: "element",
tagName: "br",
properties: {},
children: []
});
else if (structure === "classic") lines.push({
type: "text",
value: "\n"
});
}
let lineNode = {
type: "element",
tagName: "span",
properties: { class: "line" },
children: []
};
let col = 0;
for (const token of line) {
let tokenNode = {
type: "element",
tagName: "span",
properties: { ...token.htmlAttrs },
children: [{
type: "text",
value: token.content
}]
};
const style = stringifyTokenStyle(token.htmlStyle || getTokenStyleObject(token));
if (style) tokenNode.properties.style = style;
for (const transformer of transformers) tokenNode = transformer?.span?.call(context, tokenNode, idx + 1, col, lineNode, token) || tokenNode;
if (structure === "inline") root.children.push(tokenNode);
else if (structure === "classic") lineNode.children.push(tokenNode);
col += token.content.length;
}
if (structure === "classic") {
for (const transformer of transformers) lineNode = transformer?.line?.call(context, lineNode, idx + 1) || lineNode;
lineNodes.push(lineNode);
lines.push(lineNode);
} else if (structure === "inline") lineNodes.push(lineNode);
});
if (structure === "classic") {
for (const transformer of transformers) codeNode = transformer?.code?.call(context, codeNode) || codeNode;
preNode.children.push(codeNode);
for (const transformer of transformers) preNode = transformer?.pre?.call(context, preNode) || preNode;
root.children.push(preNode);
} else if (structure === "inline") {
const syntheticLines = [];
let currentLine = {
type: "element",
tagName: "span",
properties: { class: "line" },
children: []
};
for (const child of root.children) if (child.type === "element" && child.tagName === "br") {
syntheticLines.push(currentLine);
currentLine = {
type: "element",
tagName: "span",
properties: { class: "line" },
children: []
};
} else if (child.type === "element" || child.type === "text") currentLine.children.push(child);
syntheticLines.push(currentLine);
let transformedCode = {
type: "element",
tagName: "code",
properties: {},
children: syntheticLines
};
for (const transformer of transformers) transformedCode = transformer?.code?.call(context, transformedCode) || transformedCode;
root.children = [];
for (let i = 0; i < transformedCode.children.length; i++) {
if (i > 0) root.children.push({
type: "element",
tagName: "br",
properties: {},
children: []
});
const line = transformedCode.children[i];
if (line.type === "element") root.children.push(...line.children);
}
}
let result = root;
for (const transformer of transformers) result = transformer?.root?.call(context, result) || result;
if (grammarState) setLastGrammarStateToMap(result, grammarState);
return result;
}
function mergeWhitespaceTokens(tokens) {
return tokens.map((line) => {
const newLine = [];
let carryOnContent = "";
let firstOffset;
line.forEach((token, idx) => {
const couldMerge = !(token.fontStyle && (token.fontStyle & FontStyle.Underline || token.fontStyle & FontStyle.Strikethrough));
if (couldMerge && RE_WHITESPACE_ONLY.test(token.content) && line[idx + 1]) {
if (firstOffset === void 0) firstOffset = token.offset;
carryOnContent += token.content;
} else if (carryOnContent) {
if (couldMerge) newLine.push({
...token,
offset: firstOffset,
content: carryOnContent + token.content
});
else newLine.push({
content: carryOnContent,
offset: firstOffset
}, token);
firstOffset = void 0;
carryOnContent = "";
} else newLine.push(token);
});
return newLine;
});
}
function splitWhitespaceTokens(tokens) {
return tokens.map((line) => {
return line.flatMap((token) => {
if (RE_WHITESPACE_ONLY.test(token.content)) return token;
const match = token.content.match(RE_LEADING_TRAILING_WHITESPACE);
if (!match) return token;
const [, leading, content, trailing] = match;
if (!leading && !trailing) return token;
const expanded = [{
...token,
offset: token.offset + leading.length,
content
}];
if (leading) expanded.unshift({
content: leading,
offset: token.offset
});
if (trailing) expanded.push({
content: trailing,
offset: token.offset + leading.length + content.length
});
return expanded;
});
});
}
function mergeAdjacentStyledTokens(tokens) {
return tokens.map((line) => {
const newLine = [];
for (const token of line) {
if (newLine.length === 0) {
newLine.push({ ...token });
continue;
}
const prevToken = newLine.at(-1);
const prevStyle = stringifyTokenStyle(prevToken.htmlStyle || getTokenStyleObject(prevToken));
const currentStyle = stringifyTokenStyle(token.htmlStyle || getTokenStyleObject(token));
const isPrevDecorated = prevToken.fontStyle && (prevToken.fontStyle & FontStyle.Underline || prevToken.fontStyle & FontStyle.Strikethrough);
const isDecorated = token.fontStyle && (token.fontStyle & FontStyle.Underline || token.fontStyle & FontStyle.Strikethrough);
if (!isPrevDecorated && !isDecorated && prevStyle === currentStyle) prevToken.content += token.content;
else newLine.push({ ...token });
}
return newLine;
});
}
//#endregion
//#region src/highlight/code-to-html.ts
const hastToHtml = toHtml;
/**
* Get highlighted code in HTML.
*/
function codeToHtml(primitive, code, options) {
const context = {
meta: {},
options,
codeToHast: (_code, _options) => codeToHast(primitive, _code, _options),
codeToTokens: (_code, _options) => codeToTokens(primitive, _code, _options)
};
let result = hastToHtml(codeToHast(primitive, code, options, context));
for (const transformer of getTransformers(options)) result = transformer.postprocess?.call(context, result, options) || result;
return result;
}
//#endregion
//#region src/constructors/highlighter.ts
/**
* Create a Shiki core highlighter instance, with no languages or themes bundled.
* Wasm and each language and theme must be loaded manually.
*
* @see http://shiki.style/guide/bundles#fine-grained-bundle
*/
async function createHighlighterCore(options) {
const primitive = await createShikiPrimitiveAsync$1(options);
return {
getLastGrammarState: (...args) => getLastGrammarState(primitive, ...args),
codeToTokensBase: (code, options) => codeToTokensBase(primitive, code, options),
codeToTokensWithThemes: (code, options) => codeToTokensWithThemes$1(primitive, code, options),
codeToTokens: (code, options) => codeToTokens(primitive, code, options),
codeToHast: (code, options) => codeToHast(primitive, code, options),
codeToHtml: (code, options) => codeToHtml(primitive, code, options),
getBundledLanguages: () => ({}),
getBundledThemes: () => ({}),
...primitive,
getInternalContext: () => primitive
};
}
/**
* Create a Shiki core highlighter instance, with no languages or themes bundled.
* Wasm and each language and theme must be loaded manually.
*
* Synchronous version of `createHighlighterCore`, which requires to provide the engine and all themes and languages upfront.
*
* @see http://shiki.style/guide/bundles#fine-grained-bundle
*/
function createHighlighterCoreSync(options) {
const internal = createShikiPrimitive$1(options);
return {
getLastGrammarState: (...args) => getLastGrammarState(internal, ...args),
codeToTokensBase: (code, options) => codeToTokensBase(internal, code, options),
codeToTokensWithThemes: (code, options) => codeToTokensWithThemes$1(internal, code, options),
codeToTokens: (code, options) => codeToTokens(internal, code, options),
codeToHast: (code, options) => codeToHast(internal, code, options),
codeToHtml: (code, options) => codeToHtml(internal, code, options),
getBundledLanguages: () => ({}),
getBundledThemes: () => ({}),
...internal,
getInternalContext: () => internal
};
}
function makeSingletonHighlighterCore(createHighlighter) {
let _shiki;
async function getSingletonHighlighterCore(options) {
if (!_shiki) {
_shiki = createHighlighter({
...options,
themes: options.themes || [],
langs: options.langs || []
});
return _shiki;
} else {
const s = await _shiki;
await Promise.all([s.loadTheme(...options.themes || []), s.loadLanguage(...options.langs || [])]);
return s;
}
}
return getSingletonHighlighterCore;
}
const getSingletonHighlighterCore = /* @__PURE__ */ makeSingletonHighlighterCore(createHighlighterCore);
//#endregion
//#region src/constructors/bundle-factory.ts
function createBundledHighlighter(options) {
const bundledLanguages = options.langs;
const bundledThemes = options.themes;
const engine = options.engine;
async function createHighlighter(options) {
function resolveLang(lang) {
if (typeof lang === "string") {
lang = options.langAlias?.[lang] || lang;
if (isSpecialLang(lang)) return [];
const bundle = bundledLanguages[lang];
if (!bundle) throw new ShikiError$1(`Language \`${lang}\` is not included in this bundle. You may want to load it from external source.`);
return bundle;
}
return lang;
}
function resolveTheme(theme) {
if (isSpecialTheme(theme)) return "none";
if (typeof theme === "string") {
const bundle = bundledThemes[theme];
if (!bundle) throw new ShikiError$1(`Theme \`${theme}\` is not included in this bundle. You may want to load it from external source.`);
return bundle;
}
return theme;
}
const _themes = (options.themes ?? []).map((i) => resolveTheme(i));
const langs = (options.langs ?? []).map((i) => resolveLang(i));
const core = await createHighlighterCore({
engine: options.engine ?? engine(),
...options,
themes: _themes,
langs
});
return {
...core,
loadLanguage(...langs) {
return core.loadLanguage(...langs.map(resolveLang));
},
loadTheme(...themes) {
return core.loadTheme(...themes.map(resolveTheme));
},
getBundledLanguages() {
return bundledLanguages;
},
getBundledThemes() {
return bundledThemes;
}
};
}
return createHighlighter;
}
function makeSingletonHighlighter(createHighlighter) {
let _shiki;
async function getSingletonHighlighter(options = {}) {
if (!_shiki) {
_shiki = createHighlighter({
...options,
themes: [],
langs: []
});
const s = await _shiki;
await Promise.all([s.loadTheme(...options.themes || []), s.loadLanguage(...options.langs || [])]);
return s;
} else {
const s = await _shiki;
await Promise.all([s.loadTheme(...options.themes || []), s.loadLanguage(...options.langs || [])]);
return s;
}
}
return getSingletonHighlighter;
}
function createSingletonShorthands(createHighlighter, config) {
const getSingletonHighlighter = makeSingletonHighlighter(createHighlighter);
async function get(code, options) {
const shiki = await getSingletonHighlighter({
langs: [options.lang],
themes: "theme" in options ? [options.theme] : Object.values(options.themes)
});
const langs = await config?.guessEmbeddedLanguages?.(code, options.lang, shiki);
if (langs) await shiki.loadLanguage(...langs);
return shiki;
}
return {
getSingletonHighlighter(options) {
return getSingletonHighlighter(options);
},
async codeToHtml(code, options) {
return (await get(code, options)).codeToHtml(code, options);
},
async codeToHast(code, options) {
return (await get(code, options)).codeToHast(code, options);
},
async codeToTokens(code, options) {
return (await get(code, options)).codeToTokens(code, options);
},
async codeToTokensBase(code, options) {
return (await get(code, options)).codeToTokensBase(code, options);
},
async codeToTokensWithThemes(code, options) {
return (await get(code, options)).codeToTokensWithThemes(code, options);
},
async getLastGrammarState(code, options) {
return (await getSingletonHighlighter({
langs: [options.lang],
themes: [options.theme]
})).getLastGrammarState(code, options);
}
};
}
//#endregion
//#region src/theme-css-variables.ts
/**
* A factory function to create a css-variable-based theme
*
* @see https://shiki.style/guide/theme-colors#css-variables-theme
*/
function createCssVariablesTheme(options = {}) {
const { name = "css-variables", variablePrefix = "--shiki-", fontStyle = true } = options;
const variable = (name) => {
if (options.variableDefaults?.[name]) return `var(${variablePrefix}${name}, ${options.variableDefaults[name]})`;
return `var(${variablePrefix}${name})`;
};
const theme = {
name,
type: "dark",
colors: {
"editor.foreground": variable("foreground"),
"editor.background": variable("background"),
"terminal.ansiBlack": variable("ansi-black"),
"terminal.ansiRed": variable("ansi-red"),
"terminal.ansiGreen": variable("ansi-green"),
"terminal.ansiYellow": variable("ansi-yellow"),
"terminal.ansiBlue": variable("ansi-blue"),
"terminal.ansiMagenta": variable("ansi-magenta"),
"terminal.ansiCyan": variable("ansi-cyan"),
"terminal.ansiWhite": variable("ansi-white"),
"terminal.ansiBrightBlack": variable("ansi-bright-black"),
"terminal.ansiBrightRed": variable("ansi-bright-red"),
"terminal.ansiBrightGreen": variable("ansi-bright-green"),
"terminal.ansiBrightYellow": variable("ansi-bright-yellow"),
"terminal.ansiBrightBlue": variable("ansi-bright-blue"),
"terminal.ansiBrightMagenta": variable("ansi-bright-magenta"),
"terminal.ansiBrightCyan": variable("ansi-bright-cyan"),
"terminal.ansiBrightWhite": variable("ansi-bright-white")
},
tokenColors: [
{
scope: [
"keyword.operator.accessor",
"meta.group.braces.round.function.arguments",
"meta.template.expression",
"markup.fenced_code meta.embedded.block"
],
settings: { foreground: variable("foreground") }
},
{
scope: "emphasis",
settings: { fontStyle: "italic" }
},
{
scope: [
"strong",
"markup.heading.markdown",
"markup.bold.markdown"
],
settings: { fontStyle: "bold" }
},
{
scope: ["markup.italic.markdown"],
settings: { fontStyle: "italic" }
},
{
scope: "meta.link.inline.markdown",
settings: {
fontStyle: "underline",
foreground: variable("token-link")
}
},
{
scope: [
"string",
"markup.fenced_code",
"markup.inline"
],
settings: { foreground: variable("token-string") }
},
{
scope: ["comment", "string.quoted.docstring.multi"],
settings: { foreground: variable("token-comment") }
},
{
scope: [
"constant.numeric",
"constant.language",
"constant.other.placeholder",
"constant.character.format.placeholder",
"variable.language.this",
"variable.other.object",
"variable.other.class",
"variable.other.constant",
"meta.property-name",
"meta.property-value",
"support"
],
settings: { foreground: variable("token-constant") }
},
{
scope: [
"keyword",
"storage.modifier",
"storage.type",
"storage.control.clojure",
"entity.name.function.clojure",
"entity.name.tag.yaml",
"support.function.node",
"support.type.property-name.json",
"punctuation.separator.key-value",
"punctuation.definition.template-expression"
],
settings: { foreground: variable("token-keyword") }
},
{
scope: "variable.parameter.function",
settings: { foreground: variable("token-parameter") }
},
{
scope: [
"support.function",
"entity.name.type",
"entity.other.inherited-class",
"meta.function-call",
"meta.instance.constructor",
"entity.other.attribute-name",
"entity.name.function",
"constant.keyword.clojure"
],
settings: { foreground: variable("token-function") }
},
{
scope: [
"entity.name.tag",
"string.quoted",
"string.regexp",
"string.interpolated",
"string.template",
"string.unquoted.plain.out.yaml",
"keyword.other.template"
],
settings: { foreground: variable("token-string-expression") }
},
{
scope: [
"punctuation.definition.arguments",
"punctuation.definition.dict",
"punctuation.separator",
"meta.function-call.arguments"
],
settings: { foreground: variable("token-punctuation") }
},
{
scope: ["markup.underline.link", "punctuation.definition.metadata.markdown"],
settings: { foreground: variable("token-link") }
},
{
scope: ["beginning.punctuation.definition.list.markdown"],
settings: { foreground: variable("token-string") }
},
{
scope: [
"punctuation.definition.string.begin.markdown",
"punctuation.definition.string.end.markdown",
"string.other.link.title.markdown",
"string.other.link.description.markdown"
],
settings: { foreground: variable("token-keyword") }
},
{
scope: [
"markup.inserted",
"meta.diff.header.to-file",
"punctuation.definition.inserted"
],
settings: { foreground: variable("token-inserted") }
},
{
scope: [
"markup.deleted",
"meta.diff.header.from-file",
"punctuation.definition.deleted"
],
settings: { foreground: variable("token-deleted") }
},
{
scope: ["markup.changed", "punctuation.definition.changed"],
settings: { foreground: variable("token-changed") }
}
]
};
if (!fontStyle) theme.tokenColors = theme.tokenColors?.map((tokenColor) => {
if (tokenColor.settings?.fontStyle) delete tokenColor.settings.fontStyle;
return tokenColor;
});
return theme;
}
//#endregion
export { ShikiError, addClassToHast, applyColorReplacements, codeToHast, codeToHtml, codeToTokens, codeToTokensBase, codeToTokensWithThemes, createBundledHighlighter, createCssVariablesTheme, createHighlighterCore, createHighlighterCoreSync, createPositionConverter, createShikiInternal, createShikiInternalSync, createShikiPrimitive, createShikiPrimitiveAsync, createSingletonShorthands, flatTokenVariants, getLastGrammarState, getSingletonHighlighterCore, getTokenStyleObject, guessEmbeddedLanguages, hastToHtml, isNoneTheme, isPlainLang, isSpecialLang, isSpecialTheme, makeSingletonHighlighter, makeSingletonHighlighterCore, normalizeGetter, normalizeTheme, resolveColorReplacements, splitLines, splitToken, splitTokens, stringifyTokenStyle, toArray, tokenizeAnsiWithTheme, tokenizeWithTheme, tokensToHast, transformerDecorations };