vitepress-plugin-llms
Version:
📜 A VitePress plugin for generating LLM-friendly documentation
1,146 lines (1,112 loc) • 43.8 kB
JavaScript
// Built with bunup (https://bunup.dev)
// src/markdown/markdown-it-plugins.ts
import Token from "markdown-it/lib/token.mjs";
function copyOrDownloadAsMarkdownButtons(md, componentName = "CopyOrDownloadAsMarkdownButtons") {
const orig = md.renderer.render.bind(md.renderer);
md.renderer.render = (tokens, options, env) => {
const len = tokens.length;
for (let i = 0;i < len; i += 1) {
const open = tokens[i];
if (open?.tag === "h1" && open.type === "heading_open") {
const closeIndex = tokens.findIndex((token, j) => j > i && token.tag === "h1" && token.type === "heading_close");
if (closeIndex !== -1) {
const htmlToken = new Token("html_block", "", 0);
htmlToken.content = `<${componentName} />`;
tokens.splice(closeIndex + 1, 0, htmlToken);
}
break;
}
}
return orig(tokens, options, env);
};
}
// src/plugin/plugin.ts
import path11 from "node:path";
import pc4 from "picocolors";
// package.json
var name = "vitepress-plugin-llms";
// src/constants.ts
var defaultLLMsTxtTemplate = `# {title}
{description}
{details}
## Table of Contents
{toc}`;
var unnecessaryFilesList = {
indexPage: ["index.md"],
blogs: ["blog/*", "blog.md"],
team: ["team.md"],
readmeMd: ["README.md"]
};
var tagRegex = (tag, type, flags) => new RegExp(`<${type === "open" ? "" : "/"}${tag}>`, flags);
var fullTagRegex = (tag, flags) => new RegExp(`${tagRegex(tag, "open").source}([\\s\\S]*?)${tagRegex(tag, "closed").source}`, flags);
// src/plugin/dev-server.ts
import fs from "node:fs";
import path from "node:path";
import pc2 from "picocolors";
// src/utils/logger.ts
import pc from "picocolors";
var logPrefix = pc.blue("llmstxt") + pc.dim(" » ");
var log = {
info: (message) => {
console.log(`${logPrefix} ${message}`);
},
success: (message) => {
console.log(`${logPrefix}${pc.green("✓")} ${message}`);
},
warn: (message) => {
console.warn(`${logPrefix}${pc.yellow("⚠")} ${pc.yellow(message)}`);
},
error: (message) => {
console.error(`${logPrefix}${pc.red("✗")} ${pc.red(message)}`);
}
};
var logger_default = log;
// src/plugin/dev-server.ts
function configureDevServer(server, config) {
logger_default.info("Dev server configured for serving plain text docs for LLMs");
server.middlewares.use((req, res, next) => {
if (req.url.endsWith(".md") || req.url.endsWith(".txt")) {
try {
const base = config.base || "/";
const urlWithoutBase = req.url.startsWith(base) ? req.url.slice(base.length) : req.url;
const filePath = path.join(config.vitepress.outDir || "dist", urlWithoutBase);
const content = fs.readFileSync(filePath, "utf8");
res.setHeader("Content-Type", "text/plain; charset=utf-8");
res.end(content);
return;
} catch {
logger_default.warn(`Failed to return ${pc2.cyan(req.url)}: File not found`);
}
}
next();
});
}
var dev_server_default = configureDevServer;
// src/plugin/hooks.ts
import matter5 from "@11ty/gray-matter";
import { millify } from "millify";
import { minimatch as minimatch2 } from "minimatch";
import fs4 from "node:fs/promises";
import path10 from "node:path";
import pc3 from "picocolors";
import { remark } from "remark";
import remarkFrontmatter from "remark-frontmatter";
import { estimateTokenCount } from "tokenx";
import { remove } from "unist-util-remove";
// src/generator/llms-full-txt.ts
import matter from "@11ty/gray-matter";
import path4 from "node:path";
// src/generator/toc.ts
import path3 from "node:path";
// src/utils/file-utils.ts
import path2 from "node:path";
var splitDirAndFile = (filepath) => ({
dir: path2.dirname(filepath),
file: path2.basename(filepath)
});
var contentFileExts = new Set([".md", ".html"]);
var stripExt = (filepath, usePosix = false) => {
const { dir, file } = splitDirAndFile(filepath);
const ext = path2.extname(file);
const base = contentFileExts.has(ext) ? path2.basename(file, ext) : file;
const joinFn = usePosix ? path2.posix.join : path2.join;
return joinFn(dir, base);
};
var stripExtPosix = (filepath) => stripExt(filepath, true);
var transformToPosixPath = (filepath) => filepath.replaceAll("\\", "/");
function getDirectoriesAtDepths(files, baseDir, maxDepth) {
const directories = new Set([baseDir]);
for (const file of files) {
const relativePath = path2.relative(baseDir, file);
const parts = relativePath.split(path2.sep);
for (let depth = 1;depth < Math.min(parts.length, maxDepth); depth += 1) {
const dirParts = parts.slice(0, depth);
const dirPath = path2.resolve(baseDir, ...dirParts);
directories.add(dirPath);
}
}
return [...directories].map((dirPath) => ({
depth: dirPath === baseDir ? 1 : path2.relative(baseDir, dirPath).split(path2.sep).length + 1,
path: dirPath,
relativePath: path2.relative(baseDir, dirPath) || "."
})).filter((dir) => dir.depth <= maxDepth).sort((one, another) => {
if (one.depth !== another.depth) {
return one.depth - another.depth;
}
return one.path.localeCompare(another.path);
});
}
// src/utils/template-utils.ts
var templateVariable = (key) => new RegExp(`(\\n\\s*\\n)?\\{${key}\\}`, "gi");
function replaceTemplateVariable(content, variable, value, fallback) {
return content.replace(templateVariable(variable), (_, prefix) => {
const val = value !== undefined && value.length > 0 ? value : fallback ?? "";
return val.length > 0 ? `${prefix ? `
` : ""}${val}` : "";
});
}
var expandTemplate = (template, variables) => {
let result = template;
for (const [key, value] of Object.entries(variables)) {
result = replaceTemplateVariable(result, key, value);
}
return result;
};
var generateLink = (urlPath, domain, extension, base) => expandTemplate("{domain}/{base}{path}{extension}", {
base: base !== undefined && base.length > 0 ? `${base.slice(base.startsWith("/") ? 1 : 0)}${base.endsWith("/") ? "" : "/"}` : "",
domain: domain ?? "",
extension,
path: transformToPosixPath(urlPath)
});
function generateMetadata(sourceFile, { domain, filePath, linksExtension, base }) {
return {
url: generateLink(stripExtPosix(filePath), domain, linksExtension ?? ".md", base),
...typeof sourceFile.data["description"] === "string" && {
description: sourceFile.data["description"]
}
};
}
// src/generator/toc.ts
var generateTOCLink = (file, domain, relativePath, extension, base) => {
const { description } = file.file.data;
return `- [${file.title}](${generateLink(stripExtPosix(relativePath), domain, extension ?? ".md", base)})${typeof description === "string" ? `: ${description.trim()}` : ""}
`;
};
async function collectPathsFromSidebarItems(items, base = "") {
return Promise.all(items.map(async (item) => {
const paths = [];
if (typeof item.link === "string") {
paths.push((item.base ?? base) + item.link);
}
if (item.items && Array.isArray(item.items)) {
const nestedPaths = await collectPathsFromSidebarItems(item.items, item.base ?? base ?? "");
paths.push(...nestedPaths);
}
return paths;
})).then((results) => results.flat());
}
function normalizeLinkPath(link) {
const normalizedPath = stripExtPosix(link);
if (path3.basename(normalizedPath) === "index") {
return path3.dirname(normalizedPath);
}
return normalizedPath;
}
function isPathMatch(filePath, sidebarPath) {
const normalizedFilePath = normalizeLinkPath(filePath);
const normalizedSidebarPath = normalizeLinkPath(sidebarPath);
return normalizedFilePath === normalizedSidebarPath || normalizedFilePath === `${normalizedSidebarPath}.md`;
}
async function resolveLeafItems(items, preparedFiles, sectionBase, base, domain, linksExtension) {
const leafItems = items.filter((item) => typeof item.link === "string");
const resolved = await Promise.all(leafItems.map(async (item) => {
const normalizedItemLink = normalizeLinkPath(path3.posix.join(base, item.base ?? sectionBase, item.link));
const basePrefix = base.endsWith("/") ? base : `${base}/`;
const matchingFile = preparedFiles.find((file) => {
const relativePath = `${basePrefix}${transformToPosixPath(stripExtPosix(file.path))}`;
return isPathMatch(relativePath, normalizedItemLink);
});
if (matchingFile) {
return generateTOCLink(matchingFile, domain, matchingFile.path, linksExtension, base);
}
logger_default.warn(`No matching file found for sidebar link: ${item.link} (normalized: ${normalizedItemLink})`);
return;
}));
return resolved.filter((item) => item !== undefined);
}
function buildHeader(sectionText, depth) {
return sectionText !== undefined ? `${"#".repeat(depth)} ${sectionText}
` : "";
}
function buildContent(linkItems, nestedSections) {
let content = "";
if (linkItems.length > 0) {
content += linkItems.join("");
}
if (linkItems.length > 0 && nestedSections.length > 0) {
content += `
`;
}
if (nestedSections.length > 0) {
content += nestedSections.join(`
`);
}
return content;
}
function assembleSectionTOC(sectionText, linkItems, nonEmptyNestedSections, depth) {
const hasContent = linkItems.length > 0 || nonEmptyNestedSections.length > 0;
if (!hasContent) {
return "";
}
return buildHeader(sectionText, depth) + buildContent(linkItems, nonEmptyNestedSections);
}
async function resolveNestedSections(items, preparedFiles, sectionBase, base, depth, domain, linksExtension) {
const nestedItems = items.filter((item) => Array.isArray(item.items) && item.items.length > 0);
const results = await Promise.all(nestedItems.map(async (item) => processSidebarSection(item, preparedFiles, domain, linksExtension, depth + 1, item.base ?? sectionBase ?? base ?? "")));
return results.filter((section_) => section_.trim() !== "");
}
async function processSidebarSection(section, preparedFiles, domain, linksExtension, depth = 3, base = "") {
if (!section.items || !Array.isArray(section.items)) {
return "";
}
const sectionBase = section.base ?? "";
const [linkItems, nonEmptyNestedSections] = await Promise.all([
resolveLeafItems(section.items, preparedFiles, sectionBase, base, domain, linksExtension),
resolveNestedSections(section.items, preparedFiles, sectionBase, base, depth, domain, linksExtension)
]);
return assembleSectionTOC(section.text, linkItems, nonEmptyNestedSections, depth);
}
function flattenSidebarConfig(sidebarConfig) {
if (Array.isArray(sidebarConfig)) {
return sidebarConfig;
}
if (typeof sidebarConfig === "object") {
return Object.values(sidebarConfig).flat();
}
return [];
}
function filterFiles(preparedFiles, directoryFilter) {
if (typeof directoryFilter !== "string") {
return preparedFiles;
}
if (directoryFilter === ".") {
return preparedFiles;
}
const normalizedFilter = transformToPosixPath(directoryFilter);
return preparedFiles.filter((file) => {
const normalizedPath = transformToPosixPath(file.path);
return normalizedPath.startsWith(`${normalizedFilter}/`) || normalizedPath === normalizedFilter;
});
}
async function generateFileEntries(files, domain, linksExtension, base = "") {
return Promise.all(files.map((file) => generateTOCLink(file, domain, file.path, linksExtension, base)));
}
function findUnsortedFiles(files, sidebarPaths) {
return files.filter((file) => {
const relativePath = `/${transformToPosixPath(stripExtPosix(file.path))}`;
return !sidebarPaths.some((sidebarPath) => isPathMatch(relativePath, sidebarPath));
});
}
async function generateSidebarTOC(sidebarConfig, files, domain, linksExtension, base = "") {
const flattenedSidebar = flattenSidebarConfig(sidebarConfig);
if (flattenedSidebar.length === 0) {
return "";
}
const sections = flattenedSidebar.filter((section) => Array.isArray(section.items) && section.items.length > 0);
const sectionResults = await Promise.all(sections.map(async (section) => processSidebarSection(section, files, domain, linksExtension, 3, base)));
let toc = `${sectionResults.join(`
`)}
`;
const sidebarPaths = await collectPathsFromSidebarItems(sections);
const unsortedFiles = findUnsortedFiles(files, sidebarPaths);
if (unsortedFiles.length > 0) {
toc += `### Other
`;
const entries = await generateFileEntries(unsortedFiles, domain, linksExtension, base);
toc += entries.join("");
}
return toc;
}
async function generateTOC(preparedFiles, options) {
const { domain, sidebarConfig, linksExtension, base, directoryFilter } = options;
const filteredFiles = filterFiles(preparedFiles, directoryFilter);
if (sidebarConfig) {
const sidebarTOC = await generateSidebarTOC(sidebarConfig, filteredFiles, domain, linksExtension, base);
if (sidebarTOC) {
return sidebarTOC;
}
}
const entries = await generateFileEntries(filteredFiles, domain, linksExtension, base);
return entries.join("");
}
// src/generator/llms-full-txt.ts
async function sortFilesBySidebar(files, sidebar) {
const flatSidebar = flattenSidebarConfig(sidebar);
const sidebarPaths = await collectPathsFromSidebarItems(flatSidebar);
const ordered = [];
const seen = new Set;
for (const sidebarPath of sidebarPaths) {
const match = files.find((file) => {
const relativePath = `/${transformToPosixPath(stripExtPosix(file.path))}`;
return isPathMatch(relativePath, sidebarPath);
});
if (match && !seen.has(match.path)) {
ordered.push(match);
seen.add(match.path);
}
}
for (const file of files) {
if (!seen.has(file.path)) {
ordered.push(file);
}
}
return ordered;
}
async function generateLLMsFullTxt(preparedFiles, options) {
const { domain, linksExtension, base, directoryFilter, sidebar } = options;
let filteredFiles = preparedFiles;
if (typeof directoryFilter === "string") {
filteredFiles = directoryFilter === "." ? preparedFiles : preparedFiles.filter((file) => {
const relativePath = file.path;
return relativePath.startsWith(directoryFilter + path4.sep) || relativePath === directoryFilter;
});
}
if (sidebar) {
filteredFiles = await sortFilesBySidebar(filteredFiles, sidebar);
}
const fileContents = await Promise.all(filteredFiles.map(async (file) => {
const metadata = generateMetadata(file.file, {
base,
domain,
filePath: file.path,
linksExtension
});
return matter.stringify(file.file.content, metadata);
}));
return fileContents.join(`
---
`);
}
// src/utils/helpers.ts
import matter2 from "@11ty/gray-matter";
import prettyBytes from "pretty-bytes";
var getHumanReadableSizeOf = (string) => prettyBytes(Buffer.byteLength(string, "utf8"));
var clearGrayMatterCache = () => {
matter2.clearCache();
};
// src/utils/markdown.ts
import markdownTitle from "markdown-title";
function extractTitle(file) {
return file.data["title"] ?? file.data["titleTemplate"] ?? markdownTitle(file.content);
}
// src/generator/llms-txt.ts
async function generateLLMsTxt(preparedFiles, {
indexMdFile,
LLMsTxtTemplate = defaultLLMsTxtTemplate,
templateVariables = {},
vitepressConfig,
domain,
sidebar,
directoryFilter
}) {
clearGrayMatterCache();
const variables = { ...templateVariables };
variables["title"] ??= indexMdFile.data["hero"]?.name ?? indexMdFile.data["title"] ?? vitepressConfig?.title ?? vitepressConfig?.titleTemplate ?? extractTitle(indexMdFile) ?? "LLMs Documentation";
variables["description"] ??= indexMdFile.data?.["hero"]?.text ?? vitepressConfig?.description ?? indexMdFile.data?.["description"] ?? indexMdFile.data?.["titleTemplate"];
if (typeof variables["description"] === "string") {
variables["description"] = `> ${variables["description"]}`;
}
variables["details"] ??= indexMdFile.data?.["hero"]?.["tagline"] ?? indexMdFile.data["tagline"] ?? (variables["description"] === undefined && "This file contains links to all documentation sections.");
variables["toc"] ??= await generateTOC(preparedFiles, {
base: vitepressConfig.base,
directoryFilter,
domain,
sidebarConfig: sidebar ?? vitepressConfig.themeConfig?.sidebar
});
return expandTemplate(LLMsTxtTemplate, variables);
}
// src/generator/page-generator.ts
import matter3 from "@11ty/gray-matter";
import fs2 from "node:fs/promises";
import path5 from "node:path";
async function generateLLMFriendlyPages(preparedFiles, outDir, domain, base) {
const tasks = preparedFiles.map(async (file) => {
try {
const mdFile = file.file;
const targetPath = path5.resolve(outDir, file.path);
await fs2.mkdir(path5.dirname(targetPath), { recursive: true });
await fs2.writeFile(targetPath, matter3.stringify(mdFile.content, generateMetadata(mdFile, {
base,
domain,
filePath: file.path,
linksExtension: ".md"
})));
logger_default.success(`Processed ${file.path}`);
} catch (error) {
logger_default.error(`Failed to process ${file.path}: ${error.message}`);
}
});
await Promise.all(tasks);
}
// src/markdown/remark-plugins/remark-please.ts
import { visit } from "unist-util-visit";
function remarkPlease(intent, tag) {
return () => (tree) => {
const ourFullTagRegex = fullTagRegex(tag);
const nodesToProcess = [];
visit(tree, "html", (node, index, parent) => {
if (!parent || typeof index !== "number") {
return;
}
nodesToProcess.push([node, index, parent]);
});
const emptyParagraphs = new Set;
for (const [node, index, parent] of [...nodesToProcess].reverse()) {
if (ourFullTagRegex.test(node.value)) {
if (intent === "remove") {
parent.children.splice(index, 1);
if (parent.type === "paragraph" && parent.children.length === 0) {
emptyParagraphs.add({ node: parent, parent });
}
continue;
} else if (intent === "unwrap") {
const match = node.value.match(ourFullTagRegex);
if (typeof match?.[1] === "string") {
node.value = match[1].trim();
}
continue;
}
}
if (tagRegex(tag, "open").test(node.value)) {
let closeIndex = index + 1;
while (closeIndex < parent.children.length) {
const closeNode = parent.children[closeIndex];
if (closeNode?.type === "html" && tagRegex(tag, "closed").test(closeNode.value)) {
break;
}
closeIndex += 1;
}
if (closeIndex < parent.children.length) {
if (intent === "remove") {
parent.children.splice(index, closeIndex - index + 1);
if (parent.type === "paragraph" && parent.children.length === 0) {
emptyParagraphs.add({ node: parent, parent });
}
} else if (intent === "unwrap") {
parent.children.splice(closeIndex, 1);
parent.children.splice(index, 1);
}
}
}
}
const paragraphsToRemove = [];
for (const { node, parent } of emptyParagraphs) {
const index = parent.children.indexOf(node);
if (index !== -1) {
paragraphsToRemove.push({ index, parent });
}
}
visit(tree, "paragraph", (node, index, parent) => {
if (!parent || typeof index !== "number") {
return;
}
const [firstChild] = node.children;
const isEmpty = node.children.length === 0 || node.children.length === 1 && firstChild?.type === "text" && firstChild.value.trim() === "";
if (isEmpty) {
paragraphsToRemove.push({ index, parent });
}
});
for (const { index, parent } of [...paragraphsToRemove].reverse()) {
parent.children.splice(index, 1);
}
return tree;
};
}
var remark_please_default = remarkPlease;
// src/markdown/remark-plugins/replace-image-urls.ts
import path6 from "node:path";
import { visit as visit2 } from "unist-util-visit";
function remarkReplaceImageUrls(map) {
return () => (tree) => {
visit2(tree, "image", (node) => {
const original = path6.posix.basename(node.url);
const hashed = map.get(original);
if (hashed !== undefined) {
node.url = `/${hashed}`;
}
});
};
}
var replace_image_urls_default = remarkReplaceImageUrls;
// src/markdown/remark-plugins/snippets.ts
import matter4 from "@11ty/gray-matter";
import { fromMarkdown } from "mdast-util-from-markdown";
import fs3 from "node:fs";
import path7 from "node:path";
import { visit as visit3 } from "unist-util-visit";
var includesRE = /<!--\s*@include:\s*(.*?)\s*-->/g;
var snippetRE = /^<<<\s*(.*?)$/gm;
var regionRE = /(#[^\s{]+)/;
var rangeRE = /\{(\d*),(\d*)\}$/;
var rawPathRegexp = /^(.+?(?:(?:\.([a-z0-9]+))?))(?:(#[\w-]+))?(?: ?(?:{(\d+(?:[,-]\d+)*)? ?(\S+)? ?(\S+)?}))? ?(?:\[(.+)\])?$/;
var markers = [
{
end: /^\s*\/\/\s*#?endregion\b\s*(.*?)\s*$/,
start: /^\s*\/\/\s*#?region\b\s*(.*?)\s*$/
},
{
end: /^\s*<!--\s*#?endregion\b\s*(.*?)\s*-->/,
start: /^\s*<!--\s*#?region\b\s*(.*?)\s*-->/
},
{
end: /^\s*\/\*\s*#endregion\b\s*(.*?)\s*\*\//,
start: /^\s*\/\*\s*#region\b\s*(.*?)\s*\*\//
},
{
end: /^\s*#[eE]nd ?[rR]egion\b\s*(.*?)\s*$/,
start: /^\s*#[rR]egion\b\s*(.*?)\s*$/
},
{
end: /^\s*#\s*#?endregion\b\s*(.*?)\s*$/,
start: /^\s*#\s*#?region\b\s*(.*?)\s*$/
},
{
end: /^\s*(?:--|::|@?REM)\s*#endregion\b\s*(.*?)\s*$/,
start: /^\s*(?:--|::|@?REM)\s*#region\b\s*(.*?)\s*$/
},
{
end: /^\s*#pragma\s+endregion\b\s*(.*?)\s*$/,
start: /^\s*#pragma\s+region\b\s*(.*?)\s*$/
},
{
end: /^\s*\(\*\s*#endregion\b\s*(.*?)\s*\*\)/,
start: /^\s*\(\*\s*#region\b\s*(.*?)\s*\*\)/
}
];
function rawPathToToken(rawPath) {
const [filepath = "", extension = "", region = "", lines = "", lang = "", attrs = "", title = ""] = (rawPathRegexp.exec(rawPath) ?? []).slice(1);
return { attrs, extension, filepath, lang, lines, region, title };
}
function findRegionStart(lines, regionName) {
for (let i = 0;i < lines.length; i += 1) {
for (const re of markers) {
if (re.start.exec(lines[i])?.[1] === regionName) {
return { re, start: i + 1 };
}
}
}
return;
}
function findRegionEnd(lines, regionName, chosen) {
let counter = 1;
for (let i = chosen.start;i < lines.length; i += 1) {
const line = lines[i];
if (chosen.re.start.exec(line)?.[1] === regionName) {
counter += 1;
}
const endRegion = chosen.re.end.exec(line)?.[1];
if ((endRegion === regionName || endRegion === "") && (counter -= 1) === 0) {
return i;
}
}
return;
}
function findRegion(lines, regionName) {
const chosen = findRegionStart(lines, regionName);
if (!chosen) {
return;
}
const end = findRegionEnd(lines, regionName, chosen);
return { ...chosen, end };
}
function dedent(text) {
const lines = text.split(`
`);
let minIndentLength = Infinity;
for (const line of lines) {
for (let i = 0;i < line.length; i += 1) {
if (line[i] !== " " && line[i] !== "\t") {
minIndentLength = Math.min(i, minIndentLength);
break;
}
}
}
if (minIndentLength < Infinity) {
return lines.map((line) => line.slice(minIndentLength)).join(`
`);
}
return text;
}
function processIncludes({ srcDir, content, filePath, includes }) {
return content.replace(includesRE, (string, m1) => {
if (m1.length === 0) {
return string;
}
const range = rangeRE.exec(m1);
const region = regionRE.exec(m1);
const hasMeta = Boolean(region ?? range);
if (hasMeta) {
const len = (region?.[0].length ?? 0) + (range?.[0].length ?? 0);
m1 = m1.slice(0, -len);
}
const atPresent = m1.startsWith("@");
try {
const includePath = atPresent ? path7.join(srcDir, m1.slice(m1[1] === "/" ? 2 : 1)) : path7.join(path7.dirname(filePath), m1);
if (!fs3.existsSync(includePath)) {
throw new Error(`File not found: ${includePath}`);
}
let content2 = fs3.readFileSync(includePath, "utf8");
if (region) {
const [regionName] = region;
const lines = content2.split(/\r?\n/);
const regionData = findRegion(lines, regionName.slice(1));
if (regionData) {
content2 = lines.slice(regionData.start, regionData.end).join(`
`);
} else {
logger_default.warn(`[remark-include] Region '${regionName}' not found in ${includePath}`);
}
}
if (range) {
const [, startLine, endLine] = range;
const lines = content2.split(/\r?\n/);
content2 = lines.slice(startLine ? Number.parseInt(startLine, 10) - 1 : undefined, endLine ? Number.parseInt(endLine, 10) : undefined).join(`
`);
}
if (!hasMeta && path7.extname(includePath) === ".md") {
({ content: content2 } = matter4(content2));
}
includes.push(includePath);
return processIncludes({ content: content2, filePath, includes, srcDir });
} catch {
logger_default.warn(`[remark-include] Include file not found: ${m1}`);
return string;
}
});
}
function processSnippets({
srcDir,
content,
filePath,
includes
}) {
let codeNode;
content.replace(snippetRE, (string, rawPath) => {
if (rawPath.length === 0) {
return string;
}
const cleanPath = rawPath.trim();
const atPresent = cleanPath.startsWith("@");
const pathToParse = atPresent ? cleanPath.slice(1) : cleanPath;
const { filepath, extension, region, lines, lang, attrs, title } = rawPathToToken(pathToParse);
try {
const snippetPath = atPresent ? path7.join(srcDir, filepath) : path7.resolve(path7.dirname(filePath), filepath);
if (!fs3.existsSync(snippetPath)) {
throw new Error(`Snippet file not found: ${snippetPath}`);
}
let codeContent = fs3.readFileSync(snippetPath, "utf8").replaceAll(`\r
`, `
`);
if (region) {
const regionName = region.slice(1);
const contentLines = codeContent.split(`
`);
const regionData = findRegion(contentLines, regionName);
if (regionData) {
codeContent = dedent(contentLines.slice(regionData.start, regionData.end).filter((line) => !(regionData.re.start.test(line) || regionData.re.end.test(line))).join(`
`));
}
}
includes.push(snippetPath);
const infoLang = lang || extension || undefined;
const infoMeta = `${lines && `{${lines}}`}${title && `[${title}]`}${attrs && ` ${attrs}`}`.trim() || undefined;
codeNode = {
lang: infoLang,
meta: infoMeta,
type: "code",
value: codeContent
};
} catch {
logger_default.warn(`[remark-include] Snippet file not found: ${rawPath}`);
}
return string;
});
return codeNode;
}
function remarkInclude({ srcDir }) {
return () => (tree, file) => {
const includes = [];
visit3(tree, (node, index, parent) => {
if (parent === undefined || typeof index !== "number") {
return;
}
const isIncludeNode = node.type === "html" && includesRE.test(node.value);
const isSnippetNode = node.type === "text" && snippetRE.test(node.value);
if (isIncludeNode || isSnippetNode) {
let processedValue;
if (isIncludeNode) {
processedValue = processIncludes({
content: node.value,
filePath: file.path,
includes,
srcDir
});
} else if (isSnippetNode) {
processedValue = processSnippets({
content: node.value,
filePath: file.path,
includes,
srcDir
});
}
if (processedValue !== undefined) {
if (typeof processedValue === "string") {
if (processedValue !== node.value) {
parent.children.splice(index, 1, ...fromMarkdown(processedValue).children);
}
} else {
parent.children.splice(index, 1, processedValue);
}
}
}
});
file.data["includes"] = includes;
};
}
var snippets_default = remarkInclude;
// src/utils/dynamic-routes.ts
var VP_PARAMS_MARKER_REGEX = /^__VP_PARAMS_START([\s\S]+?)__VP_PARAMS_END__/;
function processVPParams(content) {
let params = {};
content = content.replace(VP_PARAMS_MARKER_REGEX, (_, paramsString) => {
params = JSON.parse(paramsString);
return "";
});
if (Object.keys(params).length > 0) {
content = content.replaceAll(/\{\{\s*\$params\.([\s\S]+?)\s*\}\}/g, (_, paramKey) => params[paramKey] ?? "");
}
return content;
}
// src/utils/ignore.ts
import { minimatch } from "minimatch";
import path8 from "node:path";
function resolveIgnorePatterns(globalPatterns, perOutputPatterns) {
if (!perOutputPatterns) {
return { negative: [], positive: [...globalPatterns] };
}
const extraNegations = perOutputPatterns.filter((pattern) => pattern.startsWith("!")).map((pattern) => pattern.slice(1));
const extraPositive = perOutputPatterns.filter((pattern) => !pattern.startsWith("!"));
const positive = [
...globalPatterns.filter((positive_) => !extraNegations.some((negative) => negative === positive_)),
...extraPositive
];
return { negative: extraNegations, positive };
}
function isIgnored(filePath, positive, negative) {
if (positive.length === 0) {
return false;
}
const matchesPositive = positive.some((positive_) => minimatch(filePath, positive_));
if (!matchesPositive) {
return false;
}
const matchesNegative = negative.some((negative_) => minimatch(filePath, negative_));
return !matchesNegative;
}
function filterPreparedFiles(files, workDir, positive, negative) {
if (positive.length === 0 && negative.length === 0) {
return files;
}
return files.filter((file) => {
const relativePath = path8.isAbsolute(file.path) ? path8.relative(workDir, file.path) : file.path;
return !isIgnored(relativePath, positive, negative);
});
}
// src/utils/vitepress-rewrites.ts
import path9 from "node:path";
import { compile, match } from "path-to-regexp";
function resolveOutputFilePath(file, workDir, rewrites = {}) {
let resolvedRewrite;
const normalizedFile = file.split(path9.sep).join(path9.posix.sep);
const normalizedWorkDir = workDir.split(path9.sep).join(path9.posix.sep);
const relativePath = path9.posix.relative(normalizedWorkDir, normalizedFile);
if (typeof rewrites === "function") {
const resolvedFilePath = rewrites(relativePath);
if (resolvedFilePath) {
resolvedRewrite = resolvedFilePath;
}
} else if (Object.keys(rewrites).length > 0) {
if (relativePath in rewrites) {
resolvedRewrite = rewrites[relativePath];
} else {
for (const [pattern, replacement] of Object.entries(rewrites)) {
if (!pattern.includes(":") && !pattern.includes("*")) {
continue;
}
try {
const matcher = match(pattern);
const result = matcher(relativePath);
if (typeof result === "object" && "params" in result) {
const compileFn = compile(replacement);
resolvedRewrite = compileFn(result.params);
break;
}
} catch {}
}
}
}
if (resolvedRewrite !== undefined) {
return path9.join(workDir, resolvedRewrite);
}
return file;
}
function resolvePageURL(url) {
const hasLeadingSlash = url.startsWith("/");
const normalized = transformToPosixPath(hasLeadingSlash ? url.slice(1) : url);
if (normalized.endsWith("/index.md") && normalized !== "index.md") {
const newUrl = `${normalized.slice(0, -"/index.md".length)}.md`;
return hasLeadingSlash ? `/${newUrl}` : newUrl;
}
return url;
}
// src/plugin/hooks.ts
async function transform(content, id, settings, setIndexMdFile, mdFiles, config) {
const orig = content;
if (!id.endsWith(".md") || !path10.resolve(id).startsWith(settings.workDir)) {
return null;
}
const resolvedOutFilePath = resolveOutputFilePath(id, settings.workDir, config.vitepress.userConfig.rewrites);
const isMainPage = path10.relative(settings.workDir, resolvedOutFilePath) === "index.md";
if (isMainPage) {
setIndexMdFile(matter5(content));
}
if (settings.ignoreFiles.length > 0) {
const shouldIgnore = await Promise.all(settings.ignoreFiles.map(async (pattern) => {
if (typeof pattern === "string") {
return minimatch2(path10.relative(settings.workDir, id), pattern);
}
return false;
}));
if (shouldIgnore.some(Boolean) && !isMainPage) {
return null;
}
}
let modifiedContent = content.replace(fullTagRegex("llm-only", "g"), "").replace(fullTagRegex("llm-exclude", "g"), "$1");
if (settings.injectLLMHint && (settings.generateLLMFriendlyDocsForEachPage || settings.generateLLMsTxt || settings.generateLLMsFullTxt)) {
clearGrayMatterCache();
modifiedContent = matter5(modifiedContent);
let llmHint = "";
const currentUrl = resolvePageURL(path10.relative(settings.workDir, resolvedOutFilePath));
const base = config.base || "/";
const basePath = base === "/" ? "" : base.replace(/\/$/, "");
if (isMainPage) {
const notices = [];
if (settings.generateLLMsTxt) {
notices.push(`${basePath}/llms.txt for optimized Markdown documentation`);
}
if (settings.generateLLMsFullTxt) {
notices.push(`${basePath}/llms-full.txt for full documentation bundle`);
}
if (notices.length > 0) {
llmHint = `Are you an LLM? View ${notices.join(", or ")}`;
}
} else if (settings.generateLLMFriendlyDocsForEachPage) {
const mdUrl = `${basePath}/${currentUrl}`;
llmHint = `Are you an LLM? You can read better optimized documentation at ${mdUrl} for this page in Markdown format`;
}
if (llmHint) {
const hintBlock = `<div style="display: none;" hidden="true" aria-hidden="true" data-nosnippet>${llmHint}</div>
`;
let { content: content2 } = modifiedContent;
const marker = "__VP_PARAMS_END__";
const idx = content2.indexOf(marker);
content2 = idx === -1 ? `${hintBlock}
${content2}` : `${content2.slice(0, idx + marker.length)}${hintBlock}
${content2.slice(idx + marker.length)}`;
modifiedContent = matter5.stringify(content2, modifiedContent.data);
} else {
modifiedContent = matter5.stringify(modifiedContent.content, modifiedContent.data);
}
}
if (!isMainPage || !settings.excludeIndexPage) {
mdFiles.set(id, content);
}
return modifiedContent === orig ? null : { code: modifiedContent, map: null };
}
async function generateBundle(bundle, settings, config, indexMdFile, mdFiles) {
const resolvedSidebar = typeof settings.sidebar === "function" ? await settings.sidebar(config.vitepress.userConfig.themeConfig?.sidebar) : settings.sidebar ?? config.vitepress.userConfig.themeConfig?.sidebar;
const outDir = config.vitepress.outDir || "dist";
try {
await fs4.access(outDir);
} catch {
logger_default.info(`Creating output directory: ${pc3.cyan(outDir)}`);
await fs4.mkdir(outDir, { recursive: true });
}
const fileCount = mdFiles.size;
if (fileCount === 0) {
logger_default.error(`No markdown files found to process. Check your \`${pc3.bold("workDir")}\` and \`${pc3.bold("ignoreFiles")}\` settings.`);
return;
}
logger_default.info(`Processing ${pc3.bold(fileCount.toString())} markdown files from ${pc3.cyan(settings.workDir)}`);
const imageMap = new Map;
if (typeof bundle === "object") {
for (const asset of Object.values(bundle)) {
if (/(png|jpe?g|gif|svg|webp)$/i.test(path10.extname(asset.fileName))) {
const name2 = path10.posix.basename(asset.fileName);
imageMap.set(name2, asset.fileName);
}
}
}
const mdFilesList = [...mdFiles];
const preparedFiles = await Promise.all(mdFilesList.map(async ([file, content]) => {
const resolvedOutFilePath = path10.relative(settings.workDir, resolveOutputFilePath(file, settings.workDir, config.vitepress.userConfig.rewrites));
const markdownProcessor = remark().use(remarkFrontmatter).use(snippets_default({ srcDir: settings.workDir })).use(remark_please_default("unwrap", "llm-only")).use(remark_please_default("remove", "llm-exclude")).use(replace_image_urls_default(imageMap));
if (settings.stripHTML) {
markdownProcessor.use(() => (tree) => {
remove(tree, { type: "html" });
return tree;
});
}
content = processVPParams(content);
const processedMarkdown = matter5(String(await markdownProcessor.process({
cwd: settings.workDir,
path: file,
value: content
})));
const title = extractTitle(processedMarkdown)?.trim() ?? "Untitled";
const filePath = path10.basename(resolvedOutFilePath) === "index.md" && path10.dirname(resolvedOutFilePath) !== "." && path10.dirname(resolvedOutFilePath) !== "" ? `${path10.dirname(resolvedOutFilePath)}.md` : resolvedOutFilePath;
return { file: processedMarkdown, path: filePath, title };
}));
preparedFiles.sort((one, another) => one.title.localeCompare(another.title));
const perOutput = settings.ignoreFilesPerOutput;
const llmsTxtPatterns = resolveIgnorePatterns(settings.ignoreFiles, perOutput.llmsTxt);
const llmsFullTxtPatterns = resolveIgnorePatterns(settings.ignoreFiles, perOutput.llmsFullTxt);
const pagesPatterns = resolveIgnorePatterns(settings.ignoreFiles, perOutput.pages);
const mdFilesKeys = [...mdFiles.keys()];
const tasks = [];
if (settings.generateLLMsTxt) {
const templateVariables = {
description: settings.description,
details: settings.details,
title: settings.title,
toc: settings.toc,
...settings.customTemplateVariables
};
const directories = getDirectoriesAtDepths(mdFilesKeys, settings.workDir, settings.experimental.depth ?? 1);
const llmsTxtFiles = filterPreparedFiles(preparedFiles, settings.workDir, llmsTxtPatterns.positive, llmsTxtPatterns.negative);
tasks.push(...directories.map(async (directory) => (async () => {
const isRoot = directory.relativePath === ".";
const directoryFilter = isRoot ? "." : directory.relativePath;
const outputFileName = isRoot ? "llms.txt" : path10.join(directory.relativePath, "llms.txt");
const llmsTxtPath = path10.resolve(outDir, outputFileName);
await fs4.mkdir(path10.dirname(llmsTxtPath), { recursive: true });
logger_default.info(`Generating ${pc3.cyan(outputFileName)}...`);
const llmsTxt = await generateLLMsTxt(llmsTxtFiles, {
LLMsTxtTemplate: settings.customLLMsTxtTemplate,
directoryFilter,
domain: settings.domain,
indexMdFile,
linksExtension: settings.generateLLMFriendlyDocsForEachPage ? undefined : ".html",
sidebar: resolvedSidebar,
templateVariables: { ...templateVariables },
vitepressConfig: config.vitepress.userConfig
});
await fs4.writeFile(llmsTxtPath, llmsTxt, "utf8");
logger_default.success(expandTemplate("Generated {file} (~{tokens} tokens, {size}) with {fileCount} documentation links", {
file: pc3.cyan(outputFileName),
fileCount: pc3.bold(llmsTxtFiles.length.toString()),
size: pc3.bold(getHumanReadableSizeOf(llmsTxt)),
tokens: pc3.bold(millify(estimateTokenCount(llmsTxt)))
}));
})()));
}
if (settings.generateLLMsFullTxt) {
const directories = getDirectoriesAtDepths(mdFilesKeys, settings.workDir, settings.experimental.depth ?? 1);
const llmsFullTxtFiles = filterPreparedFiles(preparedFiles, settings.workDir, llmsFullTxtPatterns.positive, llmsFullTxtPatterns.negative);
tasks.push(...directories.map(async (directory) => (async () => {
const isRoot = directory.relativePath === ".";
const directoryFilter = isRoot ? "." : directory.relativePath;
const outputFileName = isRoot ? "llms-full.txt" : path10.join(directory.relativePath, "llms-full.txt");
const llmsFullTxtPath = path10.resolve(outDir, outputFileName);
await fs4.mkdir(path10.dirname(llmsFullTxtPath), { recursive: true });
logger_default.info(`Generating full documentation bundle (${pc3.cyan(outputFileName)})...`);
const llmsFullTxt = await generateLLMsFullTxt(llmsFullTxtFiles, {
base: config.base,
directoryFilter,
domain: settings.domain,
linksExtension: settings.generateLLMFriendlyDocsForEachPage ? undefined : ".html",
sidebar: resolvedSidebar
});
await fs4.writeFile(llmsFullTxtPath, llmsFullTxt, "utf8");
logger_default.success(expandTemplate("Generated {file} (~{tokens} tokens, {size}) with {fileCount} markdown files", {
file: pc3.cyan(outputFileName),
fileCount: pc3.bold(llmsFullTxtFiles.length.toString()),
size: pc3.bold(getHumanReadableSizeOf(llmsFullTxt)),
tokens: pc3.bold(millify(estimateTokenCount(llmsFullTxt)))
}));
})()));
}
if (settings.generateLLMFriendlyDocsForEachPage) {
const pagesFiles = filterPreparedFiles(preparedFiles, settings.workDir, pagesPatterns.positive, pagesPatterns.negative);
tasks.push(generateLLMFriendlyPages(pagesFiles, outDir, settings.domain, config.base));
}
if (tasks.length > 0) {
await Promise.all(tasks);
}
}
// src/plugin/plugin.ts
var PLUGIN_NAME = name;
function llmstxt(userSettings = {}) {
const settings = {
customLLMsTxtTemplate: defaultLLMsTxtTemplate,
excludeBlog: true,
excludeIndexPage: true,
excludeTeam: true,
excludeUnnecessaryFiles: true,
experimental: {
depth: 1,
...userSettings.experimental
},
generateLLMFriendlyDocsForEachPage: true,
generateLLMsFullTxt: true,
generateLLMsTxt: true,
ignoreFiles: [],
ignoreFilesPerOutput: {},
injectLLMHint: true,
stripHTML: true,
workDir: undefined,
...userSettings
};
let config;
const mdFiles = new Map;
let indexMdFile = undefined;
let isSsrBuild = false;
return [
{
enforce: "pre",
name: `${PLUGIN_NAME}:llm-tags`,
async transform(content, id) {
return transform(content, id, settings, (file) => {
indexMdFile = file;
}, mdFiles, config);
}
},
{
name: PLUGIN_NAME,
enforce: "post",
configResolved(resolvedConfig) {
config = resolvedConfig;
settings.workDir = settings.workDir ? path11.resolve(config.vitepress.srcDir, settings.workDir) : path11.resolve(config.vitepress.srcDir);
if (settings.excludeUnnecessaryFiles) {
settings.excludeIndexPage && settings.ignoreFiles.push(...unnecessaryFilesList.indexPage);
settings.excludeBlog && settings.ignoreFiles.push(...unnecessaryFilesList.blogs);
settings.excludeTeam && settings.ignoreFiles.push(...unnecessaryFilesList.team);
}
isSsrBuild = Boolean(resolvedConfig.build.ssr);
logger_default.info(`${pc4.bold(PLUGIN_NAME)} initialized ${isSsrBuild ? pc4.dim("(SSR build)") : pc4.dim("(client build)")} with workDir: ${pc4.cyan(settings.workDir)}`);
},
configureServer(server) {
dev_server_default(server, config);
},
buildStart() {
mdFiles.clear();
logger_default.info("Build started, file collection cleared");
},
async generateBundle(_options, bundle) {
if (isSsrBuild) {
logger_default.info("Skipping LLMs docs generation in SSR build");
return;
}
if (settings.generateLLMsTxt && indexMdFile === undefined) {
throw new Error("index.md file was not found during build");
} else {
await generateBundle(bundle, settings, config, indexMdFile, mdFiles);
}
}
}
];
}
export {
llmstxt as default,
copyOrDownloadAsMarkdownButtons
};