vitepress-plugin-llms
Version:
๐ A VitePress plugin for generating LLM-friendly documentation
772 lines (741 loc) โข 30.1 kB
JavaScript
// Built with bunup (https://bunup.dev)
import {
cleanUrl
} from "./shared/chunk-8kwyv1sq.js";
// src/markdown/markdown-it-plugins.ts
import Token from "markdown-it/lib/token.mjs";
function copyOrDownloadAsMarkdownButtons(md, componentName = "CopyOrDownloadAsMarkdownButtons") {
const orig = md.renderer.render.bind(md.renderer.render);
md.renderer.render = (tokens, options, env) => {
for (let i = 0;i < tokens.length; i++) {
if (tokens[i].tag === "h1" && tokens[i].type === "heading_open") {
for (let j = i + 1;j < tokens.length; j++) {
if (tokens[j].tag === "h1" && tokens[j].type === "heading_close") {
const htmlToken = new Token("html_block", "", 0);
htmlToken.content = `<${componentName} />`;
tokens.splice(j + 1, 0, htmlToken);
break;
}
}
break;
}
}
return orig(tokens, options, env);
};
}
// src/plugin/plugin.ts
import path9 from "node:path";
import pc4 from "picocolors";
// package.json
var name = "vitepress-plugin-llms";
// src/constants.ts
var defaultLLMsTxtTemplate = `# {title}
{description}
{details}
## Table of Contents
{toc}`;
var unnecessaryFilesList = {
indexPage: ["index.md"],
blogs: ["blog/*", "blog.md"],
team: ["team.md"],
readmeMd: ["README.md"]
};
var tagRegex = (tag, type, flags) => {
return new RegExp(`<${type === "open" ? "" : "/"}${tag}>`, flags);
};
var fullTagRegex = (tag, flags) => new RegExp(`${tagRegex(tag, "open").source}([\\s\\S]*?)${tagRegex(tag, "closed").source}`, flags);
// src/plugin/dev-server.ts
import fs from "node:fs/promises";
import path2 from "node:path";
import pc2 from "picocolors";
// src/utils/file-utils.ts
import path from "node:path";
var splitDirAndFile = (filepath) => ({
dir: path.dirname(filepath),
file: path.basename(filepath)
});
var contentFileExts = new Set([".md", ".html"]);
var stripExt = (filepath, usePosix = false) => {
const { dir, file } = splitDirAndFile(filepath);
const ext = path.extname(file);
const base = contentFileExts.has(ext) ? path.basename(file, ext) : file;
const joinFn = usePosix ? path.posix.join : path.join;
return joinFn(dir, base);
};
var stripExtPosix = (filepath) => stripExt(filepath, true);
var transformToPosixPath = (filepath) => filepath.replace(/\\/g, "/");
function getDirectoriesAtDepths(files, baseDir, maxDepth) {
const directories = new Set;
directories.add(baseDir);
for (const file of files) {
const relativePath = path.relative(baseDir, file);
const parts = relativePath.split(path.sep);
for (let depth = 1;depth < Math.min(parts.length, maxDepth); depth++) {
const dirParts = parts.slice(0, depth);
const dirPath = path.resolve(baseDir, ...dirParts);
directories.add(dirPath);
}
}
return Array.from(directories).map((dirPath) => ({
path: dirPath,
depth: dirPath === baseDir ? 1 : path.relative(baseDir, dirPath).split(path.sep).length + 1,
relativePath: path.relative(baseDir, dirPath) || "."
})).filter((dir) => dir.depth <= maxDepth).sort((a, b) => {
if (a.depth !== b.depth)
return a.depth - b.depth;
return a.path.localeCompare(b.path);
});
}
// src/utils/logger.ts
import pc from "picocolors";
var logPrefix = pc.blue("llmstxt") + pc.dim(" ยป ");
var log = {
info: (message) => console.log(`${logPrefix} ${message}`),
success: (message) => console.log(`${logPrefix}${pc.green("โ")} ${message}`),
warn: (message) => console.warn(`${logPrefix}${pc.yellow("โ ")} ${pc.yellow(message)}`),
error: (message) => console.error(`${logPrefix}${pc.red("โ")} ${pc.red(message)}`)
};
var logger_default = log;
// src/plugin/dev-server.ts
async function configureDevServer(server, config) {
logger_default.info("Dev server configured for serving plain text docs for LLMs");
server.middlewares.use(async (req, res, next) => {
if (req.url?.endsWith(".md") || req.url?.endsWith(".txt")) {
try {
const filePath = path2.resolve(config.vitepress?.outDir ?? "dist", `${stripExt(req.url)}.md`);
const content = await fs.readFile(filePath, "utf-8");
res.setHeader("Content-Type", "text/plain; charset=utf-8");
res.end(content);
return;
} catch (_error) {
logger_default.warn(`Failed to return ${pc2.cyan(req.url)}: File not found`);
next();
}
}
next();
});
}
// src/plugin/hooks.ts
import fs4 from "node:fs/promises";
import path8 from "node:path";
import matter4 from "gray-matter";
import { millify } from "millify";
import { minimatch } from "minimatch";
import pc3 from "picocolors";
import { remark } from "remark";
import remarkFrontmatter from "remark-frontmatter";
import { approximateTokenSize } from "tokenx";
import { remove } from "unist-util-remove";
// src/generator/llms-full-txt.ts
import path3 from "node:path";
import matter from "gray-matter";
// src/utils/template-utils.ts
var templateVariable = (key) => new RegExp(`(\\n\\s*\\n)?\\{${key}\\}`, "gi");
function replaceTemplateVariable(content, variable, value, fallback) {
return content.replace(templateVariable(variable), (_, prefix) => {
const val = value?.length ? value : fallback?.length ? fallback : "";
return val ? `${prefix ? `
` : ""}${val}` : "";
});
}
var expandTemplate = (template, variables) => {
return Object.entries(variables).reduce((result, [key, value]) => replaceTemplateVariable(result, key, value), template);
};
var generateLink = (urlPath, domain, extension, base) => expandTemplate("{domain}/{base}{path}{extension}", {
domain: domain || "",
base: base ? `${base.slice(base.startsWith("/") ? 1 : 0) + (!base.endsWith("/") ? "/" : "")}` : "",
path: transformToPosixPath(urlPath),
extension
});
function generateMetadata(sourceFile, { domain, filePath, linksExtension, base }) {
return {
url: generateLink(stripExtPosix(filePath), domain, linksExtension ?? ".md", base),
...sourceFile.data?.description && { description: sourceFile.data.description }
};
}
// src/generator/llms-full-txt.ts
async function generateLLMsFullTxt(preparedFiles, options) {
const { domain, linksExtension, base, directoryFilter } = options;
const filteredFiles = directoryFilter ? directoryFilter === "." ? preparedFiles : preparedFiles.filter((file) => {
const relativePath = file.path;
return relativePath.startsWith(directoryFilter + path3.sep) || relativePath === directoryFilter;
}) : preparedFiles;
const fileContents = await Promise.all(filteredFiles.map(async (file) => {
const metadata = generateMetadata(file.file, {
domain,
filePath: file.path,
linksExtension,
base
});
return matter.stringify(file.file.content, metadata);
}));
return fileContents.join(`
---
`);
}
// src/generator/llms-txt.ts
import fs2 from "node:fs/promises";
import matter2 from "gray-matter";
// src/generator/toc.ts
import path4 from "node:path";
var generateTOCLink = (file, domain, relativePath, extension, base) => {
const description = file.file.data.description;
return `- [${file.title}](${generateLink(stripExtPosix(relativePath), domain, extension ?? ".md", base)})${description ? `: ${description.trim()}` : ""}
`;
};
async function collectPathsFromSidebarItems(items, base = "") {
return Promise.all(items.map(async (item) => {
const paths = [];
if (item.link) {
paths.push((item.base ?? base) + item.link);
}
if (item.items && Array.isArray(item.items)) {
const nestedPaths = await collectPathsFromSidebarItems(item.items, item.base ?? base ?? "");
paths.push(...nestedPaths);
}
return paths;
})).then((results) => results.flat());
}
function normalizeLinkPath(link) {
const normalizedPath = stripExtPosix(link);
if (path4.basename(normalizedPath) === "index") {
return path4.dirname(normalizedPath);
}
return normalizedPath;
}
function isPathMatch(filePath, sidebarPath) {
const normalizedFilePath = normalizeLinkPath(filePath);
const normalizedSidebarPath = normalizeLinkPath(sidebarPath);
return normalizedFilePath === normalizedSidebarPath || normalizedFilePath === `${normalizedSidebarPath}.md`;
}
async function processSidebarSection(section, preparedFiles, outDir, domain, linksExtension, depth = 3, base = "") {
let sectionTOC = "";
if (section.items && Array.isArray(section.items)) {
const [linkItems, nestedSections] = await Promise.all([
Promise.all(section.items.filter((item) => typeof item.link === "string").map(async (item) => {
const normalizedItemLink = normalizeLinkPath((item.base ?? section.base ?? base ?? "") + item.link);
const matchingFile = preparedFiles.find((file) => {
const relativePath = `/${transformToPosixPath(stripExtPosix(file.path))}`;
return isPathMatch(relativePath, normalizedItemLink);
});
if (matchingFile) {
const relativePath = matchingFile.path;
return generateTOCLink(matchingFile, domain, relativePath, linksExtension, base);
}
return null;
})).then((items) => items.filter((item) => item !== null)),
Promise.all(section.items.filter((item) => Array.isArray(item.items) && item.items.length > 0).map((item) => processSidebarSection(item, preparedFiles, outDir, domain, linksExtension, depth + 1, item.base ?? section.base ?? base ?? "")))
]);
const nonEmptyNestedSections = nestedSections.filter((section2) => section2.trim() !== "");
const hasContent = linkItems.length > 0 || nonEmptyNestedSections.length > 0;
if (hasContent && section.text) {
sectionTOC += `${"#".repeat(depth)} ${section.text}
`;
}
if (linkItems.length > 0) {
sectionTOC += linkItems.join("");
}
if (linkItems.length > 0 && nonEmptyNestedSections.length > 0) {
sectionTOC += `
`;
}
if (nonEmptyNestedSections.length > 0) {
sectionTOC += nonEmptyNestedSections.join(`
`);
}
}
return sectionTOC;
}
function flattenSidebarConfig(sidebarConfig) {
if (Array.isArray(sidebarConfig)) {
return sidebarConfig;
}
if (typeof sidebarConfig === "object") {
return Object.values(sidebarConfig).flat();
}
return [];
}
async function generateTOC(preparedFiles, options) {
const { outDir, domain, sidebarConfig, linksExtension, base, directoryFilter } = options;
let tableOfContent = "";
const filteredFiles = directoryFilter ? directoryFilter === "." ? preparedFiles : preparedFiles.filter((file) => {
const normalizedPath = transformToPosixPath(file.path);
const normalizedFilter = transformToPosixPath(directoryFilter);
return normalizedPath.startsWith(`${normalizedFilter}/`) || normalizedPath === normalizedFilter;
}) : preparedFiles;
if (sidebarConfig) {
const flattenedSidebarConfig = flattenSidebarConfig(sidebarConfig);
if (flattenedSidebarConfig.length > 0) {
const sectionResults = await Promise.all(flattenedSidebarConfig.map((section) => processSidebarSection(section, filteredFiles, outDir, domain, linksExtension, 3, base)));
tableOfContent += `${sectionResults.join(`
`)}
`;
const allSidebarPaths = await collectPathsFromSidebarItems(flattenedSidebarConfig);
const unsortedFiles = filteredFiles.filter((file) => {
const relativePath = `/${transformToPosixPath(stripExtPosix(file.path))}`;
return !allSidebarPaths.some((sidebarPath) => isPathMatch(relativePath, sidebarPath));
});
if (unsortedFiles.length > 0) {
tableOfContent += `### Other
`;
const tocEntries = [];
await Promise.all(unsortedFiles.map(async (file) => {
const relativePath = file.path;
tocEntries.push(generateTOCLink(file, domain, relativePath, linksExtension, base));
}));
tableOfContent += tocEntries.join("");
}
return tableOfContent;
}
}
if (filteredFiles.length > 0) {
const tocEntries = await Promise.all(filteredFiles.map(async (file) => {
const relativePath = file.path;
return generateTOCLink(file, domain, relativePath, linksExtension, base);
}));
tableOfContent += tocEntries.join("");
}
return tableOfContent;
}
// src/utils/markdown.ts
import markdownTitle from "markdown-title";
function extractTitle(file) {
return file.data?.title || file.data?.titleTemplate || markdownTitle(file.content);
}
// src/generator/llms-txt.ts
async function generateLLMsTxt(preparedFiles, {
indexMd,
outDir,
LLMsTxtTemplate = defaultLLMsTxtTemplate,
templateVariables = {},
vitepressConfig,
domain,
sidebar,
directoryFilter,
base
}) {
matter2.clearCache();
const indexMdContent = await fs2.readFile(indexMd, "utf-8");
const indexMdFile = matter2(indexMdContent);
templateVariables.title ??= indexMdFile.data?.hero?.name || indexMdFile.data?.title || vitepressConfig?.title || vitepressConfig?.titleTemplate || extractTitle(indexMdFile) || "LLMs Documentation";
templateVariables.description ??= indexMdFile.data?.hero?.text || vitepressConfig?.description || indexMdFile?.data?.description || indexMdFile.data?.titleTemplate;
if (templateVariables.description) {
templateVariables.description = `> ${templateVariables.description}`;
}
templateVariables.details ??= indexMdFile.data?.hero?.tagline || indexMdFile.data?.tagline || !templateVariables.description && "This file contains links to all documentation sections.";
templateVariables.toc ??= await generateTOC(preparedFiles, {
outDir,
domain,
sidebarConfig: sidebar || vitepressConfig?.themeConfig?.sidebar,
directoryFilter,
base
});
return expandTemplate(LLMsTxtTemplate, templateVariables);
}
// src/generator/page-generator.ts
import fs3 from "node:fs/promises";
import path5 from "node:path";
import matter3 from "gray-matter";
async function generateLLMFriendlyPages(preparedFiles, outDir, domain, base) {
const tasks = preparedFiles.map(async (file) => {
try {
const mdFile = file.file;
const targetPath = path5.resolve(outDir, file.path);
await fs3.mkdir(path5.dirname(targetPath), { recursive: true });
await fs3.writeFile(targetPath, matter3.stringify(mdFile.content, generateMetadata(mdFile, {
domain,
filePath: file.path,
linksExtension: ".md",
base
})));
logger_default.success(`Processed ${file.path}`);
} catch (error) {
logger_default.error(`Failed to process ${file.path}: ${error.message}`);
}
});
await Promise.all(tasks);
}
// src/markdown/remark-plugins.ts
import path6 from "node:path";
import { visit } from "unist-util-visit";
function remarkPlease(intent, tag) {
return () => (tree) => {
const ourFullTagRegex = fullTagRegex(tag);
const nodesToProcess = [];
visit(tree, "html", (node, index, parent) => {
if (!parent || typeof index !== "number")
return;
nodesToProcess.push([node, index, parent]);
});
const emptyParagraphs = new Set;
for (const [node, index, parent] of nodesToProcess.reverse()) {
if (ourFullTagRegex.test(node.value)) {
if (intent === "remove") {
parent.children.splice(index, 1);
if (parent.type === "paragraph" && parent.children.length === 0) {
emptyParagraphs.add({ node: parent, parent });
}
continue;
}
if (intent === "unwrap") {
const match = node.value.match(ourFullTagRegex);
if (match?.[1]) {
node.value = match[1].trim();
}
continue;
}
}
if (tagRegex(tag, "open").test(node.value)) {
let closeIndex = index + 1;
while (closeIndex < parent.children.length) {
const closeNode = parent.children[closeIndex];
if (closeNode.type === "html" && tagRegex(tag, "closed").test(closeNode.value)) {
break;
}
closeIndex++;
}
if (closeIndex < parent.children.length) {
if (intent === "remove") {
parent.children.splice(index, closeIndex - index + 1);
if (parent.type === "paragraph" && parent.children.length === 0) {
emptyParagraphs.add({ node: parent, parent });
}
} else if (intent === "unwrap") {
parent.children.splice(closeIndex, 1);
parent.children.splice(index, 1);
}
}
}
}
const paragraphsToRemove = [];
for (const { node, parent } of emptyParagraphs) {
const index = parent.children.indexOf(node);
if (index !== -1) {
paragraphsToRemove.push({ index, parent });
}
}
visit(tree, "paragraph", (node, index, parent) => {
if (!parent || typeof index !== "number")
return;
const isEmpty = node.children.length === 0 || node.children.length === 1 && node.children[0].type === "text" && node.children[0].value.trim() === "";
if (isEmpty) {
paragraphsToRemove.push({ index, parent });
}
});
for (const { index, parent } of paragraphsToRemove.reverse()) {
parent.children.splice(index, 1);
}
return tree;
};
}
function remarkReplaceImageUrls(map) {
return () => (tree) => {
visit(tree, "image", (node) => {
const original = path6.posix.basename(node.url);
const hashed = map.get(original);
if (hashed) {
node.url = `/${hashed}`;
}
});
};
}
// src/utils/helpers.ts
import byteSize from "byte-size";
var getHumanReadableSizeOf = (string) => byteSize(new Blob([string]).size).toString();
// src/utils/vitepress-rewrites.ts
import path7 from "node:path";
import { compile, match } from "path-to-regexp";
function resolveOutputFilePath(file, workDir, rewrites = {}) {
let resolvedRewrite;
if (typeof rewrites === "function") {
const resolvedFilePath = rewrites(file);
if (resolvedFilePath)
resolvedRewrite = resolvedFilePath;
} else if (rewrites && typeof rewrites === "object") {
if (file in rewrites) {
resolvedRewrite = rewrites[file];
} else {
for (const [pattern, replacement] of Object.entries(rewrites)) {
if (!pattern.includes(":") && !pattern.includes("*")) {
continue;
}
try {
const matcher = match(pattern);
const result = matcher(file);
if (result) {
const compileFn = compile(replacement);
resolvedRewrite = compileFn(result.params);
break;
}
} catch (_error) {}
}
}
}
if (resolvedRewrite) {
return path7.join(workDir, resolvedRewrite);
}
return file;
}
function resolveSourceFilePath(outputPath, workDir, rewrites = {}) {
if (typeof rewrites === "function") {
return outputPath;
}
if (rewrites && typeof rewrites === "object") {
for (const [source, target] of Object.entries(rewrites)) {
if (target === outputPath) {
return path7.join(workDir, source);
}
}
for (const [sourcePattern, targetPattern] of Object.entries(rewrites)) {
if (!targetPattern.includes(":") && !targetPattern.includes("*")) {
continue;
}
try {
const matcher = match(targetPattern);
const result = matcher(outputPath);
if (result) {
const compileFn = compile(sourcePattern);
const resolvedSource = compileFn(result.params);
return path7.join(workDir, resolvedSource);
}
} catch (_error) {}
}
}
return path7.join(workDir, outputPath);
}
// src/plugin/hooks.ts
async function transform(content, id, settings, mdFiles, config) {
const orig = content;
if (!id.endsWith(".md") || !path8.resolve(id).startsWith(settings.workDir)) {
return null;
}
const resolvedOutFilePath = resolveOutputFilePath(id, settings.workDir, config.vitepress.userConfig?.rewrites);
const isMainPage = path8.relative(settings.workDir, resolvedOutFilePath) === "index.md";
if (settings.ignoreFiles?.length) {
const shouldIgnore = await Promise.all(settings.ignoreFiles.map(async (pattern) => {
if (typeof pattern === "string") {
return minimatch(path8.relative(settings.workDir, id), pattern);
}
return false;
}));
if (shouldIgnore.some((result) => result === true) && !isMainPage) {
return null;
}
}
let modifiedContent = content.replace(fullTagRegex("llm-only", "g"), "").replace(fullTagRegex("llm-exclude", "g"), "$1");
if (settings.injectLLMHint && (settings.generateLLMFriendlyDocsForEachPage || settings.generateLLMsTxt || settings.generateLLMsFullTxt)) {
matter4.clearCache();
modifiedContent = matter4(modifiedContent);
let llmHint = "";
const currentCleanUrl = cleanUrl(path8.relative(settings.workDir, resolvedOutFilePath));
const base = config.base || "/";
const basePath = base === "/" ? "" : base.replace(/\/$/, "");
if (isMainPage) {
const notices = [];
if (settings.generateLLMsTxt) {
notices.push(`${basePath}/llms.txt for optimized Markdown documentation`);
}
if (settings.generateLLMsFullTxt) {
notices.push(`${basePath}/llms-full.txt for full documentation bundle`);
}
if (notices.length > 0) {
llmHint = `Are you an LLM? View ${notices.join(", or ")}`;
}
} else {
if (settings.generateLLMFriendlyDocsForEachPage) {
const mdUrl = `${basePath}/${currentCleanUrl}`;
llmHint = `Are you an LLM? You can read better optimized documentation at ${mdUrl} for this page in Markdown format`;
}
}
llmHint = `<div style="display: none;" hidden="true" aria-hidden="true">${llmHint}</div>
`;
modifiedContent = matter4.stringify(llmHint + modifiedContent.content, modifiedContent.data);
}
if (!isMainPage) {
mdFiles.add(id);
}
return modifiedContent !== orig ? { code: modifiedContent, map: null } : null;
}
async function generateBundle(bundle, settings, config, mdFiles, isSsrBuild) {
if (isSsrBuild) {
logger_default.info("Skipping LLMs docs generation in SSR build");
return;
}
const resolvedSidebar = settings.sidebar instanceof Function ? await settings.sidebar(config?.vitepress?.userConfig?.themeConfig?.sidebar) : settings.sidebar;
const outDir = config.vitepress?.outDir ?? "dist";
try {
await fs4.access(outDir);
} catch {
logger_default.info(`Creating output directory: ${pc3.cyan(outDir)}`);
await fs4.mkdir(outDir, { recursive: true });
}
const mdFilesList = Array.from(mdFiles);
const fileCount = mdFilesList.length;
if (fileCount === 0) {
logger_default.warn(`No markdown files found to process. Check your \`${pc3.bold("workDir")}\` and \`${pc3.bold("ignoreFiles")}\` settings.`);
return;
}
logger_default.info(`Processing ${pc3.bold(fileCount.toString())} markdown files from ${pc3.cyan(settings.workDir)}`);
const imageMap = new Map;
if (bundle) {
for (const asset of Object.values(bundle)) {
if (asset && typeof asset === "object" && "type" in asset && asset.type === "asset" && "fileName" in asset && typeof asset.fileName === "string" && /(png|jpe?g|gif|svg|webp)$/i.test(path8.extname(asset.fileName))) {
const name2 = path8.posix.basename(asset.name || asset.fileName);
imageMap.set(name2, asset.fileName);
}
}
}
const preparedFiles = await Promise.all(mdFilesList.map(async (file) => {
const resolvedOutFilePath = path8.relative(settings.workDir, resolveOutputFilePath(file, settings.workDir, config.vitepress.userConfig?.rewrites));
const content = await fs4.readFile(file, "utf-8");
const markdownProcessor = remark().use(remarkFrontmatter).use(remarkPlease("unwrap", "llm-only")).use(remarkPlease("remove", "llm-exclude")).use(remarkReplaceImageUrls(imageMap));
if (settings.stripHTML) {
markdownProcessor.use(() => {
return (tree) => {
remove(tree, { type: "html" });
return tree;
};
});
}
const processedMarkdown = matter4(String(await markdownProcessor.process(content)));
const title = extractTitle(processedMarkdown)?.trim() || "Untitled";
const filePath = path8.basename(resolvedOutFilePath) === "index.md" && path8.dirname(resolvedOutFilePath) !== settings.workDir ? `${path8.dirname(resolvedOutFilePath)}.md` : resolvedOutFilePath;
return { path: filePath, title, file: processedMarkdown };
}));
preparedFiles.sort((a, b) => a.title.localeCompare(b.title));
const tasks = [];
if (settings.generateLLMsTxt) {
const templateVariables = {
title: settings.title,
description: settings.description,
details: settings.details,
toc: settings.toc,
...settings.customTemplateVariables
};
const directories = getDirectoriesAtDepths(mdFilesList, settings.workDir, settings.experimental?.depth ?? 1);
tasks.push(...directories.map((directory) => (async () => {
const isRoot = directory.relativePath === ".";
const directoryFilter = isRoot ? "." : directory.relativePath;
const outputFileName = isRoot ? "llms.txt" : path8.join(directory.relativePath, "llms.txt");
const llmsTxtPath = path8.resolve(outDir, outputFileName);
await fs4.mkdir(path8.dirname(llmsTxtPath), { recursive: true });
logger_default.info(`Generating ${pc3.cyan(outputFileName)}...`);
const llmsTxt = await generateLLMsTxt(preparedFiles, {
indexMd: path8.resolve(settings.workDir, resolveSourceFilePath("index.md", settings.workDir, config.vitepress.userConfig?.rewrites)),
outDir: settings.workDir,
LLMsTxtTemplate: settings.customLLMsTxtTemplate || defaultLLMsTxtTemplate,
templateVariables,
vitepressConfig: config?.vitepress?.userConfig,
domain: settings.domain,
sidebar: resolvedSidebar,
linksExtension: !settings.generateLLMFriendlyDocsForEachPage ? ".html" : undefined,
base: config.base,
directoryFilter
});
await fs4.writeFile(llmsTxtPath, llmsTxt, "utf-8");
logger_default.success(expandTemplate("Generated {file} (~{tokens} tokens, {size}) with {fileCount} documentation links", {
file: pc3.cyan(outputFileName),
tokens: pc3.bold(millify(approximateTokenSize(llmsTxt))),
size: pc3.bold(getHumanReadableSizeOf(llmsTxt)),
fileCount: pc3.bold(fileCount.toString())
}));
})()));
}
if (settings.generateLLMsFullTxt) {
const directories = getDirectoriesAtDepths(mdFilesList, settings.workDir, settings.experimental?.depth ?? 1);
tasks.push(...directories.map((directory) => (async () => {
const isRoot = directory.relativePath === ".";
const directoryFilter = isRoot ? "." : directory.relativePath;
const outputFileName = isRoot ? "llms-full.txt" : path8.join(directory.relativePath, "llms-full.txt");
const llmsFullTxtPath = path8.resolve(outDir, outputFileName);
await fs4.mkdir(path8.dirname(llmsFullTxtPath), { recursive: true });
logger_default.info(`Generating full documentation bundle (${pc3.cyan(outputFileName)})...`);
const llmsFullTxt = await generateLLMsFullTxt(preparedFiles, {
domain: settings.domain,
linksExtension: !settings.generateLLMFriendlyDocsForEachPage ? ".html" : undefined,
base: config.base,
directoryFilter
});
await fs4.writeFile(llmsFullTxtPath, llmsFullTxt, "utf-8");
logger_default.success(expandTemplate("Generated {file} (~{tokens} tokens, {size}) with {fileCount} markdown files", {
file: pc3.cyan(outputFileName),
tokens: pc3.bold(millify(approximateTokenSize(llmsFullTxt))),
size: pc3.bold(getHumanReadableSizeOf(llmsFullTxt)),
fileCount: pc3.bold(fileCount.toString())
}));
})()));
}
if (settings.generateLLMFriendlyDocsForEachPage) {
tasks.push(generateLLMFriendlyPages(preparedFiles, outDir, settings.domain, config.base));
}
if (tasks.length) {
await Promise.all(tasks);
}
}
// src/plugin/plugin.ts
var PLUGIN_NAME = name;
function llmstxt(userSettings = {}) {
const settings = {
generateLLMsTxt: true,
generateLLMsFullTxt: true,
generateLLMFriendlyDocsForEachPage: true,
ignoreFiles: [],
excludeUnnecessaryFiles: true,
excludeIndexPage: true,
excludeBlog: true,
excludeTeam: true,
injectLLMHint: true,
workDir: undefined,
stripHTML: true,
experimental: {
depth: 1,
...userSettings.experimental
},
...userSettings
};
let config;
const mdFiles = new Set;
let isSsrBuild = false;
return [
{
enforce: "pre",
name: `${PLUGIN_NAME}:llm-tags`,
async transform(content, id) {
return transform(content, id, settings, mdFiles, config);
}
},
{
name: PLUGIN_NAME,
enforce: "post",
configResolved(resolvedConfig) {
config = resolvedConfig;
if (settings.workDir) {
settings.workDir = path9.resolve(config.vitepress.srcDir, settings.workDir);
} else {
settings.workDir = path9.resolve(config.vitepress.srcDir);
}
if (settings.excludeUnnecessaryFiles) {
settings.excludeIndexPage && settings.ignoreFiles.push(...unnecessaryFilesList.indexPage);
settings.excludeBlog && settings.ignoreFiles.push(...unnecessaryFilesList.blogs);
settings.excludeTeam && settings.ignoreFiles.push(...unnecessaryFilesList.team);
}
isSsrBuild = !!resolvedConfig.build?.ssr;
logger_default.info(`${pc4.bold(PLUGIN_NAME)} initialized ${isSsrBuild ? pc4.dim("(SSR build)") : pc4.dim("(client build)")} with workDir: ${pc4.cyan(settings.workDir)}`);
},
async configureServer(server) {
await configureDevServer(server, config);
},
buildStart() {
mdFiles.clear();
logger_default.info("Build started, file collection cleared");
},
async generateBundle(_options, bundle) {
await generateBundle(bundle, settings, config, mdFiles, isSsrBuild);
}
}
];
}
export {
llmstxt as default,
copyOrDownloadAsMarkdownButtons
};