vitepress-plugin-llmstxt
Version:
VitePress plugin to generate llms.txt files automatically
320 lines (308 loc) • 9.72 kB
JavaScript
import { createContentLoader } from 'vitepress';
import { mkdir, access, constants, stat, writeFile } from 'node:fs/promises';
import { join, dirname } from 'node:path';
import { styleText } from 'node:util';
const name = "vitepress-plugin-llmstxt";
const PLUGIN_NAME = name;
const joinUrl = (...parts) => {
parts = parts.map((part) => part.replace(/^\/+|\/+$/g, ""));
return parts.join("/");
};
const overrideFrontmatter = (markdown, frontmatter) => {
const toYAML = (obj, indent = 0) => {
const pad = " ".repeat(indent);
return Object.entries(obj).map(([key, value]) => {
if (Array.isArray(value)) {
return `${pad}${key}:
` + value.map((item) => {
if (typeof item === "object" && item !== null) {
const nested = toYAML(item, indent + 2);
return `${pad} - ${nested.trimStart().replace(/^/gm, `${pad} `).replace(`${pad} `, "")}`;
} else {
return `${pad} - ${JSON.stringify(item)}`;
}
}).join("\n");
} else if (typeof value === "object" && value !== null) {
return `${pad}${key}:
${toYAML(value, indent + 1)}`;
} else {
return `${pad}${key}: ${JSON.stringify(value)}`;
}
}).join("\n");
};
const frontmatterBlock = `---
${toYAML(frontmatter)}
---
`;
const cleanedMarkdown = markdown.replace(/^---\n[\s\S]*?\n---\n*/, "");
return frontmatterBlock + cleanedMarkdown.trimStart();
};
const removeFrontmatter = (markdown) => {
const match = markdown.match(/^---\n([\s\S]*?)\n---\n?/);
if (!match) return markdown;
return markdown.slice(match[0].length);
};
const getMDTitleLine = (markdown) => {
try {
const match = markdown.match(/^# .*/m);
return match ? match[0].replace("#", "").trim() : void 0;
} catch (_) {
return void 0;
}
};
async function existsDir(path) {
try {
await access(path, constants.F_OK);
const stats = await stat(path);
return stats.isDirectory();
} catch (_error) {
return false;
}
}
const ensureDir = async (path) => {
const exist = await existsDir(path);
if (!exist) await mkdir(path, { recursive: true });
};
const green = (v) => styleText("green", v);
const bold = (v) => styleText("bold", v);
const red = (v) => styleText("red", v);
const yellow = (v) => styleText("yellow", v);
const log = {
success: (v) => console.log(green("\u2713 " + bold(PLUGIN_NAME) + " " + v)),
error: (v) => console.log(red("\u2717 " + bold(PLUGIN_NAME) + " " + v)),
warn: (v) => console.log(yellow("\u26A0 " + bold(PLUGIN_NAME) + " " + v)),
info: (v) => console.log("i " + bold(PLUGIN_NAME) + " " + v)
};
const LLM_FILENAME = "llms.txt";
const LLM_FULL_FILENAME = "llms-full.txt";
const getPages = async (config) => {
const loader = createContentLoader("**/*.md", {
includeSrc: true,
excerpt: true,
globOptions: config?.ignore ? { ignore: [
"node_modules",
"dist",
...config.ignore
] } : void 0
});
const pages = await loader.load();
return pages;
};
const transformPages = async (pages, config, vpConfig) => {
if (!config?.transform) return pages;
for (const key in pages) {
const tRes = await config?.transform({
page: pages[key],
pages,
vpConfig,
utils: {
getIndexTOC: (type) => getIndex(pages, { llmsFile: { indexTOC: type } }, vpConfig),
removeFrontmatter
}
});
if (tRes) pages[key] = tRes;
}
return pages;
};
const getIndex = (pages, config, vpConfig) => {
try {
let res = "";
const indextoc = typeof config?.llmsFile === "object" ? config?.llmsFile?.indexTOC : config?.llmsFile;
if (!indextoc) return res;
const indexP = pages.find((d) => d.path === "/" + LLM_FILENAME);
if (!indexP) return res;
const title = getMDTitleLine(indexP.content);
const h = "#".repeat(title && title !== "" ? 2 : 1);
const webLinks = pages.filter((d) => !d.path.endsWith(".txt")).map((p) => `- [${p.title}](${p.url})`).join("\n");
const llmLinks = pages.filter((d) => !d.path.endsWith(".txt")).map((p) => `- [${p.title}](${p.llmUrl})`).join("\n");
res += `${h} Table of contents
${vpConfig?.userConfig.description ? "\n" + vpConfig?.userConfig.description : ""}`;
if (indextoc === "only-web") res += `
${h}# Web links
${webLinks}`;
else if (indextoc === "only-web-links") res = webLinks;
else if (indextoc === "only-llms") res += `
${h}# LLMs links
${llmLinks}`;
else if (indextoc === "only-llms-links") res = llmLinks;
else res += `
${h}# Web links
${webLinks}
${h}# LLMs links
${llmLinks}`;
return res;
} catch (_) {
return "";
}
};
const setIndex = (pages, config, vpConfig) => {
const indextoc = typeof config?.llmsFile === "object" ? config?.llmsFile?.indexTOC : config?.llmsFile;
if (!indextoc) return pages;
return pages.map((d) => {
if (d.path !== "/" + LLM_FILENAME) return d;
const index = getIndex(pages, config, vpConfig);
if (index && index !== "") d.content += `
${index}`;
return d;
});
};
const getPagesData = async (pages, originURL, config, vpConfig) => {
let res = [], fullContent = "";
if (config?.llmsFile) {
const path = "/" + LLM_FILENAME;
const extra = {
URL: join(originURL, path),
LLMS_URL: join(originURL, path)
};
res.push({
path,
url: extra.URL,
llmUrl: extra.LLMS_URL,
content: fullContent,
title: getMDTitleLine(fullContent) || "",
frontmatter: extra
});
}
if (config?.mdFiles) {
for (const page of pages.slice().reverse()) {
const content = page.src;
const route = page.url;
const pathname = page.url.replace(".html", "");
const path = join((pathname === "/" ? "/index" : pathname.endsWith("/") ? pathname.slice(0, -1) : pathname) + ".md");
const URL2 = joinUrl(originURL, route);
const LLMS_URL = joinUrl(originURL, path);
const frontmatter = {
URL: URL2,
LLMS_URL,
...page.frontmatter
};
const finalContent = overrideFrontmatter(content || "", frontmatter);
res.push({
path,
url: URL2,
llmUrl: LLMS_URL,
content: finalContent,
title: page.frontmatter.title || getMDTitleLine(finalContent) || page.frontmatter.layout || "",
frontmatter
});
fullContent += `${finalContent}
`;
}
}
if (config?.llmsFullFile) {
const path = "/" + LLM_FULL_FILENAME;
const extra = {
URL: join(originURL, path),
LLMS_URL: join(originURL, path)
};
res.push({
path,
url: extra.URL,
llmUrl: extra.LLMS_URL,
content: fullContent,
title: getMDTitleLine(fullContent) || "",
frontmatter: extra
});
}
const resT = await transformPages(res, config, vpConfig);
const resI = setIndex(resT, config, vpConfig);
return resI;
};
const addVPConfigLllmData = (data, vpConfig) => {
if (vpConfig) vpConfig.site.themeConfig.llmstxt = {
pageData: data
};
};
const llmstxtPlugin = (config) => {
const {
llmsFullFile = true,
llmsFile = true,
mdFiles = true,
hostname = "/"
} = config || {};
const c = {
...config,
llmsFullFile,
llmsFile,
mdFiles,
hostname
};
let vpConfig = void 0;
return {
name: PLUGIN_NAME,
enforce: "pre",
async configureServer(server) {
const pages = await getPages(c);
const data = await getPagesData(
pages,
c.hostname,
c,
vpConfig
);
addVPConfigLllmData(data, vpConfig);
server.middlewares.use(async (req, res, next) => {
const urlPath = req?.url;
if (!urlPath || !(urlPath.endsWith(".txt") || urlPath.endsWith(".md"))) return next();
const url = await (async () => new URL(joinUrl(server.resolvedUrls?.local[0] || process.env.HOST || "localhost", urlPath)))().catch(void 0);
if (!url) return next();
try {
const data2 = await getPagesData(
pages,
c.hostname,
c,
vpConfig
);
for (const d of data2) {
const llmRoute = [
join("/", d.path),
join("/", d.path, "index.md"),
join("/", d.path + ".md"),
join("/", d.path + ".html"),
join("/", d.path + ".html", "index.md")
];
if (llmRoute.includes(url.pathname)) {
res.setHeader("Content-Type", "text/markdown");
res.end(d.content);
return;
}
}
} catch (e) {
log.warn(e instanceof Error ? e.message : "Unexpected error");
}
next();
});
},
async configResolved(params) {
if (vpConfig) return;
vpConfig = "vitepress" in params ? params.vitepress : void 0;
if (!vpConfig) return;
const pages = await getPages(c);
const data = await getPagesData(
pages,
c.hostname,
c,
vpConfig
);
addVPConfigLllmData(data, vpConfig);
const selfBuildEnd = vpConfig.buildEnd;
const outDir = vpConfig.outDir;
vpConfig.buildEnd = async (siteConfig) => {
await selfBuildEnd?.(siteConfig);
const pages2 = await getPages(c);
const data2 = await getPagesData(
pages2,
c.hostname,
c,
vpConfig
);
for (const page of data2) {
const dir = join(outDir, dirname(page.path));
await ensureDir(dir);
await writeFile(join(outDir, page.path), page.content, "utf-8");
}
log.success("LLM routes builded susccesfully \u2728\n");
};
}
};
};
export { llmstxtPlugin as default, llmstxtPlugin };