vitepress-plugin-llmstxt/dist/index.js

VitePress plugin to generate llms.txt files automatically

// src/pages.ts
import { createContentLoader } from "vitepress";

// src/utils.ts
import {
  access,
  stat,
  writeFile,
  constants,
  mkdir
} from "fs/promises";
import {
  join,
  dirname
} from "path";
import { styleText } from "util";

// package.json
var name = "vitepress-plugin-llmstxt";

// src/utils.ts
var PLUGIN_NAME = name;
var joinUrl = (...parts) => {
  parts = parts.map((part) => part.replace(/^\/+|\/+$/g, ""));
  return parts.join("/");
};
var overrideFrontmatter = (markdown, frontmatter) => {
  const toYAML = (obj, indent = 0) => {
    const pad = "  ".repeat(indent);
    return Object.entries(obj).map(([key, value]) => {
      if (Array.isArray(value)) {
        return `${pad}${key}:
` + value.map((item) => {
          if (typeof item === "object" && item !== null) {
            const nested = toYAML(item, indent + 2);
            return `${pad}  - ${nested.trimStart().replace(/^/gm, `${pad}    `).replace(`${pad}    `, "")}`;
          } else {
            return `${pad}  - ${JSON.stringify(item)}`;
          }
        }).join("\n");
      } else if (typeof value === "object" && value !== null) {
        return `${pad}${key}:
${toYAML(value, indent + 1)}`;
      } else {
        return `${pad}${key}: ${JSON.stringify(value)}`;
      }
    }).join("\n");
  };
  const frontmatterBlock = `---
${toYAML(frontmatter)}
---

`;
  const cleanedMarkdown = markdown.replace(/^---\n[\s\S]*?\n---\n*/, "");
  return frontmatterBlock + cleanedMarkdown.trimStart();
};
var removeFrontmatter = (markdown) => {
  const match = markdown.match(/^---\n([\s\S]*?)\n---\n?/);
  if (!match) return markdown;
  return markdown.slice(match[0].length);
};
var markdownPathToUrlRoute = (path) => {
  const route = path.endsWith(".md") ? path.slice(0, -3) : path;
  return route.startsWith("/") ? route : "/" + route;
};
var getMDTitleLine = (markdown) => {
  try {
    const match = markdown.match(/^# .*/m);
    return match ? match[0].replace("#", "").trim() : void 0;
  } catch (_) {
    return void 0;
  }
};
async function existsDir(path) {
  try {
    await access(path, constants.F_OK);
    const stats = await stat(path);
    return stats.isDirectory();
  } catch (_error) {
    return false;
  }
}
var ensureDir = async (path) => {
  const exist = await existsDir(path);
  if (!exist) await mkdir(path, { recursive: true });
};
var green = (v) => styleText("green", v);
var bold = (v) => styleText("bold", v);
var red = (v) => styleText("red", v);
var yellow = (v) => styleText("yellow", v);
var log = {
  success: (v) => console.log(green("\u2713 " + bold(PLUGIN_NAME) + " " + v)),
  error: (v) => console.log(red("\u2717 " + bold(PLUGIN_NAME) + " " + v)),
  warn: (v) => console.log(yellow("\u26A0 " + bold(PLUGIN_NAME) + " " + v)),
  info: (v) => console.log("i " + bold(PLUGIN_NAME) + " " + v)
};

// src/pages.ts
var LLM_FILENAME = "llms.txt";
var LLM_FULL_FILENAME = "llms-full.txt";
var replaceMarkdownTemplate = (template, params, frontmatter, content) => {
  try {
    return template.replace(/\{\{\s*\$params\.(\w+)\s*\}\}/g, (_, key) => params[key] ?? "").replace(/\{\{\s*\$frontmatter\.(\w+)\s*\}\}/g, (_, key) => frontmatter[key] ?? "").replace(/<!--\s*@content\s*-->/g, content ?? "");
  } catch (_e) {
    return template;
  }
};
var orderContent = (content) => content.sort((a, b) => b.url.localeCompare(a.url));
var transformPages = async (pages, config, vpConfig) => {
  if (!config?.transform) return pages;
  const utils = {
    getIndexTOC: (type) => getIndex(pages, { llmsFile: { indexTOC: type } }, vpConfig),
    removeFrontmatter
  };
  for (const key in pages) {
    const tRes = await config?.transform({
      page: pages[key],
      pages,
      vpConfig,
      utils
    });
    if (tRes) pages[key] = tRes;
  }
  return pages;
};
var getIndex = (pages, config, vpConfig) => {
  try {
    let res = "";
    const indextoc = typeof config?.llmsFile === "object" ? config?.llmsFile?.indexTOC : config?.llmsFile;
    if (!indextoc) return res;
    const h = "#".repeat(1);
    const webLinks = pages.filter((d) => !d.path.endsWith(".txt")).map((p) => `- [${p.title}](${p.url})`).join("\n");
    const llmLinks = pages.filter((d) => !d.path.endsWith(".txt")).map((p) => `- [${p.title}](${p.llmUrl})`).join("\n");
    res += `${h} Table of contents
${vpConfig?.userConfig.description ? "\n" + vpConfig?.userConfig.description.trimEnd() + "\n" : ""}`;
    if (indextoc === "only-web") res += `
${h}# Web links

${webLinks}`;
    else if (indextoc === "only-web-links") res = webLinks;
    else if (indextoc === "only-llms" && config?.mdFiles) res += `
${h}# LLMs links

${llmLinks}`;
    else if (indextoc === "only-llms-links" && config?.mdFiles) res = llmLinks;
    else res += `
${h}# Web links

${webLinks}${config?.mdFiles ? `

${h}# LLMs links

${llmLinks}` : ""}`;
    return res;
  } catch (_) {
    return "";
  }
};
var getPages = async (config, vpConfig) => {
  const loader = createContentLoader("**/*.md", {
    includeSrc: true,
    excerpt: true,
    globOptions: config?.ignore ? { ignore: [
      "node_modules",
      "dist",
      ...config.ignore
    ] } : void 0
  });
  const pages = await loader.load();
  if (!vpConfig?.dynamicRoutes.routes || config?.dynamicRoutes === false) return orderContent(pages);
  const dynamicPaths = [];
  for (const key in vpConfig?.dynamicRoutes.routes) {
    const page = vpConfig?.dynamicRoutes.routes[key];
    const route = markdownPathToUrlRoute(page.route);
    const content = pages.find((p) => p.url.replace(".html", "") === route);
    if (!content || !content.src) continue;
    dynamicPaths.push(content.url);
    pages.push({
      excerpt: void 0,
      frontmatter: {},
      html: void 0,
      url: markdownPathToUrlRoute(page.path),
      src: replaceMarkdownTemplate(content.src, page.params, {}, page.content)
    });
  }
  const res = dynamicPaths.length ? pages.filter((p) => dynamicPaths.includes(p.url) ? void 0 : p) : pages;
  return orderContent(res);
};
var getPagesData = async (config, vpConfig) => {
  const pages = await getPages(config, vpConfig);
  const originURL = config.hostname;
  const mdFiles = [];
  const allFiles = [];
  for (const page of pages.slice().reverse()) {
    const route = page.url;
    const pathname = route.replace(".html", "");
    const path = join((pathname === "/" ? "/index" : pathname.endsWith("/") ? pathname.slice(0, -1) : pathname) + ".md");
    const URL2 = joinUrl(originURL, route);
    const LLMS_URL = joinUrl(originURL, path);
    const frontmatter = {
      URL: URL2,
      LLMS_URL,
      ...page.frontmatter
    };
    const content = overrideFrontmatter(page.src || "", frontmatter);
    mdFiles.push({
      path,
      url: URL2,
      llmUrl: LLMS_URL,
      content,
      title: page.frontmatter.title || getMDTitleLine(content) || page.frontmatter.layout || "",
      frontmatter
    });
  }
  if (config?.llmsFullFile) {
    const path = "/" + LLM_FULL_FILENAME;
    const extra = {
      URL: join(originURL, path),
      LLMS_URL: join(originURL, path)
    };
    const content = mdFiles.map((d) => d.content).join("\n\n");
    allFiles.push({
      path,
      url: extra.URL,
      llmUrl: extra.LLMS_URL,
      content,
      title: getMDTitleLine(content) || "",
      frontmatter: extra
    });
  }
  if (config.mdFiles) allFiles.push(...mdFiles);
  if (config?.llmsFile) {
    const path = "/" + LLM_FILENAME;
    const extra = {
      URL: join(originURL, path),
      LLMS_URL: join(originURL, path)
    };
    const content = getIndex(mdFiles, config, vpConfig).trim();
    allFiles.push({
      path,
      url: extra.URL,
      llmUrl: extra.LLMS_URL,
      content,
      title: getMDTitleLine(content) || "",
      frontmatter: extra
    });
  }
  const res = await transformPages(allFiles, config, vpConfig);
  return res;
};

// src/index.ts
var addVPConfigLllmData = (data, vpConfig) => {
  if (!vpConfig) return;
  const config = { pageData: data?.map((d) => ({
    path: d.path,
    url: d.url,
    llmUrl: d.llmUrl
  })) };
  vpConfig.site.themeConfig.llmstxt = config;
};
var llmstxtPlugin = (config) => {
  const {
    llmsFullFile = true,
    llmsFile = true,
    mdFiles = true,
    hostname = "/",
    dynamicRoutes = true,
    watch = false
  } = config || {};
  const c = {
    ...config,
    dynamicRoutes,
    llmsFullFile,
    llmsFile,
    mdFiles,
    hostname,
    watch
  };
  let vpConfig = void 0, data = void 0;
  return {
    name: PLUGIN_NAME,
    /**
     * **ALERT:**
     * Do not add 'enforce' because it gives unexpected errors with other plugins and alters the order of these plugins
     * Tested at vitepress@1.6.3 and vue@3.5.18
     */
    // enforce : 'pre',
    /**
     * **NOTE:**
     * 'buildStart' runs in server mode too!
     * That's why it's not necessary to add this function in 'configureServer'
     *
     * @see https://vite.dev/guide/api-plugin.html#universal-hooks
     * @see https://rollupjs.org/plugin-development/#buildstart
     */
    async buildStart() {
      if (!vpConfig) return;
      if (data) return;
      data = await getPagesData(
        c,
        vpConfig
      );
      addVPConfigLllmData(data, vpConfig);
    },
    /**
     * Called when a watched file changes during development.
     */
    watchChange: async (path) => {
      if (!vpConfig) return;
      if (!c.watch) return;
      if (!(path.endsWith(".md") || path.endsWith(".txt"))) return;
      data = await getPagesData(
        c,
        vpConfig
      );
      addVPConfigLllmData(data, vpConfig);
    },
    /**
     * Configures the Vite dev server middleware.
     * Adds support to serve `.txt` and `.md` files dynamically for matched routes.
     */
    async configureServer(server) {
      server.middlewares.use(async (req, res, next) => {
        const urlPath = req?.url;
        if (!urlPath || !(urlPath.endsWith(".txt") || urlPath.endsWith(".md"))) return next();
        const url = await (async () => new URL(joinUrl(server.resolvedUrls?.local[0] || "localhost", urlPath)))().catch(void 0);
        if (!url) return next();
        try {
          if (!data) {
            data = await getPagesData(
              c,
              vpConfig
            );
            addVPConfigLllmData(data, vpConfig);
          }
          for (const d of data) {
            const llmRoute = [
              join("/", d.path),
              join("/", d.path, "index.md"),
              join("/", d.path + ".md"),
              join("/", d.path + ".html"),
              join("/", d.path + ".html", "index.md")
            ];
            if (llmRoute.includes(url.pathname)) {
              res.setHeader("Content-Type", "text/markdown");
              res.end(d.content);
              return;
            }
          }
        } catch (e) {
          log.warn(e instanceof Error ? e.message : "Unexpected error");
        }
        next();
      });
    },
    /**
     * Called once the final Vite config is resolved.
     *
     * This hook is used to attach a `buildEnd` hook dynamically to the VitePress config,
     * which will generate static `.md` files for all collected LLM routes.
     *
     */
    async configResolved(params) {
      if (vpConfig) return;
      vpConfig = "vitepress" in params ? params.vitepress : void 0;
      if (!vpConfig) return;
      const selfBuildEnd = vpConfig.buildEnd;
      vpConfig.buildEnd = async (siteConfig) => {
        await selfBuildEnd?.(siteConfig);
        const outDir = siteConfig.outDir;
        if (!data) {
          data = await getPagesData(
            c,
            vpConfig
          );
          addVPConfigLllmData(data, siteConfig);
        }
        for (const page of data) {
          const dir = join(outDir, dirname(page.path));
          await ensureDir(dir);
          await writeFile(join(outDir, page.path), page.content, "utf-8");
        }
        log.success("LLM routes builded susccesfully \u2728");
      };
    }
  };
};
var index_default = llmstxtPlugin;
export {
  index_default as default,
  llmstxtPlugin
};