docusaurus-plugin-llms-txt/lib/parseDocuments.js

A Docusaurus plugin for creating llms.txt and llms-full.txt files

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.extractRoutesMetadata = extractRoutesMetadata;
const utils_1 = require("@docusaurus/utils");
const gray_matter_1 = __importDefault(require("gray-matter"));
const fs_extra_1 = __importDefault(require("fs-extra"));
const logger_1 = __importDefault(require("@docusaurus/logger"));
const path_1 = __importDefault(require("path"));
function extractUrl(route, siteConfig) {
    return path_1.default.join(siteConfig.url, route.path);
}
function extractTitle(content, frontMatter) {
    let title = frontMatter.data?.title;
    // If no title in frontmatter, try to get first header
    if (!title) {
        const headerMatch = frontMatter.content.match(/^#\s+(.+)$/m);
        if (headerMatch) {
            title = headerMatch[1].trim();
        }
    }
    return title || "";
}
function extractSectionName(route, siteConfig) {
    const navbars = siteConfig.themeConfig.navbar || {};
    const matching_sidebars = navbars.items.filter((item) => item.sidebarId === route.sidebar);
    if (matching_sidebars.length == 1) {
        return matching_sidebars[0].label;
    }
    else {
        logger_1.default.error('Multiple sidebars found for route ' + route.path);
        throw new Error('Multiple sidebars found for route ' + route.path);
    }
}
function extractSingleRouteMetadata(route, siteConfig) {
    let description = null;
    let content = "";
    let section_name = "";
    let url = "";
    let title = "";
    if (route.metadata?.sourceFilePath) {
        try {
            const source = route.metadata.sourceFilePath;
            content = fs_extra_1.default.readFileSync(source, 'utf8');
        }
        catch (error) {
            console.error(`Error reading file ${route.metadata.sourceFilePath}:`, error);
        }
        try {
            const frontMatter = (0, gray_matter_1.default)(content);
            title = extractTitle(content, frontMatter);
            description = frontMatter.data?.description;
            section_name = extractSectionName(route, siteConfig);
            url = extractUrl(route, siteConfig);
        }
        catch (error) {
            console.error(`Error parsing markdown file ${route.metadata.sourceFilePath}:`, error);
        }
    }
    return {
        title: title,
        description: description,
        content: content,
        section_name: section_name,
        url: url
    };
}
function extractRoutesMetadata(routes, siteConfig) {
    const flatRoutes = (0, utils_1.flattenRoutes)(routes);
    const filteredRoutes = flatRoutes.filter((route) => route.metadata?.sourceFilePath?.endsWith('.md') || route.metadata?.sourceFilePath?.endsWith('.mdx'));
    const routesMetadata = filteredRoutes.map((route) => {
        return extractSingleRouteMetadata(route, siteConfig);
    });
    return routesMetadata;
}