vuepress-plugin-llms
Version:
📜 VuePress plugin for generating documentation friendly to Large Language Models (LLMs) | 📜 VuePress 插件，用于生成对大语言模型(LLMs)友好的文档。
232 lines (231 loc) • 12.1 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.default = llmstxt;
const promises_1 = __importDefault(require("node:fs/promises"));
const node_path_1 = __importDefault(require("node:path"));
const gray_matter_1 = __importDefault(require("gray-matter"));
const minimatch_1 = require("minimatch");
const picocolors_1 = __importDefault(require("picocolors"));
const remark_1 = require("remark");
const remark_frontmatter_1 = __importDefault(require("remark-frontmatter"));
const unist_util_remove_1 = require("unist-util-remove");
const PLUGIN_NAME = 'vuepress-plugin-llms';
const constants_js_1 = require("./constants.js");
const index_js_1 = require("./helpers/index.js");
const logger_js_1 = __importDefault(require("./helpers/logger.js"));
const utils_js_1 = require("./helpers/utils.js");
/**
* [VuePress](https://v2.vuepress.vuejs.org/) 插件，用于生成原始的 Markdown 格式文档，这种格式对**大语言模型(LLMs)**更轻量且更高效
* [VuePress](https://v2.vuepress.vuejs.org/) plugin for generating raw documentation for **LLMs** in Markdown format which is much lighter and more efficient for LLMs
*
* @param userSettings - Plugin settings. | 插件设置。
* @see https://github.com/guoqw7/vuepress-plugin-llms
* @see https://llmstxt.org/
*/
function llmstxt(userSettings = {}) {
// Create a settings object with defaults explicitly merged
// 创建一个设置对象，明确合并默认值
const settings = {
generateLLMsTxt: true,
generateLLMsFullTxt: true,
stripHTML: true,
ignoreFiles: [],
workDir: '',
...userSettings,
// Ensure workDir is set after merging
};
// Set to store all markdown file paths
// 用于存储所有markdown文件路径的集合
const mdFiles = new Set();
return {
name: PLUGIN_NAME,
/** Set up plugin on initialization | 在初始化时设置插件 */
onInitialized(app) {
if (settings.workDir) {
settings.workDir = node_path_1.default.resolve(app.dir.source(), settings.workDir);
}
else {
settings.workDir = app.dir.source();
}
logger_js_1.default.info(`${picocolors_1.default.bold(PLUGIN_NAME)} initialized with workDir: ${picocolors_1.default.cyan(settings.workDir)}`);
},
/** Register middleware to serve markdown files as plain text during dev | 在开发模式下注册中间件以纯文本形式提供markdown文件 */
onPrepared(app) {
if (app.env.isDev) {
// In dev mode, add middleware to serve .md and .txt files as plain text
// This would require custom code to integrate with VuePress dev server
// 在开发模式下，添加中间件以纯文本形式提供.md和.txt文件
// 这需要自定义代码与VuePress开发服务器集成
logger_js_1.default.info('Dev server configured for serving plain text docs for LLMs');
}
},
/** Process files after they've been prepared | 在文件准备好后处理它们 */
async onGenerated(app) {
// Reset file collection
// 重置文件集合
mdFiles.clear();
logger_js_1.default.info('Starting markdown file collection');
// Collect all markdown files in the source directory
// 收集源目录中的所有markdown文件
const collectMarkdownFiles = async (dir) => {
const entries = await promises_1.default.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = node_path_1.default.join(dir, entry.name);
// Skip files outside workDir if configured
// 如果配置了workDir，跳过workDir之外的文件
if (!fullPath.startsWith(settings.workDir)) {
continue;
}
// Process directories recursively
// 递归处理目录
if (entry.isDirectory()) {
await collectMarkdownFiles(fullPath);
continue;
}
// Skip non-markdown files
// 跳过非markdown文件
if (!entry.name.endsWith('.md')) {
continue;
}
// Check if file should be ignored
// 检查文件是否应该被忽略
if (settings.ignoreFiles?.length) {
const relPath = node_path_1.default.relative(settings.workDir, fullPath);
const shouldIgnore = settings.ignoreFiles.some(pattern => {
if (typeof pattern === 'string') {
return (0, minimatch_1.minimatch)(relPath, pattern);
}
return false;
});
if (shouldIgnore) {
continue;
}
}
// Add markdown file to collection
// 将markdown文件添加到集合中
mdFiles.add(fullPath);
}
};
await collectMarkdownFiles(settings.workDir);
const mdFilesList = Array.from(mdFiles);
const fileCount = mdFilesList.length;
// Skip if no files found
// 如果没有找到文件则跳过
if (fileCount === 0) {
logger_js_1.default.warn(`No markdown files found to process. Check your \`${picocolors_1.default.bold('workDir')}\` and \`${picocolors_1.default.bold('ignoreFiles')}\` settings.`);
return;
}
logger_js_1.default.info(`Processing ${picocolors_1.default.bold(fileCount.toString())} markdown files from ${picocolors_1.default.cyan(settings.workDir)}`);
// Prepare files for processing
// 准备文件以进行处理
const preparedFiles = await Promise.all(mdFilesList.map(async (file) => {
const content = await promises_1.default.readFile(file, 'utf-8');
let mdFile;
if (settings.stripHTML) {
const cleanedMarkdown = await (0, remark_1.remark)()
.use(remark_frontmatter_1.default)
.use(() => {
// Strip HTML tags
// 去除HTML标签
return (tree) => {
(0, unist_util_remove_1.remove)(tree, { type: 'html' });
return tree;
};
})
.process(content);
mdFile = (0, gray_matter_1.default)(String(cleanedMarkdown));
}
else {
mdFile = (0, gray_matter_1.default)(content);
}
// Extract title from frontmatter or use the first heading
// 从frontmatter中提取标题或使用第一个标题
const title = (0, utils_js_1.extractTitle)(mdFile)?.trim() || 'Untitled';
const filePath = node_path_1.default.basename(file) === 'index.md' &&
node_path_1.default.dirname(file) !== settings.workDir
? `${node_path_1.default.dirname(file)}.md`
: file;
return { path: filePath, title, file: mdFile };
}));
// Sort files by title for better organization
// 按标题排序文件以获得更好的组织
preparedFiles.sort((a, b) => a.title.localeCompare(b.title));
const tasks = [];
const outDir = app.dir.dest();
// Create output directory if it doesn't exist
// 如果输出目录不存在则创建
try {
await promises_1.default.access(outDir);
}
catch {
logger_js_1.default.info(`Creating output directory: ${picocolors_1.default.cyan(outDir)}`);
await promises_1.default.mkdir(outDir, { recursive: true });
}
// Generate llms.txt
// 生成 llms.txt
if (settings.generateLLMsTxt) {
const llmsTxtPath = node_path_1.default.resolve(outDir, 'llms.txt');
const templateVariables = {
title: settings.title,
description: settings.description,
details: settings.details,
toc: settings.toc,
...settings.customTemplateVariables,
};
tasks.push((async () => {
logger_js_1.default.info(`Generating ${picocolors_1.default.cyan('llms.txt')}...`);
const siteConfig = app.siteData;
// Find index.md file
// 查找 index.md 文件
const indexMdPath = node_path_1.default.resolve(settings.workDir, 'index.md');
let indexMdExists = true;
try {
await promises_1.default.access(indexMdPath);
}
catch {
indexMdExists = false;
logger_js_1.default.warn('index.md not found in workDir, using fallback values');
}
const content = await (0, index_js_1.generateLLMsTxt)(preparedFiles, {
indexMd: indexMdExists ? indexMdPath : preparedFiles[0]?.path || '',
srcDir: settings.workDir,
LLMsTxtTemplate: settings.customLLMsTxtTemplate || constants_js_1.defaultLLMsTxtTemplate,
templateVariables: templateVariables,
siteConfig: {
title: siteConfig.title,
description: siteConfig.description,
},
domain: settings.domain,
linksExtension: '.md',
cleanUrls: false,
});
await promises_1.default.writeFile(llmsTxtPath, content, 'utf-8');
logger_js_1.default.success(`Generated ${picocolors_1.default.cyan('llms.txt')} (${picocolors_1.default.bold(content.length.toString())} bytes)`);
})());
}
// Generate llms-full.txt
// 生成 llms-full.txt
if (settings.generateLLMsFullTxt) {
const llmsFullTxtPath = node_path_1.default.resolve(outDir, 'llms-full.txt');
tasks.push((async () => {
logger_js_1.default.info(`Generating ${picocolors_1.default.cyan('llms-full.txt')}...`);
const content = await (0, index_js_1.generateLLMsFullTxt)(preparedFiles, {
srcDir: settings.workDir,
domain: settings.domain,
linksExtension: '.md',
cleanUrls: false,
});
await promises_1.default.writeFile(llmsFullTxtPath, content, 'utf-8');
logger_js_1.default.success(`Generated ${picocolors_1.default.cyan('llms-full.txt')} (${picocolors_1.default.bold(content.length.toString())} bytes)`);
})());
}
// Wait for all tasks to complete
// 等待所有任务完成
await Promise.all(tasks);
logger_js_1.default.success(`${picocolors_1.default.bold(PLUGIN_NAME)} completed all tasks`);
}
};
}