@signalwire/docusaurus-plugin-llms-txt
Version:
Generate Markdown versions of Docusaurus HTML pages and an llms.txt index file
139 lines (138 loc) • 6.51 kB
JavaScript
/**
* Simplified route processing coordination
* Uses focused modules for validation, context management, and processing
*/
import path from 'path';
import pMap from 'p-map';
import { CacheManager } from '../cache/cache';
import { validateCliContext } from '../cache/cache-strategy';
import { hashFile } from '../cache/cache-validation';
import { getEffectiveConfigForRoute, getContentConfig } from '../config';
import { ERROR_MESSAGES } from '../constants';
import { getErrorMessage } from '../errors';
import { processHtmlFileWithContext } from '../transformation/html-file-processor';
import { analyzeProcessingContext } from './processing-context';
import { validateRoutesForProcessing } from './route-validator';
/**
* Type guard to check if a route is a complete RouteConfig
*/
function isCompleteRouteConfig(route) {
return 'component' in route && typeof route.component === 'string';
}
/**
* Process a single route with error handling
*/
async function processSingleRoute(route, cachedRoute, config, directories, logger, siteUrl, outDir, routeLookup) {
if (!cachedRoute.htmlPath) {
logger.debug(`No HTML path for route: ${route.path}`);
return {};
}
try {
const fullHtmlPath = path.join(directories.docsDir, cachedRoute.htmlPath);
const effectiveConfig = getEffectiveConfigForRoute(route.path, config);
const doc = await processHtmlFileWithContext(fullHtmlPath, route.path, effectiveConfig, directories.mdOutDir, logger, siteUrl, outDir, routeLookup);
if (doc) {
const hash = await hashFile(fullHtmlPath);
const contentConfig = getContentConfig(config);
// Note: This is a temporary CacheManager just for the update method
// We don't have siteConfig here, but it's not needed for updateCachedRouteWithDoc
const cacheManager = new CacheManager('', '', config, logger, outDir);
const updatedCachedRoute = cacheManager.updateCachedRouteWithDoc(cachedRoute, doc, hash, contentConfig.enableMarkdownFiles);
logger.debug(`Processed route: ${route.path}`);
return { doc, updatedCachedRoute };
}
return {};
}
catch (error) {
const errorMessage = getErrorMessage(error);
logger.reportRouteError(ERROR_MESSAGES.ROUTE_PROCESSING_FAILED(route.path, errorMessage));
return {};
}
}
/**
* Simplified route processing using focused modules
*/
async function processRoutesStream(routes, generatedFilesDir, docsDir, mdOutDir, siteDir, options, logger, siteUrl, useCache = true, existingCachedRoutes, outDir, siteConfig) {
// Setup cache and directories
const cacheManager = new CacheManager(siteDir, generatedFilesDir, options, logger, outDir, siteConfig);
const cachedRoutes = existingCachedRoutes ?? cacheManager.createCachedRouteInfo(routes);
const directories = { docsDir, mdOutDir };
// Create route lookup table for link resolution
const routeLookup = new Map();
for (const route of cachedRoutes) {
routeLookup.set(route.path, route);
}
// Validate routes for processing
const validatedRoutes = validateRoutesForProcessing(routes, cachedRoutes, options, logger);
// Process routes concurrently with p-map
const results = await pMap(validatedRoutes, async (routeData, index) => {
if (!routeData) {
return { originalIndex: index };
}
const { route, cachedRoute, isValid } = routeData;
if (!isValid) {
return { originalIndex: index };
}
// Only process routes that have a complete RouteConfig (not Partial<RouteConfig>)
if (!isCompleteRouteConfig(route)) {
logger.debug(`Skipping incomplete route: ${route.path}`);
return { originalIndex: index };
}
// Check if we can use cached data
let canUseCache = false;
try {
canUseCache =
useCache &&
(await cacheManager.isCachedRouteValid(cachedRoute, options));
}
catch (error) {
const errorMessage = getErrorMessage(error);
logger.debug(`Cache validation failed for route ${route.path}, falling back to processing: ${errorMessage}`);
canUseCache = false;
}
if (canUseCache) {
const doc = cacheManager.cachedRouteToDocInfo(cachedRoute);
if (doc) {
logger.debug(`Using cached data for route: ${route.path}`);
return { doc, originalIndex: index };
}
}
// Process the route - TypeScript now knows route is a complete RouteConfig
const result = await processSingleRoute(route, cachedRoute, options, directories, logger, siteUrl, outDir, routeLookup);
return {
doc: result.doc,
updatedCachedRoute: result.updatedCachedRoute,
originalIndex: index,
};
}, { concurrency: 10 });
// Collect results safely
const docs = results
.map((r) => r.doc)
.filter((doc) => doc !== undefined);
const updatedCachedRoutes = [...cachedRoutes];
results.forEach((result) => {
if (result.updatedCachedRoute) {
updatedCachedRoutes[result.originalIndex] = result.updatedCachedRoute;
}
});
return { docs, cachedRoutes: updatedCachedRoutes };
}
/**
* Unified document processing using focused context management
*/
export async function processDocuments(routes, cache, docsDir, mdOutDir, siteDir, config, logger, siteUrl, useCache = true, outDir, generatedFilesDir, siteConfig) {
logger.debug(`Processing: useCache=${useCache}, routes=${routes.length}`);
// Analyze processing context
const context = analyzeProcessingContext(routes, cache, logger);
logger.debug(context.description);
// Validate CLI context if needed
if (context.mode === 'cli') {
const cacheHasRoutes = cache.routes.length > 0;
const configMatches = true; // This will be validated by the cache strategy
validateCliContext(cacheHasRoutes, configMatches, logger);
}
// Process using unified pipeline
const result = await processRoutesStream(context.routesToProcess, generatedFilesDir ?? '', docsDir, mdOutDir, siteDir, config, logger, siteUrl, useCache, context.mode === 'cli' || useCache ? [...cache.routes] : undefined, outDir, siteConfig);
logger.debug(`Processed ${result.docs.length} documents`);
return result;
}