@mintlify/link-rot
Version:
Static checking for broken internal links
166 lines (165 loc) • 8.33 kB
JavaScript
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
import { cleanHeadingId, COMPONENTS_WITH_CLEANED_ID, COMPONENTS_WITHOUT_ID, coreRemark, extractImportSources, generateParamFieldId, isMintIgnored, preprocessCustomHeadingIds, remarkComponentIds, remarkExtractTableOfContents, } from '@mintlify/common';
import { getMintIgnore } from '@mintlify/prebuild';
import fs from 'fs-extra';
import path from 'path';
import { visit } from 'unist-util-visit';
import { Graph, Wrapper } from '../graph.js';
import { getLinkPaths, getPagePaths } from '../prebuild.js';
import { getNavigationHrefs } from './getNavigationHrefs.js';
import { getOpenApiPagePaths } from './getOpenApiPagePaths.js';
import { getRedirects } from './getRedirects.js';
export const flattenTableOfContentsSlugs = (sections) => {
const slugs = new Set();
for (const section of sections) {
slugs.add(cleanHeadingId(section.slug));
for (const slug of flattenTableOfContentsSlugs(section.children)) {
slugs.add(slug);
}
}
return slugs;
};
const PARAM_FIELD_NAMES = new Set(['ParamField', 'Param', 'ResponseField']);
const PARAM_FIELD_ATTR_NAMES = new Set(['query', 'path', 'body', 'header', 'name']);
export const extractComponentAnchorIds = (tree) => {
const ids = new Set();
const paramCounts = new Map();
visit(tree, 'mdxJsxFlowElement', (node) => {
var _a, _b, _c, _d;
if (PARAM_FIELD_NAMES.has((_a = node.name) !== null && _a !== void 0 ? _a : '')) {
const nameAttr = node.attributes.find((attr) => 'name' in attr && PARAM_FIELD_ATTR_NAMES.has(attr.name));
if (nameAttr && typeof nameAttr.value === 'string' && nameAttr.value) {
const currentCount = (_b = paramCounts.get(nameAttr.value)) !== null && _b !== void 0 ? _b : 0;
paramCounts.set(nameAttr.value, currentCount + 1);
ids.add(generateParamFieldId(nameAttr.value, currentCount));
}
return;
}
if (COMPONENTS_WITHOUT_ID.has((_c = node.name) !== null && _c !== void 0 ? _c : '')) {
return;
}
const idAttr = node.attributes.find((attr) => 'name' in attr && attr.name === 'id');
if (idAttr && typeof idAttr.value === 'string' && idAttr.value) {
const id = COMPONENTS_WITH_CLEANED_ID.has((_d = node.name) !== null && _d !== void 0 ? _d : '')
? cleanHeadingId(idAttr.value)
: idAttr.value;
ids.add(id);
}
});
return ids;
};
export const decorateGraphNodeFromPageContent = (graphNode, content, options) => __awaiter(void 0, void 0, void 0, function* () {
const mdxExtracts = {};
const visitLinks = () => {
return (tree) => {
if (options === null || options === void 0 ? void 0 : options.checkSnippets) {
for (const source of extractImportSources(tree)) {
graphNode.addPath(source);
}
}
visit(tree, (node) => {
var _a, _b;
if (node.type === 'link' || node.type === 'image') {
graphNode.addPath(node.url, Wrapper.MD);
return;
}
const mdxJsxFlowElement = node;
if (mdxJsxFlowElement.name === 'img' || mdxJsxFlowElement.name === 'source') {
const srcAttrIndex = mdxJsxFlowElement.attributes.findIndex((attr) => attr.type === 'mdxJsxAttribute' && attr.name === 'src');
const nodeUrl = (_a = mdxJsxFlowElement.attributes[srcAttrIndex]) === null || _a === void 0 ? void 0 : _a.value;
if (typeof nodeUrl === 'string') {
graphNode.addPath(nodeUrl, Wrapper.SRC);
return;
}
}
else if (mdxJsxFlowElement.name === 'a' || mdxJsxFlowElement.name === 'Card') {
const hrefAttrIndex = mdxJsxFlowElement.attributes.findIndex((attr) => attr.type === 'mdxJsxAttribute' && attr.name === 'href');
const nodeUrl = (_b = mdxJsxFlowElement.attributes[hrefAttrIndex]) === null || _b === void 0 ? void 0 : _b.value;
if (typeof nodeUrl === 'string') {
graphNode.addPath(nodeUrl, Wrapper.HREF);
return;
}
}
});
return tree;
};
};
const preprocessed = preprocessCustomHeadingIds(content);
const processor = coreRemark()
.use(visitLinks)
.use(remarkComponentIds)
.use(remarkExtractTableOfContents, mdxExtracts);
const tree = processor.parse(preprocessed);
yield processor.run(tree);
const tocSlugs = mdxExtracts.tableOfContents
? flattenTableOfContentsSlugs(mdxExtracts.tableOfContents)
: new Set();
const componentIds = extractComponentAnchorIds(tree);
graphNode.headingSlugs = new Set([...tocSlugs, ...componentIds]);
});
export const buildGraph = (repoPath, options) => __awaiter(void 0, void 0, void 0, function* () {
const baseDir = repoPath ? repoPath : process.cwd();
const mintIgnoreGlobs = yield getMintIgnore(baseDir);
const graph = new Graph(baseDir, mintIgnoreGlobs);
const allFilenames = getLinkPaths(baseDir);
const filenames = allFilenames.filter((file) => !isMintIgnored(file, mintIgnoreGlobs));
graph.addNodes(filenames);
const redirectMappings = yield getRedirects(baseDir);
graph.setRedirects(redirectMappings);
graph.addAliasNodes();
try {
const openApiPagePaths = yield getOpenApiPagePaths(baseDir);
graph.addVirtualNodes(openApiPagePaths);
}
catch (err) {
console.warn(`Warning: Failed to extract OpenAPI page paths: ${err}`);
}
const navResult = yield getNavigationHrefs(baseDir);
if (navResult) {
const configNode = graph.addNode(navResult.configFile);
for (const href of navResult.hrefs) {
const normalized = href === '/' || href === '' ? '/index' : href.replace(/^\/#/, '/index#');
configNode.addPath(normalized);
}
}
const allSitePages = getPagePaths(baseDir);
const sitePages = allSitePages.filter((file) => !isMintIgnored(file, mintIgnoreGlobs));
const BUILD_GRAPH_CONCURRENCY = 8;
let cursor = 0;
const processNext = () => __awaiter(void 0, void 0, void 0, function* () {
while (cursor < sitePages.length) {
const filePath = sitePages[cursor++];
if (filePath === undefined)
continue;
const fileNode = graph.getNode(filePath);
if (!fileNode)
continue;
const fileContent = fs.readFileSync(path.join(baseDir, filePath)).toString();
try {
yield decorateGraphNodeFromPageContent(fileNode, fileContent, {
checkSnippets: options === null || options === void 0 ? void 0 : options.checkSnippets,
});
}
catch (err) {
throw new Error(`Syntax error - Unable to parse ${filePath} - ${err}`);
}
}
});
yield Promise.all(Array.from({ length: Math.min(BUILD_GRAPH_CONCURRENCY, sitePages.length) }, processNext));
return graph;
});
export const getBrokenInternalLinks = (repoPath, options) => __awaiter(void 0, void 0, void 0, function* () {
const graph = yield buildGraph(repoPath, {
checkSnippets: options === null || options === void 0 ? void 0 : options.checkSnippets,
});
graph.precomputeFileResolutions();
return graph.getBrokenInternalLinks(options);
});