next-validate-link
Version:
An utility to validate links in markdown file
665 lines (664 loc) • 21 kB
JavaScript
import picocolors from "picocolors";
import * as fs$1 from "node:fs/promises";
import fs from "node:fs/promises";
import { glob } from "tinyglobby";
import { parse } from "yaml";
import * as path from "node:path";
import { remark } from "remark";
import remarkGfm from "remark-gfm";
import remarkMdx from "remark-mdx";
import { visit } from "unist-util-visit";
//#region src/print.ts
/**
* Print validation errors
*/
function printErrors(results, throwError = false) {
let totalErrors = 0;
const logs = [];
for (const result of results) {
logs.push(picocolors.bold(picocolors.redBright(`Invalid URLs in ${result.file}:`)));
for (const error of result.errors) {
const message = error.reason instanceof Error ? error.reason.message : error.reason;
logs.push(`${picocolors.bold(error.url)}: ${message} at ${result.file}:${error.line}:${error.column}`);
}
logs.push(picocolors.dim("------"));
totalErrors += result.errors.length;
}
const summary = `${results.length} errored file, ${totalErrors} errors`;
logs.push(picocolors.bold(totalErrors > 0 ? picocolors.redBright(summary) : picocolors.greenBright(summary)));
if (throwError && totalErrors > 0) {
console.error(logs.join("\n"));
process.exit(1);
} else console.log(logs.join("\n"));
}
//#endregion
//#region src/utils/frontmatter.ts
/**
* Inspired by https://github.com/jonschlinkert/gray-matter
*/
const regex = /^---\r?\n(.+?)\r?\n---\r?\n?/s;
/**
* parse frontmatter, it supports only yaml format
*/
function frontmatter(input) {
const output = {
matter: "",
data: {},
content: input
};
const match = regex.exec(input);
if (!match) return output;
output.matter = match[0];
output.content = input.slice(match[0].length);
output.data = parse(match[1]) ?? {};
return output;
}
//#endregion
//#region src/sample.ts
async function readFileFromPath(file, pathToUrl) {
const content = await fs.readFile(file, "utf-8");
const parsed = frontmatter(content);
return {
path: file,
data: parsed.data,
content: "\n".repeat(countLine(content) - countLine(parsed.content)) + parsed.content,
url: pathToUrl ? pathToUrl(file) : void 0
};
}
async function readFiles(patterns, options = {}) {
const files = await glob(patterns);
return await Promise.all(files.map((file) => readFileFromPath(file, options.pathToUrl)));
}
function countLine(s) {
let out = 0;
for (const c of s) if (c === "\n") out++;
return out;
}
//#endregion
//#region src/presets/shared.ts
const defaultPopulate = [{}];
const OPTIONAL_CATCH_ALL = /^\[\[\.\.\.(.+)\]\]$/;
const CALCH_ALL = /^\[\.\.\.(.+)\]$/;
function parseSegments(segments) {
const path = [];
const params = [];
for (let i = 0; i < segments.length; i++) {
const segment = segments[i];
if (segment.startsWith("(") && segment.endsWith(")")) continue;
let match = OPTIONAL_CATCH_ALL.exec(segment);
if (match) {
params.push("optional");
path.push(match[1]);
continue;
}
match = CALCH_ALL.exec(segment);
if (match) {
params.push("required");
path.push(match[1]);
continue;
}
if (segment.startsWith("[") && segment.endsWith("]")) {
params.push("required");
path.push(segment.slice(1, -1));
continue;
}
params.push(null);
path.push(segment);
}
return {
path,
params
};
}
function populate(segments, options) {
const parsed = parseSegments(segments);
const countParams = parsed.params.filter((param) => param !== null).length;
if (countParams === 0) {
const meta = options.meta?.[segments.length === 0 ? "/" : segments.join("/")];
return [{
url: `/${parsed.path.join("/")}`,
meta
}];
}
const out = [];
let params;
if (options.populate) {
params = options.populate["/"];
const searchPath = [...segments];
while (!params && searchPath.length > 0) {
params = options.populate[searchPath.join("/")];
searchPath.pop();
}
}
params ??= defaultPopulate;
for (const param of params) {
let url = [...parsed.path];
if (countParams > 1 && (Array.isArray(param.value) || typeof param.value === "string")) console.warn(`path ${segments.join("/")} requires multiple params, an object value for populate is expected.`);
let isFallback = false;
for (let i = 0; i < parsed.params.length; i++) {
if (parsed.params[i] === null) continue;
const name = parsed.path[i];
let value;
if (Array.isArray(param.value) || typeof param.value === "string") value = param.value;
else if (param.value && name in param.value) value = param.value[name];
if (value) {
url[i] = typeof value === "string" ? value : value.join("/");
continue;
}
if (parsed.params[i] === "optional") {
if (i !== parsed.params.length - 1) throw new Error("Invalid position of optional catch-all");
out.push({
url: `/${url.slice(0, -1).join("/")}`,
meta: param
});
}
url[i] = "(.+)";
isFallback = true;
}
url = url.filter(Boolean);
out.push({
url: isFallback ? new RegExp(`^\\/${url.join("\\/")}$`) : `/${url.join("/")}`,
meta: param
});
}
return out;
}
/**
* Populate Next-like route file paths
*
* ```
* docs/page
* docs/[slug]/[nested]
* docs/(group)/[[...optional_catch_all]]
* docs/(group)/[...catch_all]
* ```
*/
function populateToScanResult(segments, options, result) {
const out = populate(segments, options);
for (const entry of out) {
if (typeof entry.url === "string") {
result.urls.set(entry.url, entry.meta ?? {});
continue;
}
result.fallbackUrls.push({
url: entry.url,
meta: entry.meta ?? {}
});
}
}
//#endregion
//#region src/presets/astro.ts
async function scanURLs$6(options = {}) {
const ext = options.extensions ?? [
"astro",
"md",
"mdx"
];
const cwd = options.cwd ?? process.cwd();
async function getFiles() {
const suffix = ext.length > 0 ? `.{${ext.join(",")}}` : "";
return (await glob(`**/*${suffix}`, { cwd: path.join(cwd, "src/pages") })).map((file) => {
const parsed = path.parse(file);
if (parsed.name === "index") return parsed.dir;
return path.join(parsed.dir, parsed.name);
});
}
const result = {
urls: /* @__PURE__ */ new Map(),
fallbackUrls: []
};
const files = options.pages ?? await getFiles();
for (const file of files) populateToScanResult(file.split(path.sep), options, result);
return result;
}
//#endregion
//#region src/utils/fs.ts
function isDirExists(dir) {
return fs$1.stat(dir).then((res) => res.isDirectory()).catch(() => false);
}
async function isFileExists(file) {
try {
await fs$1.access(file);
return true;
} catch (_error) {
return false;
}
}
//#endregion
//#region src/presets/next.ts
async function scanURLs$5(options = {}) {
const ext = options.extensions ?? [
"js",
"jsx",
"tsx",
"md",
"mdx"
];
const cwd = options.cwd ?? process.cwd();
async function getFiles() {
const suffix = ext.length > 0 ? `.{${ext.join(",")}}` : "";
const appFiles = await glob(`**/page${suffix}`, { cwd: await isDirExists(path.join(cwd, "src/app")) ? path.join(cwd, "src/app") : path.join(cwd, "app") });
const pagesFiles = await glob(`**/*${suffix}`, { cwd: await isDirExists(path.join(cwd, "src/pages")) ? path.join(cwd, "src/pages") : path.join(cwd, "pages") });
if (options.pages) appFiles.push(...options.pages);
return [...appFiles.map((file) => {
const dir = path.dirname(file);
return dir === "." ? "" : dir;
}), ...pagesFiles.map((file) => {
const parsed = path.parse(file);
if (parsed.name === "index") return parsed.dir;
return path.join(parsed.dir, parsed.name);
})];
}
if (options.meta) for (const key of Object.keys(options.meta)) {
if (!key.endsWith("page.tsx")) continue;
let newKey = path.dirname(key);
if (newKey === ".") newKey = "/";
options.meta[newKey] = options.meta[key];
delete options.meta[key];
}
const result = {
urls: /* @__PURE__ */ new Map(),
fallbackUrls: []
};
const files = await getFiles();
for (const file of files) populateToScanResult(file.length === 0 ? [] : file.split(path.sep), options, result);
return result;
}
//#endregion
//#region src/presets/nuxt.ts
async function scanURLs$4(options = {}) {
const ext = options.extensions ?? [
"vue",
"md",
"mdx"
];
const cwd = options.cwd ?? process.cwd();
async function getFiles() {
const suffix = ext.length > 0 ? `.{${ext.join(",")}}` : "";
return (await glob(`**/*${suffix}`, { cwd: await isDirExists(path.join(cwd, "src/pages")) ? path.join(cwd, "src/pages") : path.join(cwd, "pages") })).map((file) => {
const parsed = path.parse(file);
if (parsed.name === "index") return parsed.dir;
return path.join(parsed.dir, parsed.name);
});
}
const result = {
urls: /* @__PURE__ */ new Map(),
fallbackUrls: []
};
const files = options.pages ?? await getFiles();
for (const file of files) populateToScanResult(file.split(path.sep), options, result);
return result;
}
//#endregion
//#region src/presets/react-router.ts
async function scanURLs$3(options) {
const { routerConfig } = options;
async function getFiles() {
if (options.pages) return options.pages;
const files = [];
const resolved = await routerConfig;
for (const route of resolved) resolveEntryFiles(files, route);
return files;
}
const result = {
urls: /* @__PURE__ */ new Map(),
fallbackUrls: []
};
for (const file of await getFiles()) populateToScanResult(file.split("/"), options, result);
return result;
}
function resolveEntryFiles(outputFiles, entry, parent) {
const fullPath = entry.path?.split("/") ?? [];
if (parent) fullPath.unshift(...parent);
if (entry.path) {
const combinations = [[]];
function pushSegment(item, newCombination) {
if (newCombination) {
const next = combinations.map((combination) => [...combination, item]);
combinations.push(...next);
} else for (const combination of combinations) combination.push(item);
}
for (let i = 0; i < fullPath.length; i++) {
let name = fullPath[i];
if (name.length === 0) continue;
const isOptional = name.endsWith("?");
if (isOptional) name = name.slice(0, -1);
if (name.startsWith(":")) {
pushSegment(`[${name.slice(1)}]`, isOptional);
continue;
}
if (name === "*") {
pushSegment(`[[...splat]]`, isOptional);
continue;
}
pushSegment(name, isOptional);
}
for (const combination of combinations) outputFiles.push(combination.join("/"));
}
if (entry.children) for (const child of entry.children) resolveEntryFiles(outputFiles, child, fullPath);
}
//#endregion
//#region src/presets/tanstack-start.ts
async function scanURLs$2(options = {}) {
const ext = options.extensions ?? [
"tsx",
"ts",
"jsx",
"js"
];
const cwd = options.cwd ?? process.cwd();
async function getFiles() {
const suffix = ext.length > 0 ? `.{${ext.join(",")}}` : "";
const routesFiles = await glob(`**/*${suffix}`, { cwd: await isDirExists(path.join(cwd, "src/routes")) ? path.join(cwd, "src/routes") : path.join(cwd, "routes") });
const outFiles = [];
for (const file of routesFiles) {
let segments = file.replaceAll("[.]", "#").split(/[/\\.]/);
segments.pop();
segments = segments.map((segment) => segment.replaceAll("#", "."));
if (segments.at(-1)?.startsWith("_")) continue;
if (segments.at(-1) === "index") segments.pop();
const outSegments = [];
for (const name of segments) {
if (name.length === 0) continue;
if (name === "$") {
outSegments.push(`[[..._splat]]`);
continue;
}
if (name.startsWith("$")) {
const paramName = name.slice(1);
outSegments.push(`[${paramName}]`);
continue;
}
outSegments.push(name);
}
outFiles.push(outSegments.join("/"));
}
return outFiles;
}
const result = {
urls: /* @__PURE__ */ new Map(),
fallbackUrls: []
};
const files = options.pages ?? await getFiles();
for (const file of files) populateToScanResult(file.split("/"), options, result);
return result;
}
//#endregion
//#region src/presets/waku.ts
async function scanURLs$1(options = {}) {
const ext = options.extensions ?? [
"tsx",
"ts",
"jsx",
"js"
];
const cwd = options.cwd ?? process.cwd();
async function getFiles() {
const suffix = ext.length > 0 ? `.{${ext.join(",")}}` : "";
const pagesFiles = await glob(`**/*${suffix}`, { cwd: await isDirExists(path.join(cwd, "src/pages")) ? path.join(cwd, "src/pages") : path.join(cwd, "pages") });
const outFiles = [];
for (const file of pagesFiles) {
const parsed = path.parse(file);
if (parsed.name.startsWith("_")) continue;
const segments = parsed.dir.split(path.sep);
if (parsed.name !== "index") segments.push(parsed.name);
const outSegments = [];
for (const name of segments) if (name.startsWith("[...") && name.endsWith("]")) outSegments.push(`[${name}]`);
else if (name.length > 0) outSegments.push(name);
outFiles.push(outSegments.join("/"));
}
return outFiles;
}
const result = {
urls: /* @__PURE__ */ new Map(),
fallbackUrls: []
};
const files = options.pages ?? await getFiles();
for (const file of files) populateToScanResult(file.split("/"), options, result);
return result;
}
//#endregion
//#region src/scan.ts
async function scanURLs(options = {}) {
switch (options.preset) {
case "astro": return scanURLs$6(options);
case "nuxt": return scanURLs$4(options);
case "react-router": return scanURLs$3(options);
case "tanstack-start": return scanURLs$2(options);
case "waku": return scanURLs$1(options);
default: return scanURLs$5(options);
}
}
//#endregion
//#region src/utils/url.ts
/**
* Split path into segments, trailing/leading slashes are removed
*/
function splitPath(path) {
return path.split("/").filter((p) => p.length > 0);
}
function resolveUrl(base, relative) {
const v1 = splitPath(base);
const v2 = splitPath(relative);
while (v2.length > 0) {
switch (v2[0]) {
case "..":
v1.pop();
break;
case ".": break;
default: v1.push(v2[0]);
}
v2.shift();
}
return v1.join("/");
}
//#endregion
//#region src/validate/markdown.ts
function createMarkdownValidator(config, detector) {
const { components = {}, remarkPlugins = [], onNode = (node) => {
if (node.type === "link") return { hrefs: [node.url] };
if ((node.type === "mdxJsxFlowElement" || node.type === "mdxJsxTextElement") && node.name && node.name in components) {
const analyze = components[node.name];
const hrefs = [];
for (const attr of node.attributes) {
if (attr.type !== "mdxJsxAttribute" || typeof attr.value !== "string") continue;
if (!analyze.attributes.includes(attr.name)) continue;
hrefs.push(attr.value);
}
return { hrefs };
}
} } = config;
const mdProcessor = remark().use(remarkGfm).use(remarkPlugins);
const mdxProcessor = remark().use(remarkMdx).use(remarkGfm).use(remarkPlugins);
return { async validate(file, resolution) {
const errors = [];
const tasks = [];
const processor = file.path.endsWith(".mdx") ? mdxProcessor : mdProcessor;
const vfile = {
path: file.path,
value: file.content
};
let tree = processor.parse(vfile);
tree = await processor.run(tree, vfile);
visit(tree, (node) => {
if (!node.position || node.type === "root") return;
const pos = node.position;
const scanned = onNode(node);
if (!scanned) return;
for (const href of scanned.hrefs) tasks.push(detector.detect(href, resolution).then((err) => {
if (!err || err.type !== "error") return;
errors.push({
url: href,
line: pos.start.line,
column: pos.start.column,
reason: err.reason
});
}).catch((err) => {
errors.push({
url: href,
line: pos.start.line,
column: pos.start.column,
reason: err
});
}));
});
await Promise.all(tasks);
return errors;
} };
}
//#endregion
//#region src/utils/external-link.ts
function externalLink(config) {
const { validate } = config;
return async (url) => {
const parsed = new URL(url);
if (validate) return validate(parsed);
if (parsed.hostname === "localhost") return { success: true };
try {
const res = await fetch(parsed, { method: "HEAD" });
if (!res.ok) {
if (res.status === 404) return {
success: false,
message: "not found"
};
if (res.status >= 300 && res.status < 400) return { success: true };
return {
success: false,
message: `${url} responded status ${res.status}`
};
}
return { success: true };
} catch (e) {
if (e instanceof Error) return {
success: false,
message: e.message
};
return { success: false };
}
};
}
//#endregion
//#region src/validate.ts
const mdExtensions = [".md", ".mdx"];
const supportedExtensions = mdExtensions;
/**
* Validate markdown files
*
* @param files - file paths or file objects
* @param config - configurations
*/
async function validateFiles(files, config) {
const detector = createDetector(config);
const markdownValidator = createMarkdownValidator(config.markdown ?? {}, detector);
const normalized = await Promise.all(files.map(async (file) => typeof file === "string" ? await readFileFromPath(file, config.pathToUrl) : file));
const defaultPathToUrl = (path) => {
for (const file of normalized) if (file.path === path && file.url) return file.url;
};
async function run(file) {
const resolution = {
baseUrl: file.url ? file.url.split("/").slice(0, -1).join("/") : config.baseUrl,
baseDir: path.dirname(file.path),
pathToUrl: config.pathToUrl ?? defaultPathToUrl
};
const ext = path.extname(file.path);
let errors = [];
if (mdExtensions.includes(ext)) errors = await markdownValidator.validate(file, resolution);
else console.warn(`format unsupported: ${ext}, supported: ${supportedExtensions.join(", ")}`);
return {
file: file.path,
errors,
get detected() {
return errors.map(generateLegacyError);
}
};
}
return (await Promise.all(normalized.map(run))).filter((err) => err.errors.length > 0);
}
function createDetector(config) {
const PathnameRegex = /^([^?#]*)(\?[^#]*)?(#.*)?$/;
const { checkRelativePaths = false, checkExternal = false, ignoreFragment = false, ignoreQuery = false, checkRelativeUrls = true, whitelist, determinatePathname = (pathname) => {
if (!pathname.startsWith(".")) return "url";
if (pathname.endsWith(".md") || pathname.endsWith(".mdx")) return "relative-file-path";
return "relative-url";
} } = config;
const externalLinkChecker = checkExternal === false ? null : externalLink(typeof checkExternal === "object" ? checkExternal : {});
let isWhiteListed;
if (typeof whitelist === "function") isWhiteListed = whitelist;
else if (Array.isArray(whitelist)) {
const whitelistSet = new Set(whitelist);
isWhiteListed = (href) => whitelistSet.has(href);
}
function parsePathname(pathname) {
const match = PathnameRegex.exec(pathname);
if (!match) return { pathname };
return {
pathname: match[1],
query: match[2]?.slice(1),
fragment: match[3]?.slice(1)
};
}
return { async detect(href, { baseDir, baseUrl, pathToUrl }) {
if (href.startsWith("mailto:") || isWhiteListed?.(href)) return;
if (href.match(/https?:\/\//)) {
if (!externalLinkChecker) return;
const result = await externalLinkChecker(href);
if (result.success) return;
return {
type: "error",
reason: result.message ? new Error(result.message) : "not-found"
};
}
let { pathname, query, fragment } = parsePathname(href);
if (pathname.length === 0 || pathname === "./") return;
switch (await determinatePathname(pathname)) {
case "relative-url":
if (!checkRelativeUrls) return;
if (!baseUrl) throw new Error(`relative URL ${pathname} detected, but 'baseUrl' option is missing.`);
pathname = resolveUrl(baseUrl, pathname);
break;
case "relative-file-path": {
if (!checkRelativePaths) return;
const filePath = path.join(baseDir ?? "", pathname);
if (checkRelativePaths === "exists") return await isFileExists(filePath) ? void 0 : {
type: "error",
reason: "not-found"
};
else if (checkRelativePaths === "as-url") {
if (!pathToUrl) throw new Error(`'checkRelativePaths: as-url' is set, but 'pathToUrl' option is missing.`);
const url = pathToUrl(filePath);
if (!url) return;
pathname = url;
}
break;
}
}
if (!pathname.startsWith("/")) pathname = `/${pathname}`;
let meta = config.scanned.urls.get(pathname);
if (!meta) meta = config.scanned.fallbackUrls.find((fallbackUrl) => {
return fallbackUrl.url.test(pathname);
})?.meta;
if (!meta) return {
type: "error",
reason: "not-found"
};
if (fragment && !ignoreFragment && meta.hashes && !meta.hashes.includes(fragment)) return {
type: "error",
reason: "invalid-fragment"
};
if (query && !ignoreQuery && meta.queries && !meta.queries.some((item) => new URLSearchParams(item).toString() === query)) return {
type: "error",
reason: "invalid-query"
};
} };
}
function generateLegacyError(v) {
return [
v.url,
v.line,
v.column,
v.reason
];
}
//#endregion
export { printErrors, readFileFromPath, readFiles, scanURLs, validateFiles };
//# sourceMappingURL=index.mjs.map