@photogabble/eleventy-plugin-interlinker
Version:
Obsidian WikiLinks, BackLinks and Embed support for 11ty
719 lines (600 loc) • 24.7 kB
JavaScript
'use strict';
var jsdom = require('jsdom');
var path = require('node:path');
var chalk = require('chalk');
var fs = require('node:fs');
var eleventy = require('@11ty/eleventy');
var entities = require('entities');
/**
* This rule will be looped through an inline token by markdown-it.
*
* @param {WikilinkParser} wikilinkParser
* @returns {(function(*, *): (boolean|undefined))|*}
*/
const wikilinkInlineRule = (wikilinkParser) => (state, silent) => {
// Have we found the start of a WikiLink Embed `![[`
if (state.src.charAt(state.pos) === '[' && state.src.charAt(state.pos + 1) !== '[') return false; // Not wikilink opening
if (state.src.charAt(state.pos) === '!' && state.src.substring(state.pos, state.pos + 3) !== '![[') return false; // Not embed opening
const matches = state.src.match(wikilinkParser.wikiLinkRegExp);
if (!matches) return false;
// We have found the start of a WikiLink (`[[`) or Embed (`![[`)
// char at state.pos will be either `[` or `!`, need to walk through state.src until `]]` is encountered
let pos = state.pos;
let text = '';
let found = false;
while (pos <= state.posMax) {
text += state.src.charAt(pos);
if (text.length > 2 && text.substring(text.length - 2) === ']]') {
found = true;
break;
}
pos++;
}
if (!found) return false;
const wikiLink = wikilinkParser.linkCache.get(text);
// By this time in the execution cycle the wikilink parser's cache should contain all
// wikilinks, including those linking to a stub. In the unlikely case that it doesn't
// we ignore the wikilink.
if (!wikiLink) return false;
if (!silent) {
// The wikilink content is HTML generated by a "Resolving Function" lets make use of
// the builtin html_inline render rule to display it.
const token = state.push('html_inline', '', 0);
token.content = wikiLink.content.trim();
}
state.pos = state.pos + text.length;
return true;
};
/**
* This rule is to catch Wikilink Embeds that are bookended by new lines e.g `\n![[test]]\n`, the inline rule
* would only get the content within the new lines with the newlines themselves getting tokenised as a paragraph.
* That was causing the bugs raised in issues #65 and #66.
*
* @param wikilinkParser
* @return {(function(*, *, *, *): boolean)|*}
*/
const wikilinkBlockRule = (wikilinkParser) => (state, startLine, endLine, silent) => {
let pos = state.bMarks[startLine] + state.tShift[startLine];
let max = state.eMarks[startLine];
// if it's indented more than 3 spaces, it should be a code block
if (state.sCount[startLine] - state.blkIndent >= 4) { return false }
// This block rule only cares about Wiki Link embeds that are at a block level (on a single line bookended by `\n`)
// therefore if the line text doesn't begin with `![[` we do not continue.
let lineText = state.src.slice(pos, max);
if (lineText.substring(0, 3) !== '![[') return false;
const wikiLink = wikilinkParser.linkCache.get(lineText);
if (!wikiLink) return false;
if (!silent) {
const token = state.push('html_block', '', 0);
token.content = wikiLink.content.trim();
}
// Block level Wikilink embeds should be on a single line, increment the line counter and continue.
state.line++;
return true;
};
const install = (md, wikilinkParser) => {
md.inline.ruler.push('inline_wikilink', wikilinkInlineRule(
wikilinkParser,
));
md.block.ruler.before('heading', 'block_wikilink', wikilinkBlockRule(
wikilinkParser,
), {
// alt contains a list of rules which can be terminated by this one
alt: ['paragraph', 'reference', 'blockquote']
});
};
class HTMLLinkParser {
/**
* This regex finds all html tags with a href that begins with / denoting they are internal links.
*
* @type {RegExp}
*/
internalLinkRegex = /href="\/(.*?)"/g;
/**
* @param { DeadLinks } deadLinks
*/
constructor(deadLinks) {
this.deadLinks = deadLinks;
}
/**
* Parses a single HTML link into the link object understood by the Interlinker. Unlike with Wikilinks we only
* care about the href so that we can look up the linked page record.
*
* @param {string} link
* @param {import('@photogabble/eleventy-plugin-interlinker').PageDirectoryService} pageDirectory
* @return {import('@photogabble/eleventy-plugin-interlinker').LinkMeta}
*/
parseSingle(link, pageDirectory) {
const meta = {
href: link
.replace(/.(md|markdown)\s?$/i, "")
.replace("\\", "")
.trim()
.split("#")[0],
isEmbed: false,
};
const {found, page} = pageDirectory.findByLink(meta);
if (!found) {
this.deadLinks.add(link);
return meta;
}
meta.exists = true;
meta.page = page;
return meta;
}
/**
* @param {Array<string>} links
* @param {import('@photogabble/eleventy-plugin-interlinker').PageDirectoryService} pageDirectory
* @return {Array<import('@photogabble/eleventy-plugin-interlinker').LinkMeta>}
*/
parseMultiple(links, pageDirectory) {
return links.map(link => this.parseSingle(link, pageDirectory));
}
/**
* Find's all internal href links within an HTML document and returns the parsed result.
* @param {string} document
* @param {import('@photogabble/eleventy-plugin-interlinker').PageDirectoryService} pageDirectory
* @return {Array<import('@photogabble/eleventy-plugin-interlinker').LinkMeta>}
*/
find(document, pageDirectory) {
const dom = new jsdom.JSDOM(document);
const anchors = dom.window.document.getElementsByTagName('a');
const toParse = [];
for (const anchor of anchors) {
// Ignore any anchor tags within either code or pre tags
if (anchor.closest('code,pre')) continue;
// Ignore any links that don't begin with / denoting internal links
if (anchor.href.startsWith('/')) toParse.push(anchor.href);
}
return this.parseMultiple(
toParse,
pageDirectory
)
}
}
class WikilinkParser {
/**
* This regex finds all WikiLink style links: [[id|optional text]] as well as WikiLink style embeds: ![[id]]
*
* @type {RegExp}
*/
wikiLinkRegExp = /(?<!!)(!?)\[\[([^|\n]+?)(\|([^\n]+?))?]]/g;
/**
* @param { import('@photogabble/eleventy-plugin-interlinker').EleventyPluginInterlinkOptions } opts
* @param { DeadLinks } deadLinks
* @param { Map } linkCache
*/
constructor(opts, deadLinks, linkCache) {
this.opts = opts;
this.deadLinks = deadLinks;
this.linkCache = linkCache;
}
/**
* Parses a single WikiLink into the link object understood by the Interlinker.
*
* @param {string} link
* @param {import('@photogabble/eleventy-plugin-interlinker').PageDirectoryService} pageDirectory
* @param {string|undefined} filePathStem
* @return {import('@photogabble/eleventy-plugin-interlinker').WikilinkMeta}
*/
parseSingle(link, pageDirectory, filePathStem = undefined) {
if (this.linkCache.has(link)) {
return this.linkCache.get(link);
}
// Wikilinks starting with a ! are considered Embeds e.g. `![[ ident ]]`
const isEmbed = link.startsWith('!');
// By default, we display the linked page's title (or alias if used for lookup). This can be overloaded by
// defining the link text prefixed by a | character, e.g. `[[ ident | custom link text ]]`
const parts = link.slice((isEmbed ? 3 : 2), -2).split("|").map(part => part.trim());
/** @var {import('@photogabble/eleventy-plugin-interlinker').WikilinkMeta} */
const meta = {
title: parts.length === 2 ? parts[1] : null,
// Strip .md and .markdown extensions from the file ident; this is so it can be used for
// filePathStem match if path lookup.
name: parts[0].replace(/.(md|markdown)\s?$/i, ""),
anchor: null,
link,
isEmbed,
isPath: false,
exists: false,
resolvingFnName: isEmbed ? 'default-embed' : 'default',
};
////
// Anchor link identification:
// This works similar to Obsidian.md except this doesn't look ahead to check if the referenced anchor exists.
// An anchor link can be referenced by a # character in the file ident, e.g. `[[ ident#anchor-id ]]`.
//
// This supports escaping by prefixing the # with a /, e.g `[[ Page about C/# ]]`
if (meta.name.includes('#')) {
const nameParts = parts[0].split('#').map(part => part.trim());
// Allow for escaping a # when prefixed with a /
if (nameParts[0].at(-1) !== '/') {
meta.name = nameParts[0];
meta.anchor = nameParts[1];
} else {
meta.name = meta.name.replace('/#', '#');
}
}
////
// Path link identification:
// This supports both relative links from the linking files path and lookup from the project root path.
meta.isPath = (meta.name.startsWith('/') || meta.name.startsWith('../') || meta.name.startsWith('./'));
// This is a relative path lookup, need to mutate name so that its absolute path from project
// root so that we can match it on a pages filePathStem.
if (meta.isPath && meta.name.startsWith('.')) {
if (!filePathStem) throw new Error('Unable to do relative path lookup of wikilink.');
const cwd = filePathStem.split('/');
const relative = meta.name.split('/');
const stepsBack = relative.filter(file => file === '..').length;
meta.name = [
...cwd.slice(0, -(stepsBack + 1)),
...relative.filter(file => file !== '..' && file !== '.')
].join('/');
}
////
// Custom Resolving Fn:
// If the author has referenced a custom resolving function via inclusion of the `:` character
// then we use that one. Otherwise, use the default resolving functions.
// As with anchor links, this supports escaping the `:` character by prefixing with `/`
if (meta.name.includes(':')) {
const parts = meta.name.split(':').map(part => part.trim());
if (parts[0].at(-1) !== '/') {
if (!this.opts.resolvingFns || this.opts.resolvingFns.has(parts[0]) === false) {
const {found} = pageDirectory.findByLink(meta);
if (!found) throw new Error(`Unable to find resolving fn [${parts[0]}] for wikilink ${link} on page [${filePathStem}]`);
} else {
meta.resolvingFnName = parts[0];
meta.name = parts[1];
}
} else {
meta.name = meta.name.replace('/:', ':');
}
}
// Lookup page data from 11ty's collection to obtain url and title if currently null
const {page, foundByAlias} = pageDirectory.findByLink(meta);
if (page) {
if (foundByAlias) {
meta.title = meta.name;
} else if (meta.title === null && page.data.title) {
meta.title = page.data.title;
}
meta.href = page.url;
meta.path = page.inputPath;
meta.exists = true;
meta.page = page;
} else if (['default', 'default-embed'].includes(meta.resolvingFnName)) {
// If this wikilink goes to a page that doesn't exist, add to deadLinks list and
// update href for stub post.
this.deadLinks.add(link);
meta.href = this.opts.stubUrl;
if (isEmbed) meta.resolvingFnName = '404-embed';
}
// Cache discovered meta to link, this cache can then be used by the Markdown render rule
// to display the link.
this.linkCache.set(link, meta);
return meta;
}
/**
* @param {Array<string>} links
* @param {import('@photogabble/eleventy-plugin-interlinker').PageDirectoryService} pageDirectory
* @param {string|undefined} filePathStem
* @return {Array<import('@photogabble/eleventy-plugin-interlinker').WikilinkMeta>}
*/
parseMultiple(links, pageDirectory, filePathStem) {
return links.map(link => this.parseSingle(link, pageDirectory, filePathStem));
}
/**
* Finds all wikilinks within a document (HTML or otherwise) and returns their
* parsed result.
*
* @param {string} document
* @param {import('@photogabble/eleventy-plugin-interlinker').PageDirectoryService} pageDirectory
* @param {string|undefined} filePathStem
* @return {Array<import('@photogabble/eleventy-plugin-interlinker').WikilinkMeta>}
*/
find(document, pageDirectory, filePathStem) {
return this.parseMultiple(
(document.match(this.wikiLinkRegExp) || []),
pageDirectory,
filePathStem
)
}
}
class DeadLinks {
constructor() {
this.gravestones = new Map;
this.fileSrc = 'unknown';
}
setFileSrc(fileSrc) {
this.fileSrc = fileSrc;
}
/**
* @param {string} link
*/
add(link) {
if (!this.fileSrc) this.fileSrc = 'unknown';
const names = this.gravestones.has(link)
? this.gravestones.get(link)
: [];
names.push(this.fileSrc);
this.gravestones.set(link, names);
}
/**
* @param {'console'|'json'} format
*/
report(format) {
if (format === 'console') {
for (const [link, files] of this.gravestones.entries()) {
console.warn(
chalk.blue('[@photogabble/wikilinks]'),
chalk.yellow('WARNING'),
`${(link.includes('href') ? 'Link' : 'Wikilink')} (${link}) found pointing to to non-existent page in:`
);
for (const file of files) {
console.warn(`\t- ${file}`);
}
}
return;
}
let obj = {};
for (const [link, files] of this.gravestones.entries()) {
obj[link] = files;
}
fs.writeFileSync(
path.join(process.env.ELEVENTY_ROOT, '.dead-links.json'),
JSON.stringify(obj)
);
}
/**
* Reset to initial state
*/
clear() {
this.fileSrc = 'unknown';
this.gravestones.clear();
}
}
/**
* Page Lookup Service:
* This wraps the 11ty all pages collection providing two methods for finding pages.
*
* @param {Array<any>} allPages
* @return {import('@photogabble/eleventy-plugin-interlinker').PageDirectoryService}
*/
const pageLookup = (allPages = []) => {
return {
findByLink: (link) => {
let foundByAlias = false;
const page = allPages.find((page) => {
// Order of lookup:
// 1. if is path link, return filePathStem match state
// 2. match file url to link href
// 3. match file slug to link slug
// 4. match file title to link identifier (name)
// 5. match file based upon alias
if (link.isPath) {
return page.filePathStem === link.name;
}
if (link.href && (page.url === link.href || page.url === `${link.href}/`)) {
return true;
}
if ((page.data.title && page.data.title === link.name) || page.fileSlug === link.name ) {
return true;
}
const aliases = ((page.data.aliases && Array.isArray(page.data.aliases))
? page.data.aliases
: (typeof page.data.aliases === 'string' ? [page.data.aliases] : [])
).reduce(function (set, alias) {
set.add(alias);
return set;
}, new Set());
foundByAlias = aliases.has(link.name);
return foundByAlias;
});
return {
found: !!page,
page,
foundByAlias,
}
},
findByFile: (file) => allPages.find((page) => page.url === file.page.url),
}
};
/**
* Interlinker:
*
*/
class Interlinker {
/**
* @param {import('@photogabble/eleventy-plugin-interlinker').EleventyPluginInterlinkOptions} opts
*/
constructor(opts) {
this.opts = opts;
// Map of WikiLinks pointing to non-existent pages
this.deadLinks = new DeadLinks();
// Map of Wikilink Meta that have been resolved by the WikilinkParser
this.linkCache = new Map();
// Instance of TemplateConfig loaded by the `eleventy.config` event
this.templateConfig = undefined;
// Instance of EleventyExtensionMap loaded by the `eleventy.extensionmap` event
this.extensionMap = undefined;
this.wikiLinkParser = new WikilinkParser(opts, this.deadLinks, this.linkCache);
this.HTMLLinkParser = new HTMLLinkParser(this.deadLinks);
}
reset() {
this.deadLinks.clear();
this.linkCache.clear();
}
/**
* This is a computed function that gets added to the global data of 11ty prompting its
* invocation for every page.
*
* @param {Object} data
* @return {Promise<Array<any>>}
*/
async compute(data) {
// 11ty will invoke this several times during its build cycle, accessing the values we
// need helps 11ty automatically detect data dependency and invoke the function only
// once they are met.
// @see https://www.11ty.dev/docs/data-computed/#declaring-your-dependencies
const dependencies = [data.title, data.page, data.collections.all];
if (dependencies[0] === undefined || !dependencies[1].inputPath || dependencies[2].length === 0) return [];
this.deadLinks.setFileSrc(data.page.inputPath);
const compilePromises = [];
const pageDirectory = pageLookup(data.collections.all);
const currentPage = pageDirectory.findByFile(data);
if (!currentPage) return [];
// TODO: 1.1.0 keep track of defined aliases and throw exception if duplicates are found (#46)
// Identify this pages outbound internal links both as wikilink _and_ regular html anchor tags. For each out-link
// lookup the other page and add this to its backlinks data value.
const template = await currentPage.template.read();
if (template?.content) {
const pageContent = template.content;
const outboundLinks = [
...this.wikiLinkParser.find(pageContent, pageDirectory, currentPage.filePathStem),
...this.HTMLLinkParser.find(pageContent, pageDirectory),
];
// Foreach link on this page, if it has its own resolving function we invoke that
// otherwise the default behaviour is to look up the page and add this page to
// its backlinks list.
for (const link of outboundLinks) {
if (link.resolvingFnName) {
const fn = this.opts.resolvingFns.get(link.resolvingFnName);
link.content = await fn(link, currentPage, this);
}
// If the linked page exists we can add the linking page to its backlinks array
if (link.exists) {
if (!link.page.data.backlinks) link.page.data.backlinks = [];
if (link.page.data.backlinks.findIndex((backlink => backlink.url === currentPage.url)) === -1) {
link.page.data.backlinks.push({
url: currentPage.url,
title: currentPage.data.title,
});
}
}
}
// Block iteration until compilation complete.
if (compilePromises.length > 0) await Promise.all(compilePromises);
// The computed outbound links for the current page.
return outboundLinks;
}
return [];
}
}
/**
* Default Resolving function for converting Wikilinks into html links.
*
* @param {import('@photogabble/eleventy-plugin-interlinker').WikilinkMeta} link
* @param {*} currentPage
* @param {import('./interlinker')} interlinker
* @return {Promise<string|undefined>}
*/
const defaultResolvingFn = async (link, currentPage, interlinker) => {
const text = entities.encodeHTML(link.title ?? link.name);
let href = link.href;
if (link.anchor) {
href = `${href}#${link.anchor}`;
}
return href === false ? link.link : `<a href="${href}">${text}</a>`;
};
/**
* Default Resolving function for converting Wikilinks into Embeds.
*
* @param {import('@photogabble/eleventy-plugin-interlinker').WikilinkMeta} link
* @param {*} currentPage
* @param {import('./interlinker')} interlinker
* @return {Promise<string|undefined>}
*/
const defaultEmbedFn = async (link, currentPage, interlinker) => {
if (!link.exists || !interlinker.templateConfig || !interlinker.extensionMap) return;
const page = link.page;
const template = await page.template.read();
const layout = (page.data.hasOwnProperty(interlinker.opts.layoutKey))
? page.data[interlinker.opts.layoutKey]
: interlinker.opts.defaultLayout;
const language = (page.data.hasOwnProperty(interlinker.opts.layoutTemplateLangKey))
? page.data[interlinker.opts.layoutTemplateLangKey]
: interlinker.opts.defaultLayoutLang === null
? page.page.templateSyntax
: interlinker.opts.defaultLayoutLang;
// TODO: (#36) the layout below is liquid, will break if content contains invalid template tags such as passing njk file src
// This is the async compile function from the RenderPlugin.js bundled with 11ty. I'm using it directly here
// to compile the embedded content.
const compiler = eleventy.EleventyRenderPlugin.String;
// Compile template.content
const contentFn = await compiler(template.content, language, {
templateConfig: interlinker.templateConfig,
extensionMap: interlinker.extensionMap
});
const content = await contentFn({...page.data});
// If we don't have an embed layout wrapping this content, return the compiled result.
if (layout === null) return content;
// The template string is just to invoke the embed layout, the content value is the
// compiled result of template.content.
const tpl = `{% layout "${layout}" %}`;
const tplFn = await compiler(tpl, language, {
templateConfig: interlinker.templateConfig,
extensionMap: interlinker.extensionMap
});
return await tplFn({content, ...page.data});
};
/**
* Some code borrowed from:
* @see https://git.sr.ht/~boehs/site/tree/master/item/html/pages/garden/garden.11tydata.js
* @see https://github.com/oleeskild/digitalgarden/blob/main/src/helpers/linkUtils.js
*
* @param { import('@11ty/eleventy/src/UserConfig') } eleventyConfig
* @param { import('@photogabble/eleventy-plugin-interlinker').EleventyPluginInterlinkOptions } options
*/
function index (eleventyConfig, options = {}) {
/** @var { import('@photogabble/eleventy-plugin-interlinker').EleventyPluginInterlinkOptions } opts */
const opts = Object.assign({
defaultLayout: null,
defaultLayoutLang: null,
layoutKey: 'embedLayout',
layoutTemplateLangKey: 'embedLayoutLanguage',
resolvingFns: new Map(),
deadLinkReport: 'console',
}, options);
// TODO: deprecate usage of unableToLocateEmbedFn in preference of using resolving fn
if (typeof opts.unableToLocateEmbedFn === 'function' && !opts.resolvingFns.has('404-embed')) {
opts.resolvingFns.set('404-embed', async (link) => opts.unableToLocateEmbedFn(link.page.fileSlug));
}
// Set default stub url if not set by author.
if (typeof opts.stubUrl === 'undefined') opts.stubUrl = '/stubs/';
// Default resolving functions for converting a Wikilink into HTML.
if (!opts.resolvingFns.has('default')) opts.resolvingFns.set('default', defaultResolvingFn);
if (!opts.resolvingFns.has('default-embed')) opts.resolvingFns.set('default-embed', defaultEmbedFn);
if (!opts.resolvingFns.has('404-embed')) opts.resolvingFns.set('404-embed', async () => '[UNABLE TO LOCATE EMBED]');
const interlinker = new Interlinker(opts);
// This populates templateConfig with an instance of TemplateConfig once 11ty has initiated it, it's
// used by the template compiler function that's exported by the EleventyRenderPlugin and used
// by the defaultEmbedFn resolving function for compiling embed templates.
eleventyConfig.on("eleventy.config", (cfg) => {
interlinker.templateConfig = cfg;
});
// This triggers on an undocumented internal 11ty event that is triggered once EleventyExtensionMap
// has been loaded. This is used by the EleventyRenderPlugin which is made user of within the
// compileTemplate method of the Interlinker class.
eleventyConfig.on("eleventy.extensionmap", (map) => {
interlinker.extensionMap = map;
});
// After 11ty has finished generating the site output a list of wikilinks that do not link to
// anything.
eleventyConfig.on('eleventy.after', () => {
if (opts.deadLinkReport !== 'none') interlinker.deadLinks.report(opts.deadLinkReport);
});
// Reset the internal state of the interlinker if running in watch mode, this stops
// the interlinker from working against out of date data.
eleventyConfig.on('eleventy.beforeWatch', () => {
interlinker.reset();
});
// Teach Markdown-It how to display MediaWiki Links.
eleventyConfig.amendLibrary('md', (md) => install(md, interlinker.wikiLinkParser));
// Add outboundLinks computed global data, this is executed before the templates are compiled and
// thus markdown parsed.
eleventyConfig.addGlobalData('eleventyComputed.outboundLinks', () => {
return async (data) => interlinker.compute(data);
});
// TODO: 1.1.0 Make Interlinker class available via global data
}
module.exports = index;