@chrisburnell/eleventy-cache-webmentions/eleventy-cache-webmentions.min.cjs

Cache webmentions using eleventy-fetch and make them available to use in collections, layouts, pages, etc. in Eleventy.

const{AssetCache}=require("@11ty/eleventy-fetch"),{styleText}=require("node:util"),sanitizeHTML=require("sanitize-html"),defaults={refresh:!1,duration:"1d",uniqueKey:"webmentions",cacheDirectory:void 0,allowedHTML:{allowedTags:["a","b","em","i","strong"],allowedAttributes:{a:["href"]}},allowlist:[],blocklist:[],urlReplacements:{},maximumHtmlLength:1e3,maximumHtmlText:"mentioned this in"},absoluteURL=(url,domain)=>{try{return new URL(url,domain).toString()}catch(error){return console.error(`Trying to convert ${styleText("bold",url)} to be an absolute url with base ${styleText("bold",domain)} and failed.`,error),url}},baseURL=url=>url.split("#")[0].split("?")[0],fixURL=(url,urlReplacements)=>Object.entries(urlReplacements).reduce((accumulator,[key,value])=>{const regex=new RegExp(key,"g");return accumulator.replace(regex,value)},url),hostname=url=>typeof url=="string"&&url.includes("//")?new URL(url).hostname:url,epoch=date=>new Date(date).getTime(),removeDuplicates=webmentions=>[...webmentions.reduce((map,webmention)=>{const key=webmention==null?webmention:getUniqueKey(webmention);return map.has(key)||map.set(key,webmention),map},new Map).values()],getPublished=webmention=>webmention?.data?.published||webmention.published||webmention["wm-received"]||webmention.verified_date,getReceived=webmention=>webmention["wm-received"]||webmention.verified_date||webmention.published||webmention?.data?.published,getContent=webmention=>webmention?.contentSanitized||webmention?.content?.html||webmention?.content?.value||webmention?.content||webmention?.data?.content||"",getSource=webmention=>webmention["wm-source"]||webmention.source||webmention?.data?.url||webmention.url,getURL=webmention=>webmention?.data?.url||webmention.url||webmention["wm-source"]||webmention.source,getTarget=webmention=>webmention["wm-target"]||webmention.target,getUniqueKey=webmention=>JSON.stringify([getSource(webmention),getTarget(webmention)]),getType=webmention=>webmention["wm-property"]||webmention?.activity?.type||webmention.type,getByTypes=(webmentions,types)=>webmentions.filter(webmention=>typeof types=="string"?types===getType(webmention):types.includes(getType(webmention))),processBlocklist=(webmentions,blocklist)=>webmentions.filter(webmention=>{let url=getSource(webmention),source=getSource(webmention);for(let blocklistURL of blocklist)if(url.includes(blocklistURL.replace(/\/?$/,"/"))||source.includes(blocklistURL.replace(/\/?$/,"/")))return!1;return!0}),processAllowlist=(webmentions,allowlist)=>webmentions.filter(webmention=>{let url=getSource(webmention),source=getSource(webmention);for(let allowlistURL of allowlist)if(url.includes(allowlistURL.replace(/\/?$/,"/"))||source.includes(allowlistURL.replace(/\/?$/,"/")))return!0;return!1}),fetchWebmentions=async(options,webmentions,url)=>await fetch(url).then(async response=>{if(!response.ok)return Promise.reject(response);const feed=await response.json();if(!(options.key in feed))return console.log(`${styleText("gray",`[${hostname(options.domain)}]`)} ${options.key} was not found as a key in the response from ${styleText("bold",hostname(options.feed))}!`),Promise.reject(response);const fetchedKeys=new Set(feed[options.key].map(getUniqueKey));return webmentions=[...feed[options.key],...webmentions.filter(wm=>!fetchedKeys.has(getUniqueKey(wm)))],webmentions=removeDuplicates(webmentions),options.blocklist.length&&(webmentions=processBlocklist(webmentions,options.blocklist)),options.allowlist.length&&(webmentions=processAllowlist(webmentions,options.allowlist)),webmentions=webmentions.sort((a,b)=>epoch(getReceived(b))-epoch(getReceived(a))),{found:feed[options.key].length,webmentions}}).catch(error=>(console.warn(`${styleText("gray",`[${hostname(options.domain)}]`)} Something went wrong with your Webmention request to ${styleText("bold",hostname(options.feed))}!`),console.warn(error instanceof Error?error.message:error),{found:0,webmentions})),retrieveWebmentions=async options=>{if(!options.domain)throw new Error("`domain` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");if(!options.feed)throw new Error("`feed` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");if(!options.key)throw new Error("`key` is a required field when attempting to retrieve Webmentions. See https://www.npmjs.com/package/@chrisburnell/eleventy-cache-webmentions#installation for more information.");let asset=new AssetCache(options.uniqueKey||`webmentions-${hostname(options.domain)}`,options.cacheDirectory),webmentions=[];asset.isCacheValid("9001y")&&!options.refresh&&(webmentions=await asset.getCachedValue());const webmentionsCachedLength=webmentions.length;if(!asset.isCacheValid(options.refresh?"0s":options.duration)){const performanceStart=process.hrtime(),since=webmentions.length?getReceived(webmentions[0]):!1,url=`${options.feed}${since?`${options.feed.includes("?")?"&":"?"}since=${since}`:""}`;if(url.includes("https://webmention.io")){const urlObject=new URL(url),perPage=Number(urlObject.searchParams.get("per-page"))||1e3;urlObject.searchParams.delete("per-page");let page=0;for(;;){const urlPaginated=urlObject.href+`&per-page=${perPage}&page=${page}`,fetched=await fetchWebmentions(options,webmentions,urlPaginated);if(!fetched&&!fetched.found&&!fetched.webmentions||fetched.found===0||(webmentions=fetched.webmentions,fetched.found<perPage))break;page+=1,await new Promise(resolve=>setTimeout(resolve,1e3))}}else webmentions=(await fetchWebmentions(options,webmentions,url)).webmentions;options.blocklist.length&&(webmentions=processBlocklist(webmentions,options.blocklist)),options.allowlist.length&&(webmentions=processAllowlist(webmentions,options.allowlist)),await asset.save(webmentions,"json");const performance=process.hrtime(performanceStart);webmentionsCachedLength<webmentions.length&&console.log(`${styleText("gray",`[${hostname(options.domain)}]`)} ${styleText("bold",String(webmentions.length-webmentionsCachedLength))} new Webmentions fetched into cache in ${styleText("bold",(performance[0]+performance[1]/1e9).toFixed(3)+" seconds")}.`)}return webmentions},WEBMENTIONS={},webmentionsByURL=async options=>(Object.keys(WEBMENTIONS).length||(await retrieveWebmentions(options)).forEach(webmention=>{let url=baseURL(fixURL(getTarget(webmention).replace(/\/?$/,"/"),options.urlReplacements));WEBMENTIONS[url]||(WEBMENTIONS[url]=[]),WEBMENTIONS[url].push(webmention)}),WEBMENTIONS),getWebmentions=async(options,url,types=[])=>{const webmentions=await webmentionsByURL(options);return url=absoluteURL(url,options.domain),!url||!webmentions||!webmentions[url]?[]:webmentions[url].filter(entry=>typeof types=="object"&&Object.keys(types).length?types.includes(getType(entry)):typeof types=="string"?types===getType(entry):!0).map(entry=>{const html=getContent(entry);return html.length&&(entry.contentSanitized=sanitizeHTML(html,options.allowedHTML),html.length>options.maximumHtmlLength&&(entry.contentSanitized=`${options.maximumHtmlText} <a href="${getSource(entry)}">${getSource(entry)}</a>`)),entry}).sort((a,b)=>epoch(getPublished(a))-epoch(getPublished(b)))},eleventyCacheWebmentions=async(eleventyConfig,options={})=>{options=Object.assign(defaults,options);const byURL=await webmentionsByURL(options),all=Object.values(byURL).reduce((array,webmentions)=>[...array,...webmentions],[]);eleventyConfig.addGlobalData("webmentionsDefaults",defaults),eleventyConfig.addGlobalData("webmentionsOptions",options),eleventyConfig.addGlobalData("webmentionsByURL",byURL),eleventyConfig.addGlobalData("webmentionsByUrl",byURL),eleventyConfig.addGlobalData("webmentionsAll",all),eleventyConfig.addLiquidFilter("getWebmentionsByType",getByTypes),eleventyConfig.addLiquidFilter("getWebmentionsByTypes",getByTypes),eleventyConfig.addLiquidFilter("getWebmentionPublished",getPublished),eleventyConfig.addLiquidFilter("getWebmentionReceived",getReceived),eleventyConfig.addLiquidFilter("getWebmentionContent",getContent),eleventyConfig.addLiquidFilter("getWebmentionSource",getSource),eleventyConfig.addLiquidFilter("getWebmentionURL",getURL),eleventyConfig.addLiquidFilter("getWebmentionTarget",getTarget),eleventyConfig.addLiquidFilter("getWebmentionType",getType),eleventyConfig.addNunjucksFilter("getWebmentionsByType",getByTypes),eleventyConfig.addNunjucksFilter("getWebmentionsByTypes",getByTypes),eleventyConfig.addNunjucksFilter("getWebmentionPublished",getPublished),eleventyConfig.addNunjucksFilter("getWebmentionReceived",getReceived),eleventyConfig.addNunjucksFilter("getWebmentionContent",getContent),eleventyConfig.addNunjucksFilter("getWebmentionSource",getSource),eleventyConfig.addNunjucksFilter("getWebmentionURL",getURL),eleventyConfig.addNunjucksFilter("getWebmentionTarget",getTarget),eleventyConfig.addNunjucksFilter("getWebmentionType",getType)};module.exports=eleventyCacheWebmentions,module.exports.defaults=defaults,module.exports.getPublished=getPublished,module.exports.getWebmentionPublished=getPublished,module.exports.getReceived=getReceived,module.exports.getWebmentionReceived=getReceived,module.exports.getContent=getContent,module.exports.getWebmentionContent=getContent,module.exports.getSource=getSource,module.exports.getWebmentionSource=getSource,module.exports.getURL=getURL,module.exports.getWebmentionURL=getURL,module.exports.getTarget=getTarget,module.exports.getWebmentionTarget=getTarget,module.exports.getUniqueKey=getUniqueKey,module.exports.getWebmentionUniqueKey=getUniqueKey,module.exports.getType=getType,module.exports.getWebmentionType=getType,module.exports.getByTypes=getByTypes,module.exports.getByType=getByTypes,module.exports.getWebmentionsByTypes=getByTypes,module.exports.getWebmentionsByType=getByTypes,module.exports.processBlocklist=processBlocklist,module.exports.processWebmentionBlocklist=processBlocklist,module.exports.processWebmentionsBlocklist=processBlocklist,module.exports.processAllowlist=processAllowlist,module.exports.processWebmentionAllowlist=processAllowlist,module.exports.processWebmentionsAllowlist=processAllowlist,module.exports.fetchWebmentions=fetchWebmentions,module.exports.retrieveWebmentions=retrieveWebmentions,module.exports.webmentionsByURL=webmentionsByURL,module.exports.webmentionsByUrl=webmentionsByURL,module.exports.filteredWebmentions=webmentionsByURL,module.exports.getWebmentions=getWebmentions;