link-enricher
Version:
Fetch oEmbed, Opeh Graph, etc info for links
420 lines (414 loc) • 14.4 kB
JavaScript
import got from 'got';
import _ from 'lodash';
import mime from 'mime';
import { URL, parse as parse$1 } from 'node:url';
import { parse } from 'content-disposition';
import { load } from 'cheerio';
const getHead = (link, options) => {
return new Promise((resolve) => {
const req = got.stream(link, options);
req.on('response', (res) => {
if (res.statusCode >= 200 && res.statusCode < 300) {
resolve(res);
}
else {
resolve(null);
}
req.destroy();
});
req.on('error', () => {
resolve(null);
});
});
};
const getHtml = async (link, reqOptions) => {
const options = { ...reqOptions, retry: 0, responseType: 'text', isStream: false };
try {
const res = await got(link, options);
return res.body;
}
catch (error) {
return null;
}
};
const getJSON = async (link, reqOptions) => {
const options = { ...reqOptions, retry: 0, responseType: 'json', isStream: false };
try {
const res = await got(link, options);
return res.body;
}
catch (error) {
return null;
}
};
const sortObjectKeys = (obj) => {
const sortedObj = {};
_.forEach(_.sortBy(_.keys(obj)), key => {
sortedObj[key] = obj[key];
});
return sortedObj;
};
const sanitizeLink = (link) => {
return link.replace(/^\/\//, 'http://');
};
const normalizeLink = (link) => {
const url = new URL(sanitizeLink(link));
const params = sortObjectKeys(Object.fromEntries(url.searchParams.entries()));
url.search = new URLSearchParams(params).toString();
return url.toString();
};
const userAgentHeader = (value) => {
return value
? { headers: { 'user-agent': value } }
: {};
};
const parseSizes = (value) => {
const matched = (value || '').match(/(\d+)x(\d+)/i);
return matched
? { width: parseInt(matched[1]), height: parseInt(matched[2]) }
: null;
};
const urlResolve = (to, from) => {
const resolvedUrl = new URL(to, new URL(from, 'resolve://'));
if (resolvedUrl.protocol === 'resolve:') {
const { pathname, search, hash } = resolvedUrl;
return pathname + search + hash;
}
return resolvedUrl.toString();
};
const getType = (link) => {
const url = new URL(sanitizeLink(link));
return mime.getType(url.pathname);
};
const file = (link, contentType, rawDisposition, rawLength) => {
const disposition = rawDisposition
? parse(rawDisposition)
: null;
const { pathname } = parse$1(link);
const size = rawLength && !isNaN(parseInt(rawLength))
? parseInt(rawLength)
: null;
const type = contentType || getType(link) || null;
const ext = type ? mime.getExtension(type) : null;
const name = _.get(disposition, 'parameters.filename', _.last(pathname.split('/'))) || null;
return {
name, type, size, ext
};
};
const webpage = (link, $) => {
const pageTitle = $('title').text().trim() || null;
let type = null;
let url = null;
let siteName = null;
let title = null;
let description = null;
let icons = [];
let images = [];
let videos = [];
let audios = [];
$('link[rel="icon"]').each((i, el) => {
const href = $(el).attr('href');
const type = $(el).attr('type');
const sizes = $(el).attr('sizes');
if (href)
icons.push({
rel: 'icon',
url: urlResolve(href, link),
type: type || getType(urlResolve(href, link)) || null,
...(sizes ? parseSizes(sizes) || {} : {})
});
});
$('link[rel="apple-touch-icon"]').each((i, el) => {
const href = $(el).attr('href');
const type = $(el).attr('type');
const sizes = $(el).attr('sizes');
if (href)
icons.push({
rel: 'apple-touch-icon',
url: urlResolve(href, link),
type: type || getType(urlResolve(href, link)) || null,
...(sizes ? parseSizes(sizes) || {} : {})
});
});
$('meta').each((i, el) => {
const attrs = $(el).attr();
const name = attrs.name || attrs.property;
const value = attrs.content;
switch (name) {
case 'og:type':
type = value;
break;
case 'og:url':
url = value;
break;
case 'og:site_name':
siteName = value;
break;
case 'title':
case 'og:title':
case 'twitter:title':
title = value || title;
break;
case 'description':
case 'og:description':
case 'twitter:description':
description = value || description;
break;
case 'og:image':
case 'og:image:url':
case 'twitter:image':
case 'twitter:image:src':
images.push({
url: urlResolve(value, link),
});
break;
case 'og:image:secure_url': {
const image = _.last(images);
if (image)
image.url = urlResolve(value, link);
break;
}
case 'og:image:type': {
const image = _.last(images);
if (image && !image.type)
image.type = value;
break;
}
case 'og:image:width':
case 'twitter:image:width': {
const image = _.last(images);
const width = parseInt(value);
if (image && !image.width && !isNaN(width))
image.width = width;
break;
}
case 'og:image:height':
case 'twitter:image:height': {
const image = _.last(images);
const height = parseInt(value);
if (image && !image.height && !isNaN(height))
image.height = height;
break;
}
case 'og:video':
case 'og:video:url':
case 'twitter:player:stream':
videos.push({
url: urlResolve(value, link),
});
break;
case 'twitter:player':
videos.push({
url: urlResolve(value, link),
type: 'text/html',
});
break;
case 'og:video:secure_url': {
const video = _.last(videos);
if (video)
video.url = urlResolve(value, link);
break;
}
case 'og:video:type':
case 'twitter:player:stream:content_type': {
const video = _.last(videos);
if (video && !video.type)
video.type = value;
break;
}
case 'og:video:width':
case 'twitter:player:width': {
const video = _.last(videos);
const width = parseInt(value);
if (video && !video.width && !isNaN(width))
video.width = width;
break;
}
case 'og:video:height':
case 'twitter:player:height': {
const video = _.last(videos);
const height = parseInt(value);
if (video && !video.height && !isNaN(height))
video.height = height;
break;
}
case 'og:audio':
case 'og:audio:url':
audios.push({
url: urlResolve(value, link),
});
break;
case 'og:audio:secure_url': {
const audio = _.last(audios);
if (audio)
audio.url = urlResolve(value, link);
break;
}
case 'og:audio:type': {
const audio = _.last(audios);
if (audio && !audio.type)
audio.type = value;
break;
}
}
});
const joinIcons = (list) => {
return _.values(_.groupBy(list, 'url')).map(items => {
return items.reduce((mem, item) => ({ ...mem, ...item }), items[0]);
});
};
const joinMedia = (list) => {
return _.values(_.groupBy(list, 'url')).map(items => {
return items.reduce((mem, item) => ({ ...mem, ...item }), items[0]);
});
};
const normalizeMedia = (item) => {
item.type = item.type || getType(item.url);
return item;
};
icons = joinIcons(icons);
images = joinMedia(images).map(normalizeMedia);
videos = joinMedia(videos).map(normalizeMedia);
audios = joinMedia(audios).map(normalizeMedia);
const webpage = {
type: type || 'website',
url: url || link,
name: siteName || null,
title: title || pageTitle || null,
description: description || null,
icons, images, videos, audios,
};
return webpage;
};
const oembedJSON = (body) => {
if (!body || !body.type)
return null;
if (!['photo', 'video', 'rich', 'link'].includes(body.type))
return null;
let html = null;
let href = null;
if (body.html) {
const $embed = load(body.html.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/gi, '$1'));
html = $embed('iframe').wrap('<div></div>').parent().html();
if (html) {
const $iframe = load(html);
href = $iframe('iframe').attr('src');
}
}
const width = body.width ? parseInt(body.width + '') : null;
const height = body.height ? parseInt(body.height + '') : null;
const thumbnail_width = body.thumbnail_width ? parseInt(body.thumbnail_width + '') : null;
const thumbnail_height = body.thumbnail_height ? parseInt(body.thumbnail_height + '') : null;
return {
type: body.type,
title: body.title || null,
author: body.author_name
? { name: body.author_name, url: body.author_url || null }
: null,
thumbnail: body.thumbnail_url
? {
url: body.thumbnail_url,
type: getType(body.thumbnail_url) || null,
width: thumbnail_width && !isNaN(thumbnail_width) ? thumbnail_width : null,
height: thumbnail_height && !isNaN(thumbnail_height) ? thumbnail_height : null,
}
: null,
provider: body.provider_name
? {
name: body.provider_name,
url: body.provider_url || null,
id: _.upperFirst(_.camelCase(body.provider_name))
}
: null,
width: width && !isNaN(width) ? width : null,
height: height && !isNaN(height) ? height : null,
href: href || body.url ? normalizeLink(href || body.url) : null,
html: html || body.html || null,
};
};
const oembedXML = (xml) => {
const $ = load(xml, { xmlMode: true });
const type = $('type').text();
if (!['photo', 'video', 'rich', 'link'].includes(type))
return null;
const body = {
type: type,
version: $('version').text() || null,
title: $('title').text() || null,
author_name: $('author_name').text() || null,
author_url: $('author_url').text() || null,
provider_name: $('provider_name').text() || null,
provider_url: $('provider_url').text() || null,
cache_age: $('cache_age').text() || null,
thumbnail_url: $('thumbnail_url').text() || null,
thumbnail_width: $('thumbnail_width').text() || null,
thumbnail_height: $('thumbnail_height').text() || null,
url: $('url').text() || null,
html: $('html').text() || null,
width: $('width').text() || null,
height: $('height').text() || null,
};
return oembedJSON(body);
};
const html = (link, body) => {
const $ = load(body);
let oembedJSON = null;
let oembedXML = null;
$('link[rel="alternate"]').each((i, el) => {
const type = $(el).attr('type') || '';
const href = $(el).attr('href');
if (href && type.includes('+oembed')) {
if (type.startsWith('application/json')) {
oembedJSON = urlResolve(href, link);
}
else if (type.startsWith('application/xml') || type.startsWith('text/xml')) {
oembedXML = urlResolve(href, link);
}
}
});
return {
webpage: webpage(link, $),
oembed: {
json: oembedJSON,
xml: oembedXML
}
};
};
const enrichLink = async (link, userAgent, reqOptions) => {
const options = { ...userAgentHeader(userAgent), ...reqOptions };
const head = await getHead(link, options);
if (!head)
return null;
const contentType = head.headers['content-type'] || 'text/plain';
const contentLength = head.headers['content-length'] || '';
const contentDisposition = head.headers['content-disposition'];
const result = {};
if (contentType.startsWith('image/')) {
result.image = file(link, contentType, contentDisposition, contentLength);
}
else if (contentType.startsWith('video/')) {
result.video = file(link, contentType, contentDisposition, contentLength);
}
else if (contentType.startsWith('text/html')) {
const body = await getHtml(link, options);
const { webpage, oembed } = await html(link, body);
if (webpage) {
result.webpage = webpage;
}
if (oembed.json || oembed.xml) {
const json = oembed.json
? oembedJSON(await getJSON(oembed.json, options))
: null;
const xml = oembed.xml
? oembedXML(await getHtml(oembed.xml, options))
: null;
result.oembed = { ...xml, ...json };
}
}
if (contentDisposition) {
result.attachment = file(link, contentType, contentDisposition, contentLength);
}
return result;
};
export { enrichLink };
//# sourceMappingURL=link-enricher.js.map