microdata-parser-ts
Version:
Microdata parser. Extract and parse microdata from any website
46 lines • 4.01 kB
JavaScript
import { load } from 'cheerio';
import { fetchHTML } from '../helpers/fetch';
import { md5 } from '../helpers/md5';
import { getPropValue } from '../helpers/parser';
export const parseHTML = (html) => {
const items = [];
const api = load(html);
api('[itemtype]').map((_, item) => {
items.push({
id: md5(api(item).html()),
name: api(item).attr('itemtype'),
properties: {},
});
});
api('[itemprop]').map((_, item) => {
const itemtypeHtml = api(item).parents('[itemtype]').html();
if (!itemtypeHtml)
return;
const property = api(item).attr('itemprop');
if (!property)
return;
const currentItem = items.find((item) => item.id === md5(itemtypeHtml));
if (!currentItem)
return;
const value = getPropValue(api, item);
if (!value)
return;
if (currentItem.properties[property]) {
if (Array.isArray(currentItem.properties[property])) {
// TODO figure out a better TS type
// @ts-expect-error this can be an array
currentItem.properties[property].push(value);
}
currentItem.properties[property] = [currentItem.properties[property]];
}
else {
currentItem.properties[property] = value;
}
});
return items;
};
export const parseURL = async (url, options) => {
const html = await fetchHTML(url, options);
return parseHTML(html);
};
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvbGliL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxJQUFJLEVBQUUsTUFBTSxTQUFTLENBQUE7QUFFOUIsT0FBTyxFQUFFLFNBQVMsRUFBRSxNQUFNLGtCQUFrQixDQUFBO0FBQzVDLE9BQU8sRUFBRSxHQUFHLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQTtBQUNwQyxPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sbUJBQW1CLENBQUE7QUFHaEQsTUFBTSxDQUFDLE1BQU0sU0FBUyxHQUFHLENBQUMsSUFBWSxFQUFFLEVBQUU7SUFDeEMsTUFBTSxLQUFLLEdBQWdCLEVBQUUsQ0FBQTtJQUM3QixNQUFNLEdBQUcsR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUE7SUFFdEIsR0FBRyxDQUFDLFlBQVksQ0FBQyxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsRUFBRSxJQUFJLEVBQUUsRUFBRTtRQUNoQyxLQUFLLENBQUMsSUFBSSxDQUFDO1lBQ1QsRUFBRSxFQUFFLEdBQUcsQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLENBQUMsSUFBSSxFQUFFLENBQUM7WUFDekIsSUFBSSxFQUFFLEdBQUcsQ0FBQyxJQUFJLENBQUMsQ0FBQyxJQUFJLENBQUMsVUFBVSxDQUFDO1lBQ2hDLFVBQVUsRUFBRSxFQUFFO1NBQ2YsQ0FBQyxDQUFBO0lBQ0osQ0FBQyxDQUFDLENBQUE7SUFFRixHQUFHLENBQUMsWUFBWSxDQUFDLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxFQUFFLElBQUksRUFBRSxFQUFFO1FBQ2hDLE1BQU0sWUFBWSxHQUFHLEdBQUcsQ0FBQyxJQUFJLENBQUMsQ0FBQyxPQUFPLENBQUMsWUFBWSxDQUFDLENBQUMsSUFBSSxFQUFFLENBQUE7UUFDM0QsSUFBSSxDQUFDLFlBQVk7WUFBRSxPQUFNO1FBRXpCLE1BQU0sUUFBUSxHQUFHLEdBQUcsQ0FBQyxJQUFJLENBQUMsQ0FBQyxJQUFJLENBQUMsVUFBVSxDQUFDLENBQUE7UUFDM0MsSUFBSSxDQUFDLFFBQVE7WUFBRSxPQUFNO1FBRXJCLE1BQU0sV0FBVyxHQUFHLEtBQUssQ0FBQyxJQUFJLENBQUMsQ0FBQyxJQUFJLEVBQUUsRUFBRSxDQUFDLElBQUksQ0FBQyxFQUFFLEtBQUssR0FBRyxDQUFDLFlBQVksQ0FBQyxDQUFDLENBQUE7UUFDdkUsSUFBSSxDQUFDLFdBQVc7WUFBRSxPQUFNO1FBRXhCLE1BQU0sS0FBSyxHQUFHLFlBQVksQ0FBQyxHQUFHLEVBQUUsSUFBSSxDQUFDLENBQUE7UUFDckMsSUFBSSxDQUFDLEtBQUs7WUFBRSxPQUFNO1FBRWxCLElBQUksV0FBVyxDQUFDLFVBQVUsQ0FBQyxRQUFRLENBQUMsRUFBRTtZQUNwQyxJQUFJLEtBQUssQ0FBQyxPQUFPLENBQUMsV0FBVyxDQUFDLFVBQVUsQ0FBQyxRQUFRLENBQUMsQ0FBQyxFQUFFO2dCQUNuRCxtQ0FBbUM7Z0JBQ25DLHdDQUF3QztnQkFDeEMsV0FBVyxDQUFDLFVBQVUsQ0FBQyxRQUFRLENBQUMsQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLENBQUE7YUFDN0M7WUFDRCxXQUFXLENBQUMsVUFBVSxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQUMsV0FBVyxDQUFDLFVBQVUsQ0FBQyxRQUFRLENBQUMsQ0FBQyxDQUFBO1NBQ3RFO2FBQU07WUFDTCxXQUFXLENBQUMsVUFBVSxDQUFDLFFBQVEsQ0FBQyxHQUFHLEtBQUssQ0FBQTtTQUN6QztJQUNILENBQUMsQ0FBQyxDQUFBO0lBRUYsT0FBTyxLQUFLLENBQUE7QUFDZCxDQUFDLENBQUE7QUFFRCxNQUFNLENBQUMsTUFBTSxRQUFRLEdBQUcsS0FBSyxFQUFFLEdBQVcsRUFBRSxPQUF3QixFQUFFLEVBQUU7SUFDdEUsTUFBTSxJQUFJLEdBQUcsTUFBTSxTQUFTLENBQUMsR0FBRyxFQUFFLE9BQU8sQ0FBQyxDQUFBO0lBQzFDLE9BQU8sU0FBUyxDQUFDLElBQUksQ0FBQyxDQUFBO0FBQ3hCLENBQUMsQ0FBQSJ9