microdata-parser-ts
Version:
Microdata parser. Extract and parse microdata from any website
51 lines • 4.19 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.parseURL = exports.parseHTML = void 0;
const cheerio_1 = require("cheerio");
const fetch_1 = require("../helpers/fetch");
const md5_1 = require("../helpers/md5");
const parser_1 = require("../helpers/parser");
const parseHTML = (html) => {
const items = [];
const api = (0, cheerio_1.load)(html);
api('[itemtype]').map((_, item) => {
items.push({
id: (0, md5_1.md5)(api(item).html()),
name: api(item).attr('itemtype'),
properties: {},
});
});
api('[itemprop]').map((_, item) => {
const itemtypeHtml = api(item).parents('[itemtype]').html();
if (!itemtypeHtml)
return;
const property = api(item).attr('itemprop');
if (!property)
return;
const currentItem = items.find((item) => item.id === (0, md5_1.md5)(itemtypeHtml));
if (!currentItem)
return;
const value = (0, parser_1.getPropValue)(api, item);
if (!value)
return;
if (currentItem.properties[property]) {
if (Array.isArray(currentItem.properties[property])) {
// TODO figure out a better TS type
// @ts-expect-error this can be an array
currentItem.properties[property].push(value);
}
currentItem.properties[property] = [currentItem.properties[property]];
}
else {
currentItem.properties[property] = value;
}
});
return items;
};
exports.parseHTML = parseHTML;
const parseURL = async (url, options) => {
const html = await (0, fetch_1.fetchHTML)(url, options);
return (0, exports.parseHTML)(html);
};
exports.parseURL = parseURL;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi8uLi9zcmMvbGliL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiI7OztBQUFBLHFDQUE4QjtBQUU5Qiw0Q0FBNEM7QUFDNUMsd0NBQW9DO0FBQ3BDLDhDQUFnRDtBQUd6QyxNQUFNLFNBQVMsR0FBRyxDQUFDLElBQVksRUFBRSxFQUFFO0lBQ3hDLE1BQU0sS0FBSyxHQUFnQixFQUFFLENBQUE7SUFDN0IsTUFBTSxHQUFHLEdBQUcsSUFBQSxjQUFJLEVBQUMsSUFBSSxDQUFDLENBQUE7SUFFdEIsR0FBRyxDQUFDLFlBQVksQ0FBQyxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsRUFBRSxJQUFJLEVBQUUsRUFBRTtRQUNoQyxLQUFLLENBQUMsSUFBSSxDQUFDO1lBQ1QsRUFBRSxFQUFFLElBQUEsU0FBRyxFQUFDLEdBQUcsQ0FBQyxJQUFJLENBQUMsQ0FBQyxJQUFJLEVBQUUsQ0FBQztZQUN6QixJQUFJLEVBQUUsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUM7WUFDaEMsVUFBVSxFQUFFLEVBQUU7U0FDZixDQUFDLENBQUE7SUFDSixDQUFDLENBQUMsQ0FBQTtJQUVGLEdBQUcsQ0FBQyxZQUFZLENBQUMsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQUUsSUFBSSxFQUFFLEVBQUU7UUFDaEMsTUFBTSxZQUFZLEdBQUcsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDLE9BQU8sQ0FBQyxZQUFZLENBQUMsQ0FBQyxJQUFJLEVBQUUsQ0FBQTtRQUMzRCxJQUFJLENBQUMsWUFBWTtZQUFFLE9BQU07UUFFekIsTUFBTSxRQUFRLEdBQUcsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQTtRQUMzQyxJQUFJLENBQUMsUUFBUTtZQUFFLE9BQU07UUFFckIsTUFBTSxXQUFXLEdBQUcsS0FBSyxDQUFDLElBQUksQ0FBQyxDQUFDLElBQUksRUFBRSxFQUFFLENBQUMsSUFBSSxDQUFDLEVBQUUsS0FBSyxJQUFBLFNBQUcsRUFBQyxZQUFZLENBQUMsQ0FBQyxDQUFBO1FBQ3ZFLElBQUksQ0FBQyxXQUFXO1lBQUUsT0FBTTtRQUV4QixNQUFNLEtBQUssR0FBRyxJQUFBLHFCQUFZLEVBQUMsR0FBRyxFQUFFLElBQUksQ0FBQyxDQUFBO1FBQ3JDLElBQUksQ0FBQyxLQUFLO1lBQUUsT0FBTTtRQUVsQixJQUFJLFdBQVcsQ0FBQyxVQUFVLENBQUMsUUFBUSxDQUFDLEVBQUU7WUFDcEMsSUFBSSxLQUFLLENBQUMsT0FBTyxDQUFDLFdBQVcsQ0FBQyxVQUFVLENBQUMsUUFBUSxDQUFDLENBQUMsRUFBRTtnQkFDbkQsbUNBQW1DO2dCQUNuQyx3Q0FBd0M7Z0JBQ3hDLFdBQVcsQ0FBQyxVQUFVLENBQUMsUUFBUSxDQUFDLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFBO2FBQzdDO1lBQ0QsV0FBVyxDQUFDLFVBQVUsQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFDLFdBQVcsQ0FBQyxVQUFVLENBQUMsUUFBUSxDQUFDLENBQUMsQ0FBQTtTQUN0RTthQUFNO1lBQ0wsV0FBVyxDQUFDLFVBQVUsQ0FBQyxRQUFRLENBQUMsR0FBRyxLQUFLLENBQUE7U0FDekM7SUFDSCxDQUFDLENBQUMsQ0FBQTtJQUVGLE9BQU8sS0FBSyxDQUFBO0FBQ2QsQ0FBQyxDQUFBO0FBdENZLFFBQUEsU0FBUyxhQXNDckI7QUFFTSxNQUFNLFFBQVEsR0FBRyxLQUFLLEVBQUUsR0FBVyxFQUFFLE9BQXdCLEVBQUUsRUFBRTtJQUN0RSxNQUFNLElBQUksR0FBRyxNQUFNLElBQUEsaUJBQVMsRUFBQyxHQUFHLEVBQUUsT0FBTyxDQUFDLENBQUE7SUFDMUMsT0FBTyxJQUFBLGlCQUFTLEVBQUMsSUFBSSxDQUFDLENBQUE7QUFDeEIsQ0FBQyxDQUFBO0FBSFksUUFBQSxRQUFRLFlBR3BCIn0=