semantic-analysis
Version:
Extensible semantic Structured Data analysis library for Microdata and JSON-LD
299 lines • 11.9 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.MicrodataExtractor = void 0;
var tslib_1 = require("tslib");
var item_1 = require("../item");
require("../../util");
var spec_1 = require("../item/spec");
var util_1 = require("../../util");
/**
* Microdata extraction according to HTML specification
* @see https://html.spec.whatwg.org/multipage/microdata.html#converting-html-to-other-formats
*/
var MicrodataExtractor = /** @class */ (function () {
/* Initialization */
function MicrodataExtractor(doc) {
var _this = this;
this.doc = doc;
/**
* Finds top-level items
* @see https://html.spec.whatwg.org/multipage/microdata.html#json
*/
this.findItems = function () { return ({
/**
* @see https://html.spec.whatwg.org/multipage/microdata.html#top-level-microdata-items
* @see https://html.spec.whatwg.org/multipage/microdata.html#microdata-and-other-namespaces
* TODO Handle only valid HTML elements
*/
items: Array.from(_this.doc.document.querySelectorAll('[itemscope]:not([itemprop])'))
.map(function (element) { return _this.createItem(element); }),
logs: [],
document: _this.doc,
}); };
/**
* Creates a new item
* @see https://html.spec.whatwg.org/multipage/microdata.html#get-the-object
* @param element Element to create item from
* @param memory Memory of used elements
*/
this.createItem = function (element, memory) {
if (memory === void 0) { memory = []; }
console.debug('Found an item at: ', element);
memory.push(element);
var item = new MicrodataItem({
type: _this.parseType(element),
id: element.getAttribute('itemid') || undefined,
meta: tslib_1.__assign(tslib_1.__assign({}, _this.meta), { element: element, logs: [] }),
});
var properties = _this.findProperties(element);
console.groupCollapsed('Property assignment');
var _loop_1 = function (property) {
console.debug('Inspecting property on:', property);
var value = void 0;
var logs = [];
if (property.hasAttribute('itemscope')) {
if (!memory.includes(property)) {
value = _this.createItem(property, memory.slice());
}
else {
console.debug('Property already in memory:', property);
console.debug('Memory:', memory);
logs.push({
message: 'Current element has already been inspected, ' +
'this might be a sign of cycles in the graph',
sort: 'Error',
phase: 'Extraction',
element: property,
});
value = 'ERROR';
}
}
else {
value = extractValue(property, _this.doc.url);
}
var names = separateTokens(property.getAttribute('itemprop'))
.filter(function (token, index, self) { return self.indexOf(token) === index; })
.map(function (token) {
var combos = item.type
.map(function (type) { return spec_1.combine(token, type); })
.filter(function (v, i, s) { return s.indexOf(v) === i; });
if (combos.length === 1) {
return combos[0];
}
if (combos.length > 1) {
var message = 'Detected property name ambiguity against the types';
console.debug(message + ", possibilities:", combos);
logs.push({
message: message,
sort: 'Error',
phase: 'Extraction',
element: property,
});
}
return token;
});
item.addProperty(names, value, tslib_1.__assign(tslib_1.__assign({}, _this.meta), { element: property, logs: logs }));
console.debug('Found properties', names);
};
for (var _i = 0, properties_1 = properties; _i < properties_1.length; _i++) {
var property = properties_1[_i];
_loop_1(property);
}
console.groupEnd();
return item;
};
/**
* Parse types of an item element
* @see https://html.spec.whatwg.org/multipage/microdata.html#item-types
* @param element Element to parse
*/
this.parseType = function (element) {
return separateTokens(element.getAttribute('itemtype'))
.filter(function (value, index, self) { return self.indexOf(value) === index; });
}; // Unique only
/**
* Find properties in root
* @see https://html.spec.whatwg.org/multipage/microdata.html#the-properties-of-an-item
* @param root Element to start
*/
this.findProperties = function (root) {
console.groupCollapsed('Searching properties...');
var results = [];
var memory = [root];
var pending = Array.from(root.children);
separateTokens(root.getAttribute('itemref'))
.forEach(function (id) {
var element = root.ownerDocument ? root.ownerDocument.getElementById(id) : undefined;
if (element) {
/* tslint:disable-next-line:no-bitwise */
var action = element.compareDocumentPosition(root) & Node.DOCUMENT_POSITION_FOLLOWING
? 'unshift' : 'push';
pending[action](element);
}
});
while (pending.length) {
var current = pending.shift();
if (memory.includes(current)) {
continue;
}
memory.push(current);
if (!current.hasAttribute('itemscope')) {
pending.unshift.apply(pending, Array.from(current.children));
}
if (separateTokens(current.getAttribute('itemprop')).length > 0) {
results.push(current);
}
}
console.debug("Found " + results.length + " properties:", results);
console.groupEnd();
return results;
};
this.meta = {
type: 'Microdata',
extractor: this,
document: doc,
};
}
/* Extractor */
/**
* Extracts items according to HTML specification in format similar to JSON
* @see https://html.spec.whatwg.org/multipage/microdata.html#json
* @returns Returns an extraction result
*/
MicrodataExtractor.prototype.extract = function () {
console.groupCollapsed('Microdata extraction');
var result = this.findItems();
console.groupEnd();
return result;
};
return MicrodataExtractor;
}());
exports.MicrodataExtractor = MicrodataExtractor;
/**
* Local implementation of item
* TODO Extract into a separate package
*/
var MicrodataItem = /** @class */ (function () {
/* Initialization */
function MicrodataItem(source) {
var _this = this;
/* Helpers */
this.parse = function (value, meta) {
if (item_1.Spec.isItem(value)) {
return new MicrodataItem(tslib_1.__assign(tslib_1.__assign({}, value), { meta: meta !== null && meta !== void 0 ? meta : _this.meta }));
}
if (item_1.Spec.isValuePrimitive(value)) {
return value;
}
throw new TypeError("Invalid value type " + typeof value);
};
this.type = source.type ? source.type.slice() : [];
this.id = source.id;
this.meta = source.meta;
if (source.propMeta) {
this.propMeta = util_1.map(source.propMeta, function (props) { return props.map(function (prop) { return ({
name: prop.name,
value: _this.parse(prop.value),
meta: prop.meta,
}); }); });
}
else if (source.properties) {
this.propMeta = util_1.map(source.properties, function (values) { return values.map(function (value) { return ({
name: name,
value: _this.parse(value),
meta: _this.meta,
}); }); });
}
else {
this.propMeta = {};
}
}
Object.defineProperty(MicrodataItem.prototype, "properties", {
get: function () {
return util_1.map(this.propMeta, function (props) { return props.map(function (prop) { return prop.value; }); });
},
enumerable: false,
configurable: true
});
/* Manipulation */
MicrodataItem.prototype.addProperty = function (name, value, meta) {
var _this = this;
(Array.isArray(name) ? name : [name])
.forEach(function (n) {
if (!_this.propMeta.hasOwnProperty(n)) {
_this.propMeta[n] = [];
}
_this.propMeta[n].push({
name: n,
value: _this.parse(value),
meta: meta,
});
});
return this;
};
return MicrodataItem;
}());
/**
* Separate tokens by whitespace
* @param value Input to parse
*/
var separateTokens = function (value) {
return (value || '').match(/\S+/g) || [];
};
/**
* Extract the value of the property
* @see https://html.spec.whatwg.org/multipage/microdata.html#values
* @param element Element to examine for value
* @param base string Base URL to apply to relative URLs
*/
function extractValue(element, base) {
var attr = function (x, name) {
return x.getAttribute(name || '') || '';
};
var text = function (x) { return (x.textContent || '').trim().replace(/\s+/g, ' '); };
/**
* @see https://html.spec.whatwg.org/multipage/text-level-semantics.html#datetime-value
* @param x
*/
var datetime = function (x) {
return x.hasAttribute('datetime')
? attr(x, 'datetime')
: text(x);
};
var completeURL = function (url) {
try {
return new URL(url, base).toString();
}
catch (e) {
return url;
}
};
var resolver = {
content: function (x) { return attr(x, 'content'); },
src: function (x) { return completeURL(attr(x, 'src')); },
href: function (x) { return completeURL(attr(x, 'href')); },
data: function (x) { return attr(x, 'data'); },
value: function (x) { return attr(x, 'value'); },
datetime: function (x) { return datetime(x); },
text: function (x) { return text(x); },
};
var tagMap = {
meta: resolver.content,
audio: resolver.src,
embed: resolver.src,
iframe: resolver.src,
img: resolver.src,
source: resolver.src,
track: resolver.src,
video: resolver.src,
a: resolver.href,
area: resolver.href,
link: resolver.href,
object: resolver.data,
meter: resolver.value,
time: resolver.datetime,
other: resolver.text,
};
return (tagMap[element.tagName.toLowerCase()] || tagMap.other)(element);
}
//# sourceMappingURL=MicrodataExtractor.js.map