site-metadata-extractor
Version:
web(site) resource metadata extractor
185 lines • 23.5 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ulToText = exports.replaceWithText = exports.replaceCharacters = exports.removeNegativescoresNodes = exports.removeFewWordsParagraphs = exports.linksToText = exports.convertToText = exports.cleanParagraphText = exports.addNewlineToBr = void 0;
const xregexp_1 = __importDefault(require("xregexp"));
const stopwords_1 = __importDefault(require("./stopwords"));
function getTagName(node) {
const element = node.get(0);
return (element === null || element === void 0 ? void 0 : element.tagName) || "";
}
const addNewlineToBr = (doc, topNode) => {
const brs = topNode.find("br");
brs.each((_index, element) => {
const br = doc(element);
br.replaceWith("\n\n");
});
return doc;
};
exports.addNewlineToBr = addNewlineToBr;
const cleanParagraphText = (rawText) => {
const text = rawText.trim();
text.replace(/[\s\t]+/g, " ");
return text;
};
exports.cleanParagraphText = cleanParagraphText;
const convertToText = (doc, topNode) => {
let texts = [];
const nodes = topNode.contents();
let hangingText = "";
nodes.each((_index, element) => {
const node = doc(element);
const firstNode = node.get(0);
const nodeType = firstNode ? firstNode.type : null;
const nodeName = getTagName(node);
if (nodeType === "text") {
hangingText += node.text();
return true;
}
else if (nodeName === "ul") {
hangingText += (0, exports.ulToText)(doc, node);
return true;
}
if (hangingText.length > 0) {
const txt = (0, exports.cleanParagraphText)(hangingText);
texts = texts.concat(txt.split(/\r?\n/));
hangingText = "";
}
let text = (0, exports.cleanParagraphText)(node.text());
text = text.replace(/(\w+\.)([A-Z]+)/, "$1 $2");
texts = texts.concat(text.split(/\r?\n/));
});
if (hangingText.length > 0) {
const text = (0, exports.cleanParagraphText)(hangingText);
texts = texts = texts.concat(text.split(/\r?\n/));
}
texts = texts.map((txt) => {
return txt.trim();
});
const regex = (0, xregexp_1.default)("[\\p{Number}\\p{Letter}]");
texts = texts.filter((txt) => {
return regex.test(txt);
});
return texts.join("\n\n");
};
exports.convertToText = convertToText;
const linksToText = (doc, topNode) => {
const nodes = topNode.find("a");
nodes.each((_index, element) => {
const htmlEl = doc(element).html();
if (htmlEl) {
doc(element).replaceWith(htmlEl);
}
});
return doc;
};
exports.linksToText = linksToText;
const removeFewWordsParagraphs = (doc, topNode, lang) => {
const allNodes = topNode.find("*");
allNodes.each((_index, element) => {
const el = doc(element);
const tag = getTagName(el);
const text = el.text();
const stopWords = (0, stopwords_1.default)(text, lang);
if ((tag !== "br" || text !== "\\r") &&
stopWords.stopWordCount < 3 &&
el.find("object").length === 0 &&
el.find("embed").length === 0) {
doc(el).remove();
}
else {
const trimmed = text.trim();
if (trimmed[0] === "(" && trimmed[trimmed.length - 1] === ")") {
doc(el).remove();
}
}
});
return doc;
};
exports.removeFewWordsParagraphs = removeFewWordsParagraphs;
const removeNegativescoresNodes = (doc, topNode) => {
const gravityItems = topNode.find("*[gravityScore]");
gravityItems.each((_index, element) => {
let score = 0;
const item = doc(element);
const gravityScore = item.attr("gravityScore");
if (gravityScore) {
score = parseInt(gravityScore, 10) || 0;
}
if (score < 1) {
doc(item).remove();
}
});
return doc;
};
exports.removeNegativescoresNodes = removeNegativescoresNodes;
const replaceCharacters = (text, html, chars) => {
let processedText = text;
// if element does not match any in map and starts with & and ends with ;, replace with empty string
const htmlEntities = {
"&": "&",
"'": "'",
"¢": "¢",
"©": "©",
"€": "€",
">": ">",
"<": "<",
" ": " ",
"£": "£",
""": '"',
"®": "®",
"¥": "¥",
};
const escapeChars = {
'"': '"',
"'": "'",
// tslint:disable-next-line: object-literal-sort-keys
"\n": " ",
"\r": " ",
};
if (html) {
for (const key of Object.keys(htmlEntities)) {
const htmlregex = new RegExp(key, "g");
processedText = processedText.replace(htmlregex, htmlEntities[key]);
}
}
if (chars) {
for (const key of Object.keys(escapeChars)) {
const escapeCharsRegex = new RegExp(key, "g");
processedText = processedText.replace(escapeCharsRegex, escapeChars[key]);
}
}
return processedText;
};
exports.replaceCharacters = replaceCharacters;
const replaceWithText = (doc, topNode) => {
const nodes = topNode.find("b, strong, i, br, sup");
nodes.each((_index, element) => {
doc(element).replaceWith(doc(element).text());
});
return doc;
};
exports.replaceWithText = replaceWithText;
const ulToText = (doc, node) => {
const nodes = node.find("li");
let text = "";
nodes.each((_index, element) => {
text = `${text}\n * ${doc(element).text()}`;
});
text = `${text}\n`;
return text;
};
exports.ulToText = ulToText;
const formatter = (doc, topNode, lang) => {
(0, exports.removeNegativescoresNodes)(doc, topNode);
(0, exports.linksToText)(doc, topNode);
(0, exports.addNewlineToBr)(doc, topNode);
(0, exports.replaceWithText)(doc, topNode);
(0, exports.removeFewWordsParagraphs)(doc, topNode, lang);
const convertedText = (0, exports.convertToText)(doc, topNode).replace(/\n/g, " ");
return (0, exports.replaceCharacters)(convertedText, false, true);
};
exports.default = formatter;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZm9ybWF0dGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL2Zvcm1hdHRlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7QUFFQSxzREFBOEI7QUFDOUIsNERBQW9DO0FBRXBDLFNBQVMsVUFBVSxDQUFDLElBQXNCO0lBQ3hDLE1BQU0sT0FBTyxHQUFHLElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUF3QixDQUFDO0lBQ25ELE9BQU8sQ0FBQSxPQUFPLGFBQVAsT0FBTyx1QkFBUCxPQUFPLENBQUUsT0FBTyxLQUFJLEVBQUUsQ0FBQztBQUNoQyxDQUFDO0FBRU0sTUFBTSxjQUFjLEdBQUcsQ0FDNUIsR0FBZSxFQUNmLE9BQXlCLEVBQ2IsRUFBRTtJQUNkLE1BQU0sR0FBRyxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7SUFFL0IsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDLE1BQWMsRUFBRSxPQUFnQixFQUFFLEVBQUU7UUFDNUMsTUFBTSxFQUFFLEdBQUcsR0FBRyxDQUFDLE9BQU8sQ0FBQyxDQUFDO1FBQ3hCLEVBQUUsQ0FBQyxXQUFXLENBQUMsTUFBTSxDQUFDLENBQUM7SUFDekIsQ0FBQyxDQUFDLENBQUM7SUFFSCxPQUFPLEdBQUcsQ0FBQztBQUNiLENBQUMsQ0FBQztBQVpXLFFBQUEsY0FBYyxrQkFZekI7QUFFSyxNQUFNLGtCQUFrQixHQUFHLENBQUMsT0FBZSxFQUFVLEVBQUU7SUFDNUQsTUFBTSxJQUFJLEdBQUcsT0FBTyxDQUFDLElBQUksRUFBRSxDQUFDO0lBQzVCLElBQUksQ0FBQyxPQUFPLENBQUMsVUFBVSxFQUFFLEdBQUcsQ0FBQyxDQUFDO0lBQzlCLE9BQU8sSUFBSSxDQUFDO0FBQ2QsQ0FBQyxDQUFDO0FBSlcsUUFBQSxrQkFBa0Isc0JBSTdCO0FBRUssTUFBTSxhQUFhLEdBQUcsQ0FDM0IsR0FBZSxFQUNmLE9BQXlCLEVBQ2pCLEVBQUU7SUFDVixJQUFJLEtBQUssR0FBYSxFQUFFLENBQUM7SUFDekIsTUFBTSxLQUFLLEdBQUcsT0FBTyxDQUFDLFFBQVEsRUFBRSxDQUFDO0lBRWpDLElBQUksV0FBVyxHQUFHLEVBQUUsQ0FBQztJQUVyQixLQUFLLENBQUMsSUFBSSxDQUFDLENBQUMsTUFBYyxFQUFFLE9BQWdCLEVBQUUsRUFBRTtRQUM5QyxNQUFNLElBQUksR0FBRyxHQUFHLENBQUMsT0FBTyxDQUFDLENBQUM7UUFDMUIsTUFBTSxTQUFTLEdBQUcsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQztRQUM5QixNQUFNLFFBQVEsR0FBRyxTQUFTLENBQUMsQ0FBQyxDQUFDLFNBQVMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQztRQUNuRCxNQUFNLFFBQVEsR0FBRyxVQUFVLENBQUMsSUFBSSxDQUFDLENBQUM7UUFFbEMsSUFBSSxRQUFRLEtBQUssTUFBTSxFQUFFLENBQUM7WUFDeEIsV0FBVyxJQUFJLElBQUksQ0FBQyxJQUFJLEVBQUUsQ0FBQztZQUMzQixPQUFPLElBQUksQ0FBQztRQUNkLENBQUM7YUFBTSxJQUFJLFFBQVEsS0FBSyxJQUFJLEVBQUUsQ0FBQztZQUM3QixXQUFXLElBQUksSUFBQSxnQkFBUSxFQUFDLEdBQUcsRUFBRSxJQUFJLENBQUMsQ0FBQztZQUNuQyxPQUFPLElBQUksQ0FBQztRQUNkLENBQUM7UUFFRCxJQUFJLFdBQVcsQ0FBQyxNQUFNLEdBQUcsQ0FBQyxFQUFFLENBQUM7WUFDM0IsTUFBTSxHQUFHLEdBQUcsSUFBQSwwQkFBa0IsRUFBQyxXQUFXLENBQUMsQ0FBQztZQUM1QyxLQUFLLEdBQUcsS0FBSyxDQUFDLE1BQU0sQ0FBQyxHQUFHLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUM7WUFDekMsV0FBVyxHQUFHLEVBQUUsQ0FBQztRQUNuQixDQUFDO1FBRUQsSUFBSSxJQUFJLEdBQUcsSUFBQSwwQkFBa0IsRUFBQyxJQUFJLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FBQztRQUMzQyxJQUFJLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxpQkFBaUIsRUFBRSxPQUFPLENBQUMsQ0FBQztRQUNoRCxLQUFLLEdBQUcsS0FBSyxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUM7SUFDNUMsQ0FBQyxDQUFDLENBQUM7SUFFSCxJQUFJLFdBQVcsQ0FBQyxNQUFNLEdBQUcsQ0FBQyxFQUFFLENBQUM7UUFDM0IsTUFBTSxJQUFJLEdBQUcsSUFBQSwwQkFBa0IsRUFBQyxXQUFXLENBQUMsQ0FBQztRQUM3QyxLQUFLLEdBQUcsS0FBSyxHQUFHLEtBQUssQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDO0lBQ3BELENBQUM7SUFFRCxLQUFLLEdBQUcsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFDLEdBQUcsRUFBRSxFQUFFO1FBQ3hCLE9BQU8sR0FBRyxDQUFDLElBQUksRUFBRSxDQUFDO0lBQ3BCLENBQUMsQ0FBQyxDQUFDO0lBRUgsTUFBTSxLQUFLLEdBQUcsSUFBQSxpQkFBTyxFQUFDLDBCQUEwQixDQUFDLENBQUM7SUFDbEQsS0FBSyxHQUFHLEtBQUssQ0FBQyxNQUFNLENBQUMsQ0FBQyxHQUFHLEVBQUUsRUFBRTtRQUMzQixPQUFPLEtBQUssQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7SUFDekIsQ0FBQyxDQUFDLENBQUM7SUFFSCxPQUFPLEtBQUssQ0FBQyxJQUFJLENBQUMsTUFBTSxDQUFDLENBQUM7QUFDNUIsQ0FBQyxDQUFDO0FBakRXLFFBQUEsYUFBYSxpQkFpRHhCO0FBRUssTUFBTSxXQUFXLEdBQUcsQ0FDekIsR0FBZSxFQUNmLE9BQXlCLEVBQ2IsRUFBRTtJQUNkLE1BQU0sS0FBSyxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7SUFDaEMsS0FBSyxDQUFDLElBQUksQ0FBQyxDQUFDLE1BQWMsRUFBRSxPQUFnQixFQUFFLEVBQUU7UUFDOUMsTUFBTSxNQUFNLEdBQUcsR0FBRyxDQUFDLE9BQU8sQ0FBQyxDQUFDLElBQUksRUFBRSxDQUFDO1FBQ25DLElBQUksTUFBTSxFQUFFLENBQUM7WUFDWCxHQUFHLENBQUMsT0FBTyxDQUFDLENBQUMsV0FBVyxDQUFDLE1BQU0sQ0FBQyxDQUFDO1FBQ25DLENBQUM7SUFDSCxDQUFDLENBQUMsQ0FBQztJQUNILE9BQU8sR0FBRyxDQUFDO0FBQ2IsQ0FBQyxDQUFDO0FBWlcsUUFBQSxXQUFXLGVBWXRCO0FBRUssTUFBTSx3QkFBd0IsR0FBRyxDQUN0QyxHQUFlLEVBQ2YsT0FBeUIsRUFDekIsSUFBWSxFQUNBLEVBQUU7SUFDZCxNQUFNLFFBQVEsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDO0lBRW5DLFFBQVEsQ0FBQyxJQUFJLENBQUMsQ0FBQyxNQUFjLEVBQUUsT0FBZ0IsRUFBRSxFQUFFO1FBQ2pELE1BQU0sRUFBRSxHQUFHLEdBQUcsQ0FBQyxPQUFPLENBQUMsQ0FBQztRQUN4QixNQUFNLEdBQUcsR0FBRyxVQUFVLENBQUMsRUFBRSxDQUFDLENBQUM7UUFDM0IsTUFBTSxJQUFJLEdBQUcsRUFBRSxDQUFDLElBQUksRUFBRSxDQUFDO1FBRXZCLE1BQU0sU0FBUyxHQUFHLElBQUEsbUJBQVMsRUFBQyxJQUFJLEVBQUUsSUFBSSxDQUFDLENBQUM7UUFDeEMsSUFDRSxDQUFDLEdBQUcsS0FBSyxJQUFJLElBQUksSUFBSSxLQUFLLEtBQUssQ0FBQztZQUNoQyxTQUFTLENBQUMsYUFBYSxHQUFHLENBQUM7WUFDM0IsRUFBRSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsQ0FBQyxNQUFNLEtBQUssQ0FBQztZQUM5QixFQUFFLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxDQUFDLE1BQU0sS0FBSyxDQUFDLEVBQzdCLENBQUM7WUFDRCxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUMsTUFBTSxFQUFFLENBQUM7UUFDbkIsQ0FBQzthQUFNLENBQUM7WUFDTixNQUFNLE9BQU8sR0FBRyxJQUFJLENBQUMsSUFBSSxFQUFFLENBQUM7WUFDNUIsSUFBSSxPQUFPLENBQUMsQ0FBQyxDQUFDLEtBQUssR0FBRyxJQUFJLE9BQU8sQ0FBQyxPQUFPLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQyxLQUFLLEdBQUcsRUFBRSxDQUFDO2dCQUM5RCxHQUFHLENBQUMsRUFBRSxDQUFDLENBQUMsTUFBTSxFQUFFLENBQUM7WUFDbkIsQ0FBQztRQUNILENBQUM7SUFDSCxDQUFDLENBQUMsQ0FBQztJQUVILE9BQU8sR0FBRyxDQUFDO0FBQ2IsQ0FBQyxDQUFDO0FBN0JXLFFBQUEsd0JBQXdCLDRCQTZCbkM7QUFFSyxNQUFNLHlCQUF5QixHQUFHLENBQ3ZDLEdBQWUsRUFDZixPQUF5QixFQUNiLEVBQUU7SUFDZCxNQUFNLFlBQVksR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLGlCQUFpQixDQUFDLENBQUM7SUFFckQsWUFBWSxDQUFDLElBQUksQ0FBQyxDQUFDLE1BQWMsRUFBRSxPQUFnQixFQUFFLEVBQUU7UUFDckQsSUFBSSxLQUFLLEdBQUcsQ0FBQyxDQUFDO1FBQ2QsTUFBTSxJQUFJLEdBQUcsR0FBRyxDQUFDLE9BQU8sQ0FBQyxDQUFDO1FBQzFCLE1BQU0sWUFBWSxHQUFHLElBQUksQ0FBQyxJQUFJLENBQUMsY0FBYyxDQUFDLENBQUM7UUFDL0MsSUFBSSxZQUFZLEVBQUUsQ0FBQztZQUNqQixLQUFLLEdBQUcsUUFBUSxDQUFDLFlBQVksRUFBRSxFQUFFLENBQUMsSUFBSSxDQUFDLENBQUM7UUFDMUMsQ0FBQztRQUVELElBQUksS0FBSyxHQUFHLENBQUMsRUFBRSxDQUFDO1lBQ2QsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDLE1BQU0sRUFBRSxDQUFDO1FBQ3JCLENBQUM7SUFDSCxDQUFDLENBQUMsQ0FBQztJQUVILE9BQU8sR0FBRyxDQUFDO0FBQ2IsQ0FBQyxDQUFDO0FBcEJXLFFBQUEseUJBQXlCLDZCQW9CcEM7QUFFSyxNQUFNLGlCQUFpQixHQUFHLENBQy9CLElBQVksRUFDWixJQUFhLEVBQ2IsS0FBYyxFQUNOLEVBQUU7SUFDVixJQUFJLGFBQWEsR0FBRyxJQUFJLENBQUM7SUFDekIsb0dBQW9HO0lBQ3BHLE1BQU0sWUFBWSxHQUEyQjtRQUMzQyxPQUFPLEVBQUUsR0FBRztRQUNaLFFBQVEsRUFBRSxHQUFHO1FBQ2IsUUFBUSxFQUFFLEdBQUc7UUFDYixRQUFRLEVBQUUsR0FBRztRQUNiLFFBQVEsRUFBRSxHQUFHO1FBQ2IsTUFBTSxFQUFFLEdBQUc7UUFDWCxNQUFNLEVBQUUsR0FBRztRQUNYLFFBQVEsRUFBRSxHQUFHO1FBQ2IsU0FBUyxFQUFFLEdBQUc7UUFDZCxRQUFRLEVBQUUsR0FBRztRQUNiLE9BQU8sRUFBRSxHQUFHO1FBQ1osT0FBTyxFQUFFLEdBQUc7S0FDYixDQUFDO0lBRUYsTUFBTSxXQUFXLEdBQTJCO1FBQzFDLEdBQUcsRUFBRSxHQUFHO1FBQ1IsR0FBRyxFQUFFLEdBQUc7UUFDUixxREFBcUQ7UUFDckQsSUFBSSxFQUFFLEdBQUc7UUFDVCxJQUFJLEVBQUUsR0FBRztLQUNWLENBQUM7SUFFRixJQUFJLElBQUksRUFBRSxDQUFDO1FBQ1QsS0FBSyxNQUFNLEdBQUcsSUFBSSxNQUFNLENBQUMsSUFBSSxDQUFDLFlBQVksQ0FBQyxFQUFFLENBQUM7WUFDNUMsTUFBTSxTQUFTLEdBQUcsSUFBSSxNQUFNLENBQUMsR0FBRyxFQUFFLEdBQUcsQ0FBQyxDQUFDO1lBQ3ZDLGFBQWEsR0FBRyxhQUFhLENBQUMsT0FBTyxDQUFDLFNBQVMsRUFBRSxZQUFZLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQztRQUN0RSxDQUFDO0lBQ0gsQ0FBQztJQUVELElBQUksS0FBSyxFQUFFLENBQUM7UUFDVixLQUFLLE1BQU0sR0FBRyxJQUFJLE1BQU0sQ0FBQyxJQUFJLENBQUMsV0FBVyxDQUFDLEVBQUUsQ0FBQztZQUMzQyxNQUFNLGdCQUFnQixHQUFHLElBQUksTUFBTSxDQUFDLEdBQUcsRUFBRSxHQUFHLENBQUMsQ0FBQztZQUM5QyxhQUFhLEdBQUcsYUFBYSxDQUFDLE9BQU8sQ0FBQyxnQkFBZ0IsRUFBRSxXQUFXLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQztRQUM1RSxDQUFDO0lBQ0gsQ0FBQztJQUVELE9BQU8sYUFBYSxDQUFDO0FBQ3ZCLENBQUMsQ0FBQztBQTdDVyxRQUFBLGlCQUFpQixxQkE2QzVCO0FBRUssTUFBTSxlQUFlLEdBQUcsQ0FDN0IsR0FBZSxFQUNmLE9BQXlCLEVBQ2IsRUFBRTtJQUNkLE1BQU0sS0FBSyxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsdUJBQXVCLENBQUMsQ0FBQztJQUNwRCxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUMsTUFBYyxFQUFFLE9BQWdCLEVBQUUsRUFBRTtRQUM5QyxHQUFHLENBQUMsT0FBTyxDQUFDLENBQUMsV0FBVyxDQUFDLEdBQUcsQ0FBQyxPQUFPLENBQUMsQ0FBQyxJQUFJLEVBQUUsQ0FBQyxDQUFDO0lBQ2hELENBQUMsQ0FBQyxDQUFDO0lBQ0gsT0FBTyxHQUFHLENBQUM7QUFDYixDQUFDLENBQUM7QUFUVyxRQUFBLGVBQWUsbUJBUzFCO0FBRUssTUFBTSxRQUFRLEdBQUcsQ0FBQyxHQUFlLEVBQUUsSUFBc0IsRUFBVSxFQUFFO0lBQzFFLE1BQU0sS0FBSyxHQUFHLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDOUIsSUFBSSxJQUFJLEdBQUcsRUFBRSxDQUFDO0lBRWQsS0FBSyxDQUFDLElBQUksQ0FBQyxDQUFDLE1BQWMsRUFBRSxPQUFnQixFQUFFLEVBQUU7UUFDOUMsSUFBSSxHQUFHLEdBQUcsSUFBSSxRQUFRLEdBQUcsQ0FBQyxPQUFPLENBQUMsQ0FBQyxJQUFJLEVBQUUsRUFBRSxDQUFDO0lBQzlDLENBQUMsQ0FBQyxDQUFDO0lBQ0gsSUFBSSxHQUFHLEdBQUcsSUFBSSxJQUFJLENBQUM7SUFDbkIsT0FBTyxJQUFJLENBQUM7QUFDZCxDQUFDLENBQUM7QUFUVyxRQUFBLFFBQVEsWUFTbkI7QUFFRixNQUFNLFNBQVMsR0FBRyxDQUNoQixHQUFlLEVBQ2YsT0FBeUIsRUFDekIsSUFBWSxFQUNKLEVBQUU7SUFDVixJQUFBLGlDQUF5QixFQUFDLEdBQUcsRUFBRSxPQUFPLENBQUMsQ0FBQztJQUN4QyxJQUFBLG1CQUFXLEVBQUMsR0FBRyxFQUFFLE9BQU8sQ0FBQyxDQUFDO0lBQzFCLElBQUEsc0JBQWMsRUFBQyxHQUFHLEVBQUUsT0FBTyxDQUFDLENBQUM7SUFDN0IsSUFBQSx1QkFBZSxFQUFDLEdBQUcsRUFBRSxPQUFPLENBQUMsQ0FBQztJQUM5QixJQUFBLGdDQUF3QixFQUFDLEdBQUcsRUFBRSxPQUFPLEVBQUUsSUFBSSxDQUFDLENBQUM7SUFDN0MsTUFBTSxhQUFhLEdBQUcsSUFBQSxxQkFBYSxFQUFDLEdBQUcsRUFBRSxPQUFPLENBQUMsQ0FBQyxPQUFPLENBQUMsS0FBSyxFQUFFLEdBQUcsQ0FBQyxDQUFDO0lBQ3RFLE9BQU8sSUFBQSx5QkFBaUIsRUFBQyxhQUFhLEVBQUUsS0FBSyxFQUFFLElBQUksQ0FBQyxDQUFDO0FBQ3ZELENBQUMsQ0FBQztBQUVGLGtCQUFlLFNBQVMsQ0FBQyIsInNvdXJjZXNDb250ZW50IjpbImltcG9ydCB0eXBlIHsgQ2hlZXJpbywgQ2hlZXJpb0FQSSB9IGZyb20gXCJjaGVlcmlvXCI7XG5pbXBvcnQgdHlwZSB7IEFueU5vZGUsIEVsZW1lbnQgfSBmcm9tIFwiZG9taGFuZGxlclwiO1xuaW1wb3J0IHhyZWdleHAgZnJvbSBcInhyZWdleHBcIjtcbmltcG9ydCBzdG9wd29yZHMgZnJvbSBcIi4vc3RvcHdvcmRzXCI7XG5cbmZ1bmN0aW9uIGdldFRhZ05hbWUobm9kZTogQ2hlZXJpbzxBbnlOb2RlPik6IHN0cmluZyB7XG4gIGNvbnN0IGVsZW1lbnQgPSBub2RlLmdldCgwKSBhcyBFbGVtZW50IHwgdW5kZWZpbmVkO1xuICByZXR1cm4gZWxlbWVudD8udGFnTmFtZSB8fCBcIlwiO1xufVxuXG5leHBvcnQgY29uc3QgYWRkTmV3bGluZVRvQnIgPSAoXG4gIGRvYzogQ2hlZXJpb0FQSSxcbiAgdG9wTm9kZTogQ2hlZXJpbzxBbnlOb2RlPixcbik6IENoZWVyaW9BUEkgPT4ge1xuICBjb25zdCBicnMgPSB0b3BOb2RlLmZpbmQoXCJiclwiKTtcblxuICBicnMuZWFjaCgoX2luZGV4OiBudW1iZXIsIGVsZW1lbnQ6IEFueU5vZGUpID0+IHtcbiAgICBjb25zdCBiciA9IGRvYyhlbGVtZW50KTtcbiAgICBici5yZXBsYWNlV2l0aChcIlxcblxcblwiKTtcbiAgfSk7XG5cbiAgcmV0dXJuIGRvYztcbn07XG5cbmV4cG9ydCBjb25zdCBjbGVhblBhcmFncmFwaFRleHQgPSAocmF3VGV4dDogc3RyaW5nKTogc3RyaW5nID0+IHtcbiAgY29uc3QgdGV4dCA9IHJhd1RleHQudHJpbSgpO1xuICB0ZXh0LnJlcGxhY2UoL1tcXHNcXHRdKy9nLCBcIiBcIik7XG4gIHJldHVybiB0ZXh0O1xufTtcblxuZXhwb3J0IGNvbnN0IGNvbnZlcnRUb1RleHQgPSAoXG4gIGRvYzogQ2hlZXJpb0FQSSxcbiAgdG9wTm9kZTogQ2hlZXJpbzxBbnlOb2RlPixcbik6IHN0cmluZyA9PiB7XG4gIGxldCB0ZXh0czogc3RyaW5nW10gPSBbXTtcbiAgY29uc3Qgbm9kZXMgPSB0b3BOb2RlLmNvbnRlbnRzKCk7XG5cbiAgbGV0IGhhbmdpbmdUZXh0ID0gXCJcIjtcblxuICBub2Rlcy5lYWNoKChfaW5kZXg6IG51bWJlciwgZWxlbWVudDogQW55Tm9kZSkgPT4ge1xuICAgIGNvbnN0IG5vZGUgPSBkb2MoZWxlbWVudCk7XG4gICAgY29uc3QgZmlyc3ROb2RlID0gbm9kZS5nZXQoMCk7XG4gICAgY29uc3Qgbm9kZVR5cGUgPSBmaXJzdE5vZGUgPyBmaXJzdE5vZGUudHlwZSA6IG51bGw7XG4gICAgY29uc3Qgbm9kZU5hbWUgPSBnZXRUYWdOYW1lKG5vZGUpO1xuXG4gICAgaWYgKG5vZGVUeXBlID09PSBcInRleHRcIikge1xuICAgICAgaGFuZ2luZ1RleHQgKz0gbm9kZS50ZXh0KCk7XG4gICAgICByZXR1cm4gdHJ1ZTtcbiAgICB9IGVsc2UgaWYgKG5vZGVOYW1lID09PSBcInVsXCIpIHtcbiAgICAgIGhhbmdpbmdUZXh0ICs9IHVsVG9UZXh0KGRvYywgbm9kZSk7XG4gICAgICByZXR1cm4gdHJ1ZTtcbiAgICB9XG5cbiAgICBpZiAoaGFuZ2luZ1RleHQubGVuZ3RoID4gMCkge1xuICAgICAgY29uc3QgdHh0ID0gY2xlYW5QYXJhZ3JhcGhUZXh0KGhhbmdpbmdUZXh0KTtcbiAgICAgIHRleHRzID0gdGV4dHMuY29uY2F0KHR4dC5zcGxpdCgvXFxyP1xcbi8pKTtcbiAgICAgIGhhbmdpbmdUZXh0ID0gXCJcIjtcbiAgICB9XG5cbiAgICBsZXQgdGV4dCA9IGNsZWFuUGFyYWdyYXBoVGV4dChub2RlLnRleHQoKSk7XG4gICAgdGV4dCA9IHRleHQucmVwbGFjZSgvKFxcdytcXC4pKFtBLVpdKykvLCBcIiQxICQyXCIpO1xuICAgIHRleHRzID0gdGV4dHMuY29uY2F0KHRleHQuc3BsaXQoL1xccj9cXG4vKSk7XG4gIH0pO1xuXG4gIGlmIChoYW5naW5nVGV4dC5sZW5ndGggPiAwKSB7XG4gICAgY29uc3QgdGV4dCA9IGNsZWFuUGFyYWdyYXBoVGV4dChoYW5naW5nVGV4dCk7XG4gICAgdGV4dHMgPSB0ZXh0cyA9IHRleHRzLmNvbmNhdCh0ZXh0LnNwbGl0KC9cXHI/XFxuLykpO1xuICB9XG5cbiAgdGV4dHMgPSB0ZXh0cy5tYXAoKHR4dCkgPT4ge1xuICAgIHJldHVybiB0eHQudHJpbSgpO1xuICB9KTtcblxuICBjb25zdCByZWdleCA9IHhyZWdleHAoXCJbXFxcXHB7TnVtYmVyfVxcXFxwe0xldHRlcn1dXCIpO1xuICB0ZXh0cyA9IHRleHRzLmZpbHRlcigodHh0KSA9PiB7XG4gICAgcmV0dXJuIHJlZ2V4LnRlc3QodHh0KTtcbiAgfSk7XG5cbiAgcmV0dXJuIHRleHRzLmpvaW4oXCJcXG5cXG5cIik7XG59O1xuXG5leHBvcnQgY29uc3QgbGlua3NUb1RleHQgPSAoXG4gIGRvYzogQ2hlZXJpb0FQSSxcbiAgdG9wTm9kZTogQ2hlZXJpbzxBbnlOb2RlPixcbik6IENoZWVyaW9BUEkgPT4ge1xuICBjb25zdCBub2RlcyA9IHRvcE5vZGUuZmluZChcImFcIik7XG4gIG5vZGVzLmVhY2goKF9pbmRleDogbnVtYmVyLCBlbGVtZW50OiBBbnlOb2RlKSA9PiB7XG4gICAgY29uc3QgaHRtbEVsID0gZG9jKGVsZW1lbnQpLmh0bWwoKTtcbiAgICBpZiAoaHRtbEVsKSB7XG4gICAgICBkb2MoZWxlbWVudCkucmVwbGFjZVdpdGgoaHRtbEVsKTtcbiAgICB9XG4gIH0pO1xuICByZXR1cm4gZG9jO1xufTtcblxuZXhwb3J0IGNvbnN0IHJlbW92ZUZld1dvcmRzUGFyYWdyYXBocyA9IChcbiAgZG9jOiBDaGVlcmlvQVBJLFxuICB0b3BOb2RlOiBDaGVlcmlvPEFueU5vZGU+LFxuICBsYW5nOiBzdHJpbmcsXG4pOiBDaGVlcmlvQVBJID0+IHtcbiAgY29uc3QgYWxsTm9kZXMgPSB0b3BOb2RlLmZpbmQoXCIqXCIpO1xuXG4gIGFsbE5vZGVzLmVhY2goKF9pbmRleDogbnVtYmVyLCBlbGVtZW50OiBBbnlOb2RlKSA9PiB7XG4gICAgY29uc3QgZWwgPSBkb2MoZWxlbWVudCk7XG4gICAgY29uc3QgdGFnID0gZ2V0VGFnTmFtZShlbCk7XG4gICAgY29uc3QgdGV4dCA9IGVsLnRleHQoKTtcblxuICAgIGNvbnN0IHN0b3BXb3JkcyA9IHN0b3B3b3Jkcyh0ZXh0LCBsYW5nKTtcbiAgICBpZiAoXG4gICAgICAodGFnICE9PSBcImJyXCIgfHwgdGV4dCAhPT0gXCJcXFxcclwiKSAmJlxuICAgICAgc3RvcFdvcmRzLnN0b3BXb3JkQ291bnQgPCAzICYmXG4gICAgICBlbC5maW5kKFwib2JqZWN0XCIpLmxlbmd0aCA9PT0gMCAmJlxuICAgICAgZWwuZmluZChcImVtYmVkXCIpLmxlbmd0aCA9PT0gMFxuICAgICkge1xuICAgICAgZG9jKGVsKS5yZW1vdmUoKTtcbiAgICB9IGVsc2Uge1xuICAgICAgY29uc3QgdHJpbW1lZCA9IHRleHQudHJpbSgpO1xuICAgICAgaWYgKHRyaW1tZWRbMF0gPT09IFwiKFwiICYmIHRyaW1tZWRbdHJpbW1lZC5sZW5ndGggLSAxXSA9PT0gXCIpXCIpIHtcbiAgICAgICAgZG9jKGVsKS5yZW1vdmUoKTtcbiAgICAgIH1cbiAgICB9XG4gIH0pO1xuXG4gIHJldHVybiBkb2M7XG59O1xuXG5leHBvcnQgY29uc3QgcmVtb3ZlTmVnYXRpdmVzY29yZXNOb2RlcyA9IChcbiAgZG9jOiBDaGVlcmlvQVBJLFxuICB0b3BOb2RlOiBDaGVlcmlvPEFueU5vZGU+LFxuKTogQ2hlZXJpb0FQSSA9PiB7XG4gIGNvbnN0IGdyYXZpdHlJdGVtcyA9IHRvcE5vZGUuZmluZChcIipbZ3Jhdml0eVNjb3JlXVwiKTtcblxuICBncmF2aXR5SXRlbXMuZWFjaCgoX2luZGV4OiBudW1iZXIsIGVsZW1lbnQ6IEFueU5vZGUpID0+IHtcbiAgICBsZXQgc2NvcmUgPSAwO1xuICAgIGNvbnN0IGl0ZW0gPSBkb2MoZWxlbWVudCk7XG4gICAgY29uc3QgZ3Jhdml0eVNjb3JlID0gaXRlbS5hdHRyKFwiZ3Jhdml0eVNjb3JlXCIpO1xuICAgIGlmIChncmF2aXR5U2NvcmUpIHtcbiAgICAgIHNjb3JlID0gcGFyc2VJbnQoZ3Jhdml0eVNjb3JlLCAxMCkgfHwgMDtcbiAgICB9XG5cbiAgICBpZiAoc2NvcmUgPCAxKSB7XG4gICAgICBkb2MoaXRlbSkucmVtb3ZlKCk7XG4gICAgfVxuICB9KTtcblxuICByZXR1cm4gZG9jO1xufTtcblxuZXhwb3J0IGNvbnN0IHJlcGxhY2VDaGFyYWN0ZXJzID0gKFxuICB0ZXh0OiBzdHJpbmcsXG4gIGh0bWw6IGJvb2xlYW4sXG4gIGNoYXJzOiBib29sZWFuLFxuKTogc3RyaW5nID0+IHtcbiAgbGV0IHByb2Nlc3NlZFRleHQgPSB0ZXh0O1xuICAvLyBpZiBlbGVtZW50IGRvZXMgbm90IG1hdGNoIGFueSBpbiBtYXAgYW5kIHN0YXJ0cyB3aXRoICYgYW5kIGVuZHMgd2l0aCA7LCByZXBsYWNlIHdpdGggZW1wdHkgc3RyaW5nXG4gIGNvbnN0IGh0bWxFbnRpdGllczogUmVjb3JkPHN0cmluZywgc3RyaW5nPiA9IHtcbiAgICBcIiZhbXA7XCI6IFwiJlwiLFxuICAgIFwiJmFwb3M7XCI6IFwiJ1wiLFxuICAgIFwiJmNlbnQ7XCI6IFwiwqJcIixcbiAgICBcIiZjb3B5O1wiOiBcIsKpXCIsXG4gICAgXCImZXVybztcIjogXCLigqxcIixcbiAgICBcIiZndDtcIjogXCI+XCIsXG4gICAgXCImbHQ7XCI6IFwiPFwiLFxuICAgIFwiJm5ic3A7XCI6IFwiIFwiLFxuICAgIFwiJnBvdW5kO1wiOiBcIsKjXCIsXG4gICAgXCImcXVvdDtcIjogJ1wiJyxcbiAgICBcIiZyZWc7XCI6IFwiwq5cIixcbiAgICBcIiZ5ZW47XCI6IFwiwqVcIixcbiAgfTtcblxuICBjb25zdCBlc2NhcGVDaGFyczogUmVjb3JkPHN0cmluZywgc3RyaW5nPiA9IHtcbiAgICAnXCInOiAnXCInLFxuICAgIFwiJ1wiOiBcIidcIixcbiAgICAvLyB0c2xpbnQ6ZGlzYWJsZS1uZXh0LWxpbmU6IG9iamVjdC1saXRlcmFsLXNvcnQta2V5c1xuICAgIFwiXFxuXCI6IFwiIFwiLFxuICAgIFwiXFxyXCI6IFwiIFwiLFxuICB9O1xuXG4gIGlmIChodG1sKSB7XG4gICAgZm9yIChjb25zdCBrZXkgb2YgT2JqZWN0LmtleXMoaHRtbEVudGl0aWVzKSkge1xuICAgICAgY29uc3QgaHRtbHJlZ2V4ID0gbmV3IFJlZ0V4cChrZXksIFwiZ1wiKTtcbiAgICAgIHByb2Nlc3NlZFRleHQgPSBwcm9jZXNzZWRUZXh0LnJlcGxhY2UoaHRtbHJlZ2V4LCBodG1sRW50aXRpZXNba2V5XSk7XG4gICAgfVxuICB9XG5cbiAgaWYgKGNoYXJzKSB7XG4gICAgZm9yIChjb25zdCBrZXkgb2YgT2JqZWN0LmtleXMoZXNjYXBlQ2hhcnMpKSB7XG4gICAgICBjb25zdCBlc2NhcGVDaGFyc1JlZ2V4ID0gbmV3IFJlZ0V4cChrZXksIFwiZ1wiKTtcbiAgICAgIHByb2Nlc3NlZFRleHQgPSBwcm9jZXNzZWRUZXh0LnJlcGxhY2UoZXNjYXBlQ2hhcnNSZWdleCwgZXNjYXBlQ2hhcnNba2V5XSk7XG4gICAgfVxuICB9XG5cbiAgcmV0dXJuIHByb2Nlc3NlZFRleHQ7XG59O1xuXG5leHBvcnQgY29uc3QgcmVwbGFjZVdpdGhUZXh0ID0gKFxuICBkb2M6IENoZWVyaW9BUEksXG4gIHRvcE5vZGU6IENoZWVyaW88QW55Tm9kZT4sXG4pOiBDaGVlcmlvQVBJID0+IHtcbiAgY29uc3Qgbm9kZXMgPSB0b3BOb2RlLmZpbmQoXCJiLCBzdHJvbmcsIGksIGJyLCBzdXBcIik7XG4gIG5vZGVzLmVhY2goKF9pbmRleDogbnVtYmVyLCBlbGVtZW50OiBBbnlOb2RlKSA9PiB7XG4gICAgZG9jKGVsZW1lbnQpLnJlcGxhY2VXaXRoKGRvYyhlbGVtZW50KS50ZXh0KCkpO1xuICB9KTtcbiAgcmV0dXJuIGRvYztcbn07XG5cbmV4cG9ydCBjb25zdCB1bFRvVGV4dCA9IChkb2M6IENoZWVyaW9BUEksIG5vZGU6IENoZWVyaW88QW55Tm9kZT4pOiBzdHJpbmcgPT4ge1xuICBjb25zdCBub2RlcyA9IG5vZGUuZmluZChcImxpXCIpO1xuICBsZXQgdGV4dCA9IFwiXCI7XG5cbiAgbm9kZXMuZWFjaCgoX2luZGV4OiBudW1iZXIsIGVsZW1lbnQ6IEFueU5vZGUpID0+IHtcbiAgICB0ZXh0ID0gYCR7dGV4dH1cXG4gKiAke2RvYyhlbGVtZW50KS50ZXh0KCl9YDtcbiAgfSk7XG4gIHRleHQgPSBgJHt0ZXh0fVxcbmA7XG4gIHJldHVybiB0ZXh0O1xufTtcblxuY29uc3QgZm9ybWF0dGVyID0gKFxuICBkb2M6IENoZWVyaW9BUEksXG4gIHRvcE5vZGU6IENoZWVyaW88QW55Tm9kZT4sXG4gIGxhbmc6IHN0cmluZyxcbik6IHN0cmluZyA9PiB7XG4gIHJlbW92ZU5lZ2F0aXZlc2NvcmVzTm9kZXMoZG9jLCB0b3BOb2RlKTtcbiAgbGlua3NUb1RleHQoZG9jLCB0b3BOb2RlKTtcbiAgYWRkTmV3bGluZVRvQnIoZG9jLCB0b3BOb2RlKTtcbiAgcmVwbGFjZVdpdGhUZXh0KGRvYywgdG9wTm9kZSk7XG4gIHJlbW92ZUZld1dvcmRzUGFyYWdyYXBocyhkb2MsIHRvcE5vZGUsIGxhbmcpO1xuICBjb25zdCBjb252ZXJ0ZWRUZXh0ID0gY29udmVydFRvVGV4dChkb2MsIHRvcE5vZGUpLnJlcGxhY2UoL1xcbi9nLCBcIiBcIik7XG4gIHJldHVybiByZXBsYWNlQ2hhcmFjdGVycyhjb252ZXJ0ZWRUZXh0LCBmYWxzZSwgdHJ1ZSk7XG59O1xuXG5leHBvcnQgZGVmYXVsdCBmb3JtYXR0ZXI7XG4iXX0=