site-metadata-extractor
Version:
web(site) resource metadata extractor
180 lines • 22.9 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ulToText = exports.replaceWithText = exports.replaceCharacters = exports.removeNegativescoresNodes = exports.removeFewWordsParagraphs = exports.linksToText = exports.convertToText = exports.cleanParagraphText = exports.addNewlineToBr = void 0;
const xregexp_1 = __importDefault(require("xregexp"));
const stopwords_1 = __importDefault(require("./stopwords"));
const addNewlineToBr = (doc, topNode) => {
const brs = topNode.find("br");
brs.each((_index, element) => {
const br = doc(element);
br.replaceWith("\n\n");
});
return doc;
};
exports.addNewlineToBr = addNewlineToBr;
const cleanParagraphText = (rawText) => {
const text = rawText.trim();
text.replace(/[\s\t]+/g, " ");
return text;
};
exports.cleanParagraphText = cleanParagraphText;
const convertToText = (doc, topNode) => {
let texts = [];
const nodes = topNode.contents();
let hangingText = "";
nodes.each((_index, element) => {
const node = doc(element);
const nodeType = node.get(0) ? node.get(0).type : null;
const nodeName = node.get(0) ? node.get(0).tagName : null;
if (nodeType === "text") {
hangingText += node.text();
return true;
}
else if (nodeName === "ul") {
hangingText += (0, exports.ulToText)(doc, node);
return true;
}
if (hangingText.length > 0) {
const txt = (0, exports.cleanParagraphText)(hangingText);
texts = texts.concat(txt.split(/\r?\n/));
hangingText = "";
}
let text = (0, exports.cleanParagraphText)(node.text());
text = text.replace(/(\w+\.)([A-Z]+)/, "$1 $2");
texts = texts.concat(text.split(/\r?\n/));
});
if (hangingText.length > 0) {
const text = (0, exports.cleanParagraphText)(hangingText);
texts = texts = texts.concat(text.split(/\r?\n/));
}
texts = texts.map((txt) => {
return txt.trim();
});
const regex = (0, xregexp_1.default)("[\\p{Number}\\p{Letter}]");
texts = texts.filter((txt) => {
return regex.test(txt);
});
return texts.join("\n\n");
};
exports.convertToText = convertToText;
const linksToText = (doc, topNode) => {
const nodes = topNode.find("a");
nodes.each((_index, element) => {
const htmlEl = doc(element).html();
if (htmlEl) {
doc(element).replaceWith(htmlEl);
}
});
return doc;
};
exports.linksToText = linksToText;
const removeFewWordsParagraphs = (doc, topNode, lang) => {
const allNodes = topNode.find("*");
allNodes.each((_index, element) => {
const el = doc(element);
const tag = el.get(0).tagName;
const text = el.text();
const stopWords = (0, stopwords_1.default)(text, lang);
if ((tag !== "br" || text !== "\\r") &&
stopWords.stopWordCount < 3 &&
el.find("object").length === 0 &&
el.find("embed").length === 0) {
doc(el).remove();
}
else {
const trimmed = text.trim();
if (trimmed[0] === "(" && trimmed[trimmed.length - 1] === ")") {
doc(el).remove();
}
}
});
return doc;
};
exports.removeFewWordsParagraphs = removeFewWordsParagraphs;
const removeNegativescoresNodes = (doc, topNode) => {
const gravityItems = topNode.find("*[gravityScore]");
gravityItems.each((_index, element) => {
let score = 0;
const item = doc(element);
const gravityScore = item.attr("gravityScore");
if (gravityScore) {
score = parseInt(gravityScore, 10) || 0;
}
if (score < 1) {
doc(item).remove();
}
});
return doc;
};
exports.removeNegativescoresNodes = removeNegativescoresNodes;
const replaceCharacters = (text, html, chars) => {
let processedText = text;
// if element does not match any in map and starts with & and ends with ;, replace with empty string
const htmlEntities = {
"&": "&",
"'": "'",
"¢": "¢",
"©": "©",
"€": "€",
">": ">",
"<": "<",
" ": " ",
"£": "£",
""": '"',
"®": "®",
"¥": "¥",
};
const escapeChars = {
'"': '"',
"'": "'",
// tslint:disable-next-line: object-literal-sort-keys
"\n": " ",
"\r": " ",
};
if (html) {
for (const key of Object.keys(htmlEntities)) {
const htmlregex = new RegExp(key, "g");
processedText = processedText.replace(htmlregex, htmlEntities[key]);
}
}
if (chars) {
for (const key of Object.keys(escapeChars)) {
const escapeCharsRegex = new RegExp(key, "g");
processedText = processedText.replace(escapeCharsRegex, escapeChars[key]);
}
}
return processedText;
};
exports.replaceCharacters = replaceCharacters;
const replaceWithText = (doc, topNode) => {
const nodes = topNode.find("b, strong, i, br, sup");
nodes.each((_index, element) => {
doc(element).replaceWith(doc(element).text());
});
return doc;
};
exports.replaceWithText = replaceWithText;
const ulToText = (doc, node) => {
const nodes = node.find("li");
let text = "";
nodes.each((_index, element) => {
text = `${text}\n * ${doc(element).text()}`;
});
text = `${text}\n`;
return text;
};
exports.ulToText = ulToText;
const formatter = (doc, topNode, lang) => {
(0, exports.removeNegativescoresNodes)(doc, topNode);
(0, exports.linksToText)(doc, topNode);
(0, exports.addNewlineToBr)(doc, topNode);
(0, exports.replaceWithText)(doc, topNode);
(0, exports.removeFewWordsParagraphs)(doc, topNode, lang);
const convertedText = (0, exports.convertToText)(doc, topNode).replace(/\n/g, " ");
return (0, exports.replaceCharacters)(convertedText, false, true);
};
exports.default = formatter;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZm9ybWF0dGVyLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL2Zvcm1hdHRlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7QUFDQSxzREFBOEI7QUFDOUIsNERBQW9DO0FBRTdCLE1BQU0sY0FBYyxHQUFHLENBQzVCLEdBQWlCLEVBQ2pCLE9BQXdCLEVBQ1YsRUFBRTtJQUNoQixNQUFNLEdBQUcsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO0lBRS9CLEdBQUcsQ0FBQyxJQUFJLENBQUMsQ0FBQyxNQUFjLEVBQUUsT0FBd0IsRUFBRSxFQUFFO1FBQ3BELE1BQU0sRUFBRSxHQUFHLEdBQUcsQ0FBQyxPQUFPLENBQUMsQ0FBQztRQUN4QixFQUFFLENBQUMsV0FBVyxDQUFDLE1BQU0sQ0FBQyxDQUFDO0lBQ3pCLENBQUMsQ0FBQyxDQUFDO0lBRUgsT0FBTyxHQUFHLENBQUM7QUFDYixDQUFDLENBQUM7QUFaVyxRQUFBLGNBQWMsa0JBWXpCO0FBRUssTUFBTSxrQkFBa0IsR0FBRyxDQUFDLE9BQWUsRUFBVSxFQUFFO0lBQzVELE1BQU0sSUFBSSxHQUFHLE9BQU8sQ0FBQyxJQUFJLEVBQUUsQ0FBQztJQUM1QixJQUFJLENBQUMsT0FBTyxDQUFDLFVBQVUsRUFBRSxHQUFHLENBQUMsQ0FBQztJQUM5QixPQUFPLElBQUksQ0FBQztBQUNkLENBQUMsQ0FBQztBQUpXLFFBQUEsa0JBQWtCLHNCQUk3QjtBQUVLLE1BQU0sYUFBYSxHQUFHLENBQzNCLEdBQWlCLEVBQ2pCLE9BQXdCLEVBQ2hCLEVBQUU7SUFDVixJQUFJLEtBQUssR0FBYSxFQUFFLENBQUM7SUFDekIsTUFBTSxLQUFLLEdBQUcsT0FBTyxDQUFDLFFBQVEsRUFBRSxDQUFDO0lBRWpDLElBQUksV0FBVyxHQUFHLEVBQUUsQ0FBQztJQUVyQixLQUFLLENBQUMsSUFBSSxDQUFDLENBQUMsTUFBYyxFQUFFLE9BQXdCLEVBQUUsRUFBRTtRQUN0RCxNQUFNLElBQUksR0FBRyxHQUFHLENBQUMsT0FBTyxDQUFDLENBQUM7UUFDMUIsTUFBTSxRQUFRLEdBQUcsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLElBQUksQ0FBQztRQUN2RCxNQUFNLFFBQVEsR0FBRyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDO1FBRTFELElBQUksUUFBUSxLQUFLLE1BQU0sRUFBRTtZQUN2QixXQUFXLElBQUksSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO1lBQzNCLE9BQU8sSUFBSSxDQUFDO1NBQ2I7YUFBTSxJQUFJLFFBQVEsS0FBSyxJQUFJLEVBQUU7WUFDNUIsV0FBVyxJQUFJLElBQUEsZ0JBQVEsRUFBQyxHQUFHLEVBQUUsSUFBSSxDQUFDLENBQUM7WUFDbkMsT0FBTyxJQUFJLENBQUM7U0FDYjtRQUVELElBQUksV0FBVyxDQUFDLE1BQU0sR0FBRyxDQUFDLEVBQUU7WUFDMUIsTUFBTSxHQUFHLEdBQUcsSUFBQSwwQkFBa0IsRUFBQyxXQUFXLENBQUMsQ0FBQztZQUM1QyxLQUFLLEdBQUcsS0FBSyxDQUFDLE1BQU0sQ0FBQyxHQUFHLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUM7WUFDekMsV0FBVyxHQUFHLEVBQUUsQ0FBQztTQUNsQjtRQUVELElBQUksSUFBSSxHQUFHLElBQUEsMEJBQWtCLEVBQUMsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDLENBQUM7UUFDM0MsSUFBSSxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsaUJBQWlCLEVBQUUsT0FBTyxDQUFDLENBQUM7UUFDaEQsS0FBSyxHQUFHLEtBQUssQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQyxDQUFDO0lBQzVDLENBQUMsQ0FBQyxDQUFDO0lBRUgsSUFBSSxXQUFXLENBQUMsTUFBTSxHQUFHLENBQUMsRUFBRTtRQUMxQixNQUFNLElBQUksR0FBRyxJQUFBLDBCQUFrQixFQUFDLFdBQVcsQ0FBQyxDQUFDO1FBQzdDLEtBQUssR0FBRyxLQUFLLEdBQUcsS0FBSyxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUM7S0FDbkQ7SUFFRCxLQUFLLEdBQUcsS0FBSyxDQUFDLEdBQUcsQ0FBQyxDQUFDLEdBQUcsRUFBRSxFQUFFO1FBQ3hCLE9BQU8sR0FBRyxDQUFDLElBQUksRUFBRSxDQUFDO0lBQ3BCLENBQUMsQ0FBQyxDQUFDO0lBRUgsTUFBTSxLQUFLLEdBQUcsSUFBQSxpQkFBTyxFQUFDLDBCQUEwQixDQUFDLENBQUM7SUFDbEQsS0FBSyxHQUFHLEtBQUssQ0FBQyxNQUFNLENBQUMsQ0FBQyxHQUFHLEVBQUUsRUFBRTtRQUMzQixPQUFPLEtBQUssQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7SUFDekIsQ0FBQyxDQUFDLENBQUM7SUFFSCxPQUFPLEtBQUssQ0FBQyxJQUFJLENBQUMsTUFBTSxDQUFDLENBQUM7QUFDNUIsQ0FBQyxDQUFDO0FBaERXLFFBQUEsYUFBYSxpQkFnRHhCO0FBRUssTUFBTSxXQUFXLEdBQUcsQ0FDekIsR0FBaUIsRUFDakIsT0FBd0IsRUFDVixFQUFFO0lBQ2hCLE1BQU0sS0FBSyxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7SUFDaEMsS0FBSyxDQUFDLElBQUksQ0FBQyxDQUFDLE1BQWMsRUFBRSxPQUF3QixFQUFFLEVBQUU7UUFDdEQsTUFBTSxNQUFNLEdBQUcsR0FBRyxDQUFDLE9BQU8sQ0FBQyxDQUFDLElBQUksRUFBRSxDQUFDO1FBQ25DLElBQUksTUFBTSxFQUFFO1lBQ1YsR0FBRyxDQUFDLE9BQU8sQ0FBQyxDQUFDLFdBQVcsQ0FBQyxNQUFNLENBQUMsQ0FBQztTQUNsQztJQUNILENBQUMsQ0FBQyxDQUFDO0lBQ0gsT0FBTyxHQUFHLENBQUM7QUFDYixDQUFDLENBQUM7QUFaVyxRQUFBLFdBQVcsZUFZdEI7QUFFSyxNQUFNLHdCQUF3QixHQUFHLENBQ3RDLEdBQWlCLEVBQ2pCLE9BQXdCLEVBQ3hCLElBQVksRUFDRSxFQUFFO0lBQ2hCLE1BQU0sUUFBUSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUM7SUFFbkMsUUFBUSxDQUFDLElBQUksQ0FBQyxDQUFDLE1BQWMsRUFBRSxPQUF3QixFQUFFLEVBQUU7UUFDekQsTUFBTSxFQUFFLEdBQUcsR0FBRyxDQUFDLE9BQU8sQ0FBQyxDQUFDO1FBQ3hCLE1BQU0sR0FBRyxHQUFHLEVBQUUsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLENBQUMsT0FBTyxDQUFDO1FBQzlCLE1BQU0sSUFBSSxHQUFHLEVBQUUsQ0FBQyxJQUFJLEVBQUUsQ0FBQztRQUV2QixNQUFNLFNBQVMsR0FBRyxJQUFBLG1CQUFTLEVBQUMsSUFBSSxFQUFFLElBQUksQ0FBQyxDQUFDO1FBQ3hDLElBQ0UsQ0FBQyxHQUFHLEtBQUssSUFBSSxJQUFJLElBQUksS0FBSyxLQUFLLENBQUM7WUFDaEMsU0FBUyxDQUFDLGFBQWEsR0FBRyxDQUFDO1lBQzNCLEVBQUUsQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLENBQUMsTUFBTSxLQUFLLENBQUM7WUFDOUIsRUFBRSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsQ0FBQyxNQUFNLEtBQUssQ0FBQyxFQUM3QjtZQUNBLEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQyxNQUFNLEVBQUUsQ0FBQztTQUNsQjthQUFNO1lBQ0wsTUFBTSxPQUFPLEdBQUcsSUFBSSxDQUFDLElBQUksRUFBRSxDQUFDO1lBQzVCLElBQUksT0FBTyxDQUFDLENBQUMsQ0FBQyxLQUFLLEdBQUcsSUFBSSxPQUFPLENBQUMsT0FBTyxDQUFDLE1BQU0sR0FBRyxDQUFDLENBQUMsS0FBSyxHQUFHLEVBQUU7Z0JBQzdELEdBQUcsQ0FBQyxFQUFFLENBQUMsQ0FBQyxNQUFNLEVBQUUsQ0FBQzthQUNsQjtTQUNGO0lBQ0gsQ0FBQyxDQUFDLENBQUM7SUFFSCxPQUFPLEdBQUcsQ0FBQztBQUNiLENBQUMsQ0FBQztBQTdCVyxRQUFBLHdCQUF3Qiw0QkE2Qm5DO0FBRUssTUFBTSx5QkFBeUIsR0FBRyxDQUN2QyxHQUFpQixFQUNqQixPQUF3QixFQUNWLEVBQUU7SUFDaEIsTUFBTSxZQUFZLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxpQkFBaUIsQ0FBQyxDQUFDO0lBRXJELFlBQVksQ0FBQyxJQUFJLENBQUMsQ0FBQyxNQUFjLEVBQUUsT0FBd0IsRUFBRSxFQUFFO1FBQzdELElBQUksS0FBSyxHQUFHLENBQUMsQ0FBQztRQUNkLE1BQU0sSUFBSSxHQUFHLEdBQUcsQ0FBQyxPQUFPLENBQUMsQ0FBQztRQUMxQixNQUFNLFlBQVksR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLGNBQWMsQ0FBQyxDQUFDO1FBQy9DLElBQUksWUFBWSxFQUFFO1lBQ2hCLEtBQUssR0FBRyxRQUFRLENBQUMsWUFBWSxFQUFFLEVBQUUsQ0FBQyxJQUFJLENBQUMsQ0FBQztTQUN6QztRQUVELElBQUksS0FBSyxHQUFHLENBQUMsRUFBRTtZQUNiLEdBQUcsQ0FBQyxJQUFJLENBQUMsQ0FBQyxNQUFNLEVBQUUsQ0FBQztTQUNwQjtJQUNILENBQUMsQ0FBQyxDQUFDO0lBRUgsT0FBTyxHQUFHLENBQUM7QUFDYixDQUFDLENBQUM7QUFwQlcsUUFBQSx5QkFBeUIsNkJBb0JwQztBQUVLLE1BQU0saUJBQWlCLEdBQUcsQ0FDL0IsSUFBWSxFQUNaLElBQWEsRUFDYixLQUFjLEVBQ04sRUFBRTtJQUNWLElBQUksYUFBYSxHQUFHLElBQUksQ0FBQztJQUN6QixvR0FBb0c7SUFDcEcsTUFBTSxZQUFZLEdBQTJCO1FBQzNDLE9BQU8sRUFBRSxHQUFHO1FBQ1osUUFBUSxFQUFFLEdBQUc7UUFDYixRQUFRLEVBQUUsR0FBRztRQUNiLFFBQVEsRUFBRSxHQUFHO1FBQ2IsUUFBUSxFQUFFLEdBQUc7UUFDYixNQUFNLEVBQUUsR0FBRztRQUNYLE1BQU0sRUFBRSxHQUFHO1FBQ1gsUUFBUSxFQUFFLEdBQUc7UUFDYixTQUFTLEVBQUUsR0FBRztRQUNkLFFBQVEsRUFBRSxHQUFHO1FBQ2IsT0FBTyxFQUFFLEdBQUc7UUFDWixPQUFPLEVBQUUsR0FBRztLQUNiLENBQUM7SUFFRixNQUFNLFdBQVcsR0FBMkI7UUFDMUMsR0FBRyxFQUFFLEdBQUc7UUFDUixHQUFHLEVBQUUsR0FBRztRQUNSLHFEQUFxRDtRQUNyRCxJQUFJLEVBQUUsR0FBRztRQUNULElBQUksRUFBRSxHQUFHO0tBQ1YsQ0FBQztJQUVGLElBQUksSUFBSSxFQUFFO1FBQ1IsS0FBSyxNQUFNLEdBQUcsSUFBSSxNQUFNLENBQUMsSUFBSSxDQUFDLFlBQVksQ0FBQyxFQUFFO1lBQzNDLE1BQU0sU0FBUyxHQUFHLElBQUksTUFBTSxDQUFDLEdBQUcsRUFBRSxHQUFHLENBQUMsQ0FBQztZQUN2QyxhQUFhLEdBQUcsYUFBYSxDQUFDLE9BQU8sQ0FBQyxTQUFTLEVBQUUsWUFBWSxDQUFDLEdBQUcsQ0FBQyxDQUFDLENBQUM7U0FDckU7S0FDRjtJQUVELElBQUksS0FBSyxFQUFFO1FBQ1QsS0FBSyxNQUFNLEdBQUcsSUFBSSxNQUFNLENBQUMsSUFBSSxDQUFDLFdBQVcsQ0FBQyxFQUFFO1lBQzFDLE1BQU0sZ0JBQWdCLEdBQUcsSUFBSSxNQUFNLENBQUMsR0FBRyxFQUFFLEdBQUcsQ0FBQyxDQUFDO1lBQzlDLGFBQWEsR0FBRyxhQUFhLENBQUMsT0FBTyxDQUFDLGdCQUFnQixFQUFFLFdBQVcsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDO1NBQzNFO0tBQ0Y7SUFFRCxPQUFPLGFBQWEsQ0FBQztBQUN2QixDQUFDLENBQUM7QUE3Q1csUUFBQSxpQkFBaUIscUJBNkM1QjtBQUVLLE1BQU0sZUFBZSxHQUFHLENBQzdCLEdBQWlCLEVBQ2pCLE9BQXdCLEVBQ1YsRUFBRTtJQUNoQixNQUFNLEtBQUssR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLHVCQUF1QixDQUFDLENBQUM7SUFDcEQsS0FBSyxDQUFDLElBQUksQ0FBQyxDQUFDLE1BQWMsRUFBRSxPQUF3QixFQUFFLEVBQUU7UUFDdEQsR0FBRyxDQUFDLE9BQU8sQ0FBQyxDQUFDLFdBQVcsQ0FBQyxHQUFHLENBQUMsT0FBTyxDQUFDLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FBQztJQUNoRCxDQUFDLENBQUMsQ0FBQztJQUNILE9BQU8sR0FBRyxDQUFDO0FBQ2IsQ0FBQyxDQUFDO0FBVFcsUUFBQSxlQUFlLG1CQVMxQjtBQUVLLE1BQU0sUUFBUSxHQUFHLENBQUMsR0FBaUIsRUFBRSxJQUFxQixFQUFVLEVBQUU7SUFDM0UsTUFBTSxLQUFLLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUM5QixJQUFJLElBQUksR0FBRyxFQUFFLENBQUM7SUFFZCxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUMsTUFBYyxFQUFFLE9BQXdCLEVBQUUsRUFBRTtRQUN0RCxJQUFJLEdBQUcsR0FBRyxJQUFJLFFBQVEsR0FBRyxDQUFDLE9BQU8sQ0FBQyxDQUFDLElBQUksRUFBRSxFQUFFLENBQUM7SUFDOUMsQ0FBQyxDQUFDLENBQUM7SUFDSCxJQUFJLEdBQUcsR0FBRyxJQUFJLElBQUksQ0FBQztJQUNuQixPQUFPLElBQUksQ0FBQztBQUNkLENBQUMsQ0FBQztBQVRXLFFBQUEsUUFBUSxZQVNuQjtBQUVGLE1BQU0sU0FBUyxHQUFHLENBQ2hCLEdBQWlCLEVBQ2pCLE9BQXdCLEVBQ3hCLElBQVksRUFDSixFQUFFO0lBQ1YsSUFBQSxpQ0FBeUIsRUFBQyxHQUFHLEVBQUUsT0FBTyxDQUFDLENBQUM7SUFDeEMsSUFBQSxtQkFBVyxFQUFDLEdBQUcsRUFBRSxPQUFPLENBQUMsQ0FBQztJQUMxQixJQUFBLHNCQUFjLEVBQUMsR0FBRyxFQUFFLE9BQU8sQ0FBQyxDQUFDO0lBQzdCLElBQUEsdUJBQWUsRUFBQyxHQUFHLEVBQUUsT0FBTyxDQUFDLENBQUM7SUFDOUIsSUFBQSxnQ0FBd0IsRUFBQyxHQUFHLEVBQUUsT0FBTyxFQUFFLElBQUksQ0FBQyxDQUFDO0lBQzdDLE1BQU0sYUFBYSxHQUFHLElBQUEscUJBQWEsRUFBQyxHQUFHLEVBQUUsT0FBTyxDQUFDLENBQUMsT0FBTyxDQUFDLEtBQUssRUFBRSxHQUFHLENBQUMsQ0FBQztJQUN0RSxPQUFPLElBQUEseUJBQWlCLEVBQUMsYUFBYSxFQUFFLEtBQUssRUFBRSxJQUFJLENBQUMsQ0FBQztBQUN2RCxDQUFDLENBQUM7QUFFRixrQkFBZSxTQUFTLENBQUMiLCJzb3VyY2VzQ29udGVudCI6WyJpbXBvcnQgY2hlZXJpbyBmcm9tIFwiY2hlZXJpb1wiO1xuaW1wb3J0IHhyZWdleHAgZnJvbSBcInhyZWdleHBcIjtcbmltcG9ydCBzdG9wd29yZHMgZnJvbSBcIi4vc3RvcHdvcmRzXCI7XG5cbmV4cG9ydCBjb25zdCBhZGROZXdsaW5lVG9CciA9IChcbiAgZG9jOiBjaGVlcmlvLlJvb3QsXG4gIHRvcE5vZGU6IGNoZWVyaW8uQ2hlZXJpbyxcbik6IGNoZWVyaW8uUm9vdCA9PiB7XG4gIGNvbnN0IGJycyA9IHRvcE5vZGUuZmluZChcImJyXCIpO1xuXG4gIGJycy5lYWNoKChfaW5kZXg6IG51bWJlciwgZWxlbWVudDogY2hlZXJpby5FbGVtZW50KSA9PiB7XG4gICAgY29uc3QgYnIgPSBkb2MoZWxlbWVudCk7XG4gICAgYnIucmVwbGFjZVdpdGgoXCJcXG5cXG5cIik7XG4gIH0pO1xuXG4gIHJldHVybiBkb2M7XG59O1xuXG5leHBvcnQgY29uc3QgY2xlYW5QYXJhZ3JhcGhUZXh0ID0gKHJhd1RleHQ6IHN0cmluZyk6IHN0cmluZyA9PiB7XG4gIGNvbnN0IHRleHQgPSByYXdUZXh0LnRyaW0oKTtcbiAgdGV4dC5yZXBsYWNlKC9bXFxzXFx0XSsvZywgXCIgXCIpO1xuICByZXR1cm4gdGV4dDtcbn07XG5cbmV4cG9ydCBjb25zdCBjb252ZXJ0VG9UZXh0ID0gKFxuICBkb2M6IGNoZWVyaW8uUm9vdCxcbiAgdG9wTm9kZTogY2hlZXJpby5DaGVlcmlvLFxuKTogc3RyaW5nID0+IHtcbiAgbGV0IHRleHRzOiBzdHJpbmdbXSA9IFtdO1xuICBjb25zdCBub2RlcyA9IHRvcE5vZGUuY29udGVudHMoKTtcblxuICBsZXQgaGFuZ2luZ1RleHQgPSBcIlwiO1xuXG4gIG5vZGVzLmVhY2goKF9pbmRleDogbnVtYmVyLCBlbGVtZW50OiBjaGVlcmlvLkVsZW1lbnQpID0+IHtcbiAgICBjb25zdCBub2RlID0gZG9jKGVsZW1lbnQpO1xuICAgIGNvbnN0IG5vZGVUeXBlID0gbm9kZS5nZXQoMCkgPyBub2RlLmdldCgwKS50eXBlIDogbnVsbDtcbiAgICBjb25zdCBub2RlTmFtZSA9IG5vZGUuZ2V0KDApID8gbm9kZS5nZXQoMCkudGFnTmFtZSA6IG51bGw7XG5cbiAgICBpZiAobm9kZVR5cGUgPT09IFwidGV4dFwiKSB7XG4gICAgICBoYW5naW5nVGV4dCArPSBub2RlLnRleHQoKTtcbiAgICAgIHJldHVybiB0cnVlO1xuICAgIH0gZWxzZSBpZiAobm9kZU5hbWUgPT09IFwidWxcIikge1xuICAgICAgaGFuZ2luZ1RleHQgKz0gdWxUb1RleHQoZG9jLCBub2RlKTtcbiAgICAgIHJldHVybiB0cnVlO1xuICAgIH1cblxuICAgIGlmIChoYW5naW5nVGV4dC5sZW5ndGggPiAwKSB7XG4gICAgICBjb25zdCB0eHQgPSBjbGVhblBhcmFncmFwaFRleHQoaGFuZ2luZ1RleHQpO1xuICAgICAgdGV4dHMgPSB0ZXh0cy5jb25jYXQodHh0LnNwbGl0KC9cXHI/XFxuLykpO1xuICAgICAgaGFuZ2luZ1RleHQgPSBcIlwiO1xuICAgIH1cblxuICAgIGxldCB0ZXh0ID0gY2xlYW5QYXJhZ3JhcGhUZXh0KG5vZGUudGV4dCgpKTtcbiAgICB0ZXh0ID0gdGV4dC5yZXBsYWNlKC8oXFx3K1xcLikoW0EtWl0rKS8sIFwiJDEgJDJcIik7XG4gICAgdGV4dHMgPSB0ZXh0cy5jb25jYXQodGV4dC5zcGxpdCgvXFxyP1xcbi8pKTtcbiAgfSk7XG5cbiAgaWYgKGhhbmdpbmdUZXh0Lmxlbmd0aCA+IDApIHtcbiAgICBjb25zdCB0ZXh0ID0gY2xlYW5QYXJhZ3JhcGhUZXh0KGhhbmdpbmdUZXh0KTtcbiAgICB0ZXh0cyA9IHRleHRzID0gdGV4dHMuY29uY2F0KHRleHQuc3BsaXQoL1xccj9cXG4vKSk7XG4gIH1cblxuICB0ZXh0cyA9IHRleHRzLm1hcCgodHh0KSA9PiB7XG4gICAgcmV0dXJuIHR4dC50cmltKCk7XG4gIH0pO1xuXG4gIGNvbnN0IHJlZ2V4ID0geHJlZ2V4cChcIltcXFxccHtOdW1iZXJ9XFxcXHB7TGV0dGVyfV1cIik7XG4gIHRleHRzID0gdGV4dHMuZmlsdGVyKCh0eHQpID0+IHtcbiAgICByZXR1cm4gcmVnZXgudGVzdCh0eHQpO1xuICB9KTtcblxuICByZXR1cm4gdGV4dHMuam9pbihcIlxcblxcblwiKTtcbn07XG5cbmV4cG9ydCBjb25zdCBsaW5rc1RvVGV4dCA9IChcbiAgZG9jOiBjaGVlcmlvLlJvb3QsXG4gIHRvcE5vZGU6IGNoZWVyaW8uQ2hlZXJpbyxcbik6IGNoZWVyaW8uUm9vdCA9PiB7XG4gIGNvbnN0IG5vZGVzID0gdG9wTm9kZS5maW5kKFwiYVwiKTtcbiAgbm9kZXMuZWFjaCgoX2luZGV4OiBudW1iZXIsIGVsZW1lbnQ6IGNoZWVyaW8uRWxlbWVudCkgPT4ge1xuICAgIGNvbnN0IGh0bWxFbCA9IGRvYyhlbGVtZW50KS5odG1sKCk7XG4gICAgaWYgKGh0bWxFbCkge1xuICAgICAgZG9jKGVsZW1lbnQpLnJlcGxhY2VXaXRoKGh0bWxFbCk7XG4gICAgfVxuICB9KTtcbiAgcmV0dXJuIGRvYztcbn07XG5cbmV4cG9ydCBjb25zdCByZW1vdmVGZXdXb3Jkc1BhcmFncmFwaHMgPSAoXG4gIGRvYzogY2hlZXJpby5Sb290LFxuICB0b3BOb2RlOiBjaGVlcmlvLkNoZWVyaW8sXG4gIGxhbmc6IHN0cmluZyxcbik6IGNoZWVyaW8uUm9vdCA9PiB7XG4gIGNvbnN0IGFsbE5vZGVzID0gdG9wTm9kZS5maW5kKFwiKlwiKTtcblxuICBhbGxOb2Rlcy5lYWNoKChfaW5kZXg6IG51bWJlciwgZWxlbWVudDogY2hlZXJpby5FbGVtZW50KSA9PiB7XG4gICAgY29uc3QgZWwgPSBkb2MoZWxlbWVudCk7XG4gICAgY29uc3QgdGFnID0gZWwuZ2V0KDApLnRhZ05hbWU7XG4gICAgY29uc3QgdGV4dCA9IGVsLnRleHQoKTtcblxuICAgIGNvbnN0IHN0b3BXb3JkcyA9IHN0b3B3b3Jkcyh0ZXh0LCBsYW5nKTtcbiAgICBpZiAoXG4gICAgICAodGFnICE9PSBcImJyXCIgfHwgdGV4dCAhPT0gXCJcXFxcclwiKSAmJlxuICAgICAgc3RvcFdvcmRzLnN0b3BXb3JkQ291bnQgPCAzICYmXG4gICAgICBlbC5maW5kKFwib2JqZWN0XCIpLmxlbmd0aCA9PT0gMCAmJlxuICAgICAgZWwuZmluZChcImVtYmVkXCIpLmxlbmd0aCA9PT0gMFxuICAgICkge1xuICAgICAgZG9jKGVsKS5yZW1vdmUoKTtcbiAgICB9IGVsc2Uge1xuICAgICAgY29uc3QgdHJpbW1lZCA9IHRleHQudHJpbSgpO1xuICAgICAgaWYgKHRyaW1tZWRbMF0gPT09IFwiKFwiICYmIHRyaW1tZWRbdHJpbW1lZC5sZW5ndGggLSAxXSA9PT0gXCIpXCIpIHtcbiAgICAgICAgZG9jKGVsKS5yZW1vdmUoKTtcbiAgICAgIH1cbiAgICB9XG4gIH0pO1xuXG4gIHJldHVybiBkb2M7XG59O1xuXG5leHBvcnQgY29uc3QgcmVtb3ZlTmVnYXRpdmVzY29yZXNOb2RlcyA9IChcbiAgZG9jOiBjaGVlcmlvLlJvb3QsXG4gIHRvcE5vZGU6IGNoZWVyaW8uQ2hlZXJpbyxcbik6IGNoZWVyaW8uUm9vdCA9PiB7XG4gIGNvbnN0IGdyYXZpdHlJdGVtcyA9IHRvcE5vZGUuZmluZChcIipbZ3Jhdml0eVNjb3JlXVwiKTtcblxuICBncmF2aXR5SXRlbXMuZWFjaCgoX2luZGV4OiBudW1iZXIsIGVsZW1lbnQ6IGNoZWVyaW8uRWxlbWVudCkgPT4ge1xuICAgIGxldCBzY29yZSA9IDA7XG4gICAgY29uc3QgaXRlbSA9IGRvYyhlbGVtZW50KTtcbiAgICBjb25zdCBncmF2aXR5U2NvcmUgPSBpdGVtLmF0dHIoXCJncmF2aXR5U2NvcmVcIik7XG4gICAgaWYgKGdyYXZpdHlTY29yZSkge1xuICAgICAgc2NvcmUgPSBwYXJzZUludChncmF2aXR5U2NvcmUsIDEwKSB8fCAwO1xuICAgIH1cblxuICAgIGlmIChzY29yZSA8IDEpIHtcbiAgICAgIGRvYyhpdGVtKS5yZW1vdmUoKTtcbiAgICB9XG4gIH0pO1xuXG4gIHJldHVybiBkb2M7XG59O1xuXG5leHBvcnQgY29uc3QgcmVwbGFjZUNoYXJhY3RlcnMgPSAoXG4gIHRleHQ6IHN0cmluZyxcbiAgaHRtbDogYm9vbGVhbixcbiAgY2hhcnM6IGJvb2xlYW4sXG4pOiBzdHJpbmcgPT4ge1xuICBsZXQgcHJvY2Vzc2VkVGV4dCA9IHRleHQ7XG4gIC8vIGlmIGVsZW1lbnQgZG9lcyBub3QgbWF0Y2ggYW55IGluIG1hcCBhbmQgc3RhcnRzIHdpdGggJiBhbmQgZW5kcyB3aXRoIDssIHJlcGxhY2Ugd2l0aCBlbXB0eSBzdHJpbmdcbiAgY29uc3QgaHRtbEVudGl0aWVzOiBSZWNvcmQ8c3RyaW5nLCBzdHJpbmc+ID0ge1xuICAgIFwiJmFtcDtcIjogXCImXCIsXG4gICAgXCImYXBvcztcIjogXCInXCIsXG4gICAgXCImY2VudDtcIjogXCLColwiLFxuICAgIFwiJmNvcHk7XCI6IFwiwqlcIixcbiAgICBcIiZldXJvO1wiOiBcIuKCrFwiLFxuICAgIFwiJmd0O1wiOiBcIj5cIixcbiAgICBcIiZsdDtcIjogXCI8XCIsXG4gICAgXCImbmJzcDtcIjogXCIgXCIsXG4gICAgXCImcG91bmQ7XCI6IFwiwqNcIixcbiAgICBcIiZxdW90O1wiOiAnXCInLFxuICAgIFwiJnJlZztcIjogXCLCrlwiLFxuICAgIFwiJnllbjtcIjogXCLCpVwiLFxuICB9O1xuXG4gIGNvbnN0IGVzY2FwZUNoYXJzOiBSZWNvcmQ8c3RyaW5nLCBzdHJpbmc+ID0ge1xuICAgICdcIic6ICdcIicsXG4gICAgXCInXCI6IFwiJ1wiLFxuICAgIC8vIHRzbGludDpkaXNhYmxlLW5leHQtbGluZTogb2JqZWN0LWxpdGVyYWwtc29ydC1rZXlzXG4gICAgXCJcXG5cIjogXCIgXCIsXG4gICAgXCJcXHJcIjogXCIgXCIsXG4gIH07XG5cbiAgaWYgKGh0bWwpIHtcbiAgICBmb3IgKGNvbnN0IGtleSBvZiBPYmplY3Qua2V5cyhodG1sRW50aXRpZXMpKSB7XG4gICAgICBjb25zdCBodG1scmVnZXggPSBuZXcgUmVnRXhwKGtleSwgXCJnXCIpO1xuICAgICAgcHJvY2Vzc2VkVGV4dCA9IHByb2Nlc3NlZFRleHQucmVwbGFjZShodG1scmVnZXgsIGh0bWxFbnRpdGllc1trZXldKTtcbiAgICB9XG4gIH1cblxuICBpZiAoY2hhcnMpIHtcbiAgICBmb3IgKGNvbnN0IGtleSBvZiBPYmplY3Qua2V5cyhlc2NhcGVDaGFycykpIHtcbiAgICAgIGNvbnN0IGVzY2FwZUNoYXJzUmVnZXggPSBuZXcgUmVnRXhwKGtleSwgXCJnXCIpO1xuICAgICAgcHJvY2Vzc2VkVGV4dCA9IHByb2Nlc3NlZFRleHQucmVwbGFjZShlc2NhcGVDaGFyc1JlZ2V4LCBlc2NhcGVDaGFyc1trZXldKTtcbiAgICB9XG4gIH1cblxuICByZXR1cm4gcHJvY2Vzc2VkVGV4dDtcbn07XG5cbmV4cG9ydCBjb25zdCByZXBsYWNlV2l0aFRleHQgPSAoXG4gIGRvYzogY2hlZXJpby5Sb290LFxuICB0b3BOb2RlOiBjaGVlcmlvLkNoZWVyaW8sXG4pOiBjaGVlcmlvLlJvb3QgPT4ge1xuICBjb25zdCBub2RlcyA9IHRvcE5vZGUuZmluZChcImIsIHN0cm9uZywgaSwgYnIsIHN1cFwiKTtcbiAgbm9kZXMuZWFjaCgoX2luZGV4OiBudW1iZXIsIGVsZW1lbnQ6IGNoZWVyaW8uRWxlbWVudCkgPT4ge1xuICAgIGRvYyhlbGVtZW50KS5yZXBsYWNlV2l0aChkb2MoZWxlbWVudCkudGV4dCgpKTtcbiAgfSk7XG4gIHJldHVybiBkb2M7XG59O1xuXG5leHBvcnQgY29uc3QgdWxUb1RleHQgPSAoZG9jOiBjaGVlcmlvLlJvb3QsIG5vZGU6IGNoZWVyaW8uQ2hlZXJpbyk6IHN0cmluZyA9PiB7XG4gIGNvbnN0IG5vZGVzID0gbm9kZS5maW5kKFwibGlcIik7XG4gIGxldCB0ZXh0ID0gXCJcIjtcblxuICBub2Rlcy5lYWNoKChfaW5kZXg6IG51bWJlciwgZWxlbWVudDogY2hlZXJpby5FbGVtZW50KSA9PiB7XG4gICAgdGV4dCA9IGAke3RleHR9XFxuICogJHtkb2MoZWxlbWVudCkudGV4dCgpfWA7XG4gIH0pO1xuICB0ZXh0ID0gYCR7dGV4dH1cXG5gO1xuICByZXR1cm4gdGV4dDtcbn07XG5cbmNvbnN0IGZvcm1hdHRlciA9IChcbiAgZG9jOiBjaGVlcmlvLlJvb3QsXG4gIHRvcE5vZGU6IGNoZWVyaW8uQ2hlZXJpbyxcbiAgbGFuZzogc3RyaW5nLFxuKTogc3RyaW5nID0+IHtcbiAgcmVtb3ZlTmVnYXRpdmVzY29yZXNOb2Rlcyhkb2MsIHRvcE5vZGUpO1xuICBsaW5rc1RvVGV4dChkb2MsIHRvcE5vZGUpO1xuICBhZGROZXdsaW5lVG9Ccihkb2MsIHRvcE5vZGUpO1xuICByZXBsYWNlV2l0aFRleHQoZG9jLCB0b3BOb2RlKTtcbiAgcmVtb3ZlRmV3V29yZHNQYXJhZ3JhcGhzKGRvYywgdG9wTm9kZSwgbGFuZyk7XG4gIGNvbnN0IGNvbnZlcnRlZFRleHQgPSBjb252ZXJ0VG9UZXh0KGRvYywgdG9wTm9kZSkucmVwbGFjZSgvXFxuL2csIFwiIFwiKTtcbiAgcmV0dXJuIHJlcGxhY2VDaGFyYWN0ZXJzKGNvbnZlcnRlZFRleHQsIGZhbHNlLCB0cnVlKTtcbn07XG5cbmV4cG9ydCBkZWZhdWx0IGZvcm1hdHRlcjtcbiJdfQ==