semantic-analysis
Version:
Extensible semantic Structured Data analysis library for Microdata and JSON-LD
98 lines • 3.76 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.CrawlAnalyzer = void 0;
var BatchAnalyzer_1 = require("./BatchAnalyzer");
var VisitLogFile_1 = require("./VisitLogFile");
/**
* Analyzer crawling through on-site links
*/
var CrawlAnalyzer = /** @class */ (function () {
/* Initialization */
function CrawlAnalyzer(options) {
var _this = this;
var _a;
this.options = options;
this.addTask = function (url, source) {
return _this.logReady
.then(function () {
_this.visitLog.add(url);
return _this.batchAnalyzer.addTask(url, source);
})
.then(function (done) {
var _a, _b, _c;
var doc = done.results.document;
var yetToDo = (((_b = (_a = _this.options) === null || _a === void 0 ? void 0 : _a.perSiteLimit) !== null && _b !== void 0 ? _b : Infinity) - ((_c = _this.visitLog.siteCount(url)) !== null && _c !== void 0 ? _c : 0));
console.debug({ yetToDo: yetToDo });
if (doc && yetToDo > 0) {
getSameSiteLinks(doc)
.filter(function (url) { return !_this.visitLog.has(url); })
.slice(0, yetToDo)
.forEach(function (url) { return _this.addTask(url); });
}
return done;
});
};
this.state = function () { return _this.batchAnalyzer.state(); };
this.dispose = function () { return _this.batchAnalyzer.dispose(); };
this.batchAnalyzer = new BatchAnalyzer_1.BatchAnalyzer(options);
this.visitLog = (_a = options === null || options === void 0 ? void 0 : options.visitLog) !== null && _a !== void 0 ? _a : new VisitLogFile_1.VisitLogFile();
this.logReady = (options === null || options === void 0 ? void 0 : options.visitLogPath) && this.visitLog instanceof VisitLogFile_1.VisitLogFile
? this.visitLog.setupFromFile(options.visitLogPath).then()
: Promise.resolve();
}
Object.defineProperty(CrawlAnalyzer.prototype, "queued", {
/* Interface */
get: function () {
return this.batchAnalyzer.queued;
},
enumerable: false,
configurable: true
});
Object.defineProperty(CrawlAnalyzer.prototype, "running", {
get: function () {
return this.batchAnalyzer.running;
},
enumerable: false,
configurable: true
});
Object.defineProperty(CrawlAnalyzer.prototype, "done", {
get: function () {
return this.batchAnalyzer.done;
},
enumerable: false,
configurable: true
});
Object.defineProperty(CrawlAnalyzer.prototype, "failed", {
get: function () {
return this.batchAnalyzer.failed;
},
enumerable: false,
configurable: true
});
return CrawlAnalyzer;
}());
exports.CrawlAnalyzer = CrawlAnalyzer;
function getSameSiteLinks(doc) {
var url;
try {
url = new URL(doc.url);
}
catch (e) {
console.error('Invalid URL observed', e);
return [];
}
return Array.from(doc.document.querySelectorAll('a[href]'))
.map(function (_a) {
var href = _a.href;
try {
return new URL(href, url);
}
catch (e) {
console.error('Cannot create target URL', e);
return undefined;
}
})
.filter(function (target) { return (target === null || target === void 0 ? void 0 : target.host) === url.host; })
.map(function (target) { return target.toString(); });
}
//# sourceMappingURL=CrawlAnalyzer.js.map