krawala
Version:
Simple javascript web-crawler with command line interface
151 lines • 6.47 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
var cheerio = require("cheerio");
var yaml = require("js-yaml");
var parseDomain = require("parse-domain");
var readline = require("readline");
var _ = require("underscore");
var parseUrl = require("url-parse");
var wordCount = require("word-count");
var PROGRESS_LINES = 8;
var PADDING = ' ';
exports.error = function (message) {
if (typeof process === 'object' && typeof process.exit === 'function') {
console.error(message); // tslint:disable-line no-console
process.exit(1);
}
else {
throw new Error(message);
}
};
exports.isSameDomain = function (url, parentUrl) {
var parentUrlInfo = parseUrl(parentUrl);
// Remove path
parentUrlInfo = parseUrl('/', parentUrlInfo.href);
var urlInfo = parseUrl(url, parentUrl);
// Remove path
urlInfo = parseUrl('/', urlInfo.href);
var parentDomainInfo = parseDomain(parentUrlInfo.href);
var domainInfo = parseDomain(urlInfo.href);
return parentDomainInfo && domainInfo ?
parentDomainInfo.domain === domainInfo.domain &&
parentDomainInfo.tld === domainInfo.tld :
false;
};
exports.resolveUrl = function (url, parentUrl) {
return parseUrl(url, parentUrl).href;
};
exports.validateBaseUrl = function (url) {
if (!url) {
return exports.error('Invalid URL. URL cannot be empty');
}
var urlInfo = parseUrl(url);
var domainInfo = parseDomain(url);
if (!urlInfo.protocol) {
return exports.error('Invalid URL. URL must have an explicit protocol e.g. \'http://\'');
}
if (!domainInfo || !domainInfo.domain || !domainInfo.tld) {
return exports.error('Invalid URL. Could not resolve a domain. ' +
'Ensure the URL has a valid protocol, domain, & TLD e.g. \'http://domain.com\'');
}
return true;
};
exports.padLeft = function (value) {
value = PADDING + value.toString();
return value.substring(value.length - PADDING.length);
};
exports.clearProgress = function () {
for (var i = 0; i < PROGRESS_LINES; i += 1) {
readline.moveCursor(process.stderr, 0, -1);
readline.clearLine(process.stderr, 0);
}
};
exports.createProgressMessage = function (options, progress, currentTask) {
return "\n--------------------- Progress ---------------------\n\nDepth: " + exports.padLeft(progress.depth) + " / " + options.depth + "\nUrls: " + exports.padLeft(progress.urlsCrawled.length) + " / " + progress.urlsToCrawl.length + "\nFailed: " + exports.padLeft(progress.failed.length) + "\n" + currentTask + "\n\n";
};
exports.updateProgress = function (options, progress, currentTask) {
if (typeof process === 'object') {
if (progress.progressMade) {
exports.clearProgress();
}
process.stderr.write(exports.createProgressMessage(options, progress, currentTask));
progress.progressMade = true;
}
};
exports.resolveLinkScriptImageOrHref = function (attribsList, url, baseUrl, linkKey) {
return attribsList.map(function (attribs) { return ({
url: attribs[linkKey],
resolved: exports.resolveUrl(attribs[linkKey], url),
references: attribsList.filter(function (otherAttribs) { return attribs[linkKey] === otherAttribs[linkKey]; }).length,
attributes: attribs,
}); });
};
exports.groupHrefs = function (resolvedHrefs, baseUrl) {
return _.groupBy(resolvedHrefs, function (href) {
if (href.url.indexOf('#') === 0) {
return 'samePage';
}
if (href.url.indexOf('tel:') === 0) {
return 'phone';
}
if (href.url.indexOf('mailto:') === 0) {
return 'email';
}
return exports.isSameDomain(href.resolved, baseUrl) ? 'internal' : 'external';
});
};
exports.collectData = function (node, text, baseUrl) {
var $ = cheerio.load(text);
var hrefAttribs = $('a[href]').toArray().map(function (element) { return element.attribs; });
var hrefResolved = exports.resolveLinkScriptImageOrHref(hrefAttribs, node.resolved, baseUrl, 'href');
var groupedHrefs = exports.groupHrefs(hrefResolved, baseUrl);
var linksAttribs = $('link[href]').toArray().map(function (element) { return element.attribs; });
var linksResolved = exports.resolveLinkScriptImageOrHref(linksAttribs, node.resolved, baseUrl, 'href');
var scriptsAttribs = $('script[src]').toArray().map(function (element) { return element.attribs; });
var scriptsResolved = exports.resolveLinkScriptImageOrHref(scriptsAttribs, node.resolved, baseUrl, 'src');
var imagesAttribs = $('img[src]').toArray().map(function (element) { return element.attribs; });
var imagesResolved = exports.resolveLinkScriptImageOrHref(imagesAttribs, node.resolved, baseUrl, 'src');
return {
title: $('title').text() || null,
wordCount: wordCount($('body').text()),
charset: $('meta[charset]').attr('charset') || null,
meta: $('meta[name],meta[property]').toArray().map(function (element) { return ({ attributes: element.attribs }); }),
links: linksResolved,
scripts: scriptsResolved,
images: imagesResolved,
content: {
h1: $('h1').first().text() || null,
h2: $('h2').first().text() || null,
h3: $('h3').first().text() || null,
p: $('p').first().text() || null,
},
hrefs: {
internal: groupedHrefs.internal || [],
external: groupedHrefs.external || [],
samePage: groupedHrefs.samePage || [],
email: groupedHrefs.email || [],
phone: groupedHrefs.phone || [],
},
};
};
exports.getOutput = function (options, pages) {
switch (options.format) {
case 'yaml':
return yaml.safeDump(pages, { indent: 2 });
case 'json':
default:
return JSON.stringify(pages, null, 2) + '\n';
}
};
exports.complete = function (options, pages) {
if (typeof process === 'object' && typeof process.stdout !== 'undefined') {
process.stdout.write(exports.getOutput(options, pages));
}
else if (typeof options.callback === 'function') {
options.callback(exports.getOutput(options, pages));
}
};
exports.getFailed = function (crawled) {
return _.toArray(crawled).filter(function (crawledNode) { return crawledNode.failed; });
};
//# sourceMappingURL=utils.js.map