krawala/dist/crawl.js

Simple javascript web-crawler with command line interface

"use strict";
var __assign = (this && this.__assign) || Object.assign || function(t) {
    for (var s, i = 1, n = arguments.length; i < n; i++) {
        s = arguments[i];
        for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
            t[p] = s[p];
    }
    return t;
};
Object.defineProperty(exports, "__esModule", { value: true });
var request = require("request");
var _ = require("underscore");
var utils_1 = require("./utils");
var MATCHES_SUCCESS_STATUS = /^[123]/;
var MATCHES_TEXT_HTML = /\btext\s*\/\s*html\b/i;
var USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' +
    '(KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36';
var REQUEST_OPTIONS = {
    method: 'GET',
    headers: {
        'User-Agent': USER_AGENT,
    },
};
var options;
var crawled = {};
var queue = [];
var progress = {
    depth: 0,
    urlsToCrawl: [],
    urlsCrawled: [],
    failed: [],
    progressMade: false,
};
var output = {
    pages: [],
    externalPages: [],
    links: [],
    scripts: [],
    images: [],
    failed: [],
};
var enqueue = function (node) {
    progress.urlsToCrawl.push(node.resolved);
    queue.push(node);
};
var storeNode = function (node) {
    crawled[node.resolved] = node;
};
var storeSubNode = function (node, href, key) {
    var url = href.url, resolved = href.resolved;
    var subNode = {
        url: url,
        resolved: resolved,
        depth: node.depth + 1,
        internal: utils_1.isSameDomain(resolved, options.resolved),
        linkedFrom: [node.resolved],
    };
    if (progress.urlsToCrawl.indexOf(resolved) < 0) {
        output[key].push(subNode);
        enqueue(subNode);
    }
    var subCrawledNode = crawled[resolved];
    if (typeof subCrawledNode === 'object') {
        subCrawledNode.linkedFrom.push(node.resolved);
    }
    else {
        storeNode(subNode);
    }
};
var crawlQueue;
var crawlNode = function (node) {
    if (node.depth > progress.depth) {
        progress.depth = node.depth;
    }
    utils_1.updateProgress(options, progress, "Crawling: " + node.resolved);
    request(node.resolved, __assign({}, REQUEST_OPTIONS, { timeout: options.timeout }), function (error, response, body) {
        progress.urlsCrawled.push(node.resolved);
        var status = (response ? response.statusCode : null) || null;
        var contentType = (response ? response.headers['content-type'] : '') || '';
        var type = Array.isArray(contentType) ? "[" + contentType.join(', ') + "]" : contentType;
        if (!error && status && MATCHES_SUCCESS_STATUS.test(status.toString())) {
            node.failed = false;
            node.type = type;
            node.status = status;
            if (node.internal && MATCHES_TEXT_HTML.test(type) && body) {
                var data = utils_1.collectData(node, body, options.resolved);
                _.extend(node, data);
                var page = node;
                if (node.depth < options.depth) {
                    page.hrefs.internal.forEach(function (href) { return storeSubNode(node, href, 'pages'); });
                    page.hrefs.external.forEach(function (href) { return storeSubNode(node, href, 'externalPages'); });
                    page.links.forEach(function (href) { return storeSubNode(node, href, 'links'); });
                    page.scripts.forEach(function (href) { return storeSubNode(node, href, 'scripts'); });
                    page.images.forEach(function (href) { return storeSubNode(node, href, 'images'); });
                }
            }
            else {
                node.failed = false;
            }
        }
        else {
            progress.failed.push(node.resolved);
            node.failed = true;
        }
        node.type = type || null;
        node.status = status || null;
        var crawledNode = crawled[node.resolved];
        if (typeof crawledNode === 'object') {
            crawledNode.failed = node.failed;
            crawledNode.type = node.type;
            crawledNode.status = node.status;
        }
        utils_1.updateProgress(options, progress, "Crawled:  " + node.resolved);
        if (progress.urlsToCrawl.length === progress.urlsCrawled.length) {
            utils_1.complete(options, __assign({}, output, { failed: utils_1.getFailed(crawled) }));
        }
        else if (options.interval) {
            setTimeout(crawlQueue, options.interval);
        }
        else {
            crawlQueue();
        }
    });
};
crawlQueue = function () {
    var node = queue.shift();
    if (node) {
        crawlNode(node);
    }
};
exports.crawl = function (tree) {
    var _a = tree.kwargs, url = _a.url, _b = _a.depth, depth = _b === void 0 ? '10' : _b, _c = _a.format, format = _c === void 0 ? 'json' : _c, interval = _a.interval, _d = _a.timeout, timeout = _d === void 0 ? '10000' : _d, _e = tree.flags.wait, wait = _e === void 0 ? false : _e;
    utils_1.validateBaseUrl(url);
    options = {
        url: url,
        resolved: utils_1.resolveUrl(url),
        depth: parseInt(depth, 10),
        format: format,
        interval: typeof interval === 'string' ? parseInt(interval, 10) : undefined,
        wait: wait,
        timeout: parseInt(timeout, 10),
    };
    var node = {
        url: url,
        resolved: utils_1.resolveUrl(url),
        depth: 0,
        internal: true,
        linkedFrom: [],
    };
    output.pages.push(node);
    enqueue(node);
    crawlQueue();
};
exports.default = exports.crawl;
//# sourceMappingURL=crawl.js.map