octopus-web-crawler/dist/crawler.js

Another web crawler.

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const types_1 = require("./types");
const utils_1 = require("./utils");
const EventEmitter = require("events");
const http = require("http");
const https = require("https");
class Crawler extends EventEmitter {
    constructor(opts) {
        super();
        this.maxRequests = 100;
        this.delay = 1000;
        this.requests = [];
        this.start = new Date().getTime();
        this.concurrency = 4;
        this.finishEventEmmited = false;
        this.next = () => {
            let todo = this.requests.filter((i) => i.status === types_1.Status.PENDING);
            let processing = this.requests.filter((i) => i.status === types_1.Status.PROCESSING);
            const done = this.requests.filter((i) => i.status === types_1.Status.DONE);
            todo = todo.slice(0, this.concurrency - processing.length);
            todo.forEach((item) => {
                this.updateRequest(item, types_1.Actions.UPDATE_STATUS, {
                    status: types_1.Status.PROCESSING,
                });
                this.processItem(item);
            });
            if (todo.length === 0 &&
                processing.length === 0 &&
                !this.finishEventEmmited) {
                this.finishEventEmmited = true;
                this.emit("finish", { requests: this.requests });
            }
        };
        this.processItem = (item) => {
            const httpModule = item.url.includes("https") ? https : http;
            httpModule
                .get(item.url, (res) => {
                this.handleResponse(item, res);
            })
                .on("error", (err) => {
                this.updateRequest(item, types_1.Actions.UPDATE_STATUS, {
                    status: types_1.Status.FAILED,
                });
                setTimeout(() => this.next(), this.delay);
            });
        };
        this.changeProtocol = (current, newProtocol) => {
            this.baseUrl = this.baseUrl.replace(current, newProtocol);
            this.requests = this.requests.map((i) => {
                const newUrl = i.url.replace(current, newProtocol);
                return Object.assign(Object.assign({}, i), { url: newUrl });
            });
        };
        this.addUrls = (item, html) => {
            let links = (0, utils_1.getUrlsFromHtml)(html);
            links = [...new Set(links)];
            let news = this.getAbsoluteUrls(links);
            news = (0, utils_1.filterExternalLinks)(news, this.baseUrl);
            news = (0, utils_1.filterAlreadyAdded)(news, this.requests);
            const items = news.map((i) => (0, utils_1.createRequestItem)(i, item.id));
            const remaining = this.maxRequests - this.requests.length;
            this.requests = [...this.requests, ...items.slice(0, remaining)];
        };
        this.getAbsoluteUrls = (links) => {
            return links.map((link) => {
                const isAbsolute = link.includes("http");
                link = isAbsolute ? link : link.replace(/^\//g, "");
                return isAbsolute ? link : `${this.baseUrl}/${link}`;
            });
        };
        this.updateRequest = (item, action, payload) => {
            switch (action) {
                case types_1.Actions.UPDATE_STATUS:
                    const idx = this.findRequestIndex(item);
                    const update = Object.assign(Object.assign({}, item), { status: payload.status });
                    this.requests[idx] = update;
                    break;
            }
        };
        this.findRequestIndex = (req) => {
            return this.requests.findIndex((i) => i.id === req.id);
        };
        let { url, delay, maxRequest, concurrency } = opts;
        url = (0, utils_1.removeLastSlashCharacter)(url);
        this.delay = delay;
        this.maxRequests = maxRequest;
        this.concurrency = concurrency;
        const props = (0, utils_1.getBaseUrl)(url);
        if (props.base === null || props.domain === null) {
            throw new Error("The url provided is not valid");
        }
        this.baseUrl = props.base;
        this.domain = props.domain;
        this.buffer = opts.buffer;
        const r = (0, utils_1.createRequestItem)(url);
        this.requests.push(r);
        this.next();
    }
    handleResponse(item, res) {
        const { statusCode } = res;
        const result = {
            url: item.url,
            content: "",
            buffer: null,
        };
        const chunks = [];
        if (statusCode !== 301 && statusCode !== 200) {
            this.updateRequest(item, types_1.Actions.UPDATE_STATUS, {
                status: types_1.Status.FAILED,
            });
        }
        else if (statusCode === 301) {
            this.updateRequest(item, types_1.Actions.UPDATE_STATUS, {
                status: types_1.Status.PENDING,
            });
            const current = item.url.includes("https") ? "https" : "http";
            const newProtocol = item.url.includes("https") ? "http" : "https";
            this.changeProtocol(current, newProtocol);
            this.next();
        }
        else if (statusCode === 200) {
            res.on("data", (data) => {
                result.content += data;
                chunks.push(Buffer.from(data, "binary"));
            });
        }
        res.on("end", () => {
            if (result.content !== "") {
                if (this.buffer) {
                    let binary = Buffer.concat(chunks);
                    result.buffer = binary;
                }
                let data = (0, utils_1.removeWhiteSpaces)(result.content);
                this.addUrls(item, data);
                this.updateRequest(item, types_1.Actions.UPDATE_STATUS, {
                    status: types_1.Status.DONE,
                });
                this.emit("request", result);
            }
            setTimeout(() => this.next(), this.delay);
        });
    }
}
module.exports = Crawler;
//# sourceMappingURL=crawler.js.map