chen-crawler/dist/queue.js

Web Crawler Provider for Chen Framework

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator.throw(value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments)).next());
    });
};
/**
 * Queue class
 */
class Queue {
    /**
     * Queue constructor
     * @param {Storage} private storage
     * @param {string} private id
     */
    constructor(storage, id) {
        this.storage = storage;
        this.id = id;
    }
    /**
     * Add to queue
     * @param {string} url
     */
    push(url) {
        return __awaiter(this, void 0, void 0, function* () {
            yield this.storage.push(`${this.id}${Queue.QUEUE_SUFFIX}`, url);
        });
    }
    /**
     * Pop from queue
     */
    shift() {
        return __awaiter(this, void 0, void 0, function* () {
            return yield this.storage.shift(`${this.id}${Queue.QUEUE_SUFFIX}`);
        });
    }
}
/**
 * Suffix
 * @type {String}
 */
Queue.QUEUE_SUFFIX = ':queue';
exports.Queue = Queue;
/**
 * ProcessingList class
 */
class ProcessingList {
    /**
     * ProcessingList constructor
     * @param {Storage} private storage
     * @param {string}  private id
     */
    constructor(storage, id) {
        this.storage = storage;
        this.id = id;
    }
    /**
     * Push url to processing list
     * @param  {string} url
     * @return {Promise<void>}
     */
    add(url) {
        return __awaiter(this, void 0, void 0, function* () {
            yield this.storage.push(`${this.id}${ProcessingList.LIST_SUFFIX}`, url);
        });
    }
    /**
     * Check if url exists in processing list
     * @param  {string} url
     * @return {Promise<boolean>}
     */
    has(url) {
        return __awaiter(this, void 0, void 0, function* () {
            return yield this.storage.has(`${this.id}${ProcessingList.LIST_SUFFIX}`, url);
        });
    }
    /**
     * Remove url in processing list
     * @param  {string} url
     * @return {Promise<number>}
     */
    remove(url) {
        return __awaiter(this, void 0, void 0, function* () {
            return yield this.storage.remove(`${this.id}${ProcessingList.LIST_SUFFIX}`, url);
        });
    }
}
/**
 * Suffix
 * @type {string}
 */
ProcessingList.LIST_SUFFIX = ':processing';
exports.ProcessingList = ProcessingList;
//# sourceMappingURL=queue.js.map