chen-crawler/dist/storage.js

Web Crawler Provider for Chen Framework

"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator.throw(value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments)).next());
    });
};
const chen_redis_1 = require('chen-redis');
/**
 * Storage class
 */
class Storage {
    /**
     * Storage constructor
     * @param {string} private prefix
     * @param {RedisConfig} config
     */
    constructor(config) {
        /**
         * [prefix description]
         * @type {string}
         */
        this.prefix = 'chen-crawler:';
        this.conn = chen_redis_1.Redis.createConnection(config);
        this.prefix = config.prefix || this.prefix;
    }
    /**
     * Get all
     * @param  {string} key
     * @return {Promise<string[]>}
     */
    all(key) {
        return __awaiter(this, void 0, void 0, function* () {
            return yield this.conn.zrange(`${this.prefix}${key}`, 0, -1);
        });
    }
    /**
     * Append to queue
     * @param  {string | number} key
     * @param  {string} url
     * @return {Promise<number>}
     */
    push(key, url) {
        return __awaiter(this, void 0, void 0, function* () {
            let score = Number(yield this.conn.zscore(`${this.prefix}${key}`, url));
            if (!score) {
                score = yield this.conn.zadd(`${this.prefix}${key}`, Date.now(), url);
            }
            return score;
        });
    }
    /**
     * Check if url exists
     * @param  {string | number} key
     * @param  {string} url
     * @return {Promise<boolean>}
     */
    has(key, url) {
        return __awaiter(this, void 0, void 0, function* () {
            let score = Number(yield this.conn.zscore(`${this.prefix}${key}`, url));
            return score ? true : false;
        });
    }
    /**
     * Pop from queue
     * @param  {string | number} key
     * @return {Promise<string>}
     */
    shift(key) {
        return __awaiter(this, void 0, void 0, function* () {
            let items = yield this.conn.zrange(`${this.prefix}${key}`, 0, 1);
            if (items && items.length) {
                yield this.remove(key, items[0]);
                return items[0];
            }
            return null;
        });
    }
    /**
     * Remove a value
     * @param  {string | number} key
     * @param  {string | number} value
     * @return {Promise<number>}
     */
    remove(key, url) {
        return __awaiter(this, void 0, void 0, function* () {
            return yield this.conn.zrem(`${this.prefix}${key}`, url);
        });
    }
    /**
     * Remove all queue
     * @return {Promise<string>}
    */
    flush() {
        return __awaiter(this, void 0, void 0, function* () {
            return yield this.conn.flushdb();
        });
    }
}
exports.Storage = Storage;
//# sourceMappingURL=storage.js.map