mwoffliner/lib/RedisStore.js

MediaWiki ZIM scraper

import { createClient } from 'redis';
import RedisKvs from './util/RedisKvs.js';
import * as logger from './Logger.js';
class RedisStore {
    static instance;
    #client;
    #storesReady;
    #filesToDownloadXPath;
    #articleDetailXId;
    #redirectsXId;
    #filesQueues;
    constructor() {
        this.#filesQueues = [];
    }
    get client() {
        return this.#client;
    }
    get filesToDownloadXPath() {
        return this.#filesToDownloadXPath;
    }
    get articleDetailXId() {
        return this.#articleDetailXId;
    }
    get redirectsXId() {
        return this.#redirectsXId;
    }
    get filesQueues() {
        return this.#filesQueues;
    }
    static getInstance() {
        if (!RedisStore.instance) {
            RedisStore.instance = new RedisStore();
        }
        return RedisStore.instance;
    }
    setOptions(redisPath, opts) {
        if (RedisStore.instance) {
            const options = { ...opts };
            const quitOnError = !(options.quitOnError === false);
            delete options.quitOnError;
            if (redisPath.startsWith('/') || redisPath.startsWith('./')) {
                options.socket = {
                    ...options.socket,
                    path: redisPath,
                };
            }
            else {
                options.url = redisPath;
            }
            this.#client = createClient(options);
            this.#client.on('error', (err) => {
                if (quitOnError) {
                    logger.error('Redis Client Error', err);
                    process.exit(3);
                }
            });
        }
        else {
            throw new Error('Redis store has not been instantiated before setting options');
        }
    }
    async connect(populateStores = true) {
        if (this.#client.isOpen) {
            return;
        }
        await this.#client.connect();
        if (populateStores) {
            await this.checkForExistingStores();
            await this.populateStores();
            this.#storesReady = true;
        }
    }
    async close() {
        if (this.#client.isReady && this.#storesReady) {
            logger.log('Flushing Redis DBs');
            await Promise.all([this.#filesToDownloadXPath.flush(), this.#articleDetailXId.flush(), this.#redirectsXId.flush(), ...this.#filesQueues.map((queue) => queue.flush)]);
        }
        if (this.#client.isOpen) {
            await this.#client.quit();
        }
    }
    async checkForExistingStores() {
        const patterns = ['*-media', '*-detail', '*-redirect', '*-files'];
        let keys = [];
        for (const pattern of patterns) {
            keys = keys.concat(await this.#client.keys(pattern));
        }
        keys.forEach(async (key) => {
            try {
                const length = await this.#client.hLen(key);
                const time = new Date(Number(key.slice(0, key.indexOf('-'))));
                logger.warn(`Deleting store from previous run from ${time} that was still in Redis: ${key} with length ${length}`);
                this.#client.del(key);
            }
            catch {
                logger.error(`Key ${key} exists in DB, and is no hash.`);
            }
        });
    }
    async populateStores() {
        this.#filesToDownloadXPath = new RedisKvs(this.#client, `${Date.now()}-media`, {
            u: 'url',
            m: 'mult',
            w: 'width',
        });
        this.#articleDetailXId = new RedisKvs(this.#client, `${Date.now()}-detail`, {
            s: 'subCategories',
            c: 'categories',
            p: 'pages',
            h: 'thumbnail',
            g: 'coordinates',
            t: 'timestamp',
            r: 'revisionId',
            i: 'internalThumbnailUrl',
            m: 'missing',
            n: 'title',
        });
        this.#redirectsXId = new RedisKvs(this.#client, `${Date.now()}-redirect`, {
            f: 'fragment',
            t: 'targetId',
            n: 'title',
        });
    }
    createRedisKvs(...args) {
        return new RedisKvs(this.#client, ...args);
    }
}
const rs = RedisStore.getInstance();
export default rs;
//# sourceMappingURL=RedisStore.js.map