@rdfc/sds-storage-writer-ts/lib/repositories/RedisRepository.js

An RDF-Connect processor to write SDS streams into a given storage system

import { getLoggerFor } from "../utils/logUtil.js";
import { Lock } from "async-await-mutex-lock";
import { createClient } from "redis";
import { DataFactory, Parser } from "n3";
const { namedNode } = DataFactory;
export class RedisRepository {
    url;
    metadata;
    data;
    index;
    client;
    logger = getLoggerFor(this);
    lock = new Lock();
    constructor(url, metadata, data, index) {
        this.url = url;
        this.metadata = metadata;
        this.data = data;
        this.index = index;
    }
    async open() {
        this.client = createClient({ url: this.url });
        await this.client.connect();
        this.logger.debug(`Connected to ${this.url}`);
    }
    async close() {
        await this.client.disconnect();
        this.logger.debug(`Closed connection to ${this.url}`);
    }
    async ingestMetadata(type, id, value) {
        await this.lock.acquire("metaRedis");
        try {
            await this.client.set(`${this.metadata}:${encodeURIComponent(type)}:${encodeURIComponent(id)}`, value);
        }
        finally {
            this.lock.release("metaRedis");
        }
    }
    async findMetadataFragmentations() {
        const keys = await this.client.keys(`${this.metadata}:fragmentation:*`);
        if (keys.length === 0) {
            return [];
        }
        const entries = await this.client.mGet(keys);
        return entries.map((entry, i) => {
            const key = keys[i].split(":");
            return {
                id: namedNode(encodeURIComponent(key[2])),
                quads: new Parser().parse(entry),
            };
        });
    }
    async createIndices() {
    }
    prepareDataBulk() {
        return [];
    }
    async ingestDataBulk(bulk) {
        await Promise.all(bulk);
    }
    async handleRecord(record, data, bulk) {
        bulk.push(this.client.set(`${this.data}:${encodeURIComponent(record.payload)}`, data), this.client.set(`${this.data}:${encodeURIComponent(record.payload)}:created`, Date.now()));
    }
    prepareIndexBulk() {
        return [];
    }
    async ingestIndexBulk(bulk) {
        await Promise.all(bulk);
    }
    async handleMember(record, bucket, bulk) {
        bulk.push(this.client.json.set(`${this.index}:${encodeURIComponent(record.stream)}:${encodeURIComponent(bucket)}`, "$.updated", Date.now(), { XX: true }));
        bulk.push(this.client.sAdd(`${this.index}:${encodeURIComponent(record.stream)}:${encodeURIComponent(bucket)}:members`, record.payload));
    }
    async handleBucket(bucket, bulk) {
        if (bucket.empty) {
            bulk.push(this.client.del(`${this.index}:${encodeURIComponent(bucket.streamId)}:${encodeURIComponent(bucket.id)}:members`));
        }
        delete bucket.empty;
        bulk.push(this.client.json.set(`${this.index}:${encodeURIComponent(bucket.streamId)}:${encodeURIComponent(bucket.id)}`, "$", { created: Date.now() }, { NX: true }));
        bulk.push(this.client.json.merge(`${this.index}:${encodeURIComponent(bucket.streamId)}:${encodeURIComponent(bucket.id)}`, "$", { updated: Date.now(), ...bucket }));
    }
    async handleRelation(relation, path, value, bulk) {
        bulk.push(this.client.json.set(`${this.index}:${encodeURIComponent(relation.stream)}:${encodeURIComponent(relation.origin)}`, "$.updated", Date.now(), { XX: true }));
        bulk.push(this.client.sAdd(`${this.index}:${encodeURIComponent(relation.stream)}:${encodeURIComponent(relation.origin)}:relations`, JSON.stringify({
            type: relation.type,
            stream: relation.stream,
            origin: relation.origin,
            bucket: relation.bucket,
            path: path,
            value: value,
        })));
    }
    async removeRelation(relation, path, value, bulk) {
        const key = `${this.index}:${encodeURIComponent(relation.stream)}:${encodeURIComponent(relation.origin)}:relations`;
        const existingRelations = await this.client.sMembers(key);
        if (!existingRelations || existingRelations.length === 0) {
            return;
        }
        for (const rel of existingRelations) {
            try {
                const parsedRel = JSON.parse(rel);
                if (parsedRel.type === relation.type &&
                    parsedRel.bucket === relation.bucket &&
                    (path === undefined || parsedRel.path === path) &&
                    (value === undefined || parsedRel.value === value)) {
                    bulk.push(this.client.sRem(key, rel));
                }
            }
            catch (error) {
                this.logger.error(`Failed to parse relation: ${rel}`);
                this.logger.debug(error);
            }
        }
        bulk.push(this.client.json.set(`${this.index}:${encodeURIComponent(relation.stream)}:${encodeURIComponent(relation.origin)}`, "$.updated", Date.now(), { XX: true }));
    }
}