@rdfc/sds-storage-writer-ts/lib/repositories/MongoDBRepository.js

An RDF-Connect processor to write SDS streams into a given storage system

import { MongoClient } from "mongodb";
import { getLoggerFor } from "../utils/logUtil.js";
import { Lock } from "async-await-mutex-lock";
import { Parser } from "n3";
export class MongoDBRepository {
    url;
    metadata;
    data;
    index;
    client;
    db;
    logger = getLoggerFor(this);
    lock = new Lock();
    constructor(url, metadata, data, index) {
        this.url = url;
        this.metadata = metadata;
        this.data = data;
        this.index = index;
    }
    async open() {
        this.client = await new MongoClient(this.url).connect();
        this.db = this.client.db();
        this.logger.debug(`Connected to ${this.url}`);
    }
    async close() {
        await this.client.close();
        this.logger.debug(`Closed connection to ${this.url}`);
    }
    async ingestMetadata(type, id, value) {
        await this.lock.acquire("metaMongoDB");
        try {
            await this.db
                .collection(this.metadata)
                .updateOne({ type, id }, { $set: { value } }, { upsert: true });
        }
        finally {
            this.lock.release("metaMongoDB");
        }
    }
    async findMetadataFragmentations() {
        return await this.db
            .collection(this.metadata)
            .find({
            type: "fragmentation",
        })
            .map((entry) => {
            return {
                id: entry.id,
                quads: new Parser().parse(entry.value),
            };
        })
            .toArray();
    }
    async createIndices() {
        await this.db.collection(this.data).createIndex({ id: 1 });
        await this.db
            .collection(this.index)
            .createIndex({ streamId: 1, id: 1 });
    }
    prepareDataBulk() {
        return [];
    }
    async ingestDataBulk(bulk) {
        await this.lock.acquire("dataMongoDB");
        try {
            await this.db.collection(this.data).bulkWrite(bulk);
        }
        finally {
            this.lock.release("dataMongoDB");
        }
    }
    async handleRecord(record, data, bulk) {
        bulk.push({
            updateOne: {
                filter: {
                    id: record.payload,
                },
                update: {
                    $setOnInsert: {
                        data: data,
                        created: Date.now(),
                    },
                },
                upsert: true,
            },
        });
    }
    prepareIndexBulk() {
        return [];
    }
    async ingestIndexBulk(bulk) {
        await this.lock.acquire("indexMongoDB");
        try {
            await this.db.collection(this.index).bulkWrite(bulk);
        }
        finally {
            this.lock.release("indexMongoDB");
        }
    }
    async handleMember(record, bucket, bulk) {
        bulk.push({
            updateOne: {
                filter: {
                    streamId: record.stream,
                    id: bucket,
                },
                update: {
                    $addToSet: { members: record.payload },
                    $set: {
                        updated: Date.now(),
                    },
                    $setOnInsert: {
                        created: Date.now(),
                    },
                },
                upsert: true,
            },
        });
    }
    async handleBucket(bucket, bulk) {
        if (bucket.empty) {
            bucket.members = [];
        }
        delete bucket.empty;
        bulk.push({
            updateOne: {
                filter: {
                    streamId: bucket.streamId,
                    id: bucket.id,
                },
                update: {
                    $set: { ...bucket, updated: Date.now() },
                    $setOnInsert: {
                        created: Date.now(),
                    },
                },
                upsert: true,
            },
        });
    }
    async handleRelation(relation, path, value, bulk) {
        bulk.push({
            updateOne: {
                filter: {
                    streamId: relation.stream,
                    id: relation.origin,
                },
                update: {
                    $addToSet: {
                        relations: {
                            bucket: relation.bucket,
                            path: path,
                            type: relation.type,
                            value: value,
                        },
                    },
                    $set: {
                        updated: Date.now(),
                    },
                    $setOnInsert: {
                        created: Date.now(),
                    },
                },
                upsert: true,
            },
        });
    }
    async removeRelation(relation, path, value, bulk) {
        const matchCriteria = {
            bucket: relation.bucket,
            type: relation.type,
        };
        if (path !== undefined) {
            matchCriteria.path = path;
        }
        if (value !== undefined) {
            matchCriteria.value = value;
        }
        bulk.push({
            updateOne: {
                filter: {
                    streamId: relation.stream,
                    id: relation.origin,
                },
                update: {
                    $pull: {
                        relations: matchCriteria,
                    },
                    $set: {
                        updated: Date.now(),
                    },
                },
            },
        });
    }
}