hamok/lib/collections/HamokEmitter.js

Lightweight Distributed Object Storage on RAFT consensus algorithm

"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.HamokEmitter = void 0;
const events_1 = require("events");
const logger_1 = require("../common/logger");
const Collections = __importStar(require("../common/Collections"));
const logger = (0, logger_1.createLogger)('HamokEmitter');
class HamokEmitter {
    connection;
    payloadsCodec;
    autoClean;
    // private readonly _subscriptions = new Map<keyof T, Set<string>>();
    subscriptions = new HamokEmitterSubscriptions();
    _emitter = new events_1.EventEmitter();
    _initializing;
    _closed = false;
    stats = {
        numberOfSubscriptions: 0,
        numberOfReceivedEventInvocations: 0,
        numberOfSentEventInvocations: 0,
    };
    constructor(connection, payloadsCodec, autoClean) {
        this.connection = connection;
        this.payloadsCodec = payloadsCodec;
        this.autoClean = autoClean;
        this.connection
            .on('InsertEntriesRequest', (request) => {
            // this is for the subscription to manage and to add the source endpoint to the list
            if (request.sourceEndpointId === undefined) {
                return logger.warn('%s InsertEntriesRequest is received without sourceEndpointId, for %s, it is impossible to add the source endpoint to the list. %o', this.connection.grid.localPeerId, this.id, request);
            }
            let responseEntries;
            for (const [event, serializedMetaData] of request.entries.entries()) {
                try {
                    if (this.subscriptions.hasPeerOnEvent(event, request.sourceEndpointId)) {
                        const metaDataUpdate = JSON.parse(serializedMetaData);
                        const updated = this.subscriptions.updatePeer(event, request.sourceEndpointId, metaDataUpdate.newMetaData, metaDataUpdate.prevMetaData);
                        if (!updated) {
                            if (!responseEntries)
                                responseEntries = new Map();
                            responseEntries.set(event, 'not-updated');
                            continue;
                        }
                    }
                    else {
                        // this is a new subscription
                        let metaData = null;
                        if (serializedMetaData !== 'null')
                            metaData = JSON.parse(serializedMetaData);
                        this.subscriptions.addPeer(event, request.sourceEndpointId, metaData);
                    }
                }
                catch (err) {
                    logger.error('Error while decoding the metadata for %s, %s, %o', this.id, event, `${err}`);
                    continue;
                }
                logger.debug('%s InsertEntriesRequest is received, %s is added to the subscription list for %s', this.connection.grid.localPeerId, request.sourceEndpointId, event);
            }
            if (request.sourceEndpointId === this.connection.grid.localPeerId) {
                this.connection.respond('InsertEntriesResponse', request.createResponse(responseEntries ?? Collections.EMPTY_MAP), request.sourceEndpointId);
            }
        })
            .on('DeleteEntriesRequest', (request) => {
            const removedPeerIds = [...request.keys];
            removedPeerIds.forEach((peerId) => this.subscriptions.removePeerFromAllEvent(peerId));
            logger.debug('DeleteEntriesRequest is received, %o is removed from the subscription list for %s', removedPeerIds, this.id);
            if (request.sourceEndpointId === this.connection.grid.localPeerId) {
                this.connection.respond('DeleteEntriesResponse', request.createResponse(new Set(removedPeerIds)), request.sourceEndpointId);
            }
        })
            .on('RemoveEntriesRequest', (request) => {
            // this is for the subscription to manage, and to remove the source endpoint from the list
            if (request.sourceEndpointId === undefined) {
                return logger.warn('%s RemoveEntriesRequest is received without sourceEndpointId, for %s, it is impossible to remove the source endpoint from the list. %o', this.connection.grid.localPeerId, this.id, request);
            }
            this.subscriptions.removePeerFromAllEvent(request.sourceEndpointId);
            if (request.sourceEndpointId === this.connection.grid.localPeerId) {
                this.connection.respond('RemoveEntriesResponse', request.createResponse(Collections.EMPTY_MAP), request.sourceEndpointId);
            }
        })
            .on('UpdateEntriesRequest', (request) => {
            // this is for the events to emit
            for (const [event, serializedPayload] of request.entries) {
                try {
                    const payloads = this.payloadsCodec?.get(event)?.decode(serializedPayload) ?? JSON.parse(serializedPayload);
                    this._emitter.emit(event, ...payloads);
                    ++this.stats.numberOfReceivedEventInvocations;
                }
                catch (err) {
                    logger.error('Error while decoding the payload for %s, %s, %o', this.id, event, `${err}`);
                }
            }
            this.connection.respond('UpdateEntriesResponse', request.createResponse(new Map([[this.connection.localPeerId, 'empty']])), request.sourceEndpointId);
        })
            .on('UpdateEntriesNotification', (notification) => {
            // this is for the events to emit
            for (const [event, serializedPayload] of notification.updatedEntries) {
                try {
                    const payloads = this.payloadsCodec?.get(event)?.decode(serializedPayload) ?? JSON.parse(serializedPayload);
                    this._emitter.emit(event, ...payloads);
                    ++this.stats.numberOfReceivedEventInvocations;
                }
                catch (err) {
                    logger.error('Error while decoding the payload for %s, %s, %o', this.id, event, `${err}`);
                }
            }
        })
            .on('ClearEntriesNotification', (request) => {
            // this is for the subscription to manage, and to remove the source endpoint from the list
            if (request.sourceEndpointId === undefined) {
                return logger.warn('%s ClearEntriesNotification is received without sourceEndpointId, for %s, it is impossible to remove the source endpoint from the list. %o', this.connection.grid.localPeerId, this.id, request);
            }
            this.subscriptions.removePeerFromAllEvent(request.sourceEndpointId);
        })
            .on('remote-peer-removed', async (remotePeerId) => {
            if (this.connection.grid.leaderId !== this.connection.localPeerId)
                return;
            if (this.connection.localPeerId === remotePeerId)
                return;
            if (!this.autoClean)
                return;
            for (let retried = 0; retried < 10; retried++) {
                try {
                    await this.cleanup();
                    break;
                }
                catch (err) {
                    if (retried < 8)
                        continue;
                    logger.error('Error while cleaning up subscriptions in emitter %s, error: %o', this.id, err);
                    break;
                }
            }
        })
            .on('leader-changed', async (leaderId) => {
            if (leaderId !== this.connection.localPeerId || !this.autoClean) {
                return;
            }
            try {
                await this.cleanup();
            }
            catch (err) {
                logger.error('Error while cleaning up subscriptions in emitter %s, error: %o', this.id, err);
            }
        })
            .on('StorageHelloNotification', (notification) => {
            // every storage needs to respond with its snapshot and the highest applied index they have
            try {
                const snapshot = this.export();
                const serializedSnapshot = JSON.stringify(snapshot);
                this.connection.notifyStorageState(serializedSnapshot, this.connection.highestSeenCommitIndex, notification.sourceEndpointId);
            }
            catch (err) {
                logger.error(`Failed to send snapshot: ${err}`);
            }
        })
            .on('remote-snapshot', (serializedSnapshot, done) => {
            try {
                const snapshot = JSON.parse(serializedSnapshot);
                this._import(snapshot);
                this.subscriptions.emit('debug', `Imported snapshot from ${JSON.stringify(snapshot)}`);
            }
            catch (err) {
                logger.error(`Failed to import to emitter ${this.id}. Error: ${err}`);
            }
            finally {
                done();
            }
        })
            .once('close', () => this.close());
        this.subscriptions
            .on('added', () => (this.stats.numberOfSubscriptions = this.subscriptions.size))
            .on('removed', () => (this.stats.numberOfSubscriptions = this.subscriptions.size));
        logger.trace('Emitter %s is created', this.id);
        process.nextTick(() => (this._initializing = this._startInitializing()));
    }
    get id() {
        return this.connection.config.storageId;
    }
    get empty() {
        return this.subscriptions.size < 1;
    }
    get ready() {
        return this._initializing ?? this.connection.grid.waitUntilCommitHead().then(() => this);
    }
    get closed() {
        return this._closed;
    }
    close() {
        if (this._closed)
            return;
        this._closed = true;
        this.connection.close();
        this._emitter.removeAllListeners();
        this.subscriptions.removeAllListeners();
    }
    /**
     * This method is used to cleanup the subscriptions by removing the endpoints that are not in the grid anymore.
     */
    async cleanup() {
        const removedPeerIds = this.subscriptions.getAllPeerIds();
        removedPeerIds.delete(this.connection.grid.localPeerId);
        for (const remotePeerId of this.connection.grid.remotePeerIds) {
            if (removedPeerIds.has(remotePeerId))
                removedPeerIds.delete(remotePeerId);
        }
        if (0 < removedPeerIds.size) {
            this.subscriptions.emit('debug', `Removing endpoints ${JSON.stringify(removedPeerIds)} from subscriptions in emitter ${this.id}`);
            return this.connection.requestDeleteEntries(removedPeerIds);
        }
    }
    async hasSubscribers(event, filterByLocalNode = false) {
        if (this._closed)
            throw new Error('Cannot check subscribers on a closed emitter');
        await this._initializing;
        await this.connection.grid.waitUntilCommitHead();
        const remotePeerIds = this.subscriptions.getPeerIds(event);
        if (!remotePeerIds)
            return false;
        else if (!filterByLocalNode)
            return true;
        else
            return remotePeerIds.has(this.connection.grid.localPeerId);
    }
    async subscribe(event, listener, metaData = null) {
        if (this._closed)
            throw new Error('Cannot subscribe on a closed emitter');
        await this._initializing;
        // if we already have a listener, we don't need to subscribe in the raft
        if (this._emitter.listenerCount(event)) {
            return (this._emitter.on(event, listener), void 0);
        }
        let serializedMetaData;
        if (metaData) {
            try {
                serializedMetaData = JSON.stringify(metaData);
            }
            catch (err) {
                logger.error('Error while serializing metadata for %s, %s, %o', this.id, event, `${err}`);
                serializedMetaData = 'null';
            }
        }
        else
            serializedMetaData = 'null';
        this._emitter.on(event, listener);
        try {
            await this.connection.requestInsertEntries(new Map([[event, serializedMetaData]]));
        }
        catch (err) {
            this._emitter.off(event, listener);
            throw err;
        }
    }
    async updateSubscriptionMetaData(event, newMetaData, prevMetaData) {
        if (this._closed)
            throw new Error('Cannot subscribe on a closed emitter');
        await this._initializing;
        // if we already have a listener, we don't need to subscribe in the raft
        if (!this._emitter.listenerCount(event)) {
            throw new Error('Cannot update a non-existing subscription');
        }
        const updatedMetaData = {
            prevMetaData,
            newMetaData,
        };
        const serializedMetaData = JSON.stringify(updatedMetaData);
        return (await this.connection.requestInsertEntries(new Map([[event, serializedMetaData]]))).get(event) === undefined;
    }
    async unsubscribe(event, listener) {
        if (this._closed)
            throw new Error('Cannot unsubscribe on a closed emitter');
        await this._initializing;
        this._emitter.off(event, listener);
        // if we still have a listener, we don't need to unsubscribe in the raft
        if (this._emitter.listenerCount(event))
            return;
        await this.connection.requestRemoveEntries(Collections.setOf(event));
    }
    clear() {
        this.connection.notifyClearEntries();
        this._emitter.removeAllListeners();
    }
    async publish(event, ...args) {
        if (this._closed)
            throw new Error('Cannot publish on a closed emitter');
        await this._initializing;
        const remotePeerIds = this.subscriptions.getPeerIds(event);
        if (!remotePeerIds || remotePeerIds.size < 1) {
            return [];
        }
        else if (remotePeerIds.size === 1 && remotePeerIds.has(this.connection.grid.localPeerId)) {
            return (this._emitter.emit(event, ...args), [this.connection.grid.localPeerId]);
        }
        const entry = [event, this.payloadsCodec?.get(event)?.encode(...args) ?? JSON.stringify(args)];
        const [respondedRemotePeerIds, isLocalPeerSubscribed] = await Promise.all([
            this.connection.requestUpdateEntries(new Map([entry]), [...remotePeerIds].filter((peerId) => peerId !== this.connection.grid.localPeerId)),
            Promise.resolve(remotePeerIds.has(this.connection.grid.localPeerId) ? this._emitter.emit(event, ...args) : false)
        ]);
        const result = [...respondedRemotePeerIds.keys()];
        if (isLocalPeerSubscribed) {
            result.push(this.connection.grid.localPeerId);
        }
        ++this.stats.numberOfSentEventInvocations;
        return result;
    }
    notify(event, ...args) {
        if (this._closed)
            throw new Error('Cannot publish on a closed emitter');
        const remotePeerIds = this.subscriptions.getPeerIds(event);
        if (!remotePeerIds || remotePeerIds.size < 1) {
            return false;
        }
        else if (remotePeerIds.size === 1 && remotePeerIds.has(this.connection.grid.localPeerId)) {
            return this._emitter.emit(event, ...args);
        }
        const entry = [event, this.payloadsCodec?.get(event)?.encode(...args) ?? JSON.stringify(args)];
        for (const remotePeerId of remotePeerIds ?? []) {
            if (remotePeerId === this.connection.grid.localPeerId) {
                this._emitter.emit(event, ...args);
                continue;
            }
            this.connection.notifyUpdateEntries(new Map([entry]), remotePeerId);
        }
        ++this.stats.numberOfSentEventInvocations;
        return true;
    }
    export() {
        if (this._closed)
            throw new Error('Cannot export a closed emitter');
        const subscriptions = [];
        for (const [event, peerMap] of this.subscriptions.entries()) {
            const subscribers = [];
            for (const [peerId, metaData] of peerMap.entries()) {
                subscribers.push({
                    peerId,
                    metaData,
                });
            }
            subscriptions.push({
                event: event,
                subscribers,
            });
        }
        return {
            emitterId: this.id,
            subscriptions,
        };
    }
    import(snapshot) {
        if (snapshot.emitterId !== this.id) {
            throw new Error(`Cannot import data from a different queue: ${snapshot.emitterId} !== ${this.id}`);
        }
        else if (this.connection.connected) {
            throw new Error('Cannot import data while connected');
        }
        this._import(snapshot);
    }
    _import(snapshot) {
        for (const subscription of snapshot.subscriptions) {
            for (const { peerId, metaData } of subscription.subscribers) {
                this.subscriptions.addPeer(subscription.event, peerId, metaData);
            }
        }
    }
    async _startInitializing() {
        try {
            await this.connection.join();
        }
        catch (err) {
            logger.error('Error while initializing emitter', err);
        }
        finally {
            this._initializing = undefined;
        }
        return this;
    }
}
exports.HamokEmitter = HamokEmitter;
class HamokEmitterSubscriptions extends events_1.EventEmitter {
    _map = new Map();
    hasEvent(event) {
        return this._map.has(event);
    }
    addPeer(event, peerId, metaData = null) {
        let peersMap = this._map.get(event);
        if (!peersMap) {
            peersMap = new Map();
            this._map.set(event, peersMap);
        }
        else if (peersMap.has(peerId))
            return false;
        peersMap.set(peerId, metaData);
        this.emit('added', event, peerId, metaData);
        return true;
    }
    updatePeer(event, peerId, metaData, prevMetaData) {
        const peersMap = this._map.get(event);
        const currentMetaData = peersMap?.get(peerId);
        if (!peersMap || currentMetaData === undefined)
            return false;
        if (prevMetaData !== undefined) {
            const serializedCurrentMetaData = JSON.stringify(currentMetaData);
            const serializedPrevMetaData = JSON.stringify(prevMetaData);
            if (serializedCurrentMetaData !== serializedPrevMetaData)
                return false;
        }
        peersMap.set(peerId, metaData);
        this.emit('updated', event, peerId, metaData, currentMetaData);
        return true;
    }
    removePeer(event, peerId) {
        const peersMap = this._map.get(event);
        const metaData = peersMap?.get(peerId);
        if (!peersMap || !peersMap.delete(peerId))
            return false;
        if (peersMap.size < 1) {
            this._map.delete(event);
        }
        this.emit('removed', event, peerId, metaData ?? null);
        return true;
    }
    removePeerFromAllEvent(peerId) {
        const events = [...this.events()];
        let removedAtLeastFromOneEvent = false;
        for (const event of events) {
            removedAtLeastFromOneEvent = this.removePeer(event, peerId) || removedAtLeastFromOneEvent;
        }
        return removedAtLeastFromOneEvent;
    }
    getEventPeersMap(event) {
        return this._map.get(event);
    }
    entries() {
        return this._map.entries();
    }
    events() {
        return this._map.keys();
    }
    hasPeerOnEvent(event, peerId) {
        const peersMap = this._map.get(event);
        return peersMap ? peersMap.has(peerId) : false;
    }
    getPeerIds(event) {
        const peersMap = this._map.get(event);
        if (!peersMap)
            return;
        else
            return new Set([...peersMap.keys()]);
    }
    getAllPeerIds() {
        const peerIds = new Set();
        for (const peersMap of this._map.values()) {
            for (const peerId of peersMap.keys()) {
                peerIds.add(peerId);
            }
        }
        return peerIds;
    }
    get [Symbol.toStringTag]() {
        return 'HamokSubscriptions';
    }
    get size() {
        return this._map.size;
    }
    get [Symbol.species]() {
        return HamokEmitterSubscriptions;
    }
}