hamok
Version:
Lightweight Distributed Object Storage on RAFT consensus algorithm
871 lines (870 loc) • 43 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.Hamok = void 0;
const events_1 = require("events");
const uuid_1 = require("uuid");
const RaftEngine_1 = require("./raft/RaftEngine");
const HamokMessage_1 = require("./messages/HamokMessage");
const MemoryStoredRaftLogs_1 = require("./raft/MemoryStoredRaftLogs");
const RaftFollowerState_1 = require("./raft/RaftFollowerState");
const HamokMap_1 = require("./collections/HamokMap");
const HamokConnection_1 = require("./collections/HamokConnection");
const OngoingRequestsNotifier_1 = require("./messages/OngoingRequestsNotifier");
const HamokCodec_1 = require("./common/HamokCodec");
const StorageCodec_1 = require("./messages/StorageCodec");
const BaseMap_1 = require("./collections/BaseMap");
const logger_1 = require("./common/logger");
const HamokGridCodec_1 = require("./messages/HamokGridCodec");
const SubmitMessage_1 = require("./messages/messagetypes/SubmitMessage");
const HamokGrid_1 = require("./HamokGrid");
const RaftEmptyState_1 = require("./raft/RaftEmptyState");
const HamokQueue_1 = require("./collections/HamokQueue");
const HamokEmitter_1 = require("./collections/HamokEmitter");
const HamokRecord_1 = require("./collections/HamokRecord");
const EndpointNotification_1 = require("./messages/messagetypes/EndpointNotification");
const JoinNotification_1 = require("./messages/messagetypes/JoinNotification");
const HamokRemoteMap_1 = require("./collections/HamokRemoteMap");
const logger = (0, logger_1.createLogger)('Hamok');
// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
class Hamok extends events_1.EventEmitter {
config;
raft;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
storages = new Map();
_closed = false;
_run = false;
// private _joining?: Promise<void>;
_raftTimer;
// private readonly _remoteHeartbeats = new Map<string, ReturnType<typeof setTimeout>>();
_remoteHeartbeats = new Map();
_codec = new HamokGridCodec_1.HamokGridCodec();
grid;
// private _lookingForRemotePeers?: {
// waiters: (() => void)[],
// timer: ReturnType<typeof setTimeout>,
// close: () => void,
// };
_joining;
constructor(providedConfig) {
super();
this.setMaxListeners(Infinity);
this._emitMessage = this._emitMessage.bind(this);
this._acceptLeaderChanged = this._acceptLeaderChanged.bind(this);
this._sendJoinMsg = this._sendJoinMsg.bind(this);
this._acceptCommit = this._acceptCommit.bind(this);
this._checkRemoteHeartbeats = this._checkRemoteHeartbeats.bind(this);
this.removeRemotePeerId = this.removeRemotePeerId.bind(this);
this.addRemotePeerId = this.addRemotePeerId.bind(this);
this.broadcastEndpointNotification = this.broadcastEndpointNotification.bind(this);
const raftLogs = providedConfig?.raftLogs ?? new MemoryStoredRaftLogs_1.MemoryStoredRaftLogs({
expirationTimeInMs: providedConfig?.logEntriesExpirationTimeInMs ?? 300000,
memorySizeHighWaterMark: 0,
});
this.raft = new RaftEngine_1.RaftEngine({
peerId: providedConfig?.peerId ?? (0, uuid_1.v4)(),
electionTimeoutInMs: providedConfig?.electionTimeoutInMs ?? 3000,
followerMaxIdleInMs: providedConfig?.followerMaxIdleInMs ?? 1000,
heartbeatInMs: providedConfig?.heartbeatInMs ?? 100,
onlyFollower: providedConfig?.onlyFollower ?? false,
}, raftLogs, this);
this.config = {
appData: providedConfig?.appData ?? {},
remoteStorageStateWaitingTimeoutInMs: providedConfig?.remoteStorageStateWaitingTimeoutInMs ?? 1000,
// autoStopOnNoRemotePeers: providedConfig?.autoStopOnNoRemotePeers ?? true,
};
this.grid = new HamokGrid_1.HamokGrid(this._emitMessage.bind(this), this.submit.bind(this), this.waitUntilCommitHead.bind(this), new OngoingRequestsNotifier_1.OngoingRequestsNotifier(providedConfig?.ongoingRequestsSendingPeriodInMs ?? 1000, (msg) => this._emitMessage(this._codec.encodeOngoingRequestsNotification(msg), msg.destinationEndpointId, HamokMessage_1.HamokMessage_MessageProtocol.GRID_COMMUNICATION_PROTOCOL)), this.raft.remotePeers, this.raft.logs, () => this.raft.localPeerId, () => this.raft.leaderId);
this.once('close', () => {
this.off('commit', this._acceptCommit);
this.off('leader-changed', this._acceptLeaderChanged);
this.off('unsynced-peer', this.removeRemotePeerId);
this.off('heartbeat', this._sendJoinMsg);
this.off('heartbeat', this._checkRemoteHeartbeats);
});
this.on('commit', this._acceptCommit);
this.on('leader-changed', this._acceptLeaderChanged);
this.on('unsynced-peer', this.removeRemotePeerId);
this.on('heartbeat', this._sendJoinMsg);
this.on('heartbeat', this._checkRemoteHeartbeats);
}
get appData() {
return this.config.appData ?? {};
}
get localPeerId() {
return this.raft.localPeerId;
}
get remotePeerIds() {
return this.raft.remotePeers;
}
get leader() {
return this.raft.leaderId === this.raft.localPeerId && this.raft.state.stateName === 'leader';
}
get ready() {
return (this._joining?.promise ?? this.waitUntilLeader()).then(() => this);
}
get state() {
return this.raft.state.stateName;
}
get run() {
return this._run;
}
get activeRemoteHeartbeats() {
return this._remoteHeartbeats.keys();
}
get closed() {
return this._closed;
}
close() {
if (this._closed)
return;
this._closed = true;
this._run = false;
this._stopRaftEngine();
this.remotePeerIds.forEach((peerId) => this.removeRemotePeerId(peerId));
this.storages.forEach((storage) => storage.close());
this.emit('close');
logger.info('%s is closed', this.localPeerId);
this.removeAllListeners();
}
get stats() {
const numberOfPendingRequests = this.grid.pendingRequests.size;
const numberOfOngoingRequests = this.grid.ongoingRequestsNotifier.activeOngoingRequests.size;
const numberOfRemotePeers = this.raft.remotePeers.size;
const numberOfPendingResponses = this.grid.pendingResponses.size;
const raftLogsBytesInMemory = this.raft.logs.bytesInMemory;
return {
/**
* Number of requests sent out from the grid, but waiting for response from remote peer
*/
numberOfPendingRequests,
/**
* Number of requests received by this peer and queued for processing (for example requests to be waited to be committed by the leader)
*/
numberOfOngoingRequests,
/**
* Number of responses received by this peer and queued for processing as the response were chunked
*/
numberOfPendingResponses,
/**
* Number of remote peers this peer is connected to
*/
numberOfRemotePeers,
/**
* Number of bytes used by the raft logs in memory
*/
raftLogsBytesInMemory,
};
}
_acceptCommit(commitIndex, message) {
logger.trace('%s accepted committed message %o', this.localPeerId, message);
// if we put this request to hold when we accepted the submit request we remove the ongoing request notification
// so from this moment it is up to the storage / pubsub to accomplish the request
if (message.requestId && this.grid.ongoingRequestsNotifier.has(message.requestId)) {
this.grid.ongoingRequestsNotifier.remove(message.requestId);
logger.trace('%s Request %s is removed ongoing requests', this.localPeerId, message.requestId);
}
this.accept(message, commitIndex);
}
addRemotePeerId(remoteEndpointId) {
if (this._closed)
throw new Error('Cannot add remote peer to a closed Hamok instance');
if (remoteEndpointId === this.localPeerId)
return;
if (this.raft.remotePeers.has(remoteEndpointId))
return;
this.raft.remotePeers.add(remoteEndpointId);
logger.debug('%s added remote peer %s', this.localPeerId, remoteEndpointId);
if (this.localPeerId === this.raft.leaderId) {
this.broadcastEndpointNotification();
}
this.emit('remote-peer-joined', remoteEndpointId);
}
removeRemotePeerId(remotePeerId) {
if (!this.raft.remotePeers.delete(remotePeerId))
return;
logger.debug('%s removed remote peer %s', this.localPeerId, remotePeerId);
// remove the peer from props
this.raft.props.nextIndex.delete(remotePeerId);
this.raft.props.matchIndex.delete(remotePeerId);
if (this.localPeerId === this.raft.leaderId) {
this.broadcastEndpointNotification();
}
// notify upstream and all connections
this.emit('remote-peer-left', remotePeerId);
for (const collection of this.storages.values()) {
collection.connection.emit('remote-peer-removed', remotePeerId);
}
}
_acceptLeaderChanged(leaderId) {
// let's just reset remote heartbeat measures.
// if we are the leader we will receive those, if we are not we don't need them
// this._remoteHeartbeats.forEach((timer) => clearTimeout(timer));
this._remoteHeartbeats.clear();
// notify all storages about the leader change
for (const collection of this.storages.values()) {
collection.connection.emit('leader-changed', leaderId);
}
if (this.localPeerId === leaderId) {
// we are the leader, we should send the endpoint notification to all remote peers
return this.broadcastEndpointNotification();
}
if (leaderId === undefined) {
if (this._closed || !this._run)
return;
logger.warn('%s detected that Leader is gone, clearing the remote peers', this.localPeerId);
// this._stopRaftEngine();
[...this.remotePeerIds].forEach((peerId) => this.removeRemotePeerId(peerId));
this.join({
fetchRemotePeerTimeoutInMs: 5000,
maxRetry: -1,
}).catch((err) => {
logger.error('Failed to rejoin the grid', err);
});
}
}
/**
* Wait until the commit head (the most recent spread commit by the leader) is reached
* @returns
*/
async waitUntilCommitHead() {
if (this._closed)
throw new Error('Cannot wait until commit head on a closed Hamok instance');
const actualCommitHead = this.raft.logs.nextIndex - 1;
if (actualCommitHead <= this.raft.logs.commitIndex)
return;
return new Promise((resolve) => {
const listener = (commitIndex) => {
if (commitIndex < actualCommitHead)
return;
this.off('commit', listener);
resolve();
};
this.on('commit', listener);
});
}
async waitUntilLeader(timeoutInMs) {
if (this._closed)
throw new Error('Cannot wait until leader on a closed Hamok instance');
if (this.raft.leaderId !== undefined)
return;
return new Promise((resolve, reject) => {
const listener = () => {
if (!this.raft.leaderId === undefined)
return;
this.off('leader-changed', listener);
resolve();
};
if (timeoutInMs) {
setTimeout(() => {
this.off('leader-changed', listener);
reject(new Error('Timeout waiting for leader'));
}, timeoutInMs);
}
this.on('leader-changed', listener);
});
}
createMap(options) {
if (this._closed)
throw new Error('Cannot create map on a closed Hamok instance');
if (this.storages.has(options.mapId))
throw new Error(`Map with id ${options.mapId} already exists`);
const connection = this._createStorageConnection({
...options,
keyCodec: options.keyCodec ?? (0, HamokCodec_1.createHamokJsonBinaryCodec)(),
valueCodec: options.valueCodec ?? (0, HamokCodec_1.createHamokJsonBinaryCodec)(),
submitting: new Set([
HamokMessage_1.HamokMessage_MessageType.CLEAR_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.INSERT_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.DELETE_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.REMOVE_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.UPDATE_ENTRIES_REQUEST,
]),
storageId: options.mapId,
});
const storage = new HamokMap_1.HamokMap(connection, options.baseMap ?? new BaseMap_1.MemoryBaseMap(), options.equalValues);
storage.once('close', () => {
this.storages.delete(storage.id);
});
this.storages.set(storage.id, storage);
return storage;
}
getOrCreateMap(options, callback) {
const storage = this.storages.get(options.mapId);
if (!storage)
return this.createMap(options);
callback?.(true);
return storage;
}
createRemoteMap(options) {
if (this._closed)
throw new Error('Cannot create remote map on a closed Hamok instance');
if (this.storages.has(options.mapId))
throw new Error(`Remote map with id ${options.mapId} already exists`);
const connection = this._createStorageConnection({
...options,
keyCodec: options.keyCodec ?? (0, HamokCodec_1.createHamokJsonBinaryCodec)(),
valueCodec: options.valueCodec ?? (0, HamokCodec_1.createHamokJsonBinaryCodec)(),
submitting: new Set([
HamokMessage_1.HamokMessage_MessageType.CLEAR_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.INSERT_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.DELETE_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.REMOVE_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.UPDATE_ENTRIES_REQUEST,
]),
storageId: options.mapId,
});
const storage = new HamokRemoteMap_1.HamokRemoteMap(connection, options.remoteMap, options.equalValues);
storage.once('close', () => {
this.storages.delete(storage.id);
});
this.storages.set(storage.id, storage);
return storage;
}
getOrCreateRemoteMap(options, callback) {
const storage = this.storages.get(options.mapId);
if (!storage)
return this.createRemoteMap(options);
callback?.(true);
return storage;
}
createRecord(options) {
if (this._closed)
throw new Error('Cannot create record on a closed Hamok instance');
if (this.storages.has(options.recordId))
throw new Error(`Record with id ${options.recordId} already exists`);
const connection = this._createStorageConnection({
...options,
submitting: new Set([
HamokMessage_1.HamokMessage_MessageType.CLEAR_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.INSERT_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.DELETE_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.REMOVE_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.UPDATE_ENTRIES_REQUEST,
]),
storageId: options.recordId,
keyCodec: (0, HamokCodec_1.createStrToUint8ArrayCodec)(),
valueCodec: (0, HamokCodec_1.createStrToUint8ArrayCodec)(),
});
const storage = new HamokRecord_1.HamokRecord(connection, {
equalValues: options.equalValues,
payloadsCodec: options.payloadCodecs,
initalObject: options.initialObject,
});
storage.once('close', () => {
this.storages.delete(storage.id);
});
this.storages.set(storage.id, storage);
return storage;
}
getOrCreateRecord(options, callback) {
const storage = this.storages.get(options.recordId);
if (!storage)
return this.createRecord(options);
callback?.(true);
return storage;
}
createQueue(options) {
if (this._closed)
throw new Error('Cannot create queue on a closed Hamok instance');
const connection = this._createStorageConnection({
...options,
keyCodec: (0, HamokCodec_1.createHamokJsonBinaryCodec)(),
valueCodec: options.codec ?? (0, HamokCodec_1.createHamokJsonBinaryCodec)(),
submitting: new Set([
HamokMessage_1.HamokMessage_MessageType.CLEAR_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.INSERT_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.REMOVE_ENTRIES_REQUEST,
]),
storageId: options.queueId,
});
const storage = new HamokQueue_1.HamokQueue(connection, options.baseMap ?? new BaseMap_1.MemoryBaseMap());
storage.once('close', () => {
this.storages.delete(storage.id);
});
this.storages.set(storage.id, storage);
return storage;
}
getOrCreateQueue(options, callback) {
const storage = this.storages.get(options.queueId);
if (!storage)
return this.createQueue(options);
callback?.(true);
return storage;
}
createEmitter(options) {
if (this._closed)
throw new Error('Cannot create emitter on a closed Hamok instance');
const connection = this._createStorageConnection({
...options,
submitting: new Set([
HamokMessage_1.HamokMessage_MessageType.CLEAR_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.INSERT_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.DELETE_ENTRIES_REQUEST,
HamokMessage_1.HamokMessage_MessageType.REMOVE_ENTRIES_REQUEST,
]),
storageId: options.emitterId,
keyCodec: (0, HamokCodec_1.createStrToUint8ArrayCodec)(),
valueCodec: (0, HamokCodec_1.createStrToUint8ArrayCodec)(),
});
const storage = new HamokEmitter_1.HamokEmitter(connection, options.payloadsCodec, options.autoClean);
connection.once('close', () => {
this.storages.delete(storage.id);
});
this.storages.set(storage.id, storage);
return storage;
}
getOrCreateEmitter(options, callback) {
const storage = this.storages.get(options.emitterId);
if (!storage)
return this.createEmitter(options);
callback?.(true);
return storage;
}
async submit(entry) {
if (this._closed)
throw new Error('Cannot submit on a closed Hamok instance');
if (!this.raft.leaderId) {
const error = new Error(`No leader is elected, cannot submit message type ${entry.type}`);
if (!this.emit('error', error))
throw error;
return;
}
entry.sourceId = this.localPeerId;
if (this.leader) {
const success = this.raft.submit(entry);
if (success && entry.requestId && entry.storageId && entry.sourceId) {
// we add the request to the ongoing requests set to prevent timeout at the follower side
// when the leader is processing the request until it commits the log entry
this.grid.ongoingRequestsNotifier.add({
remotePeerId: entry.sourceId,
requestId: entry.requestId,
storageId: entry.storageId,
});
logger.trace('%s Request %s is added to ongoing requests set', this.localPeerId, entry.requestId);
}
return;
}
const request = new SubmitMessage_1.SubmitMessageRequest((0, uuid_1.v4)(), this.localPeerId, entry, this.raft.leaderId);
const message = this._codec.encodeSubmitMessageRequest(request);
message.protocol = HamokMessage_1.HamokMessage_MessageProtocol.GRID_COMMUNICATION_PROTOCOL;
const response = (await this.grid.request({
message,
neededResponses: 1,
targetPeerIds: [this.raft.leaderId],
timeoutInMs: 5000,
})).map((msg) => this._codec.decodeSubmitMessageResponse(msg))?.[0];
if (response?.success === false &&
response.leaderId &&
response.leaderId !== request.destinationEndpointId &&
response.leaderId === this.raft.leaderId) {
// the leader changed meanwhuile the submit request was sent
logger.debug('Leader changed from %s to %s, submit will be resend to the new leader', this.raft.leaderId, response.leaderId);
return this.submit(message);
}
if (!response?.success) {
throw new Error('Failed to submit message');
}
}
accept(message, commitIndex) {
if (this._closed)
return;
if (message.destinationId && message.destinationId !== this.localPeerId) {
return logger.trace('%s Received message address is not matching with the local peer %o', this.localPeerId, message);
}
// if (message.protocol !== HamokMessageProtocol.RAFT_COMMUNICATION_PROTOCOL)
logger.trace('%s received message %o', this.localPeerId, message);
switch (message.type) {
case HamokMessage_1.HamokMessage_MessageType.GET_ENTRIES_RESPONSE:
case HamokMessage_1.HamokMessage_MessageType.GET_KEYS_RESPONSE:
case HamokMessage_1.HamokMessage_MessageType.GET_SIZE_RESPONSE:
case HamokMessage_1.HamokMessage_MessageType.CLEAR_ENTRIES_RESPONSE:
case HamokMessage_1.HamokMessage_MessageType.DELETE_ENTRIES_RESPONSE:
case HamokMessage_1.HamokMessage_MessageType.REMOVE_ENTRIES_RESPONSE:
case HamokMessage_1.HamokMessage_MessageType.INSERT_ENTRIES_RESPONSE:
case HamokMessage_1.HamokMessage_MessageType.UPDATE_ENTRIES_RESPONSE:
case HamokMessage_1.HamokMessage_MessageType.SUBMIT_MESSAGE_RESPONSE:
// raft relatd messages are different animal
// case HamokMessageType.RAFT_APPEND_ENTRIES_RESPONSE
// case HamokMessageType.
return this.grid.processResponse(message);
}
switch (message.protocol) {
case HamokMessage_1.HamokMessage_MessageProtocol.RAFT_COMMUNICATION_PROTOCOL:
switch (message.type) {
case HamokMessage_1.HamokMessage_MessageType.RAFT_APPEND_ENTRIES_REQUEST_CHUNK:
case HamokMessage_1.HamokMessage_MessageType.RAFT_APPEND_ENTRIES_RESPONSE:
case HamokMessage_1.HamokMessage_MessageType.RAFT_VOTE_REQUEST:
case HamokMessage_1.HamokMessage_MessageType.RAFT_VOTE_RESPONSE:
this._acceptKeepAliveHamokMessage(message);
break;
}
this.raft.transport.receive(message);
break;
// case undefined:
case HamokMessage_1.HamokMessage_MessageProtocol.GRID_COMMUNICATION_PROTOCOL:
this._acceptGridMessage(message);
break;
case HamokMessage_1.HamokMessage_MessageProtocol.STORAGE_COMMUNICATION_PROTOCOL: {
const storage = this.storages.get(message.storageId ?? '');
if (!storage) {
if (message.type === HamokMessage_1.HamokMessage_MessageType.STORAGE_HELLO_NOTIFICATION) {
// we reply to this in any case
return (this._emitMessage(new HamokMessage_1.HamokMessage({
protocol: HamokMessage_1.HamokMessage_MessageProtocol.STORAGE_COMMUNICATION_PROTOCOL,
type: HamokMessage_1.HamokMessage_MessageType.STORAGE_STATE_NOTIFICATION,
sourceId: this.localPeerId,
destinationId: message.sourceId,
storageId: message.storageId,
raftCommitIndex: -1,
})), void 0);
}
return logger.trace('Received message for unknown collection %s', message.storageId);
}
return storage.connection.accept(message, commitIndex);
}
// case HamokMessageProtocol.PUBSUB_COMMUNICATION_PROTOCOL:
// this._dispatchToPubSub(message);
default:
return logger.warn('%s Unknown protocol %s, message: %o', this.localPeerId, message.protocol, message);
}
}
async leave() {
if (this._closed)
throw new Error('Cannot leave the network on a closed hamok');
if (!this._run)
return;
logger.debug('%s Leaving the network', this.localPeerId);
try {
if (this._joining) {
this._joining.aborted = true;
await this._joining.promise.catch((err) => logger.warn(`${err}`));
}
this._run = false;
this._stopRaftEngine();
[...this.remotePeerIds].forEach((peerId) => this.removeRemotePeerId(peerId));
this.emit('left');
logger.info('%s Left the network', this.localPeerId);
}
finally {
this._run = false;
}
}
async join(params) {
if (this._closed)
throw new Error('Cannot execute join on a closed hamok');
if (this._joining)
return this._joining.promise;
if (this.raft.leaderId !== undefined)
return logger.warn('Already joined the network as %s', this.localPeerId);
try {
if (this._run) {
this.emit('rejoining');
}
this._run = true;
// let's synchronize the remote peers and the no heartbeat peers
this._remoteHeartbeats.clear();
this._joining = {
promise: this._join({
fetchRemotePeerTimeoutInMs: params?.fetchRemotePeerTimeoutInMs ?? 5000,
maxRetry: params?.maxRetry ?? 3,
}),
aborted: false,
};
await this._joining.promise;
this.emit('joined');
}
finally {
this._joining = undefined;
}
}
async _join(params, retried = 0) {
if (this._closed)
throw new Error('Cannot join the network on a closed hamok');
const { fetchRemotePeerTimeoutInMs, maxRetry, } = params ?? {};
if (this._joining?.aborted || this._closed) {
return Promise.reject(new Error('Joining process is aborted or the hamok is closed'));
}
// this will start a heartbeat timer, which will start sending join messages
if (!this._raftTimer)
this._startRaftEngine();
try {
await this.waitUntilLeader(fetchRemotePeerTimeoutInMs);
if (this._joining?.aborted || this._closed) {
return Promise.reject(new Error('Joining process is aborted or the hamok is closed'));
}
}
catch (err) {
if (this._joining?.aborted || this._closed) {
return Promise.reject(new Error('Joining process is aborted or the hamok is closed'));
}
else if (0 < maxRetry && maxRetry <= retried) {
throw err;
}
else if (0 < this.raft.remotePeers.size) {
logger.debug('%s Failed to join the network, but we have remote peers, so we will continue', this.localPeerId);
return this._join(params, retried);
}
logger.warn('%s No remote peers found, retrying %s/%s', this.localPeerId, retried, maxRetry < 0 ? '∞' : maxRetry);
return this._join(params, retried + 1);
}
}
_startRaftEngine() {
if (this._raftTimer) {
return logger.debug('Hamok is already running');
}
const raftEngine = this.raft;
raftEngine.transport.on('message', this._emitMessage);
raftEngine.state = (0, RaftFollowerState_1.createRaftFollowerState)({
raftEngine,
});
this._raftTimer = setInterval(() => {
raftEngine.state.run();
this.emit('heartbeat');
}, raftEngine.config.heartbeatInMs);
}
_stopRaftEngine() {
if (!this._raftTimer) {
return;
}
const raftEngine = this.raft;
logger.debug('%s Stopping the raft engine', this.localPeerId);
clearInterval(this._raftTimer);
this._raftTimer = undefined;
raftEngine.state = (0, RaftEmptyState_1.createRaftEmptyState)({
raftEngine,
});
raftEngine.transport.off('message', this._emitMessage);
this._remoteHeartbeats.forEach((timer) => clearTimeout(timer));
this._remoteHeartbeats.clear();
}
_sendEndpointNotification(destinationIdPeerId, snapshot, requestId) {
const notification = new EndpointNotification_1.EndpointStatesNotification(this.localPeerId, destinationIdPeerId, this.raft.props.currentTerm, this.raft.logs.commitIndex, this.leader ? this.raft.logs.nextIndex : -1, this.raft.logs.size, this.raft.remotePeers, snapshot, requestId);
logger.trace('%s Sending endpoint state notification %o, snapshot: %o', this.localPeerId, notification, snapshot);
const message = this._codec.encodeEndpointStateNotification(notification);
this._emitMessage(message, destinationIdPeerId);
}
_acceptGridMessage(message) {
if (this._closed)
return;
switch (message.type) {
case HamokMessage_1.HamokMessage_MessageType.HELLO_NOTIFICATION: {
logger.debug('%s Received hello notification from %s', this.localPeerId, message.sourceId);
break;
}
case HamokMessage_1.HamokMessage_MessageType.JOIN_NOTIFICATION: {
const notification = this._codec.decodeJoinNotification(message);
if (notification.sourcePeerId === this.localPeerId) {
logger.trace('%s Received join notification from itself %o', this.localPeerId, notification);
break;
}
if (this.raft.remotePeers.has(notification.sourcePeerId)) {
logger.trace('%s Received join notification from %s, but it is already in the remote peers', this.localPeerId, notification.sourcePeerId);
break;
}
this.addRemotePeerId(notification.sourcePeerId);
if (this.raft.leaderId === this.localPeerId) {
this._sendEndpointNotification(notification.sourcePeerId, undefined);
}
break;
}
case HamokMessage_1.HamokMessage_MessageType.ENDPOINT_STATES_NOTIFICATION: {
const endpointStateNotification = this._codec.decodeEndpointStateNotification(message);
if (endpointStateNotification.sourceEndpointId === this.localPeerId) {
return logger.trace('%s Received endpoint state notification from itself %o', this.localPeerId, endpointStateNotification);
}
logger.debug('%s Received endpoint state notification %o, activeEndpointIds: %s', this.localPeerId, endpointStateNotification, [...(endpointStateNotification.activeEndpointIds ?? [])].join(', '));
for (const peerId of this.remotePeerIds) {
logger.trace('%s Remote peer %s is in the active endpoints', this.localPeerId, peerId);
if (endpointStateNotification.activeEndpointIds?.has(peerId))
continue;
if (endpointStateNotification.sourceEndpointId === peerId)
continue;
logger.debug('%s Received endpoint state notification from %s (supposed to be the leader), and in that it does not have %s in its active endpoints, therefore we need to remove it', this.localPeerId, endpointStateNotification.sourceEndpointId, peerId);
this.removeRemotePeerId(peerId);
}
let foundLocalPeerId = false;
for (const peerId of endpointStateNotification.activeEndpointIds ?? []) {
if (this.remotePeerIds.has(peerId))
continue;
if (peerId === this.localPeerId) {
foundLocalPeerId = true;
continue;
}
logger.debug('%s Received endpoint state notification from %s (supposed to be the leader), and in that it has %s in its active endpoints, therefore we need to add it', this.localPeerId, endpointStateNotification.sourceEndpointId, peerId);
this.addRemotePeerId(peerId);
}
if (!this.remotePeerIds.has(endpointStateNotification.sourceEndpointId)) {
this.addRemotePeerId(endpointStateNotification.sourceEndpointId);
}
if (!foundLocalPeerId) {
// we need to add the local peer id to the active endpoints of the leader
const joinMsg = this._codec.encodeJoinNotification(new JoinNotification_1.JoinNotification(this.localPeerId, endpointStateNotification.sourceEndpointId));
this._emitMessage(joinMsg, endpointStateNotification.sourceEndpointId);
break;
}
// we add 2 becasue the nextIndex of the leader has not been reserved, and
const possibleLowestIndex = endpointStateNotification.leaderNextIndex - endpointStateNotification.numberOfLogs + 2;
if (0 < endpointStateNotification.commitIndex &&
this.raft.logs.nextIndex < endpointStateNotification.leaderNextIndex &&
this.raft.logs.firstIndex < possibleLowestIndex) {
// we make a warn message only if it is not the first join
logger.warn('%s Commit index of this peer (%d) is lower than the smallest commit index (%d) from remote peers resetting the logs', this.localPeerId, this.raft.logs.commitIndex, possibleLowestIndex);
this.raft.logs.reset(possibleLowestIndex);
}
break;
}
case HamokMessage_1.HamokMessage_MessageType.ONGOING_REQUESTS_NOTIFICATION: {
const ongoingRequestNotification = this._codec.decodeOngoingRequestsNotification(message);
for (const requestId of ongoingRequestNotification.requestIds) {
const pendingRequest = this.grid.pendingRequests.get(requestId);
if (!pendingRequest) {
logger.warn('%s Received ongoing request notification for unknown request %s', this.localPeerId, requestId);
continue;
}
pendingRequest.postponeTimeout();
}
break;
}
case HamokMessage_1.HamokMessage_MessageType.SUBMIT_MESSAGE_REQUEST: {
const request = this._codec.decodeSubmitMessageRequest(message);
const entry = request.entry;
const success = this.leader ? this.raft.submit(request.entry) : false;
const response = request.createResponse(success, this.raft.leaderId);
logger.trace('%s Received submit message request %o. success: %s, leader: %s leaderId: %s, raftState: %s', this.localPeerId, request, success, this.leader, this.raft.leaderId, this.raft.state.stateName);
if (success && this.leader && entry.requestId && entry.storageId && entry.sourceId) {
// we add the request to the ongoing requests set to prevent timeout at the follower side
// when the leader is processing the request until it commits the log entry
this.grid.ongoingRequestsNotifier.add({
remotePeerId: entry.sourceId,
requestId: entry.requestId,
storageId: entry.storageId,
});
logger.trace('%s Request %s is added to ongoing requests set', this.localPeerId, entry.requestId);
}
this.grid.sendMessage(this._codec.encodeSubmitMessageResponse(response), message.sourceId);
break;
}
}
}
broadcastEndpointNotification() {
for (const remotePeerId of this.remotePeerIds) {
this._sendEndpointNotification(remotePeerId);
}
}
_emitMessage(message, destinationPeerIds, protocol) {
message.sourceId = this.localPeerId;
if (protocol) {
message.protocol = protocol;
}
// if (message.type === HamokMessageType.ENTRY_UPDATED_NOTIFICATION)
// logger.warn('%s sending message %o', this.localPeerId, message);
if (!destinationPeerIds) {
return this.emit('message', message);
}
let remotePeers;
if (typeof destinationPeerIds === 'string') {
remotePeers = new Set([destinationPeerIds]);
}
else if (Array.isArray(destinationPeerIds)) {
remotePeers = new Set(destinationPeerIds);
}
else {
remotePeers = destinationPeerIds;
}
if (remotePeers.size < 1) {
return logger.warn('Empty set of destination has been provided for request %o', message);
}
if (remotePeers.size === 1) {
return [...remotePeers].forEach((destinationId) => {
message.destinationId = destinationId;
if (message.destinationId === this.localPeerId) {
return this.accept(message);
}
this.emit('message', message);
});
}
for (const destinationId of remotePeers) {
if (destinationId === this.localPeerId) {
return this.accept(message);
}
this.emit('message', new HamokMessage_1.HamokMessage({
...message,
destinationId
}));
}
}
_createStorageConnection(options) {
const storageCodec = new StorageCodec_1.StorageCodec(options.keyCodec ?? (0, HamokCodec_1.createHamokJsonBinaryCodec)(), options.valueCodec ?? (0, HamokCodec_1.createHamokJsonBinaryCodec)());
const connection = new HamokConnection_1.HamokConnection({
requestTimeoutInMs: options.requestTimeoutInMs ?? 5000,
storageId: options.storageId,
neededResponse: 0,
maxOutboundKeys: options.maxOutboundKeys ?? 0,
maxOutboundValues: options.maxOutboundKeys ?? 0,
remoteStorageStateWaitingTimeoutInMs: options.remoteStorageStateWaitingTimeoutInMs ?? 1000,
submitting: options.submitting,
}, storageCodec, this.grid, this.waitUntilCommitHead.bind(this));
const messageListener = (message, submitting) => {
if (submitting)
return this.submit(message);
else
this.emit('message', message);
};
connection.once('close', () => {
connection.off('message', messageListener);
});
connection.on('message', messageListener);
return connection;
}
_acceptKeepAliveHamokMessage(message) {
if (this.raft.leaderId !== this.raft.localPeerId)
return;
if (!message.sourceId || message.sourceId === this.localPeerId)
return;
const remotePeerId = message.sourceId;
this._remoteHeartbeats.set(remotePeerId, Date.now());
// this._addNoHeartbeatTimer(remotePeerId);
}
_checkRemoteHeartbeats() {
if (!this.leader) {
return;
}
else if (this._joining) {
// we don't do this when we are joining. essentially we should not have anything while we are joining
return;
}
const now = Date.now();
for (const [remotePeerId, lastUpdate] of this._remoteHeartbeats) {
if (now - lastUpdate < this.raft.config.electionTimeoutInMs)
continue;
if (!this.raft.remotePeers.has(remotePeerId)) {
this._remoteHeartbeats.delete(remotePeerId);
logger.debug('%s No heartbeat from %s, but it is not registered in remotePeers', this.localPeerId, remotePeerId);
continue;
}
logger.info('%s No heartbeat from %s', this.localPeerId, remotePeerId);
this.emit('no-heartbeat-from', remotePeerId);
this.removeRemotePeerId(remotePeerId);
}
}
// we want to delete this function as we will send join messages if there is no leader
// private _acceptStateChange(newState: RaftStateName, prevState: RaftStateName): void {
// if (prevState === 'follower' && newState === 'candidate') {
// const joinMsg = new JoinNotification(this.localPeerId);
// this._emitMessage(this._codec.encodeJoinNotification(joinMsg));
// }
// }
_sendJoinMsg() {
// if there is no remote peers we want to join to the grid
if (this.raft.remotePeers.size < 1) {
const message = this._codec.encodeJoinNotification(new JoinNotification_1.JoinNotification(this.localPeerId));
return this._emitMessage(message);
}
// if we have remote peers, but we do not have a leader, then
// we need to keep send the join notification
if (this.raft.leaderId === undefined) {
const message = this._codec.encodeJoinNotification(new JoinNotification_1.JoinNotification(this.localPeerId));
return this._emitMessage(message);
}
}
}
exports.Hamok = Hamok;