hamok
Version:
Lightweight Distributed Object Storage on RAFT consensus algorithm
394 lines (393 loc) • 18 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.HamokRemoteMap = void 0;
const events_1 = require("events");
const logger_1 = require("../common/logger");
const Collections = __importStar(require("../common/Collections"));
const ConcurrentExecutor_1 = require("../common/ConcurrentExecutor");
const logger = (0, logger_1.createLogger)('HamokRemoteMap');
/**
* A remote map is a map that is stored on a remote endpoint and magaged by Hamok instances
*/
// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
class HamokRemoteMap extends events_1.EventEmitter {
connection;
remoteMap;
_closed = false;
equalValues;
_executor = new ConcurrentExecutor_1.ConcurrentExecutor(1);
/**
* Flag indicate if the storage emit events and notify other storage to emit events (if this is the leader)
*/
emitEvents = true;
_initializing;
/**
* The last commit index that was applied to the map
*/
_appliedCommitIndex = -1;
/**
* Whether this endpoint is the leader
*/
_leader = false;
/**
*
* @param supplier the supplied action has to be executed if this endpoint is the leader
* @param commitIndex the commit index that the action is associated with
* @param onCompleted callback if this endpoint is the leader and the action is executed
* @returns
*/
async _executeIfLeader(supplier, commitIndex, onCompleted) {
if (this._closed)
throw new Error(`Cannot execute on a closed storage (${this.id})`);
logger.trace('Executing action for %s, appliedCommitIndex: %d, commitIndex: %d', this.id, this._appliedCommitIndex, commitIndex);
if (commitIndex === undefined) {
return logger.warn('Cannot execute action in storage %s becasue the provided commit index undefined', this.id);
}
else if (!this._leader) {
return logger.trace('Not the leader for %s', this.id);
}
logger.trace('Executing action for %s, commitIndex: %d', this.id, commitIndex);
try {
const input = await this._executor.execute(supplier);
if (commitIndex <= this._appliedCommitIndex) {
logger.warn('Commit index is less than the applied commit index for %s. appliedCommitIndex: %d, commitIndex: %d', this.id, this._appliedCommitIndex, commitIndex);
}
this._appliedCommitIndex = commitIndex;
onCompleted?.(input);
}
catch (err) {
logger.error('Error executing on %s', this.id, err);
}
}
constructor(connection, remoteMap, equalValues) {
super();
this.connection = connection;
this.remoteMap = remoteMap;
this.setMaxListeners(Infinity);
this.equalValues = equalValues ?? ((a, b) => {
// logger.info('Comparing values: %o (%s), %o (%s)', a, b, JSON.stringify(a), JSON.stringify(b));
return JSON.stringify(a) === JSON.stringify(b);
});
this.connection
// StorageAppliedCommitNotification is deprecated in favor of StorageState which contains the applied commit index
// .on('StorageAppliedCommitNotification', (notification) => {
// if (!this._leader) {
// this._appliedCommitIndex = notification.appliedCommitIndex;
// }
// })
.on('ClearEntriesRequest', (request, commitIndex) => {
this._executeIfLeader(() => this.remoteMap.clear(), commitIndex, () => {
this.connection.respond('ClearEntriesResponse', request.createResponse(), request.sourceEndpointId);
if (this.emitEvents) {
this.connection.notifyClearEntries(this.connection.grid.remotePeerIds);
this.emit('clear');
}
});
})
.on('ClearEntriesNotification', () => this.emit('clear'))
.on('DeleteEntriesRequest', (request, commitIndex) => {
this._executeIfLeader(() => this.remoteMap.removeAll(request.keys.values()), commitIndex, (removedEntries) => {
this.connection.respond('DeleteEntriesResponse', request.createResponse(new Set(removedEntries.keys())), request.sourceEndpointId);
if (this.emitEvents) {
this.connection.notifyEntriesRemoved(removedEntries, this.connection.grid.remotePeerIds);
removedEntries.forEach((v, k) => this.emit('remove', k, v));
}
});
})
.on('InsertEntriesRequest', (request, commitIndex) => {
this._executeIfLeader(async () => {
const existingEntries = await this.remoteMap.getAll(request.entries.keys());
const insertedEntries = new Map();
for (const [key, value] of request.entries) {
if (existingEntries.has(key)) {
continue;
}
insertedEntries.set(key, value);
}
await this.remoteMap.setAll(request.entries, () => void 0);
return insertedEntries;
}, commitIndex, (insertedEntries) => {
this.connection.respond('InsertEntriesResponse', request.createResponse(new Map()), request.sourceEndpointId);
if (this.emitEvents) {
this.connection.notifyEntriesInserted(insertedEntries, this.connection.grid.remotePeerIds);
insertedEntries.forEach((value, key) => this.emit('insert', key, value));
}
});
})
.on('EntriesInsertedNotification', (insertedEntries) => insertedEntries.entries.forEach((v, k) => this.emit('insert', k, v)))
.on('RemoveEntriesRequest', (request, commitIndex) => {
this._executeIfLeader(() => this.remoteMap.removeAll(request.keys.values()), commitIndex, (removedEntries) => {
this.connection.respond('RemoveEntriesResponse', request.createResponse(removedEntries), request.sourceEndpointId);
if (this.emitEvents) {
this.connection.notifyEntriesRemoved(removedEntries, this.connection.grid.remotePeerIds);
removedEntries.forEach((v, k) => this.emit('remove', k, v));
}
});
})
.on('EntriesRemovedNotification', (removedEntries) => {
removedEntries.entries.forEach((v, k) => this.emit('remove', k, v));
})
.on('UpdateEntriesRequest', (request, commitIndex) => {
// logger.warn('Accepting UpdateEntriesRequest %s, commitIndex: %d', request.requestId, commitIndex);
this._executeIfLeader(async () => {
// logger.warn('%s UpdateEntriesRequest: %o, %s, prevValue: %o', this.connection.grid.localPeerId, request, [ ...request.entries ].join(', '), request.prevValue);
const updatedEntries = [];
const insertedEntries = [];
if (request.prevValue !== undefined) {
// this is a conditional update
if (request.entries.size !== 1) {
// we let the request to timeout
logger.trace('Conditional update request must have only one entry: %o', request);
return { insertedEntries, updatedEntries };
}
const [key, value] = [...request.entries][0];
const existingValue = await this.remoteMap.get(key);
// logger.warn('Conditional update request: %s, %s, %s, %s', key, value, existingValue, request.prevValue);
if (existingValue && this.equalValues(existingValue, request.prevValue)) {
await this.remoteMap.set(key, value, () => void 0);
updatedEntries.push([key, existingValue, value]);
// logger.warn('Conditional update request: %s, %s, %s, %s', key, value, existingValue, request.prevValue);
}
}
else {
await this.remoteMap.setAll(request.entries, ({ inserted, updated }) => {
insertedEntries.push(...inserted);
updatedEntries.push(...updated);
});
}
return { insertedEntries, updatedEntries };
}, commitIndex, ({ insertedEntries, updatedEntries }) => {
this.connection.respond('UpdateEntriesResponse', request.createResponse(new Map(updatedEntries.map(([key, oldValue]) => [key, oldValue]))), request.sourceEndpointId);
if (this.emitEvents) {
logger.trace('Emitting events for %s, insertedEntries: %s, updatedEntries: %s', this.connection.grid.localPeerId, insertedEntries.map(([key, value]) => `${key}=${value}`).join(', '), updatedEntries.map(([key, oldValue, newValue]) => `${key}=${JSON.stringify(oldValue)}=>${JSON.stringify(newValue)}`).join(', '));
insertedEntries.forEach(([key, value]) => this.emit('insert', key, value));
this.connection.notifyEntriesInserted(new Map(insertedEntries), this.connection.grid.remotePeerIds);
updatedEntries.forEach(([key, oldValue, newValue]) => {
this.emit('update', key, oldValue, newValue);
this.connection.notifyEntryUpdated(key, oldValue, newValue, this.connection.grid.remotePeerIds);
});
}
});
})
.on('EntryUpdatedNotification', ({ key, newValue, oldValue }) => {
this.emit('update', key, oldValue, newValue);
})
.on('leader-changed', (leaderId) => {
// this is all we need to know
this._leader = leaderId === this.connection.grid.localPeerId;
})
.on('StorageHelloNotification', (notification) => {
// we only reply to it if this endpoint is the leader
// if there is no leader in the grid the fetch on the remote endpoint should retry
if (!this._leader)
return;
try {
const snapshot = this.export();
const serializedSnapshot = JSON.stringify(snapshot);
this.connection.notifyStorageState(serializedSnapshot,
// and this is the trick here since this storage is async.
// the connection has it's own pace to emit commits, but we execute it async, so we need to know
// where this storage is at the moment.
this._appliedCommitIndex, notification.sourceEndpointId);
}
catch (err) {
logger.error('Failed to send snapshot', err);
}
})
.on('remote-snapshot', (serializedSnapshot, done) => {
try {
const snapshot = JSON.parse(serializedSnapshot);
this._import(snapshot);
}
catch (err) {
logger.error(`Failed to import to record ${this.id}. Error: ${err}`);
}
finally {
done();
}
})
.once('close', () => this.close());
this._initializing = new Promise((resolve) => setTimeout(resolve, 20))
.then(() => this.connection.join())
.then(() => this)
.catch((err) => {
logger.error('Error while initializing remote map', err);
return this;
})
.finally(() => (this._initializing = undefined));
}
get id() {
return this.connection.config.storageId;
}
get ready() {
return this._initializing ?? this.connection.grid.waitUntilCommitHead().then(() => this);
}
get closed() {
return this._closed;
}
close() {
if (this._closed)
return;
this._closed = true;
this.connection.close();
if (this.emitEvents) {
this.emit('close');
}
this.removeAllListeners();
}
size() {
return this.remoteMap.size();
}
async isEmpty() {
return await this.remoteMap.size() === 0;
}
keys() {
return this.remoteMap.keys();
}
async clear() {
if (this._closed)
throw new Error(`Cannot clear a closed storage (${this.id})`);
return this.connection.requestClearEntries();
}
async get(key) {
if (this._closed)
throw new Error(`Cannot get entries from a closed storage (${this.id})`);
return this.remoteMap.get(key);
}
async getAll(keys) {
if (this._closed)
throw new Error(`Cannot get entries from a closed storage (${this.id})`);
if (Array.isArray(keys))
return this.remoteMap.getAll(keys.values());
else
return this.remoteMap.getAll(keys);
}
async set(key, value) {
if (this._closed)
throw new Error(`Cannot set an entry on a closed storage (${this.id})`);
const result = await this.setAll(Collections.mapOf([key, value]));
return result.get(key);
}
async setAll(entries) {
if (this._closed)
throw new Error(`Cannot set entries on a closed storage (${this.id})`);
if (entries.size < 1) {
return Collections.emptyMap();
}
return this.connection.requestUpdateEntries(entries);
}
async insert(key, value) {
const result = await this.insertAll(Collections.mapOf([key, value]));
return result.get(key);
}
async insertAll(entries) {
if (this._closed)
throw new Error(`Cannot insert entries on a closed storage (${this.id})`);
if (Array.isArray(entries)) {
if (entries.length < 1)
return Collections.emptyMap();
entries = Collections.mapOf(...entries);
}
if (entries.size < 1) {
return Collections.emptyMap();
}
return this.connection.requestInsertEntries(entries);
}
async delete(key) {
const result = await this.deleteAll(Collections.setOf(key));
return result.has(key);
}
async deleteAll(keys) {
if (this._closed)
throw new Error(`Cannot delete entries on a closed storage (${this.id})`);
if (Array.isArray(keys)) {
if (keys.length < 1)
return Collections.emptySet();
keys = Collections.setOf(...keys);
}
if (keys.size < 1) {
return Collections.emptySet();
}
return this.connection.requestDeleteEntries(keys);
}
async remove(key) {
const result = await this.removeAll(Collections.setOf(key));
return result.has(key);
}
async removeAll(keys) {
if (this._closed)
throw new Error(`Cannot remove entries on a closed storage (${this.id})`);
if (Array.isArray(keys)) {
if (keys.length < 1)
return Collections.emptyMap();
keys = Collections.setOf(...keys);
}
if (keys.size < 1) {
return Collections.emptyMap();
}
return this.connection.requestRemoveEntries(keys);
}
async updateIf(key, value, oldValue) {
if (this._closed)
throw new Error(`Cannot update an entry on a closed storage (${this.id})`);
logger.trace('%s UpdateIf: %s, %s, %s', this.connection.grid.localPeerId, key, value, oldValue);
return (await this.connection.requestUpdateEntries(Collections.mapOf([key, value]), undefined, oldValue)).get(key) !== undefined;
}
iterator() {
return this.remoteMap.iterator();
}
/**
* Exports the storage data
*/
export() {
if (this._closed) {
throw new Error(`Cannot export data on a closed storage (${this.id})`);
}
const result = {
mapId: this.id,
appliedCommitIndex: this._appliedCommitIndex,
};
return result;
}
import(data) {
if (data.mapId !== this.id) {
throw new Error(`Cannot import data from a different storage: ${data.mapId} !== ${this.id}`);
}
else if (this.connection.connected) {
throw new Error('Cannot import data while connected');
}
else if (this._closed) {
throw new Error(`Cannot import data on a closed storage (${this.id})`);
}
}
_import(snapshot) {
if (this._closed) {
throw new Error(`Cannot import data on a closed storage (${this.id})`);
}
this._appliedCommitIndex = snapshot.appliedCommitIndex;
}
}
exports.HamokRemoteMap = HamokRemoteMap;