hamok
Version:
Lightweight Distributed Object Storage on RAFT consensus algorithm
354 lines (353 loc) • 15.4 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.HamokMap = void 0;
const events_1 = require("events");
const logger_1 = require("../common/logger");
const Collections = __importStar(require("../common/Collections"));
const HamokCodec_1 = require("../common/HamokCodec");
const logger = (0, logger_1.createLogger)('HamokMap');
/**
* Replicated storage replicates all entries on all distributed storages
*/
// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
class HamokMap extends events_1.EventEmitter {
connection;
baseMap;
_closed = false;
equalValues;
_initializing;
constructor(connection, baseMap, equalValues) {
super();
this.connection = connection;
this.baseMap = baseMap;
this.setMaxListeners(Infinity);
this.equalValues = equalValues ?? ((a, b) => {
// logger.info('Comparing values: %o (%s), %o (%s)', a, b, JSON.stringify(a), JSON.stringify(b));
return JSON.stringify(a) === JSON.stringify(b);
});
this.connection
.on('ClearEntriesRequest', (request) => {
this.baseMap.clear();
if (request.sourceEndpointId === this.connection.grid.localPeerId) {
this.connection.respond('ClearEntriesResponse', request.createResponse(), request.sourceEndpointId);
}
this.emit('clear');
})
.on('DeleteEntriesRequest', (request) => {
const removedEntries = this.baseMap.removeAll(request.keys.values());
if (request.sourceEndpointId === this.connection.grid.localPeerId) {
this.connection.respond('DeleteEntriesResponse', request.createResponse(new Set(removedEntries.keys())), request.sourceEndpointId);
}
removedEntries.forEach((v, k) => this.emit('remove', k, v));
})
.on('GetEntriesRequest', (request) => {
// only requested by the sync process when the storage enters to the grid
const foundEntries = this.baseMap.getAll(request.keys.values());
this.connection.respond('GetEntriesResponse', request.createResponse(foundEntries), request.sourceEndpointId);
})
.on('InsertEntriesRequest', (request) => {
logger.debug('%s InsertEntriesRequest: %o, %s', this.connection.grid.localPeerId, request, [...request.entries].join(', '));
const existingEntries = this.baseMap.insertAll(request.entries);
if (request.sourceEndpointId === this.connection.grid.localPeerId) {
this.connection.respond('InsertEntriesResponse', request.createResponse(existingEntries), request.sourceEndpointId);
}
request.entries.forEach((v, k) => existingEntries.has(k) || this.emit('insert', k, v));
})
.on('RemoveEntriesRequest', (request) => {
if (request.prevValue !== undefined) {
// this is a conditional remove
if (request.keys.size !== 1) {
// we let the request to timeout
return logger.warn('Conditional remove request must have only one entry: %o', request);
}
const key = [...request.keys][0];
const existingValue = this.baseMap.get(key);
logger.trace('Conditional remove request: %s, %s, %s', key, existingValue, request.prevValue);
if (!existingValue || !this.equalValues(existingValue, request.prevValue)) {
return;
}
}
const removedEntries = this.baseMap.removeAll(request.keys.values());
if (request.sourceEndpointId === this.connection.grid.localPeerId) {
this.connection.respond('RemoveEntriesResponse', request.createResponse(removedEntries), request.sourceEndpointId);
}
removedEntries.forEach((v, k) => this.emit('remove', k, v));
})
.on('UpdateEntriesRequest', (request) => {
logger.trace('%s UpdateEntriesRequest: %o, %s', this.connection.grid.localPeerId, request, [...request.entries].join(', '));
const updatedEntries = [];
const insertedEntries = [];
if (request.prevValue !== undefined) {
// this is a conditional update
if (request.entries.size !== 1) {
// we let the request to timeout
return logger.trace('Conditional update request must have only one entry: %o', request);
}
const [key, value] = [...request.entries][0];
const existingValue = this.baseMap.get(key);
logger.trace('Conditional update request: %s, %s, %s, %s', key, value, existingValue, request.prevValue);
if (existingValue && this.equalValues(existingValue, request.prevValue)) {
this.baseMap.set(key, value);
updatedEntries.push([key, existingValue, value]);
}
}
else {
this.baseMap.setAll(request.entries, ({ inserted, updated }) => {
insertedEntries.push(...inserted);
updatedEntries.push(...updated);
});
}
if (request.sourceEndpointId === this.connection.grid.localPeerId) {
this.connection.respond('UpdateEntriesResponse', request.createResponse(new Map(updatedEntries.map(([key, oldValue]) => [key, oldValue]))), request.sourceEndpointId);
}
insertedEntries.forEach(([key, value]) => this.emit('insert', key, value));
updatedEntries.forEach(([key, oldValue, newValue]) => this.emit('update', key, oldValue, newValue));
})
.on('StorageHelloNotification', (notification) => {
// every storage needs to respond with its snapshot and the highest applied index they have
try {
const snapshot = this.export();
const serializedSnapshot = JSON.stringify(snapshot);
this.connection.notifyStorageState(serializedSnapshot, this.connection.highestSeenCommitIndex, notification.sourceEndpointId);
}
catch (err) {
logger.error('Failed to send snapshot', err);
}
})
.on('remote-snapshot', (serializedSnapshot, done) => {
try {
const snapshot = JSON.parse(serializedSnapshot);
this._import(snapshot,
// emit events if we are not initializing
Boolean(this._initializing) === false);
}
catch (err) {
logger.error(`Failed to import to map ${this.id}. Error: ${err}`);
}
finally {
done();
}
})
.once('close', () => this.close());
const entries = new Map([...this.baseMap.entries()]);
// clear the initial entries
this.baseMap.clear();
this._initializing = new Promise((resolve) => setTimeout(resolve, 20))
.then(() => this.connection.join())
.then(async () => {
// initializing
logger.debug('%s Initializing record %d', this.connection.localPeerId, this.id);
if (entries.size < 1)
return this;
await this.connection.requestInsertEntries(entries).then(() => void 0);
logger.debug('%s Initialization for record %d is complete', this.connection.localPeerId, this.id);
return this;
})
.catch((err) => {
logger.error('Failed to initialize record %s %o', this.id, err);
return this;
})
.finally(() => {
this._initializing = undefined;
});
}
get id() {
return this.connection.config.storageId;
}
get ready() {
return this._initializing ?? this.connection.grid.waitUntilCommitHead().then(() => this);
}
get closed() {
return this._closed;
}
close() {
if (this._closed)
return;
this._closed = true;
this.connection.close();
this.emit('close');
this.removeAllListeners();
}
get size() {
return this.baseMap.size;
}
get isEmpty() {
return this.baseMap.size === 0;
}
keys() {
return this.baseMap.keys();
}
async clear() {
if (this._closed)
throw new Error(`Cannot clear a closed storage (${this.id})`);
await this._initializing;
return this.connection.requestClearEntries();
}
get(key) {
return this.baseMap.get(key);
}
getAll(keys) {
if (this._closed)
throw new Error(`Cannot get entries from a closed storage (${this.id})`);
if (Array.isArray(keys))
return this.baseMap.getAll(keys.values());
else
return this.baseMap.getAll(keys);
}
async set(key, value) {
if (this._closed)
throw new Error(`Cannot set an entry on a closed storage (${this.id})`);
const result = await this.setAll(Collections.mapOf([key, value]));
return result.get(key);
}
async setAll(entries) {
if (this._closed)
throw new Error(`Cannot set entries on a closed storage (${this.id})`);
await this._initializing;
if (entries.size < 1) {
return Collections.emptyMap();
}
return this.connection.requestUpdateEntries(entries);
}
async insert(key, value) {
const result = await this.insertAll(Collections.mapOf([key, value]));
return result.get(key);
}
async insertAll(entries) {
if (this._closed)
throw new Error(`Cannot insert entries on a closed storage (${this.id})`);
await this._initializing;
if (Array.isArray(entries)) {
if (entries.length < 1)
return Collections.emptyMap();
entries = Collections.mapOf(...entries);
}
if (entries.size < 1) {
return Collections.emptyMap();
}
return this.connection.requestInsertEntries(entries);
}
async delete(key) {
const result = await this.deleteAll(Collections.setOf(key));
return result.has(key);
}
async deleteAll(keys) {
if (this._closed)
throw new Error(`Cannot delete entries on a closed storage (${this.id})`);
await this._initializing;
if (Array.isArray(keys)) {
if (keys.length < 1)
return Collections.emptySet();
keys = Collections.setOf(...keys);
}
if (keys.size < 1) {
return Collections.emptySet();
}
return this.connection.requestDeleteEntries(keys);
}
async removeIf(key, oldValue) {
if (this._closed)
throw new Error(`Cannot update an entry on a closed storage (${this.id})`);
await this._initializing;
logger.trace('%s RemoveIf: %s, %s, %s', this.connection.grid.localPeerId, key, oldValue, oldValue);
return (await this.connection.requestRemoveEntries(Collections.setOf(key), undefined, oldValue)).get(key) !== undefined;
}
async remove(key) {
const result = await this.removeAll(Collections.setOf(key));
return result.has(key);
}
async removeAll(keys) {
if (this._closed)
throw new Error(`Cannot remove entries on a closed storage (${this.id})`);
await this._initializing;
if (Array.isArray(keys)) {
if (keys.length < 1)
return Collections.emptyMap();
keys = Collections.setOf(...keys);
}
if (keys.size < 1) {
return Collections.emptyMap();
}
return this.connection.requestRemoveEntries(keys);
}
async updateIf(key, value, oldValue) {
if (this._closed)
throw new Error(`Cannot update an entry on a closed storage (${this.id})`);
await this._initializing;
logger.trace('%s UpdateIf: %s, %s, %s', this.connection.grid.localPeerId, key, value, oldValue);
return (await this.connection.requestUpdateEntries(Collections.mapOf([key, value]), undefined, oldValue)).get(key) !== undefined;
}
[Symbol.iterator]() {
return this.baseMap[Symbol.iterator]();
}
entries() {
return this.baseMap.entries();
}
values() {
return this.baseMap.values();
}
/**
* Exports the storage data
*/
export() {
const [keys, values] = this.connection.codec.encodeEntries(this.baseMap);
const result = {
mapId: this.id,
keys: HamokMap.uint8ArrayToStringCodec.encode(keys),
values: HamokMap.uint8ArrayToStringCodec.encode(values),
};
return result;
}
import(data, eventing) {
if (data.mapId !== this.id) {
throw new Error(`Cannot import data from a different storage: ${data.mapId} !== ${this.id}`);
}
else if (this.connection.connected) {
throw new Error('Cannot import data while connected');
}
else if (this._closed) {
throw new Error(`Cannot import data on a closed storage (${this.id})`);
}
this._import(data, eventing);
}
_import(data, eventing) {
const keys = HamokMap.uint8ArrayToStringCodec.decode(data.keys);
const values = HamokMap.uint8ArrayToStringCodec.decode(data.values);
const entries = this.connection.codec.decodeEntries(keys, values);
this.baseMap.setAll(entries, ({ inserted, updated }) => {
if (eventing) {
inserted.forEach(([key, value]) => this.emit('insert', key, value));
updated.forEach(([key, oldValue, newValue]) => this.emit('update', key, oldValue, newValue));
}
});
}
static uint8ArrayToStringCodec = (0, HamokCodec_1.createHamokCodec)((array) => {
return array.map((item) => Buffer.from(item).toString('utf8'));
}, (array) => {
return array.map((item) => Buffer.from(item, 'utf8'));
});
}
exports.HamokMap = HamokMap;