hamok
Version:
Lightweight Distributed Object Storage on RAFT consensus algorithm
288 lines (287 loc) • 11.4 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.MemoryStoredRaftLogs = void 0;
const events_1 = require("events");
const RaftLogs_1 = require("./RaftLogs");
// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
class MemoryStoredRaftLogs extends events_1.EventEmitter {
config;
/**
* index of highest log entry applied to state
* machine (initialized to 0, increases
* monotonically)
*/
_firstIndex = 0;
/**
* The next log index
*/
_nextIndex = 0;
/**
* index of highest log entry known to be
* committed (initialized to 0, increases
* monotonically)
*/
_commitIndex = -1;
_mssingEntriesLogged = false;
_memoryEstimateBytesLength = 0;
_entries;
constructor(config, entries) {
super();
this.config = config;
this.setMaxListeners(Infinity);
this._entries = entries ?? new Map();
}
/**
* index of highest log entry known to be
* committed (initialized to 0, increases
* monotonically)
*/
get commitIndex() {
return this._commitIndex;
}
/**
* The next index for the logs to be used if an entry is added or submitted
* @return
*/
get nextIndex() {
return this._nextIndex;
}
get firstIndex() {
return this._firstIndex;
}
get size() {
return this._entries.size;
}
get bytesInMemory() {
return this._memoryEstimateBytesLength;
}
commitUntil(newCommitIndex) {
if (newCommitIndex <= this._commitIndex) {
RaftLogs_1.logger.warn('Requested to commit until %d index, but the actual commitIndex is %d', newCommitIndex, this._commitIndex);
return [];
}
const committedEntries = [];
while (this._commitIndex < newCommitIndex) {
if (this._nextIndex <= this._commitIndex + 1) {
RaftLogs_1.logger.warn(`Cannot commit, because there is no next entry to commit. commitIndex:
${this._commitIndex},
nextIndex: ${this._nextIndex},
newCommitIndex: ${newCommitIndex}`);
break;
}
const nextCommitIndex = this._commitIndex + 1;
const logEntry = this._entries.get(nextCommitIndex);
RaftLogs_1.logger.trace('Committing log entry for index %d, %o', nextCommitIndex, logEntry);
if (logEntry == undefined) {
RaftLogs_1.logger.warn(`LogEntry for nextCommitIndex ${nextCommitIndex} is null. it supposed not to be null.`);
break;
}
this._commitIndex = nextCommitIndex;
committedEntries.push(logEntry);
this.emit('committed', this._commitIndex, logEntry.entry);
}
if (0 < committedEntries.length) {
this._expire();
}
return committedEntries;
}
submit(term, entry) {
const now = Date.now();
const logEntry = {
index: this._nextIndex,
term,
entry,
timestamp: now,
};
this._entries.set(logEntry.index, logEntry);
++this._nextIndex;
this._memoryEstimateBytesLength += entry.values.length + entry.keys.length;
if (0 < this.config.memorySizeHighWaterMark && this.config.memorySizeHighWaterMark < this._memoryEstimateBytesLength) {
this.emit('highWaterMark', this._memoryEstimateBytesLength);
}
return logEntry.index;
}
compareAndOverride(index, expectedTerm, entry) {
if (this.nextIndex <= index) {
return;
}
const now = Date.now();
const oldLogEntry = this._entries.get(index);
if (oldLogEntry == undefined) {
const newLogEntry = {
index,
term: expectedTerm,
entry,
timestamp: now,
};
this._entries.set(newLogEntry.index, newLogEntry);
this._memoryEstimateBytesLength += entry.values.length + entry.keys.length;
return;
}
if (expectedTerm == oldLogEntry.term) {
// theoretically identical
return;
}
const newLogEntry = {
index,
term: expectedTerm,
entry,
timestamp: now,
};
this._entries.set(newLogEntry.index, newLogEntry);
this._memoryEstimateBytesLength -= oldLogEntry.entry.values.length + oldLogEntry.entry.keys.length;
this._memoryEstimateBytesLength += entry.values.length + entry.keys.length;
return oldLogEntry;
}
compareAndAdd(expectedNextIndex, term, entry) {
if (this._nextIndex != expectedNextIndex) {
return false;
}
const logEntry = {
index: this._nextIndex,
term,
entry,
timestamp: Date.now(),
};
this._memoryEstimateBytesLength += entry.values.length + entry.keys.length;
this._entries.set(logEntry.index, logEntry);
++this._nextIndex;
return true;
}
get(index) {
return this._entries.get(index);
}
collectEntries(startIndex, endIndex) {
const result = [];
let missingEntries = 0;
if (endIndex == undefined) {
endIndex = this._nextIndex;
}
else if (endIndex < startIndex) {
RaftLogs_1.logger.warn('Requested to collect entries, startIndex: %d, endIndex: %d, but endIndex is smaller than startIndex.', startIndex, endIndex);
return [];
}
else if (this._nextIndex < endIndex) {
RaftLogs_1.logger.warn('Requested to collect entries, startIndex: %d, endIndex: %d, but endIndex is higher than the nextIndex.', startIndex, endIndex);
endIndex = this._nextIndex;
}
else {
endIndex = Math.min(endIndex, this._nextIndex);
}
for (let logIndex = startIndex; logIndex < endIndex; ++logIndex) {
const logEntry = this._entries.get(logIndex);
if (logEntry == undefined) {
// we don't have it anymore
++missingEntries;
continue;
}
result.push(logEntry);
}
if (0 < missingEntries) {
if (!this._mssingEntriesLogged) {
RaftLogs_1.logger.warn('Requested to collect entries, startIndex: %d, endIndex: %d, but missing %d entries.', startIndex, this.nextIndex, missingEntries);
this._mssingEntriesLogged = true;
}
}
else if (this._mssingEntriesLogged) {
this._mssingEntriesLogged = false;
}
return result;
}
/**
* Direct iterator for the logs. Starts with commitIndex + 1, and iterates the logs until nextIndex.
* if a log is comitted after this iterator has been created but before the next is called on this iterator
* @return
*/
*[Symbol.iterator]() {
for (let index = this._commitIndex + 1; index < this._nextIndex; ++index) {
const logEntry = this._entries.get(index);
if (logEntry) {
yield logEntry;
}
}
}
reset(newCommitIndex) {
this._entries.clear();
this._commitIndex = newCommitIndex;
this._nextIndex = newCommitIndex + 1;
this._firstIndex = newCommitIndex;
this._memoryEstimateBytesLength = 0;
RaftLogs_1.logger.warn(`Logs are reset. new values: commitIndex: ${this._commitIndex}, nextIndex: ${this._nextIndex}, lastApplied: ${this._firstIndex}`);
}
removeUntil(newFirstIndex) {
if (newFirstIndex <= this._firstIndex) {
return;
}
else if (this._commitIndex <= newFirstIndex) {
return RaftLogs_1.logger.warn('Requested to remove until index {}, but the commitIndex is {}.', newFirstIndex, this._commitIndex);
}
let removed = 0;
for (let index = this._firstIndex; index < this.commitIndex; ++index) {
const logEntry = this._entries.get(index);
if (logEntry == undefined || !this._entries.delete(index)) {
// already purged?
RaftLogs_1.logger.trace(`LastApplied is set to ${index + 1}, because for index ${index} logEntry does not exists`);
this._firstIndex = index + 1;
continue;
}
this._memoryEstimateBytesLength -= logEntry.entry.values.length + logEntry.entry.keys.length;
++removed;
this.emit('removed', index, logEntry.entry);
}
this._firstIndex = newFirstIndex;
RaftLogs_1.logger.trace(`Set the lastApplied to ${this._firstIndex} and removed ${removed} items`);
}
_expire() {
if (this.config.expirationTimeInMs < 1) {
// infinite
return;
}
RaftLogs_1.logger.trace('Expiring logs %o', this.config);
const thresholdInMs = Date.now() - this.config.expirationTimeInMs;
let expiredLogIndex = -1;
for (let index = this._firstIndex; index < this.commitIndex; ++index) {
const logEntry = this._entries.get(index);
if (logEntry == undefined) {
// already purged?
RaftLogs_1.logger.trace(`LastApplied is set to
${index + 1}, because for index
${index} logEntry does not exists`);
this._firstIndex = index + 1;
continue;
}
RaftLogs_1.logger.trace('Log %d created at %d, elapsedMs %d threshold %d', index, logEntry.timestamp, Date.now() - logEntry.timestamp, thresholdInMs);
if (thresholdInMs <= logEntry.timestamp) {
break;
}
expiredLogIndex = index;
}
RaftLogs_1.logger.trace('Expired log index is %d', expiredLogIndex);
if (expiredLogIndex < 0) {
return;
}
RaftLogs_1.logger.trace('nextIndex %d, expiredLogIndex %d, firstIndex %d', this._nextIndex, expiredLogIndex, this._firstIndex);
if (this._nextIndex <= expiredLogIndex || expiredLogIndex < this._firstIndex) {
return;
}
if (this._commitIndex < expiredLogIndex) {
RaftLogs_1.logger.warn('expired log index is higher than the commit index. This is a problem! increase the expiration timeout, because it leads to a potential inconsistency issue.');
}
let removed = 0;
for (let index = this._firstIndex; index < expiredLogIndex; ++index) {
const logEntry = this._entries.get(index);
if (this._entries.delete(index)) {
// logger.warn("Removed entry index", index);
++removed;
if (logEntry) {
this._memoryEstimateBytesLength -= logEntry.entry.values.length + logEntry.entry.keys.length;
this.emit('expired', index, logEntry.entry);
this.emit('removed', index, logEntry.entry);
}
}
}
this._firstIndex = expiredLogIndex;
RaftLogs_1.logger.trace(`Set the lastApplied to ${this._firstIndex} and removed ${removed} items`);
}
}
exports.MemoryStoredRaftLogs = MemoryStoredRaftLogs;