hamok
Version:
Lightweight Distributed Object Storage on RAFT consensus algorithm
222 lines (221 loc) • 11.2 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.createRaftLeaderState = createRaftLeaderState;
const logger_1 = require("../common/logger");
const RaftAppendEntries_1 = require("../messages/messagetypes/RaftAppendEntries");
const RaftFollowerState_1 = require("./RaftFollowerState");
const uuid_1 = require("uuid");
const logger = (0, logger_1.createLogger)('RaftLeaderState');
function createRaftLeaderState(context) {
const { raftEngine } = context;
const messageEmitter = raftEngine.transport;
const props = raftEngine.props;
const localPeerId = raftEngine.localPeerId;
const logs = raftEngine.logs;
const remotePeers = raftEngine.remotePeers;
const currentTerm = props.currentTerm;
/**
* leaders should track the sent index per peers. the reason behinf that is if
* the response to the append request chunks arrives slower than the updateFollower is called,
* then the same chunks is sent to follower making it slower to respond, making this leader sending
* the same append request with the same entries more, making the follower even slower than before,
* and the system explode. This tracking preventing to sending the same chunk of request twice
* until the follower does not respond normally.
*/
const sentRequests = new Map();
const unsyncedRemotePeers = new Map();
let follow = () => void 0;
let closed = false;
const appendEntriesRequestListener = (request) => {
if (request.term < currentTerm) {
return;
}
if (request.leaderId === undefined) {
return logger.warn(`Append Request Chunk is received without leaderId ${request}`);
}
if (request.leaderId === localPeerId) {
// loopback message?
return;
}
if (!remotePeers.has(request.leaderId)) {
return logger.warn(`%s received an append entries request from an unknown peer: ${request.leaderId}`, localPeerId);
}
if (currentTerm < request.term) {
logger.warn('%s received a request from a leader with a higher term. Request: %o', localPeerId, request);
return follow();
}
// terms are equal
logger.warn(`Append Request Chunk is received from another leader in the same term. Selecting one leader in this case who has higher id. ${request}`);
if (localPeerId.localeCompare(request.leaderId) < 0) {
// only one can remain!
return follow();
}
};
const appendEntriesResponseListener = (response) => {
if (response.term < currentTerm) {
// this response comes from a previous term, I should not apply it in any way
return;
}
if (currentTerm < response.term) {
// I am not the leader anymore, so it is best to go back to a follower state
return follow();
}
// now we are talking in my term...
logger.trace('Received RaftAppendEntriesResponse %o', response);
// if (localPeerId !== response.sourcePeerId) {
// remotePeers.touch(response.sourcePeerId);
// }
// processed means the remote peer processed all the chunks for the request
if (!response.processed) {
return;
}
// success means that the other end successfully accepted the request
if (!response.success) {
// having unsuccessful response, but proceeded all of the chunks
// means we should or can send a request again if it was a complex one.
return sentRequests.delete(response.sourcePeerId);
}
const sourcePeerId = response.sourcePeerId;
const sentRequest = sentRequests.delete(sourcePeerId);
if (sentRequest === undefined) {
// most likely a response to a keep alive or expired request
return;
}
const peerNextIndex = response.peerNextIndex;
props.nextIndex.set(sourcePeerId, peerNextIndex);
props.matchIndex.set(sourcePeerId, peerNextIndex - 1);
let maxCommitIndex = -1;
for (const logEntry of logs) {
if (peerNextIndex <= logEntry.index) {
break;
}
// is this good here? so we will never commit things not created by our term?
if (logEntry.term !== currentTerm) {
continue;
}
let matchCount = 1;
for (const peerId of remotePeers) {
const matchIndex = props.matchIndex.get(peerId) ?? -1;
if (logEntry.index <= matchIndex) {
++matchCount;
}
}
logger.trace('logIndex: %d, matchCount: %d, remotePeerIds: %d, commit: %s', logEntry.index, matchCount, remotePeers.size, remotePeers.size + 1 < matchCount * 2);
if (remotePeers.size + 1 < matchCount * 2) {
maxCommitIndex = Math.max(maxCommitIndex, logEntry.index);
}
}
if (0 <= maxCommitIndex) {
logger.trace('%s Committing index until %d at leader state', localPeerId, maxCommitIndex);
// setTimeout(() => {
for (const committedLogEnty of logs.commitUntil(maxCommitIndex)) {
raftEngine.events.emit('commit', committedLogEnty.index, committedLogEnty.entry);
}
// }, 10000);
}
};
const voteRequestListener = (request) => {
logger.warn('%s received a vote request from %s, but it is in a leader state', raftEngine.localPeerId, request.peerId);
};
const voteResponseListener = (response) => {
logger.warn('%s received a vote response from %s, but it is in a leader state', raftEngine.localPeerId, response.sourcePeerId);
};
const close = () => {
if (closed)
return;
closed = true;
messageEmitter.off('RaftVoteRequest', voteRequestListener);
messageEmitter.off('RaftVoteResponse', voteResponseListener);
messageEmitter.off('RaftAppendEntriesRequestChunk', appendEntriesRequestListener);
messageEmitter.off('RaftAppendEntriesResponse', appendEntriesResponseListener);
logger.debug('%s LeaderState is closed', localPeerId);
};
messageEmitter.on('RaftVoteRequest', voteRequestListener);
messageEmitter.on('RaftVoteResponse', voteResponseListener);
messageEmitter.on('RaftAppendEntriesRequestChunk', appendEntriesRequestListener);
messageEmitter.on('RaftAppendEntriesResponse', appendEntriesResponseListener);
const run = () => {
if (remotePeers.size < 1) {
logger.warn('Leader endpoint should become a follower because no remote endpoint is available');
return follow();
}
// const config = this.config;
const now = Date.now();
for (const peerId of remotePeers) {
const peerNextIndex = props.nextIndex.get(peerId) ?? 0;
const prevLogIndex = peerNextIndex - 1;
let prevLogTerm = -1;
if (0 <= prevLogIndex) {
const logEntry = logs.get(prevLogIndex);
if (logEntry != null) {
prevLogTerm = logEntry.term;
}
}
const entries = logs.collectEntries(peerNextIndex);
logger.trace('%s Collected %d entries for peer %s', localPeerId, entries.length, peerId);
if (peerNextIndex < logs.firstIndex) {
const startedUnsynced = unsyncedRemotePeers.get(peerId);
if (!startedUnsynced) {
unsyncedRemotePeers.set(peerId, Date.now());
logger.warn('%s Peer %s is unsynced, logs.nextIndex: %d, peerNextIndex: %d', localPeerId, peerId, logs.nextIndex, peerNextIndex);
// logger.warn(`Collected ${entries.length} entries, but peer ${peerId} should need ${logs.nextIndex - peerNextIndex}. logs.nextIndex: ${logs.nextIndex}, peerNextIndex: ${peerNextIndex}`);
}
else if (30000 < now - startedUnsynced) {
// we should kick the peer out of the cluster
logger.warn('%s Peer %s is unsynced for a long time, we remove it from the cluster');
raftEngine.events.emit('unsynced-peer', peerId);
unsyncedRemotePeers.delete(peerId);
}
}
else if (0 < unsyncedRemotePeers.size) {
if (unsyncedRemotePeers.delete(peerId)) {
logger.info('%s Peer %s is synced, logs.nextIndex: %d, peerNextIndex: %d', localPeerId, peerId, logs.nextIndex, peerNextIndex);
}
}
let sentRequest = sentRequests.get(peerId);
if (sentRequest !== undefined) {
const [, requestCreated] = sentRequest;
// we kill the sent request if it is older than the threshold
if (requestCreated < now - 30000) {
sentRequests.delete(peerId);
sentRequest = undefined;
}
}
const requestId = (0, uuid_1.v4)();
// we should only sent an entryfull request if the remote peer does not have one, and we have something to add
logger.trace('%s Sending append entries request to %s with requestId %s, sentRequest: [%s, %s], entries: %d', localPeerId, peerId, requestId, sentRequest?.[0] ?? 'undefined', sentRequest ? new Date(sentRequest?.[1] ?? 0).toISOString() : 'undefined', entries.length);
if (sentRequest === undefined && entries !== undefined && 0 < entries.length) {
for (let sequence = 0; sequence < entries.length; ++sequence) {
const entry = entries[sequence];
const request = new RaftAppendEntries_1.RaftAppendEntriesRequestChunk(requestId, peerId, localPeerId, logs.commitIndex, logs.nextIndex, prevLogIndex, prevLogTerm, currentTerm, sequence, sequence == entries.length - 1, entry.entry);
// logger.info("Sending", request);
messageEmitter.send(request);
}
sentRequest = [requestId, now];
sentRequests.set(peerId, sentRequest);
}
else { // no entries
const appendEntries = new RaftAppendEntries_1.RaftAppendEntriesRequestChunk(requestId, peerId, localPeerId, logs.commitIndex, logs.nextIndex, prevLogIndex, prevLogTerm, currentTerm, 0, // sequence
true, // last message
undefined);
messageEmitter.send(appendEntries);
}
}
};
follow = () => {
raftEngine.leaderId = undefined;
raftEngine.state = (0, RaftFollowerState_1.createRaftFollowerState)({
raftEngine,
});
};
const init = () => {
// we need to assign it here after the state is changed
raftEngine.leaderId = localPeerId;
};
return {
stateName: 'leader',
run,
close,
init,
};
}