@aurory/nestjs-k8s-leader-election
Version:
Nestjs k8s leader election
323 lines (322 loc) • 13.6 kB
JavaScript
"use strict";
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
var LeaderElectionService_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.LeaderElectionService = void 0;
const common_1 = require("@nestjs/common");
const client_node_1 = require("@kubernetes/client-node");
const event_emitter_1 = require("@nestjs/event-emitter");
const leader_election_options_interface_1 = require("./leader-election-options.interface");
let LeaderElectionService = LeaderElectionService_1 = class LeaderElectionService {
options;
eventEmitter;
logger = new common_1.Logger(LeaderElectionService_1.name);
kubeClient;
watch;
leaseName;
namespace;
renewalInterval;
durationInSeconds;
isLeader = false;
logAtLevel;
leaseRenewalTimeout = null;
awaitLeadership;
LEADER_IDENTITY = `nestjs-${process.env.HOSTNAME}`;
constructor(options, eventEmitter) {
this.options = options;
this.eventEmitter = eventEmitter;
const kubeConfig = new client_node_1.KubeConfig();
kubeConfig.loadFromDefault();
this.kubeClient = kubeConfig.makeApiClient(client_node_1.CoordinationV1Api);
this.watch = new client_node_1.Watch(kubeConfig);
this.leaseName = options.leaseName ?? 'nestjs-leader-election';
this.namespace = options.namespace ?? 'default';
this.renewalInterval = options.renewalInterval ?? 10000;
this.durationInSeconds = 2 * (this.renewalInterval / 1000);
this.logAtLevel = options.logAtLevel ?? 'log';
this.awaitLeadership = options.awaitLeadership ?? false;
process.on('SIGINT', () => this.gracefulShutdown());
process.on('SIGTERM', () => this.gracefulShutdown());
}
async onApplicationBootstrap() {
if (!process.env.KUBERNETES_SERVICE_HOST) {
this.logger[this.logAtLevel]('Not running in Kubernetes, assuming leadership...');
this.isLeader = true;
this.emitLeaderElectedEvent();
}
else {
this.watchLeaseObject(); // This should start right away to catch any events.
if (this.awaitLeadership) {
// If awaitLeadership is true, block until leader election is complete.
await this.runLeaderElectionProcess();
}
else {
// Otherwise, run the leader election process in the background.
this.runLeaderElectionProcess().catch((error) => {
this.logger.error({
message: 'Leader election process failed',
error,
});
});
}
}
}
async runLeaderElectionProcess() {
// Attempt to become a leader.
await this.tryToBecomeLeader();
// If not successful, retry up to two more times.
for (let attempt = 0; attempt < 2; attempt++) {
if (this.isLeader)
break; // Break early if leadership is acquired.
// Wait for half the lease duration before retrying.
await new Promise((resolve) => setTimeout(resolve, this.durationInSeconds * 500));
// Try to become the leader again.
await this.tryToBecomeLeader();
}
}
async tryToBecomeLeader() {
this.logger[this.logAtLevel]('Trying to become leader...');
try {
let lease = await this.getLease();
if (this.isLeaseExpired(lease) || !lease.spec.holderIdentity) {
this.logger[this.logAtLevel]('Lease expired or not held. Attempting to become leader...');
lease = await this.acquireLease(lease);
}
if (this.isLeaseHeldByUs(lease)) {
this.becomeLeader();
}
}
catch (error) {
this.logger.error({
message: 'Error while trying to become leader',
error,
});
}
}
async acquireLease(lease) {
// Set this instance as the holder of the lease
lease.spec.holderIdentity = this.LEADER_IDENTITY;
lease.spec.leaseDurationSeconds = this.durationInSeconds;
lease.spec.acquireTime = new client_node_1.V1MicroTime(new Date());
lease.spec.renewTime = new client_node_1.V1MicroTime(new Date());
const params = { name: this.leaseName, namespace: this.namespace, body: lease };
try {
const body = await this.kubeClient.replaceNamespacedLease(params);
this.logger[this.logAtLevel]('Successfully acquired lease');
return body;
}
catch (error) {
this.logger.error({ message: 'Error while acquiring lease', error });
throw error;
}
}
async renewLease() {
try {
let lease = await this.getLease();
if (this.isLeaseHeldByUs(lease)) {
this.logger[this.logAtLevel]('Renewing lease...');
lease.spec.renewTime = new client_node_1.V1MicroTime(new Date());
const params = { name: this.leaseName, namespace: this.namespace, body: lease };
const body = await this.kubeClient.replaceNamespacedLease(params);
this.logger[this.logAtLevel]('Successfully renewed lease');
return body;
}
else {
this.loseLeadership();
}
}
catch (error) {
this.logger.error({ message: 'Error while renewing lease', error });
this.loseLeadership();
}
}
async getLease() {
const params = { name: this.leaseName, namespace: this.namespace };
try {
return await this.kubeClient.readNamespacedLease(params);
}
catch (error) {
if (error.response && error.response.statusCode === 404) {
this.logger[this.logAtLevel]('Lease not found. Creating lease...');
return this.createLease();
}
else {
throw error;
}
}
}
async createLease() {
const lease = {
metadata: {
name: this.leaseName,
namespace: this.namespace,
},
spec: {
holderIdentity: this.LEADER_IDENTITY,
leaseDurationSeconds: this.durationInSeconds,
acquireTime: new client_node_1.V1MicroTime(new Date()),
renewTime: new client_node_1.V1MicroTime(new Date()),
},
};
const params = { namespace: this.namespace, body: lease };
try {
const body = await this.kubeClient.createNamespacedLease(params);
this.logger[this.logAtLevel]('Successfully created lease');
return body;
}
catch (error) {
this.logger.error({ message: 'Failed to create lease', error });
throw error;
}
}
isLeaseExpired(lease) {
const renewTime = lease.spec.renewTime
? new Date(lease.spec.renewTime).getTime()
: 0;
const leaseDurationMs = (lease.spec.leaseDurationSeconds || this.durationInSeconds) * 1000;
return Date.now() > renewTime + leaseDurationMs;
}
isLeaseHeldByUs(lease) {
return lease.spec.holderIdentity === this.LEADER_IDENTITY;
}
async gracefulShutdown() {
this.logger[this.logAtLevel]('Graceful shutdown initiated');
if (this.isLeader) {
await this.releaseLease();
}
}
async releaseLease() {
try {
let lease = await this.getLease();
if (lease && this.isLeaseHeldByUs(lease)) {
lease.spec.holderIdentity = null;
lease.spec.renewTime = null;
const params = { name: this.leaseName, namespace: this.namespace, body: lease };
await this.kubeClient.replaceNamespacedLease(params);
this.logger[this.logAtLevel](`Lease for ${this.leaseName} released.`);
}
}
catch (error) {
this.logger.error({ message: 'Failed to release lease', error });
}
}
emitLeaderElectedEvent() {
this.eventEmitter.emit(leader_election_options_interface_1.LeaderElectedEvent, { leaseName: this.leaseName });
this.logger[this.logAtLevel](`Instance became the leader for lease: ${this.leaseName}`);
}
emitLeadershipLostEvent() {
this.eventEmitter.emit(leader_election_options_interface_1.LeaderLostEvent, { leaseName: this.leaseName });
this.logger[this.logAtLevel](`Instance lost the leadership for lease: ${this.leaseName}`);
}
becomeLeader() {
this.isLeader = true;
this.emitLeaderElectedEvent();
this.scheduleLeaseRenewal();
}
loseLeadership() {
if (this.isLeader) {
this.isLeader = false;
if (this.leaseRenewalTimeout) {
clearTimeout(this.leaseRenewalTimeout);
this.leaseRenewalTimeout = null;
}
this.emitLeadershipLostEvent();
}
}
async watchLeaseObject() {
const path = `/apis/coordination.k8s.io/v1/namespaces/${this.namespace}/leases`;
try {
await this.watch.watch(path, {}, (type, apiObj) => {
if (apiObj && apiObj.metadata.name === this.leaseName) {
this.logger[this.logAtLevel](`Watch event type: ${type} for lease: ${this.leaseName}`);
switch (type) {
case 'ADDED':
case 'MODIFIED':
setTimeout(() => this.handleLeaseUpdate(apiObj), 2000);
break;
case 'DELETED':
setTimeout(() => this.handleLeaseDeletion(), 2000);
break;
}
}
}, (err) => {
if (err) {
this.logger.error({
message: `Watch for lease ended with error: ${err}, trying again in 5 seconds`,
error: err,
});
}
else {
this.logger[this.logAtLevel]('Watch for lease gracefully closed');
}
// Restart the watch after a delay
setTimeout(() => this.watchLeaseObject(), 5000);
});
}
catch (err) {
this.logger.error(`Failed to start watch for lease: ${err}, trying again in 5 seconds`);
// Retry starting the watch after a delay
setTimeout(() => this.watchLeaseObject(), 5000);
}
}
scheduleLeaseRenewal() {
// Clear any existing lease renewal timeout.
if (this.leaseRenewalTimeout) {
clearTimeout(this.leaseRenewalTimeout);
}
// Schedule the lease renewal to happen at the renewalInterval.
// The renewal should occur before the lease duration expires.
this.leaseRenewalTimeout = setTimeout(async () => {
if (this.isLeader) {
try {
await this.renewLease();
}
catch (error) {
this.logger.error({ message: 'Error while renewing lease', error });
// If lease renewal fails, consider handling it by attempting to re-acquire leadership or similar.
}
}
}, this.renewalInterval);
}
handleLeaseUpdate(leaseObj) {
this.logger.debug({ message: 'Detecting lease update', leaseObj });
if (this.isLeaseHeldByUs(leaseObj)) {
if (!this.isLeader) {
setTimeout(() => {
this.becomeLeader();
}, 2000); // Wait for 2 seconds before becoming the leader
}
this.scheduleLeaseRenewal();
}
else if (this.isLeader) {
this.loseLeadership();
}
}
handleLeaseDeletion() {
if (!this.isLeader) {
this.tryToBecomeLeader().catch((error) => {
this.logger.error({
message: 'Error while trying to become leader after lease deletion',
error,
});
});
}
}
};
exports.LeaderElectionService = LeaderElectionService;
exports.LeaderElectionService = LeaderElectionService = LeaderElectionService_1 = __decorate([
(0, common_1.Injectable)(),
__param(0, (0, common_1.Inject)('LEADER_ELECTION_OPTIONS')),
__metadata("design:paramtypes", [Object, event_emitter_1.EventEmitter2])
], LeaderElectionService);