consul-resolver
Version:
A load balancer for Consul services with Redis-based metrics
594 lines (593 loc) • 25.1 kB
JavaScript
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const dns_query_1 = require("dns-query");
const consul_1 = __importDefault(require("consul"));
const types_1 = require("./types");
const utils_1 = require("@brimble/utils");
class ConsulResolver {
constructor(config) {
this.config = config;
this.currentIndex = 0;
this.debug = config.debug || false;
this.cachePrefix = config.cachePrefix;
this.cacheEnabled = config.cacheEnabled;
this.weights = config.weights || types_1.DEFAULT_WEIGHTS;
this.metrics = config.metrics || types_1.DEFAULT_METRICS;
this.cacheTTL = Math.floor((config.cacheTTL || 60 * 1000) / 1000);
this.consul = new consul_1.default({
host: config.host,
port: config.port,
secure: config.secure,
defaults: {
...(config.token ? { token: config.token } : {}),
},
agent: config.agent,
});
if (this.cacheEnabled && config.redis) {
this.redis = config.redis;
this.cacheEnabled = true;
}
}
getConnectionKey(serviceId) {
return `${this.cachePrefix}:connections:${serviceId}`;
}
getDNSCacheKey(service) {
return `${this.cachePrefix}:dns:${service}`;
}
getHealthCacheKey(service) {
return `${this.cachePrefix}:health:${service}`;
}
async resolveDNS(service) {
var _a, _b, _c, _d;
const cacheKey = this.getDNSCacheKey(service);
if (this.cacheEnabled) {
try {
const cachedData = await Promise.race([
(_a = this.redis) === null || _a === void 0 ? void 0 : _a.get(cacheKey),
new Promise((_, reject) => setTimeout(() => reject(new Error('Redis get timeout')), 200))
]);
if (cachedData) {
if (this.debug) {
utils_1.log.debug(`DNS cache hit for ${service}`);
}
return JSON.parse(cachedData);
}
}
catch (e) {
if (this.debug)
utils_1.log.error('Redis get error or timeout:', e);
}
}
try {
const result = await (0, dns_query_1.query)({
question: {
type: 'SRV',
name: service
}
}, {
endpoints: this.config.dnsEndpoints,
timeout: (_b = this.config.dnsTimeout) !== null && _b !== void 0 ? _b : 1500,
retries: (_c = this.config.dnsRetries) !== null && _c !== void 0 ? _c : 2
});
if (this.debug) {
utils_1.log.debug("DNS QUERY RESULT", result);
}
if (!result.answers || result.answers.length === 0) {
if (this.debug) {
utils_1.log.debug(`No SRV records found for ${service}`);
}
return [];
}
const additionalsByName = {};
if (result.additionals) {
for (const additional of result.additionals) {
if (additional.type === 'A') {
additionalsByName[additional.name] = additional;
}
}
}
const records = result.answers.map((answer) => {
const target = answer.data.target;
const aRecord = additionalsByName[target];
return {
name: target,
ip: (aRecord === null || aRecord === void 0 ? void 0 : aRecord.data) || '',
port: answer.data.port,
priority: answer.data.priority,
weight: answer.data.weight
};
}).filter(record => record.ip);
if (this.cacheEnabled) {
try {
await Promise.race([
(_d = this.redis) === null || _d === void 0 ? void 0 : _d.set(cacheKey, JSON.stringify(records), 'EX', this.cacheTTL),
new Promise((_, reject) => setTimeout(() => reject(new Error('Redis set timeout')), 200))
]);
}
catch (e) {
if (this.debug)
utils_1.log.error('Redis set error or timeout:', e);
}
}
return records;
}
catch (error) {
if (this.debug) {
utils_1.log.error('DNS resolution error:', error);
}
return [];
}
}
combineHealthAndDNSWeights(service, dnsWeight, maxDNSWeight) {
const healthScore = this.calculateHealthScore(service);
const normalizedDNSWeight = dnsWeight / maxDNSWeight;
return (healthScore * 0.7) + (normalizedDNSWeight * 0.3);
}
/**
* Select the optimal service based on the specified algorithm
*/
async selectOptimalService(service, algorithm = types_1.SelectionAlgorithm.RoundRobin) {
var _a;
try {
const [healthChecks, dnsRecords] = await Promise.all([
this.getHealthChecks(service),
this.resolveDNS(service)
]);
if ((!healthChecks || healthChecks.length === 0) && dnsRecords.length === 0) {
return { selected: null, services: [] };
}
const sortedByPriority = this.sortByPriority(dnsRecords);
const lowestPriorityValue = (_a = sortedByPriority[0]) === null || _a === void 0 ? void 0 : _a.priority;
const highestPriorityRecords = sortedByPriority.filter(record => record.priority === lowestPriorityValue);
if (!healthChecks || healthChecks.length === 0) {
const selected = this.selectFromSrvRecords(highestPriorityRecords, algorithm);
if (!selected) {
return { selected: null, services: [] };
}
await this.updateSelectionMetrics(selected.name);
return {
selected: {
ip: selected.ip,
port: selected.port,
},
services: sortedByPriority.map(record => ({
ip: record.ip,
port: record.port,
})),
};
}
const dnsWeights = new Map(dnsRecords.map((record) => [record.ip, {
weight: record.weight,
port: record.port,
priority: record.priority
}]));
const matchedHealthChecks = healthChecks.filter(check => dnsWeights.has(check.Service.Address));
if (matchedHealthChecks.length === 0) {
if (this.debug) {
utils_1.log.debug('No matching services found between DNS and Consul health checks');
}
const selected = this.selectFromSrvRecords(highestPriorityRecords, algorithm);
if (!selected) {
return { selected: null, services: [] };
}
await this.updateSelectionMetrics(selected.name);
return {
selected: {
ip: selected.ip,
port: selected.port,
},
services: sortedByPriority.map(record => ({
ip: record.ip,
port: record.port,
})),
};
}
const highPriorityIPs = new Set(highestPriorityRecords.map(record => record.ip));
const highPriorityHealthChecks = matchedHealthChecks.filter(check => highPriorityIPs.has(check.Service.Address));
const targetHealthChecks = highPriorityHealthChecks.length > 0
? highPriorityHealthChecks
: matchedHealthChecks;
const maxDNSWeight = Math.max(...dnsRecords.map((r) => r.weight || 1));
const enhancedHealthChecks = targetHealthChecks.map(check => {
var _a;
return ({
...check,
dnsWeight: this.combineHealthAndDNSWeights(check, ((_a = dnsWeights.get(check.Service.Address)) === null || _a === void 0 ? void 0 : _a.weight) || 0, maxDNSWeight)
});
});
const metrics = await this.getServicesMetrics(targetHealthChecks);
let selectedService;
switch (algorithm) {
case types_1.SelectionAlgorithm.RoundRobin:
selectedService = this.roundRobinSelection(enhancedHealthChecks);
break;
case types_1.SelectionAlgorithm.LeastConnection:
selectedService = this.leastConnectionSelection(enhancedHealthChecks, metrics);
break;
case types_1.SelectionAlgorithm.WeightedRoundRobin:
const rankedServices = await this.rankServices(enhancedHealthChecks, metrics);
rankedServices.forEach(ranked => {
const dnsInfo = dnsWeights.get(ranked.service.Service.Address);
if (dnsInfo) {
ranked.score *= (1 + (dnsInfo.weight / maxDNSWeight));
}
});
selectedService = this.weightedRandomSelection(rankedServices);
break;
}
await this.updateSelectionMetrics(selectedService.id);
const selectedDNSInfo = dnsWeights.get(selectedService.service.Service.Address);
return {
selected: {
ip: selectedService.service.Service.Address,
port: (selectedDNSInfo === null || selectedDNSInfo === void 0 ? void 0 : selectedDNSInfo.port) || selectedService.service.Service.Port,
},
services: matchedHealthChecks.map(check => {
const dnsInfo = dnsWeights.get(check.Service.Address);
return {
ip: check.Service.Address,
port: (dnsInfo === null || dnsInfo === void 0 ? void 0 : dnsInfo.port) || check.Service.Port,
};
}),
};
}
catch (error) {
if (this.debug) {
utils_1.log.error('Error selecting optimal service:', error);
}
return { selected: null, services: [] };
}
}
/**
* Get health checks from Consul with caching
*/
async getHealthChecks(service) {
var _a, _b;
const cacheKey = this.getHealthCacheKey(service);
if (this.cacheEnabled) {
const cachedHealth = await ((_a = this.redis) === null || _a === void 0 ? void 0 : _a.get(cacheKey));
if (cachedHealth) {
return JSON.parse(cachedHealth);
}
}
try {
const healthChecks = await this.consul.health.service(service);
if (this.cacheEnabled) {
await ((_b = this.redis) === null || _b === void 0 ? void 0 : _b.set(cacheKey, JSON.stringify(healthChecks), 'EX', this.cacheTTL));
}
return healthChecks;
}
catch (error) {
if (this.debug) {
utils_1.log.error(`Error fetching health checks for ${service}:`, error);
}
return [];
}
}
/**
* Sort SRV records by priority (lower number is higher priority)
*/
sortByPriority(records) {
return [...records].sort((a, b) => (a.priority || 0) - (b.priority || 0));
}
/**
* Select a service from SRV records using the specified algorithm
*/
selectFromSrvRecords(records, algorithm) {
if (!records || records.length === 0) {
return null;
}
switch (algorithm) {
case types_1.SelectionAlgorithm.RoundRobin:
const selected = records[this.currentIndex % records.length];
this.currentIndex = (this.currentIndex + 1) % records.length;
return selected;
case types_1.SelectionAlgorithm.WeightedRoundRobin:
return this.selectWeightedSrvRecord(records);
case types_1.SelectionAlgorithm.LeastConnection:
const selectedLC = records[this.currentIndex % records.length];
this.currentIndex = (this.currentIndex + 1) % records.length;
return selectedLC;
default:
return records[0];
}
}
/**
* Select SRV record using weighted random selection
*/
selectWeightedSrvRecord(records) {
if (!records || records.length === 0) {
return null;
}
const hasNonZeroWeights = records.some(record => (record.weight || 0) > 0);
if (!hasNonZeroWeights) {
return records[this.currentIndex++ % records.length];
}
const totalWeight = records.reduce((sum, record) => sum + (record.weight || 1), 0);
let random = Math.random() * totalWeight;
for (const record of records) {
random -= (record.weight || 1);
if (random <= 0) {
return record;
}
}
return records[0];
}
roundRobinSelection(services) {
const healthyServices = services.filter((service) => service.Checks.every((check) => check.Status === "passing"));
if (healthyServices.length === 0) {
throw new Error("No healthy services available");
}
const service = healthyServices[this.currentIndex % healthyServices.length];
this.currentIndex = (this.currentIndex + 1) % healthyServices.length;
return {
id: service.Service.ID,
service,
};
}
leastConnectionSelection(services, metrics) {
const healthyServices = services
.filter((service) => service.Checks.every((check) => check.Status === "passing"))
.map((service) => {
const serviceMetrics = metrics.get(service.Service.ID) || this.metrics;
return {
service,
connections: serviceMetrics.activeConnections || 0,
};
});
if (healthyServices.length === 0) {
throw new Error("No healthy services available");
}
const selectedService = healthyServices.reduce((min, current) => current.connections < min.connections ? current : min);
return {
id: selectedService.service.Service.ID,
service: selectedService.service,
};
}
async getServicesMetrics(services) {
var _a;
const metricsMap = new Map();
const pipeline = (_a = this.redis) === null || _a === void 0 ? void 0 : _a.pipeline();
if (this.cacheEnabled) {
services.forEach((service) => {
pipeline === null || pipeline === void 0 ? void 0 : pipeline.get(service.Service.ID);
pipeline === null || pipeline === void 0 ? void 0 : pipeline.get(this.getConnectionKey(service.Service.ID));
});
}
try {
const results = await (pipeline === null || pipeline === void 0 ? void 0 : pipeline.exec());
if (!results) {
services.forEach((service) => {
metricsMap.set(service.Service.ID, { ...this.metrics });
});
return metricsMap;
}
services.forEach((service, index) => {
const serviceId = service.Service.ID;
const metricsResult = results[index * 2];
const connectionsResult = results[index * 2 + 1];
let metrics;
try {
if (metricsResult === null || metricsResult === void 0 ? void 0 : metricsResult[1]) {
metrics = JSON.parse(metricsResult[1]);
}
else {
metrics = { ...this.metrics };
}
const connections = (connectionsResult === null || connectionsResult === void 0 ? void 0 : connectionsResult[1])
? parseInt(connectionsResult[1])
: 0;
metrics.activeConnections = connections;
metricsMap.set(serviceId, metrics);
}
catch (error) {
if (this.debug) {
utils_1.log.error(`Error processing metrics for service ${serviceId}:`, error);
}
metricsMap.set(serviceId, { ...this.metrics });
}
});
}
catch (error) {
if (this.debug) {
utils_1.log.error("Error executing Redis pipeline:", error);
}
services.forEach((service) => {
metricsMap.set(service.Service.ID, { ...this.metrics });
});
}
return metricsMap;
}
async rankServices(services, metrics) {
return services
.map((service) => {
const serviceId = service.Service.ID;
const serviceMetrics = metrics.get(serviceId);
if (!serviceMetrics) {
throw new Error(`No metrics found for service ${serviceId}`);
}
const healthScore = this.calculateHealthScore(service);
const responseTimeScore = this.normalizeScore(serviceMetrics.responseTime, 500, true);
const errorRateScore = this.normalizeScore(serviceMetrics.errorRate, 100, true);
const resourceScore = this.calculateResourceScore(serviceMetrics);
const connectionScore = this.normalizeScore(serviceMetrics.activeConnections, 1000, true);
const distributionScore = this.calculateDistributionScore(serviceMetrics.lastSelectedTime);
const totalScore = healthScore * this.weights.health +
responseTimeScore * this.weights.responseTime +
errorRateScore * this.weights.errorRate +
resourceScore * this.weights.resources +
connectionScore * this.weights.connections +
distributionScore * this.weights.distribution;
return {
score: totalScore,
id: serviceId,
service,
};
})
.sort((a, b) => b.score - a.score);
}
calculateHealthScore(service) {
const checks = service.Checks;
const totalChecks = checks.length;
if (totalChecks === 0)
return 0;
const passingChecks = checks.filter((check) => check.Status === "passing").length;
return passingChecks / totalChecks;
}
calculateResourceScore(metrics) {
const cpuScore = this.normalizeScore(metrics.cpuUsage, 100, true);
const memoryScore = this.normalizeScore(metrics.memoryUsage, 100, true);
return (cpuScore + memoryScore) / 2;
}
calculateDistributionScore(lastSelectedTime) {
if (!lastSelectedTime)
return 1;
const timeSinceLastSelection = Date.now() - lastSelectedTime;
return Math.min(timeSinceLastSelection / (5 * 60 * 1000), 1);
}
normalizeScore(value, max, inverse = false) {
const normalized = Math.max(0, Math.min(1, value / max));
return inverse ? 1 - normalized : normalized;
}
weightedRandomSelection(rankedServices) {
if (rankedServices.length === 0) {
throw new Error("No services available for selection");
}
const totalScore = rankedServices.reduce((sum, service) => sum + service.score, 0);
if (totalScore <= 0) {
return {
id: rankedServices[0].id,
service: rankedServices[0].service,
};
}
let random = Math.random() * totalScore;
for (const service of rankedServices) {
random -= service.score;
if (random <= 0) {
return {
id: service.id,
service: service.service,
};
}
}
return {
id: rankedServices[0].id,
service: rankedServices[0].service,
};
}
async incrementConnections(serviceId) {
var _a, _b;
try {
const connectionKey = this.getConnectionKey(serviceId);
const existingMetrics = await ((_a = this.redis) === null || _a === void 0 ? void 0 : _a.get(connectionKey));
let metrics;
if (existingMetrics) {
metrics = JSON.parse(existingMetrics);
metrics.activeConnections = (metrics.activeConnections || 0) + 1;
}
else {
metrics = {
...this.metrics,
activeConnections: 1,
};
}
if (this.cacheEnabled) {
await ((_b = this.redis) === null || _b === void 0 ? void 0 : _b.set(connectionKey, JSON.stringify(metrics), "EX", 24 * 60 * 60));
}
}
catch (error) {
if (this.debug) {
utils_1.log.error(`Failed to increment connections for service ${serviceId}:`, error);
}
}
}
async decrementConnections(serviceId) {
var _a, _b;
try {
const connectionKey = this.getConnectionKey(serviceId);
const existingMetrics = await ((_a = this.redis) === null || _a === void 0 ? void 0 : _a.get(connectionKey));
let metrics;
if (existingMetrics) {
metrics = JSON.parse(existingMetrics);
metrics.activeConnections = Math.max(0, (metrics.activeConnections || 1) - 1);
}
else {
metrics = {
...this.metrics,
activeConnections: 0,
};
}
if (this.cacheEnabled) {
await ((_b = this.redis) === null || _b === void 0 ? void 0 : _b.set(connectionKey, JSON.stringify(metrics), "EX", 24 * 60 * 60));
}
}
catch (error) {
if (this.debug) {
utils_1.log.error(`Failed to decrement connections for service ${serviceId}:`, error);
}
}
}
async updateSelectionMetrics(serviceId) {
var _a, _b;
try {
const connectionKey = this.getConnectionKey(serviceId);
const existingMetrics = await ((_a = this.redis) === null || _a === void 0 ? void 0 : _a.get(connectionKey));
let metrics;
if (existingMetrics) {
metrics = JSON.parse(existingMetrics);
}
else {
metrics = { ...this.metrics };
}
metrics.lastSelectedTime = Date.now();
if (this.cacheEnabled) {
await ((_b = this.redis) === null || _b === void 0 ? void 0 : _b.set(connectionKey, JSON.stringify(metrics), "EX", 24 * 60 * 60));
}
}
catch (error) {
if (this.debug) {
utils_1.log.error(`Failed to update selection metrics for service ${serviceId}:`, error);
}
}
}
async getSelectionMetrics(serviceId) {
var _a;
try {
if (!this.cacheEnabled) {
return null;
}
const metrics = await ((_a = this.redis) === null || _a === void 0 ? void 0 : _a.get(this.getConnectionKey(serviceId)));
return metrics ? JSON.parse(metrics) : null;
}
catch (error) {
if (this.debug) {
utils_1.log.error("Error getting service metrics:", error);
}
return null;
}
}
async refresh() {
var _a, _b;
try {
if (!this.cacheEnabled) {
if (this.debug) {
utils_1.log.debug("Cache is disabled, no need to refresh");
}
return;
}
const pattern = `${this.cachePrefix}:*`;
const keys = await ((_a = this.redis) === null || _a === void 0 ? void 0 : _a.keys(pattern));
if (keys && keys.length > 0) {
await ((_b = this.redis) === null || _b === void 0 ? void 0 : _b.del(...keys));
}
}
catch (error) {
console.log("Error refreshing Redis caches:", error);
throw new Error(`Failed to refresh caches: ${error.message}`);
}
}
}
exports.default = ConsulResolver;