UNPKG

@qrvey/health-checker

Version:

![install size](https://packagephobia.com/badge?p=@qrvey/health-checker) ![coverage](https://img.shields.io/badge/unit_test_coverage-87%25-brightgreen)

1,243 lines (1,224 loc) 47.4 kB
import { FetchService } from '@qrvey/fetch'; import { createClient } from 'redis'; import { format, createLogger, transports } from 'winston'; import { Client } from 'pg'; import { fromNodeProviderChain } from '@aws-sdk/credential-providers'; import { SignatureV4 } from '@smithy/signature-v4'; import { Sha256 } from '@aws-crypto/sha256-js'; import { HttpRequest } from '@smithy/protocol-http'; import os from 'os'; import v8 from 'v8'; import { monitorEventLoopDelay } from 'perf_hooks'; var __defProp = Object.defineProperty; var __defProps = Object.defineProperties; var __getOwnPropDescs = Object.getOwnPropertyDescriptors; var __getOwnPropSymbols = Object.getOwnPropertySymbols; var __hasOwnProp = Object.prototype.hasOwnProperty; var __propIsEnum = Object.prototype.propertyIsEnumerable; var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value; var __spreadValues = (a, b) => { for (var prop in b || (b = {})) if (__hasOwnProp.call(b, prop)) __defNormalProp(a, prop, b[prop]); if (__getOwnPropSymbols) for (var prop of __getOwnPropSymbols(b)) { if (__propIsEnum.call(b, prop)) __defNormalProp(a, prop, b[prop]); } return a; }; var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b)); var __name = (target, value) => __defProp(target, "name", { value, configurable: true }); var __objRest = (source, exclude) => { var target = {}; for (var prop in source) if (__hasOwnProp.call(source, prop) && exclude.indexOf(prop) < 0) target[prop] = source[prop]; if (source != null && __getOwnPropSymbols) for (var prop of __getOwnPropSymbols(source)) { if (exclude.indexOf(prop) < 0 && __propIsEnum.call(source, prop)) target[prop] = source[prop]; } return target; }; // src/registry.ts var registered = /* @__PURE__ */ new Set(); function registerHealthCheck(dependency) { const deps = Array.isArray(dependency) ? dependency : [ dependency ]; deps.forEach((d) => registered.add(d)); } __name(registerHealthCheck, "registerHealthCheck"); function getRegisteredHealthChecks() { return Array.from(registered); } __name(getRegisteredHealthChecks, "getRegisteredHealthChecks"); function clearRegistry() { registered.clear(); } __name(clearRegistry, "clearRegistry"); var _SystemStatusGateway = class _SystemStatusGateway { static async getHealthReport(body) { const endpoint = `/api/system/v1/status`; const data = await FetchService.post(endpoint, body, { privateDomain: true, useApiKey: true }); return data; } }; __name(_SystemStatusGateway, "SystemStatusGateway"); var SystemStatusGateway = _SystemStatusGateway; var { combine, timestamp, printf, colorize } = format; var logFormat = printf(({ level, message, timestamp: timestamp2 }) => { return `${timestamp2} [${level}]: ${message}`; }); var baseLogger = createLogger({ level: "info", format: combine(timestamp({ format: "YYYY-MM-DD HH:mm:ss" }), colorize(), logFormat), transports: [ new transports.Console() ] }); function formatMessage(context, data) { if (!data) return context; let detail = ""; if (data instanceof Error) { detail = data.stack || data.message; } else if (typeof data === "object") { try { detail = JSON.stringify(data); } catch (e) { detail = "[Unserializable object]"; } } else { detail = String(data); } return `${context} ${detail}`; } __name(formatMessage, "formatMessage"); var logger = { info: (msg, data) => baseLogger.info(formatMessage(msg, data)), warn: (msg, data) => baseLogger.warn(formatMessage(msg, data)), error: (msg, data) => baseLogger.error(formatMessage(msg, data)), debug: (msg, data) => baseLogger.debug(formatMessage(msg, data)) }; var logger_default = logger; // src/utils/requireEnv.ts function requireEnv(varName) { const value = process.env[varName]; if (!value) { logger_default.error(`[HealthCheck] Missing env variable: ${varName}`); throw new Error(`[HealthCheck] Missing required environment variable: ${varName}`); } return value; } __name(requireEnv, "requireEnv"); // src/utils/constants.ts var DEFAULT_HEALTH_CHECK_TIMEOUT = 5e3; var OK = "OK"; var FAILED = "FAILED"; var DEFAULT_HEALTH_STATUS = OK; var DEFAULT_SERVICE_NAME_HEADER = "q-service-name"; var HTTP_METHOD_OPTIONS = "OPTIONS"; var HTTP_STATUS_TOO_MANY_REQUESTS = 429; var RUNTIME_HEALTH_ERROR_REASON = "runtime_health_error"; var SYSTEM_STATUS_GATEWAY_CONTEXT = "system_status_gateway"; var DEFAULT_AWS_REGION = process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || "us-east-1"; function getNumberFromEnv(envVarName, fallbackValue) { const rawValue = process.env[envVarName]; if (rawValue === void 0) return fallbackValue; const parsed = parseInt(rawValue, 10); return Number.isNaN(parsed) || parsed < 0 ? fallbackValue : parsed; } __name(getNumberFromEnv, "getNumberFromEnv"); function getBooleanFromEnv(envVarName, fallbackValue) { const rawValue = process.env[envVarName]; if (rawValue === void 0) return fallbackValue; return rawValue.trim().toLowerCase() !== "false"; } __name(getBooleanFromEnv, "getBooleanFromEnv"); var VOID = "VOID"; var QRVEY_PRODUCT_TYPE_ULTRA = "A3"; function isUltraLicense() { return process.env.QRVEY_PRODUCT_TYPE === QRVEY_PRODUCT_TYPE_ULTRA; } __name(isUltraLicense, "isUltraLicense"); // src/services/dependencies/cache/redisHealthChecker.service.ts function createRedisClient() { const redisUrl = requireEnv("REDIS_URL"); return createClient({ url: redisUrl, socket: { connectTimeout: DEFAULT_HEALTH_CHECK_TIMEOUT } }); } __name(createRedisClient, "createRedisClient"); async function connectAndPing(client) { await client.connect(); await client.ping(); } __name(connectAndPing, "connectAndPing"); async function closeConnection(client) { try { await client.quit(); } catch (error) { logger_default.warn("[RedisHealthChecker] Failed to close Redis connection", error); } } __name(closeConnection, "closeConnection"); var RedisHealthChecker = { dependency: "cache", async check() { const client = createRedisClient(); try { await connectAndPing(client); if (process.env.NODE_ENV === "test") { logger_default.info("[RedisHealthCheck] check executed successfully"); } } catch (error) { logger_default.error("[RedisHealthChecker] Connection failed", error); throw error; } finally { await closeConnection(client); } } }; var PG_ENV_VAR = "MULTIPLATFORM_PG_CONNECTION_STRING"; async function connectAndQuery(client) { await client.connect(); await client.query("SELECT 1"); } __name(connectAndQuery, "connectAndQuery"); async function closeConnection2(client) { try { await client.end(); } catch (error) { logger_default.warn("[PostgreSQLHealthChecker] Failed to close connection", error); } } __name(closeConnection2, "closeConnection"); var PostgreSQLHealthChecker = { dependency: "database", async check() { const connectionString = requireEnv(PG_ENV_VAR); const client = new Client({ connectionString }); try { await connectAndQuery(client); if (process.env.NODE_ENV === "test") { logger_default.info("[PostgreSQLHealthChecker] check executed successfully"); } } catch (error) { logger_default.error("[PostgreSQLHealthChecker] Failed to connect or query", error); throw error; } finally { await closeConnection2(client); } } }; function getRabbitCredentials() { const user = requireEnv("RABBITMQ_USER"); const pass = requireEnv("RABBITMQ_PASSWORD"); return { user, pass }; } __name(getRabbitCredentials, "getRabbitCredentials"); function getRabbitHttpHost() { const raw = requireEnv("RABBITMQ_HTTP_HOST"); const url = new URL(raw); const basePath = url.pathname.replace(/\/$/, ""); return `${url.protocol}//${url.host}${basePath}`; } __name(getRabbitHttpHost, "getRabbitHttpHost"); function buildRabbitHttpConfig() { const { user, pass } = getRabbitCredentials(); const basicAuth = Buffer.from(`${user}:${pass}`).toString("base64"); const authHeader = `Basic ${basicAuth}`; return { baseDomain: getRabbitHttpHost(), authHeader }; } __name(buildRabbitHttpConfig, "buildRabbitHttpConfig"); async function ping() { const { baseDomain, authHeader } = buildRabbitHttpConfig(); const res = await fetch(`${baseDomain}/api/overview`, { headers: { Authorization: authHeader } }); if (!res.ok) { throw new Error(`[RabbitMQHealthChecker] Management API ping failed: ${res.statusText}`); } } __name(ping, "ping"); function listConsumers() { const { baseDomain, authHeader } = buildRabbitHttpConfig(); return FetchService.get("/api/consumers", { baseDomain, headers: { Authorization: authHeader } }); } __name(listConsumers, "listConsumers"); async function getConsumersSet() { return new Set((await listConsumers()).map((c) => { var _a, _b, _c; return `${(_a = c == null ? void 0 : c.consumer_tag) != null ? _a : ""}::${(_c = (_b = c == null ? void 0 : c.queue) == null ? void 0 : _b.name) != null ? _c : ""}`; })); } __name(getConsumersSet, "getConsumersSet"); async function validateQueues(queues, consumerTag, consumersSet) { const missingQueues = queues.filter((queue) => !consumersSet.has(`${consumerTag}::${queue}`)); if (missingQueues.length > 0) { throw new Error(`[RabbitMQHealthChecker] Missing subscriptions for queues: ${missingQueues.join(", ")} for consumerTag ${consumerTag}`); } } __name(validateQueues, "validateQueues"); async function handleConsumersCheck(param, metadata) { var _a, _b, _c, _d; const shouldLoadConsumers = ((_a = param == null ? void 0 : param.queues) == null ? void 0 : _a.length) || (param == null ? void 0 : param.returnConsumersSet); if (!shouldLoadConsumers) return; const consumersSet = (_b = param == null ? void 0 : param.consumersSet) != null ? _b : await getConsumersSet(); if ((_c = param == null ? void 0 : param.queues) == null ? void 0 : _c.length) { const consumerTag = (_d = param.hostName) != null ? _d : requireEnv("HOSTNAME"); await validateQueues(param.queues, consumerTag, consumersSet); } if (param == null ? void 0 : param.returnConsumersSet) { metadata.consumersSet = consumersSet; } } __name(handleConsumersCheck, "handleConsumersCheck"); async function performCheck(param) { const metadata = {}; if (!(param == null ? void 0 : param.omitPing)) await ping(); await handleConsumersCheck(param, metadata); if (process.env.NODE_ENV === "test") { logger_default.info("[RabbitMQHealthChecker] check executed successfully"); } return { status: OK, metadata }; } __name(performCheck, "performCheck"); var RabbitMQHealthChecker = { dependency: "eventBroker", async check(param) { try { return await performCheck(param); } catch (error) { logger_default.error("[RabbitMQHealthChecker] Health check failed", error); throw error; } } }; async function signAWSRequest(url, method = "GET", body, region = DEFAULT_AWS_REGION, service = "es") { try { const parsedUrl = new URL(url); const request = new HttpRequest({ method, protocol: parsedUrl.protocol, hostname: parsedUrl.hostname, port: parsedUrl.port ? Number(parsedUrl.port) : void 0, path: parsedUrl.pathname + parsedUrl.search, headers: { "Content-Type": "application/json", host: parsedUrl.hostname }, body }); const credentialsProvider = fromNodeProviderChain(); const credentials = await credentialsProvider(); const signer = new SignatureV4({ credentials, region, service, sha256: Sha256 }); const signedRequest = await signer.sign(request); return signedRequest.headers; } catch (error) { throw new Error(`Failed to sign AWS request: ${error instanceof Error ? error.message : "Unknown error"}`); } } __name(signAWSRequest, "signAWSRequest"); // src/services/dependencies/warehouse/utils/defaultConstants.ts var DEFAULT_ES_HEALTH_CPU_USAGE_PERCENT = 95; var DEFAULT_ES_HEALTH_JVM_HEAP_PERCENT = 90; var DEFAULT_ES_HEALTH_DISK_USAGE_PERCENT = 90; var DEFAULT_ES_HEALTH_CIRCUIT_BREAKER_PERCENT = 80; var DEFAULT_ES_HEALTH_SEARCH_QUEUE_THRESHOLD = 1; var envNumber = /* @__PURE__ */ __name((value, fallback) => { const parsed = Number(value); return Number.isFinite(parsed) ? parsed : fallback; }, "envNumber"); var DEFAULT_HEALTH_THRESHOLDS = { CPU_USAGE_PERCENT: envNumber(process.env.ES_HEALTH_CPU_USAGE_PERCENT, DEFAULT_ES_HEALTH_CPU_USAGE_PERCENT), JVM_HEAP_PERCENT: envNumber(process.env.ES_HEALTH_JVM_HEAP_PERCENT, DEFAULT_ES_HEALTH_JVM_HEAP_PERCENT), DISK_USAGE_PERCENT: envNumber(process.env.ES_HEALTH_DISK_USAGE_PERCENT, DEFAULT_ES_HEALTH_DISK_USAGE_PERCENT), CIRCUIT_BREAKER_PERCENT: envNumber(process.env.ES_HEALTH_CIRCUIT_BREAKER_PERCENT, DEFAULT_ES_HEALTH_CIRCUIT_BREAKER_PERCENT), SEARCH_QUEUE_THRESHOLD: envNumber(process.env.ES_HEALTH_SEARCH_QUEUE_THRESHOLD, DEFAULT_ES_HEALTH_SEARCH_QUEUE_THRESHOLD) }; // src/services/dependencies/warehouse/utils/constants.ts var CLUSTER_HEALTH_ENDPOINT = "/_cluster/health"; var NODES_STATS_ENDPOINT = "/_nodes/stats"; var CLUSTER_STATS_ENDPOINT = "/_cluster/stats"; function shouldUseBasicAuth() { var _a, _b; const hasAuthUser = (_a = process.env.ELASTICSEARCH_AUTH_USER) == null ? void 0 : _a.trim(); const hasAuthPassword = (_b = process.env.ELASTICSEARCH_AUTH_PASSWORD) == null ? void 0 : _b.trim(); return Boolean(hasAuthUser && hasAuthPassword); } __name(shouldUseBasicAuth, "shouldUseBasicAuth"); function getElasticsearchConfig(params) { var _a, _b; if ((_a = params == null ? void 0 : params.connection) == null ? void 0 : _a.host) { return { host: params.connection.host, port: params.connection.port, timeout: params.connection.timeout || parseInt(process.env.ELASTICSEARCH_TIMEOUT || "5000", 10), authType: "basic", username: params.connection.username, password: params.connection.password, ssl: { rejectUnauthorized: false } }; } const useBasicAuth = shouldUseBasicAuth(); const host = (_b = process.env.ELASTICSEARCH_HOST) != null ? _b : ""; const baseConfig = { host, port: parseInt(process.env.ELASTICSEARCH_PORT || "9200", 10), timeout: parseInt(process.env.ELASTICSEARCH_TIMEOUT || "5000", 10) }; if (useBasicAuth) { return __spreadProps(__spreadValues({}, baseConfig), { authType: "basic", username: process.env.ELASTICSEARCH_AUTH_USER, password: process.env.ELASTICSEARCH_AUTH_PASSWORD, ssl: { rejectUnauthorized: false } }); } return __spreadProps(__spreadValues({}, baseConfig), { authType: "aws", useAWSAuth: true }); } __name(getElasticsearchConfig, "getElasticsearchConfig"); function getThresholds(params) { var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j; return { cpuUsagePercent: (_b = (_a = params == null ? void 0 : params.thresholds) == null ? void 0 : _a.cpuUsagePercent) != null ? _b : DEFAULT_HEALTH_THRESHOLDS.CPU_USAGE_PERCENT, jvmHeapPercent: (_d = (_c = params == null ? void 0 : params.thresholds) == null ? void 0 : _c.jvmHeapPercent) != null ? _d : DEFAULT_HEALTH_THRESHOLDS.JVM_HEAP_PERCENT, diskUsagePercent: (_f = (_e = params == null ? void 0 : params.thresholds) == null ? void 0 : _e.diskUsagePercent) != null ? _f : DEFAULT_HEALTH_THRESHOLDS.DISK_USAGE_PERCENT, circuitBreakerPercent: (_h = (_g = params == null ? void 0 : params.thresholds) == null ? void 0 : _g.circuitBreakerPercent) != null ? _h : DEFAULT_HEALTH_THRESHOLDS.CIRCUIT_BREAKER_PERCENT, searchQueueThreshold: (_j = (_i = params == null ? void 0 : params.thresholds) == null ? void 0 : _i.searchQueueThreshold) != null ? _j : DEFAULT_HEALTH_THRESHOLDS.SEARCH_QUEUE_THRESHOLD }; } __name(getThresholds, "getThresholds"); function buildElasticsearchUrl(config) { if (config.host.includes(":") || !config.port) { return config.host; } return `${config.host}:${config.port}`; } __name(buildElasticsearchUrl, "buildElasticsearchUrl"); function buildRequestHeaders(config) { const headers = { "Content-Type": "application/json" }; if (config.authType === "basic" && config.username && config.password) { const auth = btoa(`${config.username}:${config.password}`); headers["Authorization"] = `Basic ${auth}`; } return headers; } __name(buildRequestHeaders, "buildRequestHeaders"); async function fetchElasticsearchData(config, endpoint) { var _a, _b; const baseDomain = buildElasticsearchUrl(config); let headers; const useAWSAuth = config.authType === "aws" || config.useAWSAuth === true; if (useAWSAuth) { const protocol = config.host.startsWith("https://") ? "" : "https://"; const fullUrl = `${protocol}${baseDomain}${endpoint}`; headers = await signAWSRequest(fullUrl); } else { headers = buildRequestHeaders(config); } const originalTlsReject = process.env.NODE_TLS_REJECT_UNAUTHORIZED; if (((_a = config.ssl) == null ? void 0 : _a.rejectUnauthorized) === false) { process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; } try { const response = await FetchService.get(endpoint, { baseDomain, headers }); return response; } catch (error) { logger_default.error(`[WarehouseHealthChecker] Request failed to ${endpoint}`, error); throw error; } finally { if (((_b = config.ssl) == null ? void 0 : _b.rejectUnauthorized) === false) { if (originalTlsReject === void 0) { delete process.env.NODE_TLS_REJECT_UNAUTHORIZED; } else { process.env.NODE_TLS_REJECT_UNAUTHORIZED = originalTlsReject; } } } } __name(fetchElasticsearchData, "fetchElasticsearchData"); function checkCircuitBreakerPercent(breakers, threshold = DEFAULT_HEALTH_THRESHOLDS.CIRCUIT_BREAKER_PERCENT) { if (!breakers) return false; const breakerTypes = [ "parent", "request", "fielddata", "in_flight_requests" ]; return breakerTypes.some((type) => { const breaker = breakers == null ? void 0 : breakers[type]; if (!breaker || typeof breaker !== "object") return false; const breakerObj = breaker; const estimatedSize = breakerObj.estimated_size_in_bytes || breakerObj.estimated_size || 0; const limitSize = breakerObj.limit_size_in_bytes || breakerObj.limit_size || breakerObj.limit || 1; const percent = estimatedSize / limitSize * 100; return percent > threshold; }); } __name(checkCircuitBreakerPercent, "checkCircuitBreakerPercent"); function buildEmptyNodeMetrics() { return { circuitBreakersException: false, threadPoolRejections: false, searchQueuedRequests: 0, cpuUsagePercent: 0, jvmHeapUsagePercent: 0, diskUsagePercent: 0, totalDiskBytes: 0, usedDiskBytes: 0, totalHeapMaxBytes: 0, totalHeapUsedBytes: 0 }; } __name(buildEmptyNodeMetrics, "buildEmptyNodeMetrics"); function buildNodeMetrics({ circuitBreakersException, threadPoolRejections, searchQueuedRequests, cpuSum, jvmHeapSum, diskSum, nodeCount, totalDiskBytes, usedDiskBytes, totalHeapMaxBytes, totalHeapUsedBytes }) { return { circuitBreakersException, threadPoolRejections, searchQueuedRequests, cpuUsagePercent: cpuSum / nodeCount, jvmHeapUsagePercent: jvmHeapSum / nodeCount, diskUsagePercent: diskSum / nodeCount, totalDiskBytes, usedDiskBytes, totalHeapMaxBytes, totalHeapUsedBytes }; } __name(buildNodeMetrics, "buildNodeMetrics"); function evaluateNodesMetrics(nodeStats, params) { var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q; if (nodeStats.length === 0) { return buildEmptyNodeMetrics(); } const thresholds = getThresholds(params); const skipRules = (params == null ? void 0 : params.skipRules) || {}; let circuitBreakersException = false; let threadPoolRejections = false; let searchQueuedRequests = 0; let cpuSum = 0; let jvmHeapSum = 0; let diskSum = 0; let totalDiskBytes = 0; let usedDiskBytes = 0; let totalHeapMaxBytes = 0; let totalHeapUsedBytes = 0; for (const node of nodeStats) { cpuSum += ((_b = (_a = node.os) == null ? void 0 : _a.cpu) == null ? void 0 : _b.percent) || 0; jvmHeapSum += ((_d = (_c = node.jvm) == null ? void 0 : _c.mem) == null ? void 0 : _d.heap_used_percent) || 0; totalHeapMaxBytes += ((_f = (_e = node.jvm) == null ? void 0 : _e.mem) == null ? void 0 : _f.heap_max_in_bytes) || 0; totalHeapUsedBytes += ((_h = (_g = node.jvm) == null ? void 0 : _g.mem) == null ? void 0 : _h.heap_used_in_bytes) || 0; searchQueuedRequests += ((_j = (_i = node.thread_pool) == null ? void 0 : _i.search) == null ? void 0 : _j.queue) || 0; const fs = (_k = node.fs) == null ? void 0 : _k.total; if (fs) { const used = fs.total_in_bytes - fs.available_in_bytes; diskSum += used / fs.total_in_bytes * 100; totalDiskBytes += fs.total_in_bytes; usedDiskBytes += used; } const bothCriticalChecksDisabled = skipRules.skipCircuitBreakerCheck && skipRules.skipThreadPoolCheck; const bothConditionsFound = circuitBreakersException && threadPoolRejections; if (bothCriticalChecksDisabled || bothConditionsFound) { continue; } if (!circuitBreakersException && !skipRules.skipCircuitBreakerCheck) { circuitBreakersException = checkCircuitBreakerPercent(node.breakers, thresholds.circuitBreakerPercent); } if (!threadPoolRejections && !skipRules.skipThreadPoolCheck) { const threadPool = node.thread_pool; threadPoolRejections = (((_l = threadPool == null ? void 0 : threadPool.search) == null ? void 0 : _l.rejected) || 0) > 0 && (((_m = threadPool == null ? void 0 : threadPool.search) == null ? void 0 : _m.queue) || 0) > 0 || (((_n = threadPool == null ? void 0 : threadPool.write) == null ? void 0 : _n.rejected) || 0) > 0 && (((_o = threadPool == null ? void 0 : threadPool.write) == null ? void 0 : _o.queue) || 0) > 0 || (((_p = threadPool == null ? void 0 : threadPool.bulk) == null ? void 0 : _p.rejected) || 0) > 0 && (((_q = threadPool == null ? void 0 : threadPool.bulk) == null ? void 0 : _q.queue) || 0) > 0; } } const nodeCount = nodeStats.length; return buildNodeMetrics({ circuitBreakersException, threadPoolRejections, searchQueuedRequests, cpuSum, jvmHeapSum, diskSum, nodeCount, totalDiskBytes, usedDiskBytes, totalHeapMaxBytes, totalHeapUsedBytes }); } __name(evaluateNodesMetrics, "evaluateNodesMetrics"); async function getClusterStats(config, params) { var _a, _b; try { const [healthData, nodesStatsData, clusterStatsData] = await Promise.all([ fetchElasticsearchData(config, CLUSTER_HEALTH_ENDPOINT), fetchElasticsearchData(config, NODES_STATS_ENDPOINT), fetchElasticsearchData(config, CLUSTER_STATS_ENDPOINT) ]); const nodeStats = Object.values(nodesStatsData.nodes || {}); const nodeMetrics = evaluateNodesMetrics(nodeStats, params); return { clusterName: healthData.cluster_name || "unknown", status: healthData.status || "red", nodes: { total: healthData.number_of_nodes || 0, successful: healthData.number_of_data_nodes || 0, failed: 0 }, cpuUsagePercent: nodeMetrics.cpuUsagePercent, jvmHeapUsagePercent: nodeMetrics.jvmHeapUsagePercent, diskUsagePercent: nodeMetrics.diskUsagePercent, activePrimaryShards: healthData.active_primary_shards || 0, activeShards: healthData.active_shards || 0, relocatingShards: healthData.relocating_shards || 0, initializingShards: healthData.initializing_shards || 0, unassignedShards: healthData.unassigned_shards || 0, circuitBreakersException: nodeMetrics.circuitBreakersException, threadPoolRejections: nodeMetrics.threadPoolRejections, searchQueuedRequests: nodeMetrics.searchQueuedRequests, indicesCount: (_b = (_a = clusterStatsData == null ? void 0 : clusterStatsData.indices) == null ? void 0 : _a.count) != null ? _b : void 0, totalDiskBytes: nodeMetrics.totalDiskBytes, usedDiskBytes: nodeMetrics.usedDiskBytes, totalHeapMaxBytes: nodeMetrics.totalHeapMaxBytes, totalHeapUsedBytes: nodeMetrics.totalHeapUsedBytes }; } catch (error) { logger_default.error("[WarehouseHealthChecker] Error fetching cluster stats", error); throw error; } } __name(getClusterStats, "getClusterStats"); function throwError(errorMessage) { const message = JSON.stringify(errorMessage, null, 2); logger_default.error(message); throw new Error(message); } __name(throwError, "throwError"); function determineHealthStatus(stats, params) { const skipRules = (params == null ? void 0 : params.skipRules) || {}; const thresholds = getThresholds(params); if (!skipRules.skipCircuitBreakerCheck && stats.circuitBreakersException) { throwError(`Circuit breakers near limit (ratio > ${thresholds.circuitBreakerPercent}%) - cluster at risk`); } if (!skipRules.skipThreadPoolCheck && stats.threadPoolRejections) { throwError("Thread pools rejecting requests - cluster saturated"); } if (!skipRules.skipSearchQueueCheck && stats.searchQueuedRequests >= thresholds.searchQueueThreshold) { throwError(`Search requests queuing up: ${stats.searchQueuedRequests} queued (threshold: ${thresholds.searchQueueThreshold}) - warehouse destabilizing`); } if (!skipRules.skipCpuCheck && stats.cpuUsagePercent >= thresholds.cpuUsagePercent) { throwError(`High CPU usage: ${stats.cpuUsagePercent}% (threshold: ${thresholds.cpuUsagePercent}%)`); } if (!skipRules.skipJvmHeapCheck && stats.jvmHeapUsagePercent >= thresholds.jvmHeapPercent) { throwError(`High JVM heap usage: ${stats.jvmHeapUsagePercent}% (threshold: ${thresholds.jvmHeapPercent}%)`); } if (!skipRules.skipDiskCheck && stats.diskUsagePercent >= thresholds.diskUsagePercent) { throwError(`High disk usage: ${stats.diskUsagePercent}% (threshold: ${thresholds.diskUsagePercent}%)`); } if (!skipRules.skipClusterStatusCheck) { const isClusterUnhealthy = (params == null ? void 0 : params.customClusterStatusCheck) ? !params.customClusterStatusCheck(stats.status) : stats.status === "red"; if (isClusterUnhealthy) { throwError(`Cluster status is ${stats.status}`); } } return "OK"; } __name(determineHealthStatus, "determineHealthStatus"); async function handleClusterStatsCheck(params, config, metadata) { var _a; const clusterStats = (_a = params == null ? void 0 : params.clusterStats) != null ? _a : await getClusterStats(config, params); if (params == null ? void 0 : params.returnClusterStats) { metadata.clusterStats = clusterStats; } return clusterStats; } __name(handleClusterStatsCheck, "handleClusterStatsCheck"); async function performCheck2(params) { const checkTimestamp = (/* @__PURE__ */ new Date()).toISOString(); try { const config = getElasticsearchConfig(params); const thresholds = getThresholds(params); logger_default.info("[WarehouseHealthChecker] Starting health check", JSON.stringify({ host: config.host, port: config.port, thresholds, skipRules: params == null ? void 0 : params.skipRules }, null, 2)); const metadata = { checkTimestamp }; const clusterStats = await handleClusterStatsCheck(params, config, metadata); let status; try { status = determineHealthStatus(clusterStats, params); } catch (ruleError) { const errorMessage = ruleError instanceof Error ? ruleError.message : "Unknown error"; logger_default.warn("[WarehouseHealthChecker] Check failed but returning cluster stats", { error: errorMessage }); if ((params == null ? void 0 : params.returnClusterStats) && !metadata.clusterStats) { metadata.clusterStats = clusterStats; } metadata.errorMessage = errorMessage; return { status: FAILED, metadata }; } logger_default.info(`[WarehouseHealthChecker] Health check completed ${JSON.stringify({ status, clusterName: clusterStats.clusterName, clusterStatus: clusterStats.status, cpuUsage: clusterStats.cpuUsagePercent, jvmHeapUsage: clusterStats.jvmHeapUsagePercent, diskUsage: clusterStats.diskUsagePercent, circuitBreakersException: clusterStats.circuitBreakersException, threadPoolRejections: clusterStats.threadPoolRejections, searchQueuedRequests: clusterStats.searchQueuedRequests })}`); const result = { status, metadata }; if ((params == null ? void 0 : params.returnClusterStats) && !metadata.clusterStats) { metadata.clusterStats = clusterStats; } return result; } catch (error) { const errorMessage = error instanceof Error ? error.message : "Unknown error"; logger_default.error("[WarehouseHealthChecker] Health check failed", { error: errorMessage, checkTimestamp, params, stack: error instanceof Error ? error.stack : void 0 }); throw error; } } __name(performCheck2, "performCheck"); var WarehouseHealthChecker = { dependency: "warehouse", async check(params) { if (!isUltraLicense()) { return { status: VOID, metadata: { checkTimestamp: (/* @__PURE__ */ new Date()).toISOString(), errorMessage: "Warehouse is not part of the current product license infrastructure" } }; } const warehouseParams = params; try { return await performCheck2(warehouseParams); } catch (error) { logger_default.error("[WarehouseHealthChecker] Health check failed", error); throw error; } } }; // src/utils/checkersMap.ts var checkersMap = { cache: RedisHealthChecker, database: PostgreSQLHealthChecker, eventBroker: RabbitMQHealthChecker, warehouse: WarehouseHealthChecker }; // src/utils/withTimeout.ts function withTimeout(context, healthCheckFn, ms = DEFAULT_HEALTH_CHECK_TIMEOUT) { return new Promise((resolve, reject) => { const timeoutId = setTimeout(() => { const message = `[${context}] HealthChecker timeout after ${ms}ms`; logger_default.error(message); reject(new Error(message)); }, ms); healthCheckFn.then((value) => { clearTimeout(timeoutId); resolve(value); }).catch((error) => { clearTimeout(timeoutId); reject(error); }); }); } __name(withTimeout, "withTimeout"); async function withTimeoutAndTiming(context, healthCheckPromise, ms = DEFAULT_HEALTH_CHECK_TIMEOUT) { const start3 = Date.now(); const result = await withTimeout(context, healthCheckPromise, ms); const durationMs = Date.now() - start3; const metadata = result && typeof result === "object" && "metadata" in result ? result.metadata : void 0; const status = result && typeof result === "object" && "status" in result ? result.status : void 0; return { durationMs, status, metadata }; } __name(withTimeoutAndTiming, "withTimeoutAndTiming"); // src/services/healthCheck.service.ts var _HealthCheckService = class _HealthCheckService { static async check(dependencies, params = {}) { const selectedDependencies = resolveDependencies(dependencies); if (!params.force) { const result = await tryFetchRemoteHealthReport(selectedDependencies, params); if (result) return result; } const results = await runDependencyChecks(selectedDependencies, params); return buildDetails(selectedDependencies, results); } }; __name(_HealthCheckService, "HealthCheckService"); var HealthCheckService = _HealthCheckService; function resolveDependencies(dependencies, skip) { const all = dependencies != null ? dependencies : getRegisteredHealthChecks(); return skip ? all.filter((dep) => !skip.includes(dep)) : all; } __name(resolveDependencies, "resolveDependencies"); async function runDependencyChecks(dependencies, params = {}) { return Promise.allSettled(dependencies.map(async (dep) => { var _a; const checker = checkersMap[dep]; if (!checker) { const msg = `[HealthCheckService] No checker found for dependency "${dep}"`; logger_default.error(msg); return Promise.reject(new Error(msg)); } const depParams = params; const dependencyParam = (_a = depParams[dep]) != null ? _a : {}; const mergedParam = __spreadProps(__spreadValues({}, dependencyParam), { hostName: params.hostName }); try { return await withTimeoutAndTiming(dep, checker.check(mergedParam)); } catch (err) { logger_default.error(`[${dep}] HealthChecker threw an error`, err); return Promise.reject(err); } })); } __name(runDependencyChecks, "runDependencyChecks"); async function tryFetchRemoteHealthReport(dependencies, params) { try { const _a = params, { skip = [], force = false, hostName } = _a, otherParams = __objRest(_a, ["skip", "force", "hostName"]); const body = { dependencies, service: hostName != null ? hostName : process.env.HOSTNAME || "unknown", skip, force, params: otherParams }; const result = await withTimeout(SYSTEM_STATUS_GATEWAY_CONTEXT, SystemStatusGateway.getHealthReport(body)); return result; } catch (err) { const message = err instanceof Error ? err.message : "Unknown error during remote check"; logger_default.warn(`[HealthCheckService] Remote health check fallback triggered: ${message}`); return null; } } __name(tryFetchRemoteHealthReport, "tryFetchRemoteHealthReport"); function buildDetails(dependencies, results) { var _a, _b, _c; const details = {}; let hasFailure = false; for (let i = 0; i < dependencies.length; i++) { const dep = dependencies[i]; const result = results[i]; if (result.status === "fulfilled") { const isVoid = result.value.status === VOID; details[dep] = { status: isVoid ? VOID : OK, durationMs: result.value.durationMs, metadata: (_a = result.value.metadata) != null ? _a : {} }; } else { details[dep] = { status: FAILED, durationMs: (_c = (_b = result.reason) == null ? void 0 : _b.durationMs) != null ? _c : 0 }; hasFailure = true; } } return { status: hasFailure ? FAILED : OK, details }; } __name(buildDetails, "buildDetails"); // src/utils/runtimeHealth.constants.ts var RUNTIME_HEALTH_CACHE_SOURCE = { computed: "computed", cached: "cached", in_flight: "in_flight" }; var DEFAULT_RUNTIME_HEALTH_WINDOW_MS = getNumberFromEnv("RUNTIME_HEALTH_WINDOW_MS", 250); var DEFAULT_RUNTIME_HEALTH_THRESHOLDS = { cpuPercent: getNumberFromEnv("RUNTIME_HEALTH_CPU_PERCENT", 90), heapPercent: getNumberFromEnv("RUNTIME_HEALTH_HEAP_PERCENT", 90), eventLoopDelayMs: getNumberFromEnv("RUNTIME_HEALTH_EVENT_LOOP_DELAY_MS", 200) }; var DEFAULT_RUNTIME_HEALTH_ENABLED_METRICS = { cpuPercent: getBooleanFromEnv("RUNTIME_HEALTH_ENABLE_CPU", true), heapPercent: getBooleanFromEnv("RUNTIME_HEALTH_ENABLE_HEAP", true), eventLoopDelayMs: getBooleanFromEnv("RUNTIME_HEALTH_ENABLE_EVENT_LOOP_DELAY", true) }; var DEFAULT_RUNTIME_HEALTH_LOG_GET = getBooleanFromEnv("RUNTIME_HEALTH_LOG_GET", false); var DEFAULT_RUNTIME_HEALTH_CACHE_TTL_MS = getNumberFromEnv("RUNTIME_HEALTH_CACHE_TTL_SECONDS", 3) * 1e3; // src/utils/runtimeHealth.utils.ts function wait(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } __name(wait, "wait"); function bytesToMb(bytes) { return Number((bytes / (1024 * 1024)).toFixed(2)); } __name(bytesToMb, "bytesToMb"); // src/services/runtimeHealth.service.ts var _RuntimeHealthService = class _RuntimeHealthService { static async getMetrics(options = {}) { var _a; const windowMs = (_a = options.windowMs) != null ? _a : DEFAULT_RUNTIME_HEALTH_WINDOW_MS; const enabledMetrics = getEnabledMetrics(); const [cpuPercent, eventLoopDelayMs] = await Promise.all([ enabledMetrics.cpuPercent ? getCpuPercent(windowMs) : null, enabledMetrics.eventLoopDelayMs ? getEventLoopDelay(windowMs) : null ]); let heapPercent = null; let heapLimitMb = null; if (enabledMetrics.heapPercent) { const heapStats = v8.getHeapStatistics(); heapLimitMb = bytesToMb(heapStats.heap_size_limit); heapPercent = heapStats.heap_size_limit > 0 ? process.memoryUsage().heapUsed / heapStats.heap_size_limit * 100 : 0; } return { cpuPercent, heapPercent, eventLoopDelayMs, heapLimitMb }; } static async get(options = {}) { const thresholds = getThresholds2(options); const enabledMetrics = getEnabledMetrics(); const metrics = await this.getMetrics(options); const failedMetrics = evaluateThresholds(metrics, thresholds, enabledMetrics); const status = failedMetrics.length > 0 ? FAILED : OK; const details = { status, metrics, enabledMetrics, thresholds, failedMetrics }; const runtimeStatus = { status, details }; if (DEFAULT_RUNTIME_HEALTH_LOG_GET) { logger_default.info("[RuntimeHealthService.get]", runtimeStatus); } return runtimeStatus; } static async getStatus(options = {}) { return this.get(options); } }; __name(_RuntimeHealthService, "RuntimeHealthService"); var RuntimeHealthService = _RuntimeHealthService; var RuntimeHealth = RuntimeHealthService; function evaluateThresholds(metrics, thresholds, enabledMetrics = DEFAULT_RUNTIME_HEALTH_ENABLED_METRICS) { const failedMetrics = []; if (enabledMetrics.cpuPercent && typeof metrics.cpuPercent === "number" && metrics.cpuPercent >= thresholds.cpuPercent) { failedMetrics.push("cpuPercent"); } if (enabledMetrics.heapPercent && typeof metrics.heapPercent === "number" && metrics.heapPercent >= thresholds.heapPercent) { failedMetrics.push("heapPercent"); } if (enabledMetrics.eventLoopDelayMs && typeof metrics.eventLoopDelayMs === "number" && metrics.eventLoopDelayMs >= thresholds.eventLoopDelayMs) { failedMetrics.push("eventLoopDelayMs"); } return failedMetrics; } __name(evaluateThresholds, "evaluateThresholds"); function getEnabledMetrics() { return __spreadValues({}, DEFAULT_RUNTIME_HEALTH_ENABLED_METRICS); } __name(getEnabledMetrics, "getEnabledMetrics"); function getThresholds2(options) { const directThresholds = {}; if (typeof (options == null ? void 0 : options.cpuPercent) === "number") { directThresholds.cpuPercent = options.cpuPercent; } if (typeof (options == null ? void 0 : options.heapPercent) === "number") { directThresholds.heapPercent = options.heapPercent; } if (typeof (options == null ? void 0 : options.eventLoopDelayMs) === "number") { directThresholds.eventLoopDelayMs = options.eventLoopDelayMs; } return __spreadValues(__spreadValues(__spreadValues({}, DEFAULT_RUNTIME_HEALTH_THRESHOLDS), directThresholds), options == null ? void 0 : options.thresholds); } __name(getThresholds2, "getThresholds"); async function getCpuPercent(windowMs) { const startUsage = process.cpuUsage(); const startTime = process.hrtime.bigint(); await wait(windowMs); const endUsage = process.cpuUsage(startUsage); const elapsedMicros = Number(process.hrtime.bigint() - startTime) / 1e3; const cpuCount = Math.max(os.cpus().length, 1); const cpuMicros = endUsage.user + endUsage.system; if (elapsedMicros <= 0) return 0; return cpuMicros / (elapsedMicros * cpuCount) * 100; } __name(getCpuPercent, "getCpuPercent"); async function getEventLoopDelay(windowMs) { const histogram = monitorEventLoopDelay({ resolution: 20 }); histogram.enable(); await wait(windowMs); histogram.disable(); return histogram.mean / 1e6; } __name(getEventLoopDelay, "getEventLoopDelay"); // src/middleware/runtimeHealth/runtimeHealth.middleware.ts var handleRuntimeHealth = /* @__PURE__ */ __name(async (req, res, options, cache) => { if (req.method === HTTP_METHOD_OPTIONS) { return true; } const resolvedOptions = resolveStartOptions(options); if (!isRequestTargeted(req, resolvedOptions)) { return true; } try { const result = await getRuntimeHealthStatusWithCache(resolvedOptions.runtimeOptions, resolvedOptions.cacheTtlMs, cache); if (result.status === FAILED) { const cacheLog = DEFAULT_RUNTIME_HEALTH_LOG_GET && result.details.cache ? { cache: result.details.cache } : {}; logger_default.warn("[RuntimeHealthMiddleware] RuntimeHealth is unhealthy", __spreadValues({ path: req.path, method: req.method, failedMetrics: result.details.failedMetrics }, cacheLog)); res.sendJson(HTTP_STATUS_TOO_MANY_REQUESTS, result); return false; } if (DEFAULT_RUNTIME_HEALTH_LOG_GET && result.details.cache && result.details.cache.source !== RUNTIME_HEALTH_CACHE_SOURCE.computed) { logger_default.info("[RuntimeHealthMiddleware] RuntimeHealth from cache", { path: req.path, method: req.method, cache: result.details.cache }); } return true; } catch (err) { logger_default.error("[RuntimeHealthMiddleware] RuntimeHealth check failed", { error: err instanceof Error ? err.message : String(err), path: req.path, method: req.method }); res.sendJson(HTTP_STATUS_TOO_MANY_REQUESTS, { status: FAILED, details: { reason: RUNTIME_HEALTH_ERROR_REASON } }); return false; } }, "handleRuntimeHealth"); function createRuntimeHealthMiddlewareCacheState() { return {}; } __name(createRuntimeHealthMiddlewareCacheState, "createRuntimeHealthMiddlewareCacheState"); function isRequestTargeted(req, resolvedOptions) { if (resolvedOptions.serviceNames.length === 0) { return true; } const serviceName = getHeaderValue(req.headers, resolvedOptions.serviceNameHeader); return !!serviceName && resolvedOptions.serviceNames.includes(serviceName); } __name(isRequestTargeted, "isRequestTargeted"); function resolveStartOptions(options) { var _a, _b, _c, _d; if (!options) { return { runtimeOptions: void 0, serviceNames: [], serviceNameHeader: DEFAULT_SERVICE_NAME_HEADER, cacheTtlMs: DEFAULT_RUNTIME_HEALTH_CACHE_TTL_MS }; } if (isRuntimeHealthMiddlewareStartOptions(options)) { return { runtimeOptions: options.runtimeOptions, serviceNames: (_a = options.serviceNames) != null ? _a : [], serviceNameHeader: (_c = (_b = options.serviceNameHeader) == null ? void 0 : _b.toLowerCase()) != null ? _c : DEFAULT_SERVICE_NAME_HEADER, cacheTtlMs: (_d = options.cacheTtlMs) != null ? _d : DEFAULT_RUNTIME_HEALTH_CACHE_TTL_MS }; } return { runtimeOptions: options, serviceNames: [], serviceNameHeader: DEFAULT_SERVICE_NAME_HEADER, cacheTtlMs: DEFAULT_RUNTIME_HEALTH_CACHE_TTL_MS }; } __name(resolveStartOptions, "resolveStartOptions"); function isRuntimeHealthMiddlewareStartOptions(options) { return "runtimeOptions" in options || "serviceNames" in options || "serviceNameHeader" in options || "cacheTtlMs" in options; } __name(isRuntimeHealthMiddlewareStartOptions, "isRuntimeHealthMiddlewareStartOptions"); function getHeaderValue(headers, headerName) { const value = headers[headerName]; if (Array.isArray(value)) { return value[0]; } return typeof value === "string" ? value : void 0; } __name(getHeaderValue, "getHeaderValue"); async function getRuntimeHealthStatusWithCache(runtimeOptions, cacheTtlMs, cache = {}) { const now = Date.now(); if (isCacheValid(cache, now, cacheTtlMs)) { return withCacheInfo(cache.cachedStatus, { source: RUNTIME_HEALTH_CACHE_SOURCE.cached, ttlMs: cacheTtlMs, ageMs: now - cache.cachedAtMs, cachedAt: toISOStringOrUndefined(cache.cachedAtMs) }); } if (cache.inFlight) { const status2 = await cache.inFlight; return withCacheInfo(status2, { source: RUNTIME_HEALTH_CACHE_SOURCE.in_flight, ttlMs: cacheTtlMs, ageMs: cache.cachedAtMs != null ? Date.now() - cache.cachedAtMs : 0, cachedAt: toISOStringOrUndefined(cache.cachedAtMs) }); } cache.inFlight = RuntimeHealthService.get(runtimeOptions).then((status2) => { cache.cachedStatus = status2; cache.cachedAtMs = Date.now(); return status2; }).finally(() => { cache.inFlight = void 0; }); const status = await cache.inFlight; return withCacheInfo(status, { source: RUNTIME_HEALTH_CACHE_SOURCE.computed, ttlMs: cacheTtlMs, ageMs: 0, cachedAt: toISOStringOrUndefined(cache.cachedAtMs) }); } __name(getRuntimeHealthStatusWithCache, "getRuntimeHealthStatusWithCache"); function isCacheValid(cache, now, cacheTtlMs) { return cache.cachedStatus != null && cache.cachedAtMs != null && now - cache.cachedAtMs < cacheTtlMs; } __name(isCacheValid, "isCacheValid"); function toISOStringOrUndefined(ms) { return ms != null ? new Date(ms).toISOString() : void 0; } __name(toISOStringOrUndefined, "toISOStringOrUndefined"); function withCacheInfo(status, cacheInfo) { return __spreadProps(__spreadValues({}, status), { details: __spreadProps(__spreadValues({}, status.details), { cache: cacheInfo }) }); } __name(withCacheInfo, "withCacheInfo"); // src/interfaces/middleware.interface.ts var toMiddlewareRequest = /* @__PURE__ */ __name((req) => { var _a, _b, _c, _d, _e; return { method: ((_a = req.method) != null ? _a : "GET").toUpperCase(), path: (_d = (_c = (_b = req.path) != null ? _b : req.originalUrl) != null ? _c : req.url) != null ? _d : "/", headers: (_e = req.headers) != null ? _e : {} }; }, "toMiddlewareRequest"); // src/middleware/runtimeHealth/runtimeHealth.express.ts var start = /* @__PURE__ */ __name((options) => { const cache = createRuntimeHealthMiddlewareCacheState(); return async (req, res, next) => { const shouldContinue = await handleRuntimeHealth(toMiddlewareRequest(req), { sendJson: (statusCode, body) => res.status(statusCode).json(body) }, options, cache); if (shouldContinue) next(); }; }, "start"); // src/middleware/runtimeHealth/runtimeHealth.fastify.ts var start2 = /* @__PURE__ */ __name((options) => { const cache = createRuntimeHealthMiddlewareCacheState(); return async (req, reply, done) => { const shouldContinue = await handleRuntimeHealth(toMiddlewareRequest(req), { sendJson: (statusCode, body) => reply.code(statusCode).send(body) }, options, cache); if (shouldContinue) done(); }; }, "start"); // src/middleware/runtimeHealth/index.ts var RuntimeHealthMiddleware = { express: { start }, fastify: { start: start2 } }; export { RedisHealthChecker as CacheHealthChecker, DEFAULT_HEALTH_STATUS, PostgreSQLHealthChecker as DatabaseHealthChecker, RabbitMQHealthChecker as EventBrokerHealthChecker, FAILED, HealthCheckService, OK, RUNTIME_HEALTH_CACHE_SOURCE, RuntimeHealth, RuntimeHealthMiddleware, RuntimeHealthService, VOID, clearRegistry, getClusterStats, getElasticsearchConfig, registerHealthCheck }; //# sourceMappingURL=out.js.map //# sourceMappingURL=index.mjs.map