@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
1,658 lines • 80.1 kB
JavaScript
/**
* Provider Health Checking System
* Prevents 500 errors by validating provider availability and configuration
*/
import { logger } from "./logger.js";
import { AIProviderName, OpenAIModels, GoogleAIModels, AnthropicModels, BedrockModels, } from "../constants/enums.js";
import { PROJECT_ID_FORMAT, satisfiesFallbacks } from "./providerConfig.js";
import { basename } from "path";
import { createProxyFetch } from "../proxy/proxyFetch.js";
import { DEFAULT_OLLAMA_MODEL } from "../providers/ollama/constants.js";
import { ProviderFactory } from "../factories/providerFactory.js";
import { PROVIDER_DESCRIPTORS } from "../factories/providerDescriptors.js";
export class ProviderHealthChecker {
static healthCache = new Map();
static DEFAULT_TIMEOUT = 5000; // 5 seconds
static DEFAULT_CACHE_AGE = 300000; // 5 minutes
/**
* Fixed backoff window for the circuit breaker below, in milliseconds.
* Deliberately NOT the per-call `maxCacheAge` option: `consecutiveFailures`
* is a single process-wide map keyed only by provider name, shared by every
* caller (e.g. `checkFallbackProviderAvailability` hard-codes a 15s
* `maxCacheAge` for its own health-status cache). Evicting the breaker
* entry using whichever caller happens to read it next would let a
* short-TTL caller silently erase a backoff another caller is relying on
* — see `checkProviderHealth`. This constant governs breaker expiry alone
* and is intentionally independent of `maxCacheAge`/`DEFAULT_CACHE_AGE`.
*/
static CIRCUIT_BREAKER_RESET_MS = 300000; // 5 minutes
static CONSECUTIVE_FAILURE_THRESHOLD = ProviderHealthChecker.getValidatedFailureThreshold();
/**
* Circuit breaker state, keyed by provider.
*
* `count` is a count of consecutive failed CONNECTIVITY PROBES (plus the
* rare case of the check itself throwing) — never of "the provider is
* unconfigured". Those are different facts and only one of them is worth
* backing off from. A missing credential is a free, local, determinate
* answer that costs nothing to re-derive and flips the instant an env var
* is set; treating it as a failure is what let a sweep of ~38 providers
* (issue #1305 grew it from 8) blacklist every provider the machine
* happens not to have credentials for, after which the sweep reported
* `isConfigured: false` and no `responseTime` for a provider that was
* merely unconfigured — and kept reporting it after the key was supplied.
*
* `lastFailureAt` bounds how long a run of failures is held against a
* provider; see `checkProviderHealth`.
*/
static consecutiveFailures = new Map();
/**
* Hard cap on simultaneous per-provider checks inside
* `checkAllProvidersHealth`. Issue #1305: the sweep used to cover only 8
* of ~38 registered providers; now every non-opted-out descriptor is
* included (see `excludeFromHealthSweep`). Firing all ~38 at once via a
* single `Promise.allSettled` would be harmless by default
* (`includeConnectivityTest` defaults to false, so most calls never
* leave the process), but a caller who opts into
* `includeConnectivityTest: true` would otherwise open up to ~38
* concurrent outbound HTTP connections to that many different vendors in
* one burst. A worker pool keeps that bounded without reducing coverage
* — every provider is still checked, just not all in the same instant,
* and without the latency a fixed batch would add (see
* `checkAllProvidersHealth`).
*/
static MAX_CONCURRENT_HEALTH_CHECKS = 8;
/**
* Validate and return a safe failure threshold value
*/
static getValidatedFailureThreshold() {
const envValue = process.env.PROVIDER_FAILURE_THRESHOLD;
if (!envValue) {
return 3; // default
}
const parsed = Number(envValue);
if (isNaN(parsed) || parsed <= 0 || parsed > 10) {
logger.warn(`Invalid PROVIDER_FAILURE_THRESHOLD: ${envValue} (must be between 1 and 10), using default: 3`);
return 3;
}
return parsed;
}
/**
* Comprehensive health check for a provider
*/
static async checkProviderHealth(providerName, options = {}) {
const { timeout = this.DEFAULT_TIMEOUT, includeConnectivityTest = false, includeModelValidation = false, cacheResults = true, maxCacheAge = this.DEFAULT_CACHE_AGE, } = options;
// Check cache first
if (cacheResults) {
const cached = this.getCachedHealth(providerName, maxCacheAge);
if (cached) {
logger.debug(`Using cached health status for ${providerName}`);
return cached;
}
}
// Circuit breaker. It suppresses outbound PROBES, not the whole check:
// for most providers the step-1 configuration audit is local and free,
// so skipping it bought nothing and cost the caller a status object of
// a different shape — no `responseTime`, and a fabricated
// `isConfigured: false` for a provider nothing had looked at.
//
// LiteLLM and Ollama are the exception: their step-1 check
// (checkLiteLLMConfig/checkOllamaConfig) makes a real outbound request,
// because for a local runtime "configured" only means something if it
// means "reachable" — both providers have zero required env vars
// (`credentialsResolvedExternally` / no `apiKey`), so with no request at
// all every install would read as configured. Auto-select
// (`NeuroLink.hasProviderEnvVars`) depends on that: it always calls with
// `includeConnectivityTest: false`, so gating this probe on that flag
// instead of the breaker would make a dead local proxy look configured
// and win auto-select over a real, working cloud provider. So this
// probe still runs in shallow mode, but — like the step-3 connectivity
// probe — it is governed by the SAME breaker below (`allowRuntimeProbe`
// passed into checkEnvironmentConfiguration): skipped while blacklisted,
// and its outcome folded into `anyProbeFailed`/`anyProbeRan` further
// down, so a dead local proxy still gets backed off instead of eating a
// full timeout on every call forever.
//
// A run of failures older than CIRCUIT_BREAKER_RESET_MS is forgotten,
// which is what makes the "will be retried after cache TTL expires"
// warning below true. Nothing else could clear it, because the
// breaker's whole effect is to suppress the probe that would disprove
// it. This is deliberately the fixed CIRCUIT_BREAKER_RESET_MS window,
// not this call's own `maxCacheAge` — `consecutiveFailures` is a single
// shared map keyed only by provider name, so reusing whichever caller
// happens to invoke `checkProviderHealth` next (some call sites pass a
// much shorter `maxCacheAge` for their own unrelated caching needs)
// would let that caller silently evict a backoff a different caller is
// relying on.
const breaker = this.consecutiveFailures.get(providerName);
if (breaker !== undefined &&
Date.now() - breaker.lastFailureAt >= this.CIRCUIT_BREAKER_RESET_MS) {
this.consecutiveFailures.delete(providerName);
}
const failureCount = this.consecutiveFailures.get(providerName)?.count ?? 0;
const blacklisted = failureCount >= this.CONSECUTIVE_FAILURE_THRESHOLD;
if (blacklisted) {
logger.warn(`Provider ${providerName} blacklisted due to consecutive failures`, { failureCount });
}
const startTime = Date.now();
const healthStatus = {
provider: providerName,
isHealthy: false,
isConfigured: false,
hasApiKey: false,
lastChecked: new Date(),
configurationIssues: [],
recommendations: [],
};
try {
// 1. Check environment configuration. For LiteLLM/Ollama this
// includes a breaker-governed runtime probe — see the comment on
// `blacklisted` above.
const runtimeProbe = await this.checkEnvironmentConfiguration(providerName, healthStatus, timeout, !blacklisted);
// 2. Check API key validity (basic format validation)
await this.checkApiKeyValidity(providerName, healthStatus);
// 3. Optional: Connectivity test. Also governed by the breaker.
const issuesBeforeProbe = healthStatus.configurationIssues.length;
const probed = includeConnectivityTest && !blacklisted;
if (probed) {
await this.checkConnectivity(providerName, healthStatus, timeout);
}
const probeFailed = probed && healthStatus.configurationIssues.length > issuesBeforeProbe;
// Fold the step-1 runtime probe (if this provider has one) together
// with the step-3 connectivity probe into one pair of breaker
// signals, so a call where BOTH run and fail still counts as exactly
// ONE failure (the breaker steps consecutive calls, not consecutive
// requests).
const anyProbeRan = probed || runtimeProbe.ran;
const anyProbeFailed = probeFailed || runtimeProbe.failed;
// 4. Optional: Model validation
if (includeModelValidation) {
await this.checkModelAvailability(providerName, healthStatus);
}
if (blacklisted) {
healthStatus.error = `Provider blacklisted after ${failureCount} consecutive failures`;
healthStatus.warning =
"Provider will be retried after cache TTL expires";
healthStatus.configurationIssues.push(`Blacklisted due to ${failureCount} consecutive failures`);
healthStatus.recommendations.push("Check provider status and configuration");
}
// 5. Determine overall health
healthStatus.isHealthy =
healthStatus.isConfigured &&
healthStatus.hasApiKey &&
healthStatus.configurationIssues.length === 0;
healthStatus.responseTime = Date.now() - startTime;
// Cache results
if (cacheResults) {
this.healthCache.set(providerName, {
status: healthStatus,
timestamp: Date.now(),
});
}
// Only a probe that actually ran moves the breaker. A check that
// issued no request is not evidence either way: it neither proves the
// provider is failing nor that it has recovered.
if (anyProbeFailed) {
this.consecutiveFailures.set(providerName, {
count: failureCount + 1,
lastFailureAt: Date.now(),
});
}
else if (anyProbeRan) {
this.consecutiveFailures.delete(providerName);
}
logger.debug(`Health check completed for ${providerName}`, {
isHealthy: healthStatus.isHealthy,
responseTime: healthStatus.responseTime,
issues: healthStatus.configurationIssues.length,
});
}
catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
healthStatus.error = errorMessage;
healthStatus.configurationIssues.push(`Health check failed: ${errorMessage}`);
healthStatus.responseTime = Date.now() - startTime;
// A check that threw failed to answer at all, which is a failure in
// the sense the breaker is for — unlike an answered "not configured".
this.consecutiveFailures.set(providerName, {
count: failureCount + 1,
lastFailureAt: Date.now(),
});
logger.warn(`Health check failed for ${providerName}`, {
error: errorMessage,
consecutiveFailures: failureCount + 1,
});
}
return healthStatus;
}
/**
* Check environment configuration for a provider
*/
static async checkEnvironmentConfiguration(providerName, healthStatus, timeout, allowRuntimeProbe) {
const requiredEnvVars = this.getRequiredEnvironmentVariables(providerName);
logger.debug(`[ProviderHealthChecker] Checking environment configuration for ${providerName}`, {
requiredEnvVars,
presentEnvVars: requiredEnvVars.map((envVar) => ({
name: envVar,
present: !!process.env[envVar],
hasValue: !!(process.env[envVar] && process.env[envVar].trim() !== ""),
})),
});
let allConfigured = true;
const missingVars = [];
for (const envVar of requiredEnvVars) {
const value = process.env[envVar];
if (!value || value.trim() === "") {
allConfigured = false;
missingVars.push(envVar);
}
}
healthStatus.isConfigured = allConfigured;
logger.debug(`[ProviderHealthChecker] Environment configuration result for ${providerName}`, {
isConfigured: allConfigured,
missingVars,
totalRequired: requiredEnvVars.length,
totalMissing: missingVars.length,
});
if (!allConfigured) {
healthStatus.configurationIssues.push(`Missing required environment variables: ${missingVars.join(", ")}`);
healthStatus.recommendations.push(`Set the following environment variables: ${missingVars.join(", ")}`);
}
// Provider-specific configuration checks
return this.checkProviderSpecificConfig(providerName, healthStatus, timeout, allowRuntimeProbe);
}
/**
* Check API key validity (format validation)
*/
static async checkApiKeyValidity(providerName, healthStatus) {
// 🎯 SPECIAL HANDLING FOR VERTEX AI: Check both auth methods
if (providerName === AIProviderName.VERTEX) {
logger.debug("Vertex AI authentication check starting", {
providerName,
});
// Derive auth-var acceptance from the vertex descriptor instead of a
// hand-maintained var list, so this stays in sync with the real
// gating logic (hasGoogleCredentials() in googleVertex/client.ts and
// googleVertex/utils.ts): extraRequired.every(...) ||
// extraRequiredFallbacks.some(...), the same pattern used by
// hasProviderEnvVars() (providerUtils.ts), setup.ts, and
// environmentManager.ts. The old two-method check here only
// recognized GOOGLE_APPLICATION_CREDENTIALS or the
// GOOGLE_AUTH_CLIENT_EMAIL + GOOGLE_AUTH_PRIVATE_KEY pair — missing
// GOOGLE_SERVICE_ACCOUNT_KEY and, most importantly,
// GOOGLE_APPLICATION_CREDENTIALS_NEUROLINK, the descriptor's
// documented first-priority fallback. A user configured via that
// var alone had a fully working Vertex provider but was reported
// unhealthy here. Note: this is a presence-only check — it no
// longer verifies the GOOGLE_APPLICATION_CREDENTIALS file actually
// exists on disk (the old file-based method did via fs.access),
// matching the presence-only semantics already used by the other
// three callers of this pattern.
const descriptor = ProviderFactory.getDescriptor(AIProviderName.VERTEX);
const { extraRequired, extraRequiredFallbacks } = descriptor?.envVars ?? {};
const hasValidAuth = (extraRequired ?? []).every((v) => !!process.env[v]) ||
satisfiesFallbacks(extraRequiredFallbacks, process.env);
logger.debug("Vertex auth final result", { hasValidAuth });
if (hasValidAuth) {
healthStatus.hasApiKey = true;
logger.debug("Vertex auth SUCCESS");
}
else {
healthStatus.hasApiKey = false;
healthStatus.configurationIssues.push("Vertex AI authentication not found: set GOOGLE_APPLICATION_CREDENTIALS, GOOGLE_APPLICATION_CREDENTIALS_NEUROLINK, GOOGLE_SERVICE_ACCOUNT_KEY, or both GOOGLE_AUTH_CLIENT_EMAIL and GOOGLE_AUTH_PRIVATE_KEY");
logger.debug("Vertex auth FAILED", {
reason: "No valid auth method found",
});
}
return;
}
// Providers that don't use API keys directly
if (providerName === AIProviderName.OLLAMA ||
providerName === AIProviderName.BEDROCK ||
providerName === AIProviderName.LITELLM) {
healthStatus.hasApiKey = true;
return;
}
// 🔧 STANDARD HANDLING FOR OTHER PROVIDERS
const apiKeyVar = this.getApiKeyEnvironmentVariable(providerName);
const apiKey = process.env[apiKeyVar];
if (!apiKey) {
healthStatus.hasApiKey = false;
healthStatus.configurationIssues.push(`API key not found in ${apiKeyVar}`);
return;
}
// Basic format validation
const isValidFormat = this.validateApiKeyFormat(providerName, apiKey);
if (!isValidFormat) {
healthStatus.hasApiKey = false;
healthStatus.configurationIssues.push(`API key format appears invalid for ${providerName}`);
healthStatus.recommendations.push(`Verify the API key format for ${providerName}`);
}
else {
healthStatus.hasApiKey = true;
}
}
/**
* Check connectivity to provider endpoints
*/
static async checkConnectivity(providerName, healthStatus, timeout) {
const endpoint = this.getProviderHealthEndpoint(providerName);
if (!endpoint) {
healthStatus.warning = "No connectivity test available for this provider";
return;
}
const headers = {
"User-Agent": "NeuroLink-HealthCheck/1.0",
...this.getConnectivityHeaders(providerName),
};
try {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeout);
try {
const proxyFetch = createProxyFetch();
let response = await proxyFetch(endpoint, {
method: "HEAD",
signal: controller.signal,
headers,
});
// Fallback to GET if HEAD returns 405 (Method Not Allowed) for restrictive gateways
if (response.status === 405) {
response = await proxyFetch(endpoint, {
method: "GET",
signal: controller.signal,
headers,
});
}
if (!response.ok) {
healthStatus.configurationIssues.push(`Connectivity test failed: HTTP ${response.status}`);
}
}
finally {
clearTimeout(timeoutId);
}
}
catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
// Provide specific error messages for common network issues
if (errorMessage.includes("abort")) {
healthStatus.configurationIssues.push(`Connectivity test timed out after ${timeout}ms`);
}
else if (errorMessage.includes("ENOTFOUND") ||
errorMessage.includes("getaddrinfo")) {
healthStatus.configurationIssues.push(`DNS resolution failed: Cannot resolve hostname for ${providerName}`);
}
else if (errorMessage.includes("ECONNREFUSED")) {
healthStatus.configurationIssues.push(`Connection refused: ${providerName} service is not accepting connections`);
}
else if (errorMessage.includes("ETIMEDOUT")) {
healthStatus.configurationIssues.push(`Connection timeout: ${providerName} service did not respond`);
}
else if (errorMessage.includes("certificate") ||
errorMessage.includes("SSL") ||
errorMessage.includes("TLS")) {
healthStatus.configurationIssues.push(`SSL/TLS certificate error: ${providerName} has certificate issues`);
}
else if (errorMessage.includes("ECONNRESET")) {
healthStatus.configurationIssues.push(`Connection reset: ${providerName} terminated the connection`);
}
else if (errorMessage.includes("network") ||
errorMessage.includes("offline")) {
healthStatus.configurationIssues.push(`Network error: Check internet connectivity and firewall settings`);
}
else {
healthStatus.configurationIssues.push(`Connectivity test failed: ${errorMessage}`);
}
}
}
static getConnectivityHeaders(providerName) {
if (providerName === AIProviderName.LITELLM) {
return {
Authorization: `Bearer ${process.env.LITELLM_API_KEY || "sk-anything"}`,
};
}
return {};
}
/**
* Check model availability (if possible without making API calls)
*/
static async checkModelAvailability(providerName, healthStatus) {
// Basic model name validation and recommendations
const commonModels = this.getCommonModelsForProvider(providerName);
if (commonModels.length > 0) {
if (providerName === AIProviderName.VERTEX) {
// Provide detailed information about dual provider architecture
healthStatus.recommendations.push(`Available models for ${providerName} (using dual provider architecture):\n` +
` Google Models (via vertex provider):\n` +
` • gemini-2.5-pro, gemini-2.5-flash, gemini-2.5-flash-lite\n` +
` • gemini-2.0-flash-001, gemini-2.0-flash-lite\n` +
` Anthropic Models (via vertexAnthropic provider):\n` +
` • claude-sonnet-4@20250514, claude-opus-4@20250514\n` +
` • claude-3-5-sonnet-20241022, claude-3-5-haiku-20241022\n` +
` • claude-3-sonnet-20240229, claude-3-haiku-20240307, claude-3-opus-20240229\n` +
` Implementation: Uses native SDKs (@google/genai for Gemini, @anthropic-ai/vertex-sdk for Claude)\n` +
` Authentication: Requires Google Cloud project with Vertex AI API enabled\n` +
` Note: Anthropic models require Anthropic integration in your Google Cloud project`);
}
else {
healthStatus.recommendations.push(`Common models for ${providerName}: ${commonModels.slice(0, 3).join(", ")}`);
}
}
}
/**
* Get required environment variables for a provider.
*
* Returns `[]` for providers with `descriptor.credentialsResolvedExternally
* === true` (Vertex, Bedrock, LiteLLM) — the check in
* checkEnvironmentConfiguration() below ANDs every entry in the returned
* list together, which can't express Vertex's file-OR-individual-creds-
* OR-service-account auth, Bedrock's AWS SDK default provider chain
* (profile / IAM role, no env vars at all), or LiteLLM's documented
* zero-config local proxy. Their real requirement is validated by
* checkProviderSpecificConfig()'s dedicated per-provider checks instead.
* Naively deriving [apiKey, ...extraRequired] from the descriptor for
* these three (as for the other providers) would make
* checkEnvironmentConfiguration() push a false "missing environment
* variables" issue — and therefore isHealthy=false — for legitimate
* fallback-based Vertex auth, AWS_PROFILE/IAM-role Bedrock auth, and
* unauthenticated local LiteLLM proxies. See hasProviderEnvVars() in
* providerUtils.ts for the equivalent OR-aware check used for
* auto-select gating. See `ProviderDescriptor.credentialsResolvedExternally`
* (types/providers.ts) for the field's full documentation.
*/
static getRequiredEnvironmentVariables(providerName) {
// Resolve the descriptor FIRST so the field check below is keyed on
// the canonical, alias-resolved `descriptor.credentialsResolvedExternally`
// — not the raw, possibly-aliased `providerName` argument. Checking the
// raw input here would let documented aliases (e.g. "googleVertex" for
// vertex, "aws" for bedrock) skip the delegation and fall through to
// the naive [apiKey, ...extraRequired] derivation below, reproducing
// the exact false "missing environment variables" regression this
// field exists to prevent.
const descriptor = ProviderFactory.getDescriptor(providerName);
if (!descriptor) {
return [];
}
if (descriptor.credentialsResolvedExternally) {
return [];
}
const { apiKey, extraRequired } = descriptor.envVars;
return [...(apiKey ? [apiKey] : []), ...(extraRequired ?? [])];
}
/**
* Get API key environment variable for a provider.
*
* Dead code at its sole call site today (checkApiKeyValidity() early-
* returns for VERTEX/OLLAMA/BEDROCK/LITELLM before reaching this), but
* now `static` (public) so a future direct caller is plausible — hence
* the two intentional diffs from the original hand-written switch are
* documented here rather than left silent:
* - vertex: old switch returned "GOOGLE_APPLICATION_CREDENTIALS"
* (one of several valid auth vars); the descriptor's `envVars.apiKey`
* is "GOOGLE_CLOUD_PROJECT_ID" instead — the actual identity var for
* the provider, not one specific credential-supply mechanism.
* - ollama: old switch returned "OLLAMA_BASE_URL"; the descriptor has
* no `envVars.apiKey` for ollama (it's a local, unauthenticated
* runtime), so this now returns "".
* Both are kept deliberately (semantically truer identity vars) rather
* than reproduced byte-for-byte from the old switch.
*/
static getApiKeyEnvironmentVariable(providerName) {
return ProviderFactory.getDescriptor(providerName)?.envVars.apiKey ?? "";
}
/**
* Validate API key format for a provider
*/
static validateApiKeyFormat(providerName, apiKey) {
// Vertex's "API key" is really a credentials file path (or an inline
// JSON blob via GOOGLE_SERVICE_ACCOUNT_KEY) — never governed by the
// shared API_KEY_FORMATS regexes. Preserved exactly from the original
// switch even though checkApiKeyValidity() early-returns before ever
// reaching this call for Vertex today; kept correct for any other/
// future caller.
if (providerName === AIProviderName.VERTEX) {
return apiKey.endsWith(".json") || apiKey.includes("type");
}
const descriptor = ProviderFactory.getDescriptor(providerName);
if (!descriptor?.apiKeyFormatPattern) {
// Providers with no documented format (or local runtimes / providers
// usable with zero configuration) are accepted as-is — matches the
// old switch's per-provider `default: true`/no-format-check branches
// (ollama, litellm).
if (descriptor?.localRuntime || descriptor?.envVars.optional) {
return true;
}
return apiKey.length > 0;
}
return descriptor.apiKeyFormatPattern.test(apiKey);
}
/**
* Get health check endpoint for connectivity testing
*/
static getProviderHealthEndpoint(providerName) {
const descriptor = ProviderFactory.getDescriptor(providerName);
if (descriptor?.healthCheck !== "models-probe") {
return null;
}
// OpenAI/LiteLLM/Ollama each need a provider-specific URL builder that
// isn't pure descriptor data (LiteLLM/Ollama depend on the configured
// base URL) — keep the existing per-provider dispatch for the 3
// models-probe providers, now gated on the descriptor instead of a
// hardcoded name list.
switch (descriptor.name) {
case AIProviderName.OPENAI:
return "https://api.openai.com/v1/models";
case AIProviderName.LITELLM:
return this.getLiteLLMModelsUrl();
case AIProviderName.OLLAMA:
return this.getOllamaTagsUrl();
default:
return null;
}
}
/**
* Provider-specific configuration checks
*/
static async checkProviderSpecificConfig(providerName, healthStatus, timeout, allowRuntimeProbe) {
switch (providerName) {
case AIProviderName.VERTEX:
await this.checkVertexAIConfig(healthStatus);
break;
case AIProviderName.BEDROCK:
await this.checkBedrockConfig(healthStatus);
break;
case AIProviderName.AZURE:
await this.checkAzureConfig(healthStatus);
break;
case AIProviderName.LITELLM:
return this.checkLiteLLMConfig(healthStatus, timeout, allowRuntimeProbe);
case AIProviderName.OLLAMA:
return this.checkOllamaConfig(healthStatus, timeout, allowRuntimeProbe);
}
// Every other provider's config check above is local (env vars, file
// existence) — never a network request, so there is no probe to report.
return { ran: false, failed: false };
}
/**
* Check Vertex AI configuration
*/
static async checkVertexAIConfig(healthStatus) {
logger.debug("Starting Vertex AI health check");
const projectId = this.getVertexProjectId();
if (!projectId) {
healthStatus.configurationIssues.push("Google Cloud project ID not set");
healthStatus.recommendations.push("Set one of: GOOGLE_VERTEX_PROJECT, GOOGLE_CLOUD_PROJECT_ID, GOOGLE_PROJECT_ID, or GOOGLE_CLOUD_PROJECT");
}
const hasValidAuth = await this.checkVertexAuthentication(healthStatus);
if (projectId && hasValidAuth) {
healthStatus.isConfigured = true;
logger.debug("Vertex AI health check PASSED");
}
else {
logger.debug("Vertex AI health check FAILED");
}
}
/**
* Get Vertex AI project ID from environment variables
*/
static getVertexProjectId() {
return (process.env.GOOGLE_PROJECT_ID ||
process.env.GOOGLE_CLOUD_PROJECT_ID ||
process.env.GOOGLE_VERTEX_PROJECT ||
process.env.GOOGLE_CLOUD_PROJECT ||
process.env.VERTEX_PROJECT_ID);
}
/**
* Check Vertex AI authentication
*/
static async checkVertexAuthentication(healthStatus) {
let hasValidAuth = false;
if (process.env.GOOGLE_APPLICATION_CREDENTIALS) {
hasValidAuth = await this.checkGoogleApplicationCredentials(healthStatus);
}
if (!hasValidAuth) {
hasValidAuth = this.checkExtraRequiredFallbackCredentials(healthStatus);
}
if (!hasValidAuth) {
healthStatus.configurationIssues.push("Google Cloud authentication not configured or credentials file missing");
healthStatus.recommendations.push("Set either GOOGLE_APPLICATION_CREDENTIALS (valid file path), GOOGLE_APPLICATION_CREDENTIALS_NEUROLINK, GOOGLE_SERVICE_ACCOUNT_KEY (base64), or both GOOGLE_AUTH_CLIENT_EMAIL and GOOGLE_AUTH_PRIVATE_KEY");
}
return hasValidAuth;
}
/**
* Check Google Application Credentials file
*/
static async checkGoogleApplicationCredentials(healthStatus) {
const credentialsPath = process.env.GOOGLE_APPLICATION_CREDENTIALS;
if (!credentialsPath) {
healthStatus.warning =
"GOOGLE_APPLICATION_CREDENTIALS environment variable not set";
return false;
}
try {
const { promises: fs } = await import("fs");
await fs.access(credentialsPath);
const fileName = basename(credentialsPath);
const jsonFilePattern = /\.json(\.\w+)?$/;
if (!jsonFilePattern.test(fileName)) {
healthStatus.warning =
"GOOGLE_APPLICATION_CREDENTIALS should point to a JSON file";
}
healthStatus.hasApiKey = true;
return true;
}
catch {
healthStatus.warning = `GOOGLE_APPLICATION_CREDENTIALS file does not exist: ${credentialsPath}`;
return false;
}
}
/**
* Check Vertex's non-file auth fallbacks (GOOGLE_APPLICATION_CREDENTIALS_
* NEUROLINK, GOOGLE_SERVICE_ACCOUNT_KEY, or the GOOGLE_AUTH_CLIENT_EMAIL +
* GOOGLE_AUTH_PRIVATE_KEY pair) via the descriptor's extraRequiredFallbacks
* instead of a hand-maintained env-var list, so this stays in sync with
* the real gating logic (hasGoogleCredentials()) the same way
* checkApiKeyValidity()'s vertex branch does. The previous hand-rolled
* version only recognized GOOGLE_SERVICE_ACCOUNT_KEY or the email+key
* pair — missing GOOGLE_APPLICATION_CREDENTIALS_NEUROLINK, the
* descriptor's documented first-priority fallback.
*/
static checkExtraRequiredFallbackCredentials(healthStatus) {
const descriptor = ProviderFactory.getDescriptor(AIProviderName.VERTEX);
const hasValidAuth = satisfiesFallbacks(descriptor?.envVars.extraRequiredFallbacks, process.env);
if (hasValidAuth) {
healthStatus.hasApiKey = true;
return true;
}
return false;
}
/**
* Check AWS Bedrock configuration
*/
static async checkBedrockConfig(healthStatus) {
logger.debug("Starting AWS Bedrock comprehensive health check");
this.checkAWSRegion(healthStatus);
this.checkAWSCredentials(healthStatus);
this.checkBedrockModels(healthStatus);
this.checkBedrockEndpoint(healthStatus);
if (healthStatus.configurationIssues.length === 0) {
healthStatus.hasApiKey = true;
logger.debug("AWS Bedrock configuration appears valid");
}
}
/**
* Check AWS region configuration
*/
static checkAWSRegion(healthStatus) {
const awsRegion = process.env.AWS_REGION;
const validBedrockRegions = [
"us-east-1",
"us-west-2",
"ap-southeast-1",
"ap-northeast-1",
"eu-central-1",
"eu-west-1",
"ap-south-1",
];
if (!awsRegion) {
healthStatus.configurationIssues.push("AWS_REGION not set");
healthStatus.recommendations.push(`Set AWS_REGION to a Bedrock-supported region: ${validBedrockRegions.join(", ")}`);
}
else if (!validBedrockRegions.includes(awsRegion)) {
healthStatus.configurationIssues.push(`AWS_REGION '${awsRegion}' may not support all Bedrock models`);
healthStatus.recommendations.push(`Consider using a primary Bedrock region: ${validBedrockRegions.slice(0, 3).join(", ")}`);
}
}
/**
* Check AWS credentials
*/
static checkAWSCredentials(healthStatus) {
const awsAccessKeyId = process.env.AWS_ACCESS_KEY_ID;
const awsSecretAccessKey = process.env.AWS_SECRET_ACCESS_KEY;
const awsProfile = process.env.AWS_PROFILE;
if (!awsAccessKeyId && !awsProfile) {
healthStatus.configurationIssues.push("No AWS credentials found");
healthStatus.recommendations.push("Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, or configure AWS_PROFILE");
}
else if (awsAccessKeyId && !awsSecretAccessKey) {
healthStatus.configurationIssues.push("AWS_ACCESS_KEY_ID set but AWS_SECRET_ACCESS_KEY missing");
healthStatus.recommendations.push("Set AWS_SECRET_ACCESS_KEY to match your AWS_ACCESS_KEY_ID");
}
}
/**
* Check Bedrock models
*/
static checkBedrockModels(healthStatus) {
const bedrockModel = process.env.BEDROCK_MODEL || process.env.BEDROCK_MODEL_ID;
const supportedModels = [
BedrockModels.CLAUDE_3_SONNET,
BedrockModels.CLAUDE_3_HAIKU,
BedrockModels.CLAUDE_3_5_SONNET,
"anthropic.claude-v2:1",
"amazon.titan-text-express-v1",
];
if (!bedrockModel) {
healthStatus.recommendations.push(`Set BEDROCK_MODEL or BEDROCK_MODEL_ID for faster startup (e.g., ${BedrockModels.CLAUDE_3_SONNET})`);
}
else if (!supportedModels.includes(bedrockModel)) {
healthStatus.recommendations.push(`Consider using a popular Bedrock model: ${supportedModels.slice(0, 3).join(", ")}`);
}
}
/**
* Check Bedrock endpoint
*/
static checkBedrockEndpoint(healthStatus) {
const bedrockEndpoint = process.env.BEDROCK_ENDPOINT_URL;
if (bedrockEndpoint && !bedrockEndpoint.startsWith("https://")) {
healthStatus.configurationIssues.push("BEDROCK_ENDPOINT_URL should use HTTPS");
healthStatus.recommendations.push("Update BEDROCK_ENDPOINT_URL to use HTTPS protocol");
}
}
/**
* Check Azure OpenAI configuration
*/
static async checkAzureConfig(healthStatus) {
const azureEndpoint = process.env.AZURE_OPENAI_ENDPOINT;
if (azureEndpoint && !azureEndpoint.startsWith("https://")) {
healthStatus.configurationIssues.push("Invalid AZURE_OPENAI_ENDPOINT format");
healthStatus.recommendations.push("Set AZURE_OPENAI_ENDPOINT to a valid URL (e.g., https://your-resource.openai.azure.com/)");
}
// Check for deployment name using the SAME logic as the Azure provider
const deploymentName = process.env.AZURE_OPENAI_MODEL ||
process.env.AZURE_OPENAI_DEPLOYMENT ||
process.env.AZURE_OPENAI_DEPLOYMENT_ID;
if (!deploymentName) {
healthStatus.configurationIssues.push("No Azure deployment specified");
healthStatus.recommendations.push("Set one of: AZURE_OPENAI_MODEL, AZURE_OPENAI_DEPLOYMENT, or AZURE_OPENAI_DEPLOYMENT_ID");
}
}
static getLiteLLMBaseUrl() {
return process.env.LITELLM_BASE_URL || "http://localhost:4000";
}
static getLiteLLMModelsUrl() {
return new URL("/v1/models", this.getLiteLLMBaseUrl()).toString();
}
static getConfiguredLiteLLMModel() {
return process.env.LITELLM_MODEL || "openai/gpt-5.4-mini";
}
static getOllamaBaseUrl() {
return (process.env.OLLAMA_BASE_URL ||
process.env.OLLAMA_API_BASE ||
"http://localhost:11434");
}
static getOllamaTagsUrl() {
return new URL("/api/tags", this.getOllamaBaseUrl()).toString();
}
static getConfiguredOllamaModel() {
return process.env.OLLAMA_MODEL || DEFAULT_OLLAMA_MODEL;
}
static async fetchJsonWithTimeout(url, options = {}) {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), options.timeout ?? this.DEFAULT_TIMEOUT);
try {
const proxyFetch = createProxyFetch();
const response = await proxyFetch(url, {
method: "GET",
headers: options.headers,
signal: controller.signal,
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
return await response.json();
}
finally {
clearTimeout(timeoutId);
}
}
static normalizeModelList(models) {
return models
.map((entry) => {
if (typeof entry === "string") {
return entry;
}
if (entry &&
typeof entry === "object" &&
"id" in entry &&
typeof entry.id === "string") {
return entry.id;
}
if (entry &&
typeof entry === "object" &&
"name" in entry &&
typeof entry.name === "string") {
return entry.name;
}
return null;
})
.filter((model) => typeof model === "string");
}
static hasRequestedModel(availableModels, requestedModel) {
const normalizedRequestedModel = requestedModel.trim();
const requiresExactMatch = /@/.test(normalizedRequestedModel);
return availableModels.some((model) => model === normalizedRequestedModel ||
(!requiresExactMatch &&
(model.startsWith(`${normalizedRequestedModel}:`) ||
model.startsWith(`${normalizedRequestedModel}@`))));
}
static async getOllamaAvailableModels(timeout = 2000) {
const payload = (await this.fetchJsonWithTimeout(this.getOllamaTagsUrl(), {
timeout,
}));
return this.normalizeModelList(payload.models ?? []);
}
static async getLiteLLMAvailableModels(timeout = 2000) {
const payload = (await this.fetchJsonWithTimeout(this.getLiteLLMModelsUrl(), {
timeout,
headers: {
Authorization: `Bearer ${process.env.LITELLM_API_KEY || "sk-anything"}`,
"Content-Type": "application/json",
},
}));
return this.normalizeModelList(payload.data ?? []);
}
static async checkOllamaAvailability(options) {
try {
const models = await this.getOllamaAvailableModels(options.timeout);
if (!this.hasRequestedModel(models, options.model)) {
return {
available: false,
reason: `Configured Ollama model '${options.model}' is not installed`,
models,
};
}
return { available: true, models };
}
catch (error) {
return {
available: false,
reason: error instanceof Error ? error.message : String(error),
models: [],
};
}
}
static async checkLiteLLMAvailability(options) {
try {
const models = await this.getLiteLLMAvailableModels(options.timeout);
if (models.length === 0) {
return {
available: false,
reason: "LiteLLM returned an empty model list",
models,
};
}
if (!this.hasRequestedModel(models, options.model)) {
return {
available: false,
reason: `Configured LiteLLM model '${options.model}' is not exposed by the proxy`,
models,
};
}
return { available: true, models };
}
catch (error) {
return {
available: false,
reason: error instanceof Error ? error.message : String(error),
models: [],
};
}
}
static async checkLiteLLMConfig(healthStatus, timeout = this.DEFAULT_TIMEOUT, allowRuntimeProbe = true) {
const liteLLMBase = this.getLiteLLMBaseUrl();
if (!liteLLMBase.startsWith("http")) {
healthStatus.isConfigured = false;
healthStatus.configurationIssues.push("Invalid LITELLM_BASE_URL format");
healthStatus.recommendations.push("Set LITELLM_BASE_URL to a valid URL (e.g., http://localhost:4000)");
return { ran: false, failed: false };
}
if (!allowRuntimeProbe) {
// Blacklisted: don't issue the request. A repeatedly unreachable
// local provider is not usable, which is what auto-select needs —
// but the caller's blacklist block (checkProviderHealth) already adds
// the error/warning/configurationIssue for this, so nothing is
// duplicated here.
healthStatus.isConfigured = false;
return { ran: false, failed: false };
}
// Only pin the availability check to a specific model when the user
// explicitly configured one. The fallback default ("openai/gpt-5.4-mini")
// is a guess, not configuration — proxies that serve a different model
// set (every self-hosted gateway) were reported "Not configured" here
// while generate/stream against them worked fine with explicit models.
const configuredModel = process.env.LITELLM_MODEL;
let failureReason;
if (configuredModel) {
const availability = await this.checkLiteLLMAvailability({
model: configuredModel,
timeout,
});
if (!availability.available) {
failureReason = availability.reason ?? "unknown error";
}
}
else {
// No configured model: a reachable proxy with any models counts.
try {
const models = await this.getLiteLLMAvailableModels(timeout);
if (models.length === 0) {
failureReason = "LiteLLM returned an empty model list";
}
}
catch (error) {
failureReason = error instanceof Error ? error.message : String(error);
}
}
if (failureReason !== undefined) {
healthStatus.isConfigured = false;
healthStatus.configurationIssues.push(`LiteLLM runtime check failed: ${failureReason}`);
healthStatus.recommendations.push("Start the LiteLLM proxy and ensure the configured model is available from /v1/models");
return { ran: true, failed: true };
}
healthStatus.isConfigured = true;
return { ran: true, failed: false };
}
/**
* Check Ollama configuration
*/
static async checkOllamaConfig(healthStatus, timeout = this.DEFAULT_TIMEOUT, allowRuntimeProbe = true) {
const ollamaBase = this.getOllamaBaseUrl();
if (!ollamaBase.startsWith("http")) {
healthStatus.isConfigured = false;
healthStatus.configurationIssues.push("Invalid OLLAMA_BASE_URL format (OLLAMA_API_BASE is still accepted as a legacy alias)");
healthStatus.recommendations.push("Set OLLAMA_BASE_URL to a valid URL (e.g., http://localhost:11434). OLLAMA_API_BASE remains supported as a legacy alias.");
return { ran: false, failed: false };
}
if (!allowRuntimeProbe) {
// Blacklisted: don't issue the request. A repeatedly unreachable
// local provider is not usable, which is what auto-select needs —
// but the caller's blacklist block (checkProviderHealth) already adds
// the error/warning/configurationIssue for this, so nothing is
// duplicated here.
healthStatus.isConfigured = false;
return { ran: false, failed: false };
}
const availability = await this.checkOllamaAvailability({
model: this.getConfiguredOllamaModel(),
timeout,
});
if (!availability.available) {
healthStatus.isConfigured = false;
healthStatus.configurationIssues.push(`Ollama runtime check failed: ${availability.reason ?? "unknown error"}`);
healthStatus.recommendations.push("Start Ollama and install the configured model before using Ollama as a fallback provider");
return { ran: true, failed: true };
}
healthStatus.isConfigured = true;
return { ran: true, failed: false };
}
/**
* Get common models for a provider
*/
static getCommonModelsForProvider(providerName) {
switch (providerName) {
case AIProviderName.ANTHROPIC:
return [
AnthropicModels.CLAUDE_3_5_SONNET,
AnthropicModels.CLAUDE_3_HAIKU,
AnthropicModels.CLAUDE_3_OPUS,
];
case AIProviderName.OPENAI:
return [
OpenAIModels.GPT_4O,
OpenAIModels.GPT_4O_MINI,
OpenAIModels.GPT_3_5_TURBO,
];
case AIProviderName.GOOGLE_AI:
return [
GoogleAIModels.GEMINI_2_5_PRO,
GoogleAIModels.GEMINI_2_5_FLASH,
GoogleAIModels.GEMINI_2_0_FLASH_001,
];
case AIProviderName.VERTEX:
return [
// Google models (via vertex provider)
GoogleAIModels.GEMINI_2_5_PRO,
GoogleAIModels.GEMINI_2_5_FLASH,
GoogleAIModels.GEMINI_2_5_FLASH_LITE,
GoogleAIModels.GEMINI_2_0_FLASH_001,
// Anthropic models (via vertexAnthropic provider)
"claude-sonnet-4@20250514",
"claude-opus-4@20250514",
AnthropicModels.CLAUDE_3_5_SONNET,
AnthropicModels.CLAUDE_3_5_HAIKU,
AnthropicModels.CLAUDE_3_SONNET,
AnthropicModels.CLAUDE_3_HAIKU,
AnthropicModels.CLAUDE_3_OPUS,
];
case AIProviderName.BEDROCK:
return [BedrockModels.CLAUDE_3_SONNET, BedrockModels.CLAUDE_3_HAIKU];
case AIProviderName.AZURE:
return [OpenAIModels.GPT_4O, OpenAIModels.GPT_4O_MINI, "gpt-35-turbo"];
case AIProviderName.LITELLM:
return [
"openai/gpt-5.4-mini",
"anthropic/claude-haiku-4-5-20251001",
"google/gemini-2.5-flash",
];
case AIProviderName.OLLAMA: {
const envModel = process.env.OLLAMA_MODEL;
const defaults = [
"llama3.2:latest",
"llama3.1:latest",
"mistral:latest",
];
return envModel ? [envModel, ...defaults] : defaults;
}
default:
return [];
}
}
/**
* Get cached health status if still valid
*/
static getCachedHealth(providerName, maxAge) {
const cached = this.healthCache.get(providerName);
if (!cached) {
return null;
}
const age = Date.now() - cached.timestamp;
if (age > maxAge) {
this.healthCache.delete(providerName);
return null;
}
return cached.status;
}
/**
* Check if Vertex AI supports Anthropic models (dual provider architecture)
*/
static async checkVertexAnthropicSupport() {
const result = {
isSupported: false,
hasCreateVertexAnthropic: false,
hasCorrectTypes: false,
hasValidProject: false,
hasRegionalSupport: false,
hasNetworkAccess: false,
hasAnthropicModels: false,
authentication: {
isValid: false,
method: "none",
issues: [],
},
projectConfiguration: {
isValid: false,
projectId: undefined,
region: undefined,
issues: [],
},
modelSupport: {
availableModels: [],
recommendedModels: [
"claude-sonnet-4@20250514",
"claude-opus-4@20250514",
"claude-3-5-sonnet-20241022",
"claude-3-5-haiku-20241022",
"claude-3-sonnet-20240229",
"claude-3-haiku-20240307",
],
deprecatedModels: [
"claude-3-opus-20240229", // Still available but newer versions preferred
],
},
recommendations: [],
troubleshooting: [],
};
logger.debug("Starting comprehensive Vertex Anthropic support verification");
try {
// 1. Check SDK module availability
logger.debug("Checking @anthropic-ai/vertex-sdk module availability");
const anthropicModule = await import("@anthropic-ai/vertex-sdk");
result.hasCreateVertexAnthropic =
typeof anthropicModule.AnthropicVertex === "function";
result.hasCorrectTypes = true; // Types are bundled with the class
if (!result.hasCreateVertexAnthropic) {
result.troubleshooting.push("📦 Install @anthropic-ai/vertex-sdk for Claude on Vertex AI support", "🔄 Run: npm install @anthropic-ai/vertex-sdk", "📖 See: https://docs.anthropic.com/en/api/claude-on-vertex-ai");
return result;
}
logger.debug("SDK module verified successfully");
// 2. Comprehensive Authentication Validation
logger.debug("Starting authentication validation");
result.authentication = await this.validateVertexAuthentication();
if (!result.authentication.isValid) {
result.troubleshooting.push("🔐 Fix authentication configuration:", " Option 1: Set GOOGLE_APPLICATION_CREDENTIALS to valid service account file", " Option 2: Set individual env vars: GOOGLE_AUTH_CLIENT_EMAIL, GOOGLE_AUTH_PRIVATE_KEY", "📖 See: https://cloud.google.com/docs/authentication/provide-credentials-adc");
}
// 3. Project Configuration Validation
logger.debug("Starting project configuration validation");
result.projectConfiguration =
await this.validateVertexProjectConfiguration();
result.hasValidProject = result.projectConfiguration.isValid;
if (!result.hasValidProject) {
result.troubleshooting.push("🏗️ Fix project configuration:", " Set GOOGLE_VERTEX_PROJECT or GOOGLE_CLOUD_PROJECT environment variable", " Ensure project exists and has Vertex AI API enabled", "📖 See: https://console.cloud.google.com/apis/library/aiplatform.googleapis.com");
}
// 4. Regional Support Validation
logger.debug("Starting regional support validation");
result.hasRegionalSupport = await this.checkVertexRegionalSupport(result.projectConfiguration.region);
if (!result.hasRegionalSupport) {
result.troubleshooting.push("🌍 Regional support issues:", " Anthropic models may not be available in your region", " Try regions: us-central1, us-east4, europe-west1, asia-southeast1", " Set GOOGLE_CLOUD_LOCATION environment variable");
}
// 5. Network Connectivity Check (non-blocking)
logger.debug("Starting network connectivity check");
result.hasNetworkAccess = await this.checkVertexNetworkConnectivity(result.projectConfiguration.region || "us-central1");
if (!result.hasNetworkAccess) {
result.troubleshooting.push("🌐 Network connectivity issues:", " Check proxy configuration if behind corporate firewall", " Verify DNS resolution for *.googleapis.com", " Ensure firewall allows HTTPS to Google Cloud endpoints");
}
// 6. Anthropic Model Integration Check
logger.debug("Starting Anthropic model integration check");
result.hasAnthropicModels = await this.checkAnthropicModelIntegration(result.projectConfiguration.projectId, result.projectConfiguration.region);
if (!result.hasAnthropicModels) {
result.troubleshooting.push("🤖 Anthropic model integration issues:", " Enable Anthropic integration in Google Cloud Console", " Navigate to: Vertex AI > Model Garden > Anthropic", " Accept terms and enable Claude model access", "📖 See: https://console.cloud.google.com/vertex-ai/publishers/anthropic");
}
// Calculate overall support status
result.isSupported =
result.hasCreateVertexAnthropic &&
result.authentication.isValid &&
result.hasValidProject &&
result.hasRegionalSupport;
// Note: Network and model integration are nice-to-have but not blocking
// Generate comprehensive recommendations
if (result.isSupported) {
result.recommendations.push("✅ Vertex Anthropic support is fully configured", "✅ Claude models are available via vertexAnthropic provider", `✅ Authentication: ${result.authentication.method}`, `✅ Project: ${result.projectConfiguration.projectId}`, `✅ Region: ${result.projectConfiguration.region}`);
if (result.hasNetworkAccess) {
result.recommendations.push("✅ Network connectivity verified");
}
else {
result.recommendations.push("⚠️ Network connectivity not verified (may still work)");
}
if (result.hasAnthropicModels) {
result.recommendations.push("✅ Anthropic model integration verified");
}
else {
result.recommendations.push("⚠️ Anthropic model integration not verified");
}
result.recommendations.push("", "🎯 Recommended Claude models:", ...result.modelSupport.recommendedModels.map((model) => ` • ${model}`), "", "📚 Usage example:", ' const vertex = new GoogleVertexProvider("claude-3-5-sonnet-20241022")', ' const result = await vertex.generate("Hello, Claude!")');
logger.info("Vertex Anthropic support verification: FULLY_SUPPORTED");
}
else {
const missingComponents = [];
if (!result.hasCreateVertexAnthropic) {
missingComponents.push("SDK module");
}
if (!result.authentication.isValid) {
missingComponents.push("authentication");
}
if (!result.hasValidProject) {
missingComponents.push("project configuration");
}
if (!result.hasRegionalSupport) {
missingComponents.push("regional support");
}
result.recommendations.push(`⚠️ Vertex Anthropic support partially available`, `❌ Missing: ${missingComponents.join(", ")}`, "", "🔧 Quick fixes needed:");
result.recommendations.push(...result.troubleshooting);
logger.warn("Vertex Anthropic support verification: PARTIALLY_SUPPORTED", {
missingComponents,
hasBasicSupport: result.hasCreateVertexAnthropic,
authenticationValid: result.authentication.isValid,
projectValid: result.hasValidProject,
});
}
}
catch (error) {
logger.error("Vertex Anthropic support check failed", {
error: error instanceof Error ? error.message : String(error),
stack: error instanceof Error ? error.stack : undefined,
});
result.recommendations.push("❌ Comprehensive Anthropic support check failed", `🐛 Error: ${error instanceof Error ? error.message : String(error)}`, "", "🔧 Troubleshooting steps:", "1. Verify @google-cloud/vertexai and @anthropic-ai/vertex-sdk are properly installed", "2. Verify Google Cloud authentication setup", "3. Check project ID and region configuration", "4. Enable Vertex AI API in Google Cloud Console", "5. Enable Anthropic integration in Vertex AI Model Garden");
}
return result;
}
/**
* Validate Vertex AI authentication configuration
*/
static async validateVertexAuthentication() {
const result = {
isValid: false,
method: "none",
issues: [],
};
try {
// Check for service account file authentication (preferred)
if (process.env.GOOGLE_APPLICATION_CREDENTIALS) {
const credentialsPath = process.env.GOOGLE_APPLICATION_CREDENTIALS;
try {
const { promises: fs } = await import("fs");
try {
await fs.access(credentialsPath);
// Validate JSON structure
const credentialsContent = await fs.readFile(credentialsPath, "utf8");
const credentials = JSON.parse(credentialsContent);
if (credentials.type === "service_account" &&
credentials.project_id &&
credentials.client_email &&
credentials.private_key) {
result.isValid = true;
result.method = "service_account_file";
return result;
}
else {
result.issues.push("Service account file missing required fields");
}
}
catch {
result.issues.push(`Service account file not found: ${credentialsPath}`);
}
}
catch (fileError) {
result.issues.push(`Service account file validation failed: ${fileError}`);
}
}
// Check for individual environment variables
if (process.env.GOOGLE_AUTH_CLIENT_EMAIL &&
process.env.GOOGLE_AUTH_PRIVATE_KEY) {
const email = process.env.GOOGLE_AUTH_CLIENT_EMAIL;
const privateKey = process.env.GOOGLE_AUTH_PRIVATE_KEY;
if (email.includes("@") && privateKey.includes("BEGIN PRIVATE KEY")) {
result.isValid = true;
result.method = "environment_variables";
return result;
}
else {
result.issues.push("Individual credentials format validation failed");
}
}
else {
result.issues.push("Missing individual credential environment variables");
}
// Check for Application Default Credentials (ADC)
try {
// This is a simple heuristic - in real implementation you'd use Google Auth library
if (process.env.GCLOUD_PROJECT || process.env.GOOGLE_CLOUD_PROJECT) {
result.method = "application_default_credentials";
result.isValid = true; // Assume valid if environment suggests ADC
return result;
}
}
catch (adcError) {
result.issues.push(`ADC check failed: ${adcError}`);
}
if (!result.isValid) {
result.issues.push("No valid authentication method found");
}
}
catch (error) {
result.issues.push(`Authentication validation error: ${error}`);
}
return result;
}
/**
* Validate Vertex AI project configuration
*/
static async validateVertexProjectConfiguration() {
const result = {
isValid: false,
projectId: undefined,
region: undefined,
issues: [],
};
// Check project ID
const projectId = process.env.GOOGLE_VERTEX_PROJECT ||
process.env.GOOGLE_CLOUD_PROJECT_ID ||
process.env.GOOGLE_PROJECT_ID ||
process.env.GOOGLE_CLOUD_PROJECT;
if (projectId) {
result.projectId = projectId;
// Validate project ID format
if (PROJECT_ID_FORMAT.PATTERN.test(projectId)) {
result.isValid = true;
}
else {
result.issues.push(`Invalid project ID format: ${projectId}`);
}
}
else {
result.issues.push("No project ID configured");
}
// Check region/location
const region = process.env.GOOGLE_CLOUD_LOCATION ||
process.env.VERTEX_LOCATION ||
process.env.GOOGLE_VERTEX_LOCATION ||
"us-central1";
result.region = region;
// Validate region format
const regionPattern = /^[a-z]+-[a-z]+\d+$/;
if (!regionPattern.test(region)) {
result.issues.push(`Invalid region format: ${region}`);
result.isValid = false;
}
return result;
}
/**
* Check if the specified region supports Anthropic models
*/
static async checkVertexRegionalSupport(region = "us-central1") {
// Based on Google Cloud documentation, these regions support Anthropic models
const supportedRegions = [
"us-central1",
"us-east4",
"us-west1",
"us-west4",
"europe-west1",
"europe-west4",
"asia-southeast1",
"asia-northeast1",
];
const isSupported = supportedRegions.includes(region);
logger.debug("Regional support check", {
region,
isSupported,
supportedRegions,
});
return isSupported;
}
/**
* Check network connectivity to Vertex AI endpoints
*/
static async checkVertexNetworkConnectivity(region = "us-central1") {
try {
const endpoint = `https://${region}-aiplatform.googleapis.com`;
// Simple connectivity check with timeout
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 5000);
const proxyFetch = createProxyFetch();
let response = await proxyFetch(endpoint, {
method: "HEAD",
signal: controller.signal,
});
// Fallback to GET if HEAD returns 405 (Method Not Allowed) for restrictive gateways
if (response.status === 405) {
response = await proxyFetch(endpoint, {
method: "GET",
signal: controller.signal,
});
}
clearTimeout(timeoutId);
// Any response (even 401/403) indicates network connectivity
const isConnected = response.status !== undefined;
logger.debug("Network connectivity check", {
endpoint,
status: response.status,
isConnected,
});
return isConnected;
}
catch (error) {
logger.debug("Network connectivity check failed", {
region,
error: error instanceof Error ? error.message : String(error),
});
return false;
}
}
/**
* Check if Anthropic model integration is enabled in the project
*/
static async checkAnthropicModelIntegration(projectId, region = "us-central1") {
if (!projectId) {
logger.debug("Cannot check Anthropic integration without project ID");
return false;
}
try {
// This is a simplified check - in a real implementation, you would:
// 1. Use Google Cloud APIs to check model availability
// 2. Verify Anthropic integration status
// 3. Check model access permissions
// For now, we'll do a basic endpoint check
const modelEndpoint = `https://${region}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/anthropic/models`;
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 3000);
const proxyFetch = createProxyFetch();
let response = await proxyFetch(modelEndpoint, {
method: "HEAD",
signal: controller.signal,
});
// Fallback to GET if HEAD returns 405 (Method Not Allowed) for restrictive gateways
if (response.status === 405) {
response = await proxyFetch(modelEndpoint, {
method: "GET",
signal: controller.signal,
});
}
clearTimeout(timeoutId);
// Status 200 or 401/403 suggests the endpoint exists (integration enabled)
// Status 404 suggests integration not enabled
const integrationEnabled = response.status !== 404;
logger.debug("Anthropic integration check", {
projectId,
region,
endpoint: modelEndpoint,
status: response.status,
integrationEnabled,
});
return integrationEnabled;
}
catch (error) {
logger.debug("Anthropic integration check failed", {
projectId,
region,
error: error instanceof Error ? error.message : String(error),
});
// Assume integration might be enabled if we can't verify
return true;
}
}
/**
* Initialize health checks in the background (NON-BLOCKING)
* Starts background health monitoring without blocking initialization
*/
static initializeBackgroundHealthChecks() {
// Run health checks in the background without awaiting
const backgroundHealthCheck = async () => {
try {
logger.debug("Starting background health check initialization");
await this.checkAllProvidersHealth({
includeConnectivityTest: false,
cacheResults: true,
timeout: 2000, // 2-second timeout for background checks
});
logger.debug("Background health check initialization completed");
}
catch (error) {
logger.warn("Background health check initialization failed", {
error: error instanceof Error ? error.message : String(error),
});
}
};
// Start background check without blocking
backgroundHealthCheck();
}
/**
* Clear health cache for a provider or all providers
*/
static clearHealthCache(providerName) {
if (providerName) {
this.healthCache.delete(providerName);
this.consecutiveFailures.delete(providerName);
}
else {
this.healthCache.clear();
this.consecutiveFailures.clear();
}
}
static async checkFallbackProviderAvailability(providerName, model) {
const provider = providerName;
if (provider === AIProviderName.OLLAMA) {
const availability = await this.checkOllamaAvailability({
model,
timeout: 2000,
});
return {
available: availability.available,
reason: availability.reason,
};
}
if (provider === AIProviderName.LITELLM) {
const availability = await this.checkLiteLLMAvailability({
model,
timeout: 2000,
});
return {
available: availability.available,
reason: availability.reason,
};
}
try {
const health = await this.checkProviderHealth(provider, {
includeConnectivityTest: false,
cacheResults: true,
maxCacheAge: 15_000,
timeout: 2000,
});
return {
available: health.isHealthy,
reason: health.error || health.configurationIssues[0] || health.warning,
};
}
catch (error) {
return {
available: false,
reason: error instanceof Error ? error.message : String(error),
};
}
}
/**
* Get the best healthy provider from a list of options (NON-BLOCKING)
* Prioritizes healthy providers over configured but unhealthy ones
* Uses fast, cached health checks to avoid blocking initialization
*/
static async getBestHealthyProvider(
// Auto-select preference comes from the descriptors too — a SEPARATE
// ordering from the sweep (local/cheap runtimes first), carried by
// autoSelectPreference rather than defaultHealthSweepPriority.
preferredProviders = PROVIDER_DESCRIPTORS.filter((d) => d.autoSelectPreference !== undefined)
.sort((a, b) => (a.autoSelectPreference ?? 0) - (b.autoSelectPreference ?? 0))
.map((d) => d.name)) {
const healthStatuses = await this.checkAllProvidersHealth({
includeConnectivityTest: false, // Quick config check only
cacheResults: true,
timeout: 1000, // Fast 1-second timeout to avoid blocking
});
// First try to find a healthy provider in order of preference
for (const provider of preferredProviders) {
const health = healthStatuses.find((h) => h.provider === provider);
if (health?.isHealthy) {
logger.debug(`Selected healthy provider: ${provider}`);
return provider;
}
}
// Fallback to first healthy provider
const firstHealthyProvider = healthStatuses.find((h) => h.isHealthy);
if (firstHealthyProvider) {
logger.info(`Using fallback healthy provider: ${firstHealthyProvider.provider}`);
return firstHealthyProvider.provider;
}
// Last resort: first configured provider
const anyConfigured = healthStatuses.find((h) => h.isConfigured);
if (anyConfigured) {
logger.warn(`Using configured but potentially unhealthy provider: ${anyConfigured.provider}`);
return anyConfigured.provider;
}
logger.error("No healthy or configured providers found");
return null;
}
/**
* Get health status for all registered providers.
*
* Membership: every descriptor in PROVIDER_DESCRIPTORS EXCEPT those that
* explicitly opt out via `excludeFromHealthSweep` (none currently do).
* Before issue #1305's fix, membership was implicit opt-IN via
* `defaultHealthSweepPriority`, which only 8 of ~38 descriptors carried —
* every other registered provider was silently invisible to this sweep,
* including anything a caller had actually configured and could use.
*
* Order: `defaultHealthSweepPriority` still decides ORDER for the
* providers that set it (lower = first — order is behaviour for
* first-healthy-wins callers like `getBestHealthyProvider`); everything
* else sorts after them, in PROVIDER_DESCRIPTORS's own declaration order
* (Array.prototype.sort is stable, so ties never reshuffle).
*
* Checks run through a worker pool of at most
* MAX_CONCURRENT_HEALTH_CHECKS, so a full sweep of every provider can't
* fire dozens of concurrent requests at once when a caller opts into
* `includeConnectivityTest`, and no check waits on a slower one beside
* it.
*/
static async checkAllProvidersHealth(options = {}) {
const providers = PROVIDER_DESCRIPTORS.filter((d) => d.excludeFromHealthSweep !== true)
.sort((a, b) => (a.defaultHealthSweepPriority ?? Number.MAX_SAFE_INTEGER) -
(b.defaultHealthSweepPriority ?? Number.MAX_SAFE_INTEGER))
.map((d) => d.name);
// A fixed pool of workers pulling from a shared cursor, NOT
// chunk-and-await. Both bound concurrency to the same number, but
// chunking also makes every batch wait for its slowest member before
// the next one starts, so a sweep costs the SUM of the per-batch
// maxima. With `includeConnectivityTest` each check can burn the whole
// `timeout`, which turned a ~1x worst case into ~ceil(n/cap)x. Here a
// slot freed by a fast check immediately takes the next provider, so
// the bound costs nothing in latency.
const results = new Array(providers.length);
let cursor = 0;
const runWorker = async () => {
// `cursor++` is atomic here: there is no await between reading and
// incrementing, so two workers can never claim the same provider.
for (let index = cursor++; index < providers.length; index = cursor++) {
try {
results[index] = {
status: "fulfilled",
value: await this.checkProviderHealth(providers[index], options),
};
}
catch (reason) {
results[index] = { status: "rejected", reason };
}
}
};
// Writing by index keeps results in provider order regardless of the
// order they complete in, which the rejected-case fallback below
// depends on. Zero providers means zero workers and an immediate
// return.
await Promise.all(Array.from({
length: Math.min(this.MAX_CONCURRENT_HEALTH_CHECKS, providers.length),
}, () => runWorker()));
return results.map((result, index) => {
if (result.status === "fulfilled") {
return result.value;
}
else {
// Return a failed health status for rejected promises
return {
provider: providers[index],
isHealthy: false,
isConfigured: false,
hasApiKey: false,
lastChecked: new Date(),
error: result.reason?.message || "Health check failed",
configurationIssues: ["Health check promise rejected"],
recommendations: [
"Check provider configuration and network connectivity",
],
};
}
});
}
/**
* Get a summary of provider health
*/
static getHealthSummary(healthStatuses) {
const healthy = healthStatuses.filter((h) => h.isHealthy);
const configured = healthStatuses.filter((h) => h.isConfigured);
const hasIssues = healthStatuses.filter((h) => h.configurationIssues.length > 0);
return {
total: healthStatuses.length,
healthy: healthy.length,
configured: configured.length,
hasIssues: hasIssues.length,
healthyProviders: healthy.map((h) => h.provider),
unhealthyProviders: healthStatuses
.filter((h) => !h.isHealthy)
.map((h) => h.provider),
};
}
}