@juspay/neurolink/dist/providers/litellm/client.d.ts

The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002

import type { AIProviderName } from "../../constants/enums.js";
import type { EmbedInput, OpenAICompatBuildBodyArgs, OpenAICompatStreamLifecycleListeners } from "../../types/index.js";
import { OpenAIChatCompletionsProvider } from "../openaiChatCompletionsBase.js";
/** Test hook: reset the discovery caches (state is module-global). */
export declare function clearLiteLLMModelLimitsCache(): void;
/**
 * Discover real per-model limits from the LiteLLM proxy's `GET /model/info`
 * and register them with the runtime resolvers: `max_input_tokens` → context
 * window, `max_output_tokens` → output ceiling. The static table only has a
 * one-size litellm `_default` (128K) while proxied models range from 8K to
 * 2M — budget checks, compaction, and max_tokens clamping need the real
 * numbers.
 *
 * Awaitable and deduped per base URL: generation pipelines await this (via
 * `LiteLLMProvider.ensureModelLimits`) BEFORE any budget math, so even the
 * FIRST call in a fresh process budgets against real windows — the previous
 * constructor-scoped fire-and-forget lost that race every time, and a
 * budget-blocked call then prevented the discovery that would have unblocked
 * it. Never rejects: any failure (endpoint absent, auth, timeout) is logged
 * at debug, leaves the static defaults in force, and allows a retry on the
 * next call.
 */
export declare function ensureLiteLLMModelLimits(baseURL: string, apiKey: string): Promise<void>;
/**
 * LiteLLM Provider — direct HTTP, no AI SDK. Talks to a LiteLLM proxy
 * server (or any deployment that speaks OpenAI chat-completions + the
 * `/v1/models` and `/v1/embeddings` endpoints).
 *
 * All request/stream/tool-loop orchestration lives in
 * `OpenAIChatCompletionsProvider`. This class adds LiteLLM-specific
 * behaviour: OTel span wrap with cost (`onStreamStart`), Gemini 2.5
 * maxTokens skip (`adjustBuildBodyOptions`), ModelAccessDeniedError on
 * 403, 10-minute model cache (`getAvailableModels`), `LITELLM_FALLBACK_MODELS`
 * env-driven fallback list, and native `/v1/embeddings`.
 */
export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
    private static modelsCache;
    private static modelsCacheTime;
    private static readonly MODELS_CACHE_DURATION;
    constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: {
        apiKey?: string;
        baseURL?: string;
    });
    /**
     * Awaitable model-limit discovery — generation pipelines call this before
     * budget checks so window/output-ceiling math uses the proxy's real
     * numbers (see BaseProvider.ensureModelLimits). Deduped and cached per
     * base URL; never rejects (failure degrades to static defaults).
     */
    ensureModelLimits(): Promise<void>;
    protected getProviderName(): AIProviderName;
    protected getDefaultModel(): string;
    protected getFallbackModelName(): string;
    protected getFallbackModels(): string[];
    /**
     * generate() rides the SSE wire for LiteLLM: deployments commonly sit
     * behind proxies/tunnels (e.g. Cloudflare, which 524s an origin that is
     * silent for ~100s), and a non-streaming completion from a slow model
     * sends nothing until it is fully done. Streaming keeps bytes flowing for
     * the whole generation while the base class aggregates into the same
     * complete result — structuredData coercion, tool calls, stopReason,
     * usage and the JSON damage flags are unchanged for callers.
     * Escape hatch: NEUROLINK_LITELLM_SSE_GENERATE=false restores the plain
     * JSON wire.
     */
    protected useStreamingWireForGenerate(): boolean;
    /**
     * Gemini 2.5 models on LiteLLM have a known compatibility issue with
     * `max_tokens` — strip it before the wire body is built. Applies to
     * both streaming and non-streaming paths.
     */
    protected adjustBuildBodyOptions(modelId: string, opts: OpenAICompatBuildBodyArgs["options"]): OpenAICompatBuildBodyArgs["options"];
    /**
     * Wrap the stream in an OTel span to capture provider-level latency,
     * token usage, finish reason, and cost. Matches the pre-migration
     * behaviour where streamText was wrapped in `neurolink.provider.streamText`.
     */
    protected onStreamStart(modelId: string): OpenAICompatStreamLifecycleListeners | undefined;
    formatProviderError(error: unknown): Error;
    /**
     * Get available models from LiteLLM proxy `/v1/models` endpoint.
     * Caches results for 10 minutes; falls back to env-driven list or a
     * minimal safe default if the API fetch fails.
     */
    getAvailableModels(): Promise<string[]>;
    private fetchModelsFromAPI;
    /**
     * Generate an embedding for a single text input via native /v1/embeddings.
     */
    embed(input: string | EmbedInput, modelName?: string): Promise<number[]>;
    /**
     * Generate embeddings for multiple text inputs via native /v1/embeddings.
     */
    embedMany(texts: string[], modelName?: string): Promise<number[][]>;
    private callEmbeddings;
}