@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
309 lines (308 loc) • 16.9 kB
TypeScript
/**
* Abstract base class for providers that talk to an OpenAI chat-completions
* shaped HTTP endpoint. Owns the entire request/stream/tool-loop pipeline
* so concrete providers only declare configuration + provider-specific
* quirks (env var names, default model, error mapping).
*
* Currently extended by:
* - OpenAICompatibleProvider (generic /v1/chat/completions backend)
* - LiteLLMProvider (LiteLLM proxy server)
* - ConfiguredOpenAICompatProvider (JSON-catalog providers, incl. deepseek)
*
* Subclasses provide:
* - getProviderName() / getDefaultModel() / formatProviderError() (abstract)
* - optional overrides: getFallbackModelName, getFallbackModels,
* adjustBuildBodyOptions, onStreamStart, getAvailableModels
*
* Nothing here imports from "ai" or "@ai-sdk/*". The base class is a
* direct HTTP client + multi-step tool-execution loop driven by SSE.
*/
import type { AIProviderName } from "../constants/enums.js";
import { BaseProvider } from "../core/baseProvider.js";
import type { LanguageModel, OpenAICompatBuildBodyArgs, OpenAICompatChatMessage, OpenAICompatChatRequest, OpenAICompatChatTool, OpenAICompatResponseFormat, OpenAICompatStreamLifecycleListeners, Schema, EnhancedGenerateResult, TextGenerationOptions, ValidationSchema, StreamOptions, StreamResult, ZodUnknownSchema } from "../types/index.js";
export declare abstract class OpenAIChatCompletionsProvider extends BaseProvider {
protected config: {
baseURL: string;
apiKey: string;
};
protected resolvedModel?: string;
constructor(providerName: AIProviderName, modelName: string | undefined, sdk: unknown, config: {
baseURL: string;
apiKey: string;
});
protected abstract getProviderName(): AIProviderName;
protected abstract getDefaultModel(): string;
protected abstract formatProviderError(error: unknown): Error;
/**
* Model name to return when `getDefaultModel()` is empty AND
* auto-discovery via `/models` finds nothing. Default "gpt-3.5-turbo".
*/
protected getFallbackModelName(): string;
/**
* Hardcoded model names returned from `getAvailableModels()` when the
* remote `/models` endpoint can't be reached. Default empty.
*/
/**
* Feed the catalog's `fallbacks` to BaseProvider's invalid-model retry, so
* a default the vendor has retired degrades to the next live model in the
* entry instead of failing the call outright.
*/
protected getModelFallbacks(): string[];
/**
* `resolvedModel` memoizes the first id this provider resolved, and
* getAISDKModel() builds its wire model from that memo rather than from
* `modelName`. Leaving it stale here silently defeats the invalid-model
* fallback: modelName advances to the next candidate while every request
* still carries the retired id, so each retry fails for the same reason
* the first attempt did.
*/
protected refreshHandlersForModel(model: string): void;
protected getFallbackModels(): string[];
/**
* Hook to mutate the `buildBody` options before the wire body is
* constructed. Default identity. Override for model-specific quirks
* (e.g. LiteLLM's Gemini 2.5 maxTokens skip).
*/
protected adjustBuildBodyOptions(_modelId: string, opts: OpenAICompatBuildBodyArgs["options"]): OpenAICompatBuildBodyArgs["options"];
/**
* Hook to adjust the OpenAI `response_format` after it's converted from the
* V3 responseFormat (non-streaming `doGenerate` path). Default identity.
* Override for providers that don't support a given format type — e.g.
* DeepSeek rejects `response_format: { type: "json_schema" }` ("This
* response_format type is unavailable now"); the `@ai-sdk/openai-compatible`
* path this replaced declared `supportsStructuredOutputs: false`, which
* downgraded `json_schema` to `json_object`. Subclasses replicate that here.
*/
protected adjustResponseFormat(rf: OpenAICompatResponseFormat | undefined, _modelId: string): OpenAICompatResponseFormat | undefined;
/**
* When true (default), `response_format` is NOT sent on requests that carry
* tools. The generate loop sets responseFormat on EVERY step of a tool
* loop, and generic/proxy backends (LiteLLM→vllm/GLM, openai-compatible, local
* servers) may silently honor it over tool calling — answering with
* final-shape JSON on step 1 instead of running the agentic loop. No error
* is raised, so the runtime tools↔schema conflict detector cannot catch it.
* The schema is still enforced post-hoc (`coerceJsonToSchema` runs against
* the final text) — the same contract as the Gemini tools↔schema exclusion.
* Mirrors the streaming path, which never sends response_format.
*
* Backends with first-party support for tools + json_schema in one request
* (OpenAI, Azure OpenAI) override this to false.
*/
protected suppressResponseFormatWithTools(): boolean;
/**
* When true, `doGenerate` puts `stream: true` on the wire and aggregates
* the SSE stream into the SAME complete result the JSON wire returns —
* callers still get one awaited result with structuredData coercion, tool
* calls, finish reason and usage intact. Bytes then flow continuously, so
* proxy/tunnel idle limits (e.g. Cloudflare's ~100s 524 on tunneled
* gateways) cannot kill a slow completion, and the request timeout is
* re-armed on every chunk (idle semantics) instead of capping total
* duration. Default false: some OpenAI-compatible backends mishandle
* `stream_options` or omit usage on streams, so each provider opts in
* deliberately. LiteLLM overrides this to true.
*/
protected useStreamingWireForGenerate(): boolean;
/**
* When true, an assistant turn's reasoning goes back to the vendor as
* `reasoning_content` on every later request of the conversation. DeepSeek
* documents this as required once tools are in play and answers 400
* without it. Default false: strict OpenAI-compatible backends reject a
* field they don't know. Read per request, never during construction.
*/
protected replayReasoningContent(): boolean;
/**
* Hook to adjust the fully-built wire request body before it is sent, on
* both the streaming and non-streaming paths. Default identity. Override for
* provider/model quirks that can't be expressed through buildBody options —
* e.g. Azure's newer reasoning deployments (o-series, gpt-5+) reject
* `max_tokens` and require `max_completion_tokens`.
*/
protected adjustRequestBody(body: OpenAICompatChatRequest, _modelId: string): OpenAICompatChatRequest;
/**
* Hook called when a request fails with HTTP 400. Return a modified body to
* retry ONCE with it; return undefined (default) to propagate the error
* unchanged. Applies on both the non-streaming doGenerate path and the
* streaming path. NVIDIA NIM uses this to strip `chat_template` /
* `chat_template_kwargs.reasoning_budget` when a model server rejects them,
* restoring the pre-migration fetch-level retry behavior.
*/
protected adjustBodyAfter400(_body: OpenAICompatChatRequest, _error: Error & {
statusCode?: number;
responseBody?: string;
}): OpenAICompatChatRequest | undefined;
/**
* Fit the outgoing `max_tokens` to what the target deployment can actually
* accept, using ONLY runtime-discovered limits — static table values are
* guesses, and hard-enforcing a guess would falsely reject requests the
* real deployment accepts:
*
* effective = min(requested, discovered output ceiling,
* discovered window − estimated input − margin)
*
* Returns the caller's value untouched when nothing was discovered, and
* `undefined` when the caller sent nothing and no ceiling is known (the
* wire then omits max_tokens and the backend applies its own default — no
* invented numbers). Throws ContextBudgetExceededError when the estimated
* input ALONE exceeds a discovered window: that request cannot succeed,
* and failing fast with honest numbers beats a guaranteed provider 400
* (plus any proxy-side fallback cascade) after a full round-trip.
*/
protected resolveWireMaxTokens(modelId: string, requested: number | undefined, messages: ReadonlyArray<OpenAICompatChatMessage>, tools: OpenAICompatChatTool[] | undefined): number | undefined;
/**
* Learn from a provider context-overflow 400 and, when possible, produce a
* corrected body for the one-shot retry slot. Two dynamic effects, zero
* static data:
*
* - the window stated in the error is registered with the runtime
* resolver, so every later budget check / compaction / max_tokens fit
* uses the backend's own number — self-healing even when a discovery
* endpoint (`/model/info`) is absent or unauthorized;
* - when the error also states the real input size (vllm/LiteLLM
* phrasing) and the body carried `max_tokens`, it is re-fit to
* `window − input − margin` and the request retried once.
*
* Returns undefined when the error is not an overflow, or when no smaller
* `max_tokens` can make the request fit (input alone too large) — the
* original error then propagates unchanged.
*/
private correctBodyAfterContextOverflow;
/**
* Hook called once at the start of every `executeStream` invocation.
* Return lifecycle listeners (onUsage / onFinish) to receive deferred
* analytics events as the stream progresses. Default returns undefined
* (no extra wiring). LiteLLM uses this for the OTel span wrap with cost.
*/
protected onStreamStart(_modelId: string): OpenAICompatStreamLifecycleListeners | undefined;
/**
* Returns true if `resolveModelName` should fall back to fetching
* `getAvailableModels()` and picking the first one when no explicit
* model is configured. Default true. Subclasses with a non-empty
* `getDefaultModel()` will never hit this branch anyway.
*/
protected shouldAutoDiscoverModel(): boolean;
/**
* Builds the chat-completions request URL for a model. Default is
* `${baseURL}/chat/completions`. Override for providers with a different
* routing scheme (e.g. Azure's deployment-based path + api-version query).
*/
protected getChatCompletionsURL(_modelId: string): string;
/**
* Auth headers merged into every request. Default is a Bearer token.
* Override for providers that authenticate differently (e.g. Azure, which
* uses an `api-key` header instead of `Authorization: Bearer`).
*/
protected getAuthHeaders(): Record<string, string>;
/**
* Health-check hook — part of the documented public provider contract
* (`docs/provider-integration/00-architecture.md`). Default returns true
* when an apiKey is configured; local providers (LM Studio, llama.cpp)
* override this to probe the server's `/models` endpoint.
*/
validateConfiguration(): Promise<boolean>;
/**
* Shared local-runtime reachability probe: GET `${baseURL}/models` with a
* short timeout, requiring at least one model entry with a non-empty id.
* Local providers (Ollama, LM Studio, llama.cpp) call this from their own
* validateConfiguration() override instead of relying on the base class's
* "apiKey is a non-empty string" default, which can't detect an
* unreachable local server.
*/
protected probeModelsEndpoint(headers?: Record<string, string>): Promise<boolean>;
/**
* Snapshot of the provider's resolved configuration — part of the documented
* public provider contract (`docs/provider-integration/00-architecture.md`).
* Subclasses inherit this; override only to expose extra fields.
*/
getConfiguration(): {
provider: AIProviderName;
model: string;
defaultModel: string;
baseURL: string;
};
/**
* Returns a minimal V3-shaped model used by BaseProvider's `generate()`
* non-streaming path. Driven by the parent's `generateText`. The
* streaming path bypasses this entirely.
*/
protected getAISDKModel(): Promise<LanguageModel>;
protected resolveModelName(): Promise<string>;
private buildDelegatingModel;
/**
* Streaming path — drives the chat-completions endpoint directly. No
* streamText, no AI SDK orchestrator. Tool calls, multi-step loops,
* telemetry, abort handling all inline.
*/
/**
* Native non-streaming generate.
*
* Drives the SAME `doGenerate` the ai loop drove — `buildDelegatingModel`'s,
* reached through `getAISDKModel()` — and supplies only the multi-step tool
* iteration around it. That matters: `doGenerate` is where the JSON-versus-SSE
* wire choice lives (`useStreamingWireForGenerate()`, false by default), along
* with the 400 retry, the context-overflow refit and the invalid-model
* fallback. An earlier attempt ran generate through the STREAMING loop
* instead and silently began sending `stream: true` on a path that had always
* sent plain JSON; ten providers returned empty content against a
* non-streaming body and a stream-rejecting backend failed outright. Loop
* around doGenerate, never around streamOneStep.
*
* Tool turns are appended in the message-builder shape, which
* `messageBuilderToOpenAI` already round-trips: an assistant message carrying
* `tool-call` parts, then one `tool` message of `tool-result` parts.
*/
generate(optionsOrPrompt: TextGenerationOptions | string, analysisSchema?: ValidationSchema): Promise<EnhancedGenerateResult | null>;
private executeNativeGenerate;
/**
* Recover structured output on a turn where tools forced `response_format`
* off the wire.
*
* Runs only after the tool turn is complete, and only when its answer does
* not already satisfy the schema. The re-ask drops the tools, which is what
* makes native `response_format` legal again on a vendor that refuses the two
* together, so this is a formatting pass over an answer the model has already
* produced rather than a second attempt at the question.
*
* The schema is deliberately NOT spelled into the tool turn's system prompt
* instead. That was tried and reverted: a model handed a JSON Schema next to a
* tool list can read it as another tool — Groq's llama-3.3 answers by calling
* one named "json", which the server rejects — turning a wrong answer into a
* hard error.
*
* Returns the merged result, or the untouched `toolPhase` whenever the re-ask
* cannot be run or cannot be trusted, so it can improve an outcome but never
* degrade one.
*/
private reformatWithoutTools;
/**
* Resolve the parsed object for a `stream({ schema })` turn, after the
* stream has been drained.
*
* Before this, streaming never sent `response_format` under any condition —
* unlike the generate path, where the gap is only tools-dependent (see
* suppressResponseFormatWithTools), so `structuredData` was never available
* to a streaming caller at all. `executeStream` now sends `response_format`
* on the wire whenever a schema is present and tools aren't suppressing it;
* this method covers what's left: the common case where the streamed text
* already parses against the schema, and the tools-suppressed case, which is
* re-asked once without tools — the same recovery `reformatWithoutTools`
* performs for generate(), reusing its acceptance rule
* (`yieldsSchemaValidObject`) and its two-strategy fallback (native
* `response_format` first, the schema spelled into the prompt if a vendor
* rejects that).
*
* Returns `undefined` on every failure path rather than throwing: a turn
* whose text the caller has already consumed and streamed to their UI must
* not be turned into an error by a follow-up they never asked for.
*/
private resolveStreamStructuredData;
protected executeStream(options: StreamOptions, _analysisSchema?: ZodUnknownSchema | Schema<unknown>): Promise<StreamResult>;
private runStreamLoop;
private streamOneStep;
private executeToolBatch;
/**
* Default implementation hits `${baseURL}/models`. Subclasses with a
* different endpoint path, caching, or fallback strategy should override.
*/
getAvailableModels(): Promise<string[]>;
getFirstAvailableModel(): Promise<string>;
}