@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
245 lines (244 loc) • 10.3 kB
JavaScript
/**
* Provider-specific token limit utilities
* Provides safe maxTokens values based on provider and model capabilities
*/
import { getRuntimeOutputCeiling } from "../constants/contextWindows.js";
import { PROVIDER_MAX_TOKENS } from "../core/constants.js";
import { logger } from "./logger.js";
import { hasRestrictedOutputLimit, RESTRICTED_OUTPUT_TOKEN_LIMIT, } from "./modelDetection.js";
// Restricted-model detection (Gemini 3.x + image-gen models capped at 32768
// output tokens) lives in modelDetection.ts. Importing the canonical helper
// keeps both call sites on one definition; the previous local copy used
// case-sensitive substring matches and could miss identifiers like
// "Gemini-3-Pro" which the canonical anchored, case-insensitive regex matches.
/**
* Get the safe maximum tokens for a provider and model
*/
export function getSafeMaxTokens(provider, model, requestedMaxTokens) {
// CRITICAL: Gemini 3 models AND image generation models have a hard limit of 32768 output tokens
// This check must happen FIRST, before any other logic, because these models
// will reject requests with maxOutputTokens > 32768
const isRestrictedModel = model ? hasRestrictedOutputLimit(model) : false;
if (isRestrictedModel) {
// Explicit undefined/null check so a caller-supplied 0 is preserved
// (truthy guards would treat 0 as "unset" and silently fall back to the cap).
if (requestedMaxTokens !== undefined &&
requestedMaxTokens !== null &&
requestedMaxTokens > RESTRICTED_OUTPUT_TOKEN_LIMIT) {
logger.warn(`Requested maxTokens ${requestedMaxTokens} exceeds ${model} limit of ${RESTRICTED_OUTPUT_TOKEN_LIMIT}. Using ${RESTRICTED_OUTPUT_TOKEN_LIMIT} instead.`);
return RESTRICTED_OUTPUT_TOKEN_LIMIT;
}
// If no maxTokens specified, use the restricted limit as default
if (requestedMaxTokens === undefined || requestedMaxTokens === null) {
return RESTRICTED_OUTPUT_TOKEN_LIMIT;
}
// Otherwise, use the requested value (it's within limits, including 0)
return requestedMaxTokens;
}
// Runtime-discovered output ceiling (e.g. LiteLLM /model/info
// max_output_tokens): the serving infrastructure's own number for the
// deployed model, authoritative over the static per-provider table. This
// both clamps over-large requests to the real cap AND replaces the static
// provider default for callers that pass no maxTokens — the litellm
// blanket default (128000) is a context-window-sized value that slashed
// usable input on total-context backends. Checked AFTER the
// restricted-model branch so hard caps proven at the origin API (Gemini 3
// / image models) still win over a proxy that over-advertises them.
const runtimeCeiling = getRuntimeOutputCeiling(provider, model);
if (runtimeCeiling !== undefined) {
if (requestedMaxTokens === undefined || requestedMaxTokens === null) {
return runtimeCeiling;
}
if (requestedMaxTokens > runtimeCeiling) {
logger.warn(`Requested maxTokens ${requestedMaxTokens} exceeds the advertised ` +
`${provider}/${model} output ceiling of ${runtimeCeiling}. Using ${runtimeCeiling} instead.`);
return runtimeCeiling;
}
return requestedMaxTokens;
}
// Get provider-specific limits
const providerLimits = PROVIDER_MAX_TOKENS[provider];
if (!providerLimits) {
logger.warn(`Unknown provider ${provider}, no token limits enforced`);
// Explicit undefined/null check so a caller-supplied 0 is preserved
// (truthy guard would silently drop 0 here as it does in the restricted branch).
if (requestedMaxTokens === undefined || requestedMaxTokens === null) {
return undefined;
}
return requestedMaxTokens;
}
// Get model-specific limit or provider default
let maxLimit;
if (model &&
typeof providerLimits === "object" &&
providerLimits[model]) {
maxLimit = providerLimits[model];
}
else if (typeof providerLimits === "object" &&
providerLimits.default) {
maxLimit = providerLimits.default;
}
else if (typeof providerLimits === "number") {
maxLimit = providerLimits;
}
else {
maxLimit = PROVIDER_MAX_TOKENS.default;
}
// If no specific maxTokens requested, return the provider limit.
// Use explicit undefined/null so a caller-supplied 0 is preserved
// (matches the restricted-model branch above).
if (requestedMaxTokens === undefined || requestedMaxTokens === null) {
return maxLimit;
}
// If requested maxTokens exceeds the limit, use the limit and warn
if (requestedMaxTokens > maxLimit) {
logger.warn(`Requested maxTokens ${requestedMaxTokens} exceeds ${provider}/${model} limit of ${maxLimit}. Using ${maxLimit} instead.`);
return maxLimit;
}
// Use the requested value if it's within limits
return requestedMaxTokens;
}
/**
* Maximum output tokens supported by a given Anthropic Claude model.
*
* The native Vertex+Claude and native Anthropic message paths send `max_tokens`
* straight to the Anthropic API, which returns 400 if the value exceeds the
* model's published output ceiling. (The AI-SDK path clamps automatically;
* these native paths do not.) This table lets those paths default to the
* model's real ceiling — 64K for Sonnet/Haiku 4.x, 32K for Opus 4.x — instead of
* the legacy 4096 that silently truncated large structured responses.
*
* Unknown identifiers fall back to a safe modern floor (8192).
*/
export function getClaudeMaxOutputTokens(model) {
const m = (model ?? "").toLowerCase();
// Claude 4.x family: Opus 4.x = 32K, Sonnet/Haiku 4.x = 64K.
if (/opus[-_.]?4/.test(m)) {
return 32000;
}
if (/sonnet[-_.]?4/.test(m) || /haiku[-_.]?4/.test(m)) {
return 64000;
}
// Claude 5.x family (Opus 5 / Opus 5.5 / Sonnet 5 / Fable 5.1 / Mythos):
// all confirmed at 128K standard max output on
// platform.claude.com/docs/en/about-claude/models/overview. The
// (?![0-9]) guard keeps this off dated 4.x ids like "opus-4-5-20251101"
// (Opus 4.5, not Opus 5), where the family word sits before the "4", not
// directly before this "5".
if (/(?:opus|sonnet|fable|mythos)[-_.]?5(?![0-9])/.test(m)) {
return 128000;
}
// Claude 3.7 Sonnet supports 64K output.
if (/3[-_.]?7[-_.]?sonnet/.test(m)) {
return 64000;
}
// Claude 3.5 Sonnet / Haiku → 8192.
if (/3[-_.]?5[-_.]?(sonnet|haiku)/.test(m)) {
return 8192;
}
// Claude 3 Opus / Sonnet / Haiku → 4096.
if (/claude-3-(opus|sonnet|haiku)/.test(m) || /3[-_.]?opus/.test(m)) {
return 4096;
}
// Bare family aliases (latest of a family) → assume the modern ceiling.
if (m.includes("opus")) {
return 32000;
}
if (m.includes("sonnet") || m.includes("haiku")) {
return 64000;
}
return 8192;
}
/**
* Resolve the `max_tokens` to send on a native Anthropic/Claude request: honour
* the caller's value but clamp it to the model's published ceiling, and default
* to that ceiling when the caller did not specify one. Prevents both silent
* truncation (the legacy 4096 default) and 400s from over-large requests.
*/
export function resolveClaudeMaxTokens(model, requested) {
const ceiling = getClaudeMaxOutputTokens(model);
if (requested !== undefined && requested !== null && requested > 0) {
return Math.min(requested, ceiling);
}
return ceiling;
}
/**
* Validate if maxTokens is safe for a provider/model combination
*/
export function validateMaxTokens(provider, model, maxTokens) {
const safeMaxTokens = getSafeMaxTokens(provider, model, maxTokens);
if (!maxTokens) {
return {
isValid: true,
recommendedMaxTokens: safeMaxTokens,
};
}
// If no limits are defined, validation always passes
if (safeMaxTokens === undefined) {
return {
isValid: true,
recommendedMaxTokens: maxTokens,
};
}
const isValid = maxTokens <= safeMaxTokens;
return {
isValid,
recommendedMaxTokens: safeMaxTokens,
warning: !isValid
? `maxTokens ${maxTokens} exceeds ${provider}/${model} limit of ${safeMaxTokens}`
: undefined,
};
}
/**
* Get provider-specific token limit recommendations
*/
export function getTokenLimitRecommendations(provider) {
const providerLimits = PROVIDER_MAX_TOKENS[provider];
if (!providerLimits || typeof providerLimits === "number") {
const limit = typeof providerLimits === "number"
? providerLimits
: PROVIDER_MAX_TOKENS.default;
return {
conservative: Math.floor(limit * 0.5),
balanced: Math.floor(limit * 0.75),
maximum: limit,
models: {},
};
}
const modelLimits = Object.entries(providerLimits)
.filter(([key]) => key !== "default")
.map(([_, limit]) => limit);
const maxLimit = Math.max(...modelLimits, providerLimits.default || 0);
const minLimit = Math.min(...modelLimits, providerLimits.default || maxLimit);
return {
conservative: Math.floor(minLimit * 0.5),
balanced: Math.floor(((minLimit + maxLimit) / 2) * 0.75),
maximum: maxLimit,
models: Object.fromEntries(Object.entries(providerLimits).filter(([key]) => key !== "default")),
};
}
/**
* Get all provider limits summary
*/
export function getAllProviderLimits() {
const result = {};
for (const [provider, limits] of Object.entries(PROVIDER_MAX_TOKENS)) {
if (provider === "default") {
continue;
}
if (typeof limits === "number") {
result[provider] = {
default: limits,
models: {},
};
}
else {
const { default: defaultLimit, ...models } = limits;
result[provider] = {
default: defaultLimit || PROVIDER_MAX_TOKENS.default,
models: models,
};
}
}
return result;
}