@sema-agent/core
Version:
Stateless, task-oriented AI agent core
874 lines (873 loc) • 41.6 kB
JavaScript
import { createAssistantMessageEventStream, } from "../internal/llm.js";
import { STREAM_TORN_WARNING } from "./stream-shared.js";
import { FLOOR_OUTPUT_TOKENS, parseContextOverflow, planOutputCapAdjustment } from "./context-overflow.js";
import { abortLikeName, BrainError, brainErrorCodeOf, classifyFetchThrow, classifyHttp, describeNetworkError, isProviderEchoShape, namesTheFailure, proxyTunnelRefusedSentence, readConnectFailure, readProviderErrorCode, readProviderRequestId, refusedTheRequest } from "./errors.js";
import { stripUrlCredentials } from "./route-adjudicator.js";
import { admittedRequestRoute } from "./request-route.js";
import { isTransientStatus, isProviderCondition, isProviderConditionSource, judgeHttpFailure, PROVIDER_CONDITION_TRAITS, resolveProviderDialect, } from "./provider-dialect.js";
import { FAST_MAX_BACKOFF_MS, providerWaitHint, retryBackoffMs } from "./retry.js";
import { bindBrainStatusEmitter, emitBrainStatus, emitBrainTelemetry, reportExtraBodyDropped } from "./status-sink.js";
import { createAttemptController, firstByteWindow, refuseRetiredTimeoutKnobs, resolveFirstByteRetries, resolveWatchdogMs, STALL_FIRST_BYTE_MS, STALL_FIRST_TOKEN_MS, STALL_IDLE_MS, } from "./timeout.js";
import { readMember, thrownText } from "../core/thrown-value.js";
import { egressText } from "../core/untrusted-egress.js";
const DEFAULT_MAX_RETRIES = 10;
const MAX_RETRIES_ENV_CEILING = 15;
const MAX_RETRIES_ENV = "SEMA_MAX_RETRIES";
export function resolveMaxRetries(configured) {
if (configured !== undefined && Number.isFinite(configured))
return configured;
const raw = process.env[MAX_RETRIES_ENV];
if (raw !== undefined && raw.trim() !== "") {
const parsed = Number(raw);
if (Number.isFinite(parsed) && parsed >= 0)
return Math.min(Math.floor(parsed), MAX_RETRIES_ENV_CEILING);
}
return DEFAULT_MAX_RETRIES;
}
const DEFAULT_DETERMINISTIC_CONNECT_RETRIES = 2;
export function resolveDeterministicConnectRetries(configured) {
if (configured !== undefined && Number.isFinite(configured) && configured >= 0)
return Math.floor(configured);
return DEFAULT_DETERMINISTIC_CONNECT_RETRIES;
}
const SHOULD_RETRY_HEADER = "x-should-retry";
function sameHeaders(a, b) {
const aKeys = Object.keys(a);
if (aKeys.length !== Object.keys(b).length)
return false;
return aKeys.every((k) => Object.prototype.hasOwnProperty.call(b, k) && a[k] === b[k]);
}
function canonicalJson(value) {
if (Array.isArray(value))
return `[${value.map(canonicalJson).join(",")}]`;
if (typeof value === "object" && value !== null) {
const rec = value;
return `{${Object.keys(rec)
.sort()
.map((k) => `${JSON.stringify(k)}:${canonicalJson(rec[k])}`)
.join(",")}}`;
}
return JSON.stringify(value) ?? "null";
}
function leafHolder(root, path) {
const parts = path.split(".");
const leaf = parts.pop();
if (leaf === undefined)
return undefined;
let cur = root;
for (const part of parts) {
if (typeof cur !== "object" || cur === null || Array.isArray(cur))
return undefined;
cur = cur[part];
}
if (typeof cur !== "object" || cur === null || Array.isArray(cur))
return undefined;
return { holder: cur, leaf };
}
export function sameRequestModuloCap(original, rebuilt, imposedCap) {
if (rebuilt.url !== original.url || !sameHeaders(rebuilt.headers, original.headers))
return false;
if (rebuilt.outputCapTokens !== imposedCap)
return false;
if (rebuilt.body === original.body)
return true;
const parse = (raw) => {
let parsed;
try {
parsed = JSON.parse(raw);
}
catch {
return undefined;
}
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed))
return undefined;
return parsed;
};
const a = parse(original.body);
const b = parse(rebuilt.body);
if (a === undefined || b === undefined)
return false;
for (const path of new Set([...(original.capDerivedPaths ?? []), ...(rebuilt.capDerivedPaths ?? [])])) {
const from = leafHolder(a, path);
const to = leafHolder(b, path);
const before = from === undefined ? undefined : from.holder[from.leaf];
const after = to === undefined ? undefined : to.holder[to.leaf];
if (before !== undefined) {
if (typeof before !== "number" || typeof after !== "number")
return false;
if (after > before)
return false;
}
if (from !== undefined)
delete from.holder[from.leaf];
if (to !== undefined)
delete to.holder[to.leaf];
}
return canonicalJson(a) === canonicalJson(b);
}
const ERROR_BODY_BYTE_CAP = 64 * 1024;
const ERROR_BODY_READ_TIMEOUT_MS = 5_000;
async function readErrorBody(res, signal) {
const body = res.body;
if (body === null)
return { text: "", endedNormally: true };
let reader;
try {
reader = body.getReader();
}
catch {
return { text: "", endedNormally: false };
}
const decoder = new TextDecoder();
let stopped = false;
const stop = () => {
stopped = true;
void reader.cancel().catch(() => undefined);
};
let timer;
let out = "";
let endedNormally = false;
try {
timer = setTimeout(stop, ERROR_BODY_READ_TIMEOUT_MS);
signal?.addEventListener("abort", stop, { once: true });
if (signal?.aborted === true)
stop();
let bytes = 0;
for (;;) {
const chunk = await reader.read();
if (chunk.done) {
endedNormally = !stopped;
break;
}
const value = chunk.value;
if (value === undefined)
continue;
const room = ERROR_BODY_BYTE_CAP - bytes;
const usable = value.byteLength <= room ? value : value.subarray(0, room);
bytes += usable.byteLength;
out += decoder.decode(usable, { stream: true });
if (bytes >= ERROR_BODY_BYTE_CAP) {
stop();
break;
}
}
out += decoder.decode();
}
catch {
endedNormally = false;
}
finally {
if (timer !== undefined)
clearTimeout(timer);
signal?.removeEventListener("abort", stop);
stop();
}
return { text: out, endedNormally };
}
function discardResponseBody(res) {
const body = res?.body;
if (body === undefined || body === null || body.locked)
return;
try {
void body.cancel().catch(() => undefined);
}
catch {
}
}
export function shouldRetryHeaderVerdict(res) {
if (res === undefined)
return undefined;
let raw = null;
try {
const headers = res.headers;
if (headers === undefined || headers === null || typeof headers.get !== "function")
return undefined;
raw = res.headers.get(SHOULD_RETRY_HEADER);
}
catch {
return undefined;
}
if (raw === "true")
return true;
if (raw === "false")
return false;
return undefined;
}
const RETRY_STATUS_SLICE_MS = 30_000;
export const WAITING_FIRST_TOKEN_AFTER_MS = 30_000;
export const WAITING_FIRST_TOKEN_EVERY_MS = 30_000;
const THINKING_RETRY_BUDGET = 2;
const EMPTY_USAGE = {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};
function reportRequestStrip(model, req) {
if (req.extraBodyDropped !== undefined && req.extraBodyDropped.length > 0)
reportExtraBodyDropped({ api: model.api, modelId: model.id, keys: req.extraBodyDropped });
}
function emptyAssistant(model) {
return {
role: "assistant",
content: [],
api: model.api,
provider: model.provider,
model: model.id,
stopReason: "stop",
usage: { ...EMPTY_USAGE, cost: { ...EMPTY_USAGE.cost } },
timestamp: Date.now(),
};
}
function isAbortError(err) {
return abortLikeName(err) !== undefined;
}
function sleep(ms, signal) {
return new Promise((resolve) => {
if (signal?.aborted)
return resolve();
let t;
const onAbort = () => {
clearTimeout(t);
resolve();
};
t = setTimeout(() => {
signal?.removeEventListener("abort", onAbort);
resolve();
}, ms);
signal?.addEventListener("abort", onAbort, { once: true });
});
}
async function sleepAnnouncingRetry(totalMs, signal, frame) {
let remaining = Math.max(0, totalMs);
for (;;) {
if (signal?.aborted)
return;
emitBrainStatus(frame(remaining));
if (remaining <= 0)
return;
const slice = Math.min(remaining, RETRY_STATUS_SLICE_MS);
await sleep(slice, signal);
remaining -= slice;
if (remaining <= 0)
return;
}
}
const RETRY_PHASE_OF_ERR_CLASS = {
connect_refused: "reconnecting",
transport: "reconnecting",
stall: "reconnecting",
rate_limit: "rate_limited",
server: "retrying",
http: "retrying",
output_cap: "retrying",
};
Object.freeze(RETRY_PHASE_OF_ERR_CLASS);
const TRANSPORT_RETRY_DETAIL = "connection lost, reconnecting";
function retryAnnouncement(failure) {
if (failure.kind !== "stalled")
return { errClass: "transport", detail: TRANSPORT_RETRY_DETAIL };
const knob = failure.allowanceMs !== undefined && failure.allowanceMs > 0
? `${failure.knob} ${failure.timeoutMs - failure.allowanceMs} + ${failure.allowanceMs} upload allowance`
: failure.knob;
return { errClass: "stall", detail: `${failure.watchdog} watchdog fired after ${failure.timeoutMs}ms (${knob}), retrying` };
}
function firstByteTimeoutMessage(windowsMs) {
const [first, ...retries] = windowsMs;
const waited = retries.length === 0 ? `within ${first ?? 0}ms` : `within ${first ?? 0}ms, then ${retries.map((w) => `${w}ms`).join(", then ")} on the retry`;
return `first-byte timeout: no response headers ${waited} (firstByteTimeoutMs) — if a proxy or gateway holds responses until they complete, raise firstByteTimeoutMs to wait longer`;
}
export function runStreamingBrain(args) {
const { model, doFetch, signal, config, httpLabel, buildRequest, makeParser, stallTimeouts } = args;
const callRetryCap = args.maxRetries !== undefined && Number.isFinite(args.maxRetries) && args.maxRetries >= 0 ? Math.floor(args.maxRetries) : Infinity;
const out = createAssistantMessageEventStream();
let cleanup;
let announcedRetry = false;
let announcedWait = false;
let terminalWaitDetail = "first token arrived";
let firstTokenArrived = false;
let terminalRetryPhase = "recovered";
let terminalRetryDetail = "recovered after retrying";
let requestIdSeen;
let attemptPartial;
let attemptOutput;
let routeSeen;
void run()
.catch((err) => {
const aborted = signal?.aborted === true || isAbortError(err);
terminalRetryPhase = "gave_up";
terminalRetryDetail = aborted ? "cancelled while retrying" : "retries exhausted";
terminalWaitDetail = firstTokenArrived
? aborted
? "cancelled after the first token arrived"
: "the call failed after the first token arrived"
: aborted
? "cancelled while waiting for the first token"
: "the first-token wait ended in failure";
const errorMsg = emptyAssistant(model);
const kept = aborted ? undefined : attemptOutput;
if (kept !== undefined) {
errorMsg.content = kept.content;
errorMsg.usage = kept.usage;
}
errorMsg.stopReason = aborted ? "aborted" : "error";
errorMsg.errorMessage = thrownText(err);
if (!aborted && brainErrorCodeOf(err) !== undefined) {
if (readMember(err, "inputTooLong") === true)
errorMsg.errorKind = "input_too_long";
else if (readMember(err, "inputTooLongRuledOut") === true)
errorMsg.inputTooLongRuledOut = true;
if (readMember(err, "errorBodyIncomplete") === true)
errorMsg.errorBodyIncomplete = true;
if (readMember(err, "apiError") === true) {
errorMsg.isApiErrorMessage = true;
const status = readMember(err, "status");
const statusNum = typeof status === "number" ? status : undefined;
if (namesTheFailure(statusNum))
errorMsg.apiErrorStatus = statusNum;
if (requestIdSeen !== undefined)
errorMsg.requestId = requestIdSeen;
if (routeSeen !== undefined)
errorMsg.apiErrorRoute = routeSeen;
const providerCode = readMember(err, "providerCode");
if ((statusNum === undefined || refusedTheRequest(statusNum)) && typeof providerCode === "string" && isProviderEchoShape(providerCode))
errorMsg.apiErrorCode = providerCode;
const condition = readMember(err, "providerCondition");
const source = readMember(err, "providerConditionSource");
if (isProviderCondition(condition) && isProviderConditionSource(source)) {
errorMsg.providerCondition = condition;
errorMsg.providerConditionSource = source;
const resetAtMs = readMember(err, "providerResetAtMs");
if (typeof resetAtMs === "number" && Number.isFinite(resetAtMs))
errorMsg.providerResetAtMs = resetAtMs;
}
}
}
if (kept === undefined || kept.usageMissing === true)
errorMsg.usageMissing = true;
const seenResponseModel = attemptPartial?.responseModel;
if (seenResponseModel !== undefined)
errorMsg.responseModel = seenResponseModel;
out.push({ type: "error", reason: aborted ? "aborted" : "error", error: errorMsg });
})
.finally(() => {
cleanup?.();
if (!announcedRetry && !announcedWait)
return;
try {
emitBrainStatus({ phase: terminalRetryPhase, detail: announcedRetry ? terminalRetryDetail : terminalWaitDetail });
}
catch {
}
});
return out;
async function run() {
let req = buildRequest();
const commit = (r) => {
r.onCommitted?.();
reportRequestStrip(model, r);
routeSeen = admittedRequestRoute(r.route);
};
commit(req);
const dialect = resolveProviderDialect(req.url, model.api);
let maxOutputTokensOverride;
const maxRetries = Math.min(resolveMaxRetries(config.maxRetries), callRetryCap);
const baseDelay = config.retryDelayMs ?? 500;
const deterministicConnectBudget = Math.min(resolveDeterministicConnectRetries(config.deterministicConnectRetries), maxRetries);
const deterministicLaneActive = deterministicConnectBudget < maxRetries;
refuseRetiredTimeoutKnobs(stallTimeouts, "advisory", "StreamOptions.stallTimeouts");
const firstByteTimeoutMs = resolveWatchdogMs(config.firstByteTimeoutMs, stallTimeouts?.firstByteMs, STALL_FIRST_BYTE_MS, "firstByteTimeoutMs");
const firstTokenTimeoutMs = resolveWatchdogMs(config.firstTokenTimeoutMs, stallTimeouts?.firstTokenMs, STALL_FIRST_TOKEN_MS, "firstTokenTimeoutMs");
const idleTimeoutMs = resolveWatchdogMs(config.idleTimeoutMs, stallTimeouts?.idleMs, STALL_IDLE_MS, "idleTimeoutMs");
const firstByteRetryBudget = resolveFirstByteRetries(config.firstByteRetries);
const armWaitDisclosure = (ceilingMs) => {
if (!(ceilingMs >= WAITING_FIRST_TOKEN_AFTER_MS))
return { stop: () => undefined, expire: () => undefined };
const startedAt = Date.now();
const emitWaitFrame = bindBrainStatusEmitter();
let timer;
let stopped = false;
let disclosed = false;
const disclose = () => {
disclosed = true;
announcedWait = true;
emitWaitFrame({ phase: "waiting_first_token", detail: "waiting for the first token", elapsedMs: Date.now() - startedAt, timeoutMs: ceilingMs });
};
const schedule = (delayMs) => {
timer = setTimeout(() => {
timer = undefined;
if (stopped || Date.now() - startedAt >= ceilingMs)
return;
disclose();
schedule(WAITING_FIRST_TOKEN_EVERY_MS);
}, delayMs);
};
schedule(WAITING_FIRST_TOKEN_AFTER_MS);
const stop = () => {
stopped = true;
if (timer)
clearTimeout(timer);
timer = undefined;
};
return {
stop,
expire: () => {
if (!stopped && !disclosed)
disclose();
stop();
},
};
};
let attempt = 0;
let thinkingRetries = 0;
let firstByteStalls = 0;
const firstByteWindowsWaited = [];
let startEmitted = false;
for (;;) {
let res;
let connect;
for (;;) {
if (signal?.aborted) {
const e = new Error("aborted");
e.name = "AbortError";
throw e;
}
const window = firstByteWindow(firstByteTimeoutMs, Buffer.byteLength(req.body), firstByteStalls);
const headerWait = armWaitDisclosure(window.windowMs);
const cc = createAttemptController(window.windowMs, signal);
let r;
let netErr;
let firstByteStall;
requestIdSeen = undefined;
attemptPartial = undefined;
attemptOutput = undefined;
const stalledAtFirstByte = () => {
firstByteWindowsWaited.push(window.windowMs);
return {
kind: "stalled",
watchdog: "first-byte",
knob: "firstByteTimeoutMs",
timeoutMs: window.windowMs,
allowanceMs: window.allowanceMs,
err: new BrainError("network", firstByteTimeoutMessage(firstByteWindowsWaited), undefined, { apiError: true }),
};
};
try {
r = await doFetch(req.url, { method: "POST", headers: req.headers, body: req.body, signal: cc.signal, redirect: "manual" });
requestIdSeen = readProviderRequestId(r);
if (cc.timedOut()) {
discardResponseBody(r);
r = undefined;
requestIdSeen = undefined;
firstByteStall = stalledAtFirstByte();
}
}
catch (e) {
if (cc.timedOut()) {
firstByteStall = stalledAtFirstByte();
}
else if (isAbortError(e) || signal?.aborted) {
cc.dispose();
throw e;
}
else if (classifyFetchThrow(e) === "request_unbuildable") {
cc.dispose();
throw new BrainError("invalid_request", `${httpLabel} request could not be built: ${egressText("same-principal", stripUrlCredentials(thrownText(e)))}`);
}
else {
netErr = e;
}
}
finally {
if (firstByteStall !== undefined)
headerWait.expire();
else
headerWait.stop();
}
if (r !== undefined && (r.type === "opaqueredirect" || (r.status >= 300 && r.status < 400))) {
const status = r.status;
discardResponseBody(r);
cc.dispose();
throw new BrainError("network", `${httpLabel} endpoint answered with a redirect${status > 0 ? ` (HTTP ${status})` : ""}; it was not followed and no credential was sent anywhere else — declare the final endpoint as the brain's baseUrl`);
}
if (r && r.ok && r.body) {
cc.clearTimer();
connect = cc;
cleanup = () => connect?.dispose();
res = r;
break;
}
cc.clearTimer();
try {
if (firstByteStall !== undefined) {
if (firstByteStalls >= firstByteRetryBudget)
throw firstByteStall.err;
firstByteStalls++;
const announcement = retryAnnouncement(firstByteStall);
const delayMs = retryBackoffMs(baseDelay, firstByteStalls - 1);
emitBrainTelemetry({ kind: "retry", attempt: firstByteStalls, phase: "connect", errClass: announcement.errClass, nextDelayMs: delayMs });
announcedRetry = true;
cc.abort();
cc.dispose();
await sleepAnnouncingRetry(delayMs, signal, (remainingMs) => ({
phase: RETRY_PHASE_OF_ERR_CLASS[announcement.errClass],
detail: announcement.detail,
retryInSec: Math.ceil(remainingMs / 1000),
retryInMs: remainingMs,
attempt: firstByteStalls,
maxRetries: firstByteRetryBudget,
errClass: announcement.errClass,
}));
continue;
}
let errorBody;
let overflowDeclineNote = "";
if (r !== undefined && r.status === 400) {
const read = await readErrorBody(r, signal);
errorBody = read;
const report = read.endedNormally ? parseContextOverflow(r.status, read.text) : undefined;
if (report !== undefined) {
const plan = planOutputCapAdjustment(report, [maxOutputTokensOverride, req.outputCapTokens]);
if (plan.outcome === "adjust" && attempt < maxRetries) {
let rebuilt;
try {
rebuilt = buildRequest({ maxOutputTokens: plan.maxOutputTokens });
}
catch {
rebuilt = undefined;
}
let acceptable = false;
try {
acceptable = rebuilt !== undefined && sameRequestModuloCap(req, rebuilt, plan.maxOutputTokens);
}
catch {
acceptable = false;
}
if (acceptable && rebuilt !== undefined) {
const previousCap = plan.current;
maxOutputTokensOverride = plan.maxOutputTokens;
req = rebuilt;
commit(req);
emitBrainTelemetry({ kind: "retry", attempt: attempt + 1, phase: "connect", errClass: "output_cap", nextDelayMs: 0 });
announcedRetry = true;
emitBrainStatus({
phase: RETRY_PHASE_OF_ERR_CLASS.output_cap,
detail: `output limit lowered from ${previousCap} to ${plan.maxOutputTokens} tokens, retrying`,
attempt: attempt + 1,
maxRetries,
errClass: "output_cap",
...(namesTheFailure(r?.status) ? { errorStatus: r.status } : {}),
});
attempt++;
continue;
}
overflowDeclineNote =
rebuilt === undefined
? ` — output-cap recovery declined: the request could not be rebuilt with a smaller cap`
: ` — output-cap recovery declined: rebuilding the request changed more than the output cap (its destination, headers, or some other body field moved), so this would no longer be the same request with a smaller cap`;
}
else {
overflowDeclineNote =
plan.outcome === "below_floor"
? ` — output-cap recovery declined: only ${plan.available} tokens are available under the context limit, below the ${FLOOR_OUTPUT_TOKENS}-token floor; the input itself must shrink`
: plan.outcome === "no_progress"
? ` — output-cap recovery declined: the recomputed cap (${plan.available}) is not below the ${plan.current} already in effect, so re-sending cannot help`
: ` — output-cap recovery declined: the retry budget (${maxRetries}) is exhausted`;
}
}
}
if (r !== undefined && errorBody === undefined && namesTheFailure(r.status) && !isTransientStatus(r.status))
errorBody = await readErrorBody(r, signal);
const judge = () => r !== undefined && namesTheFailure(r.status) ? judgeHttpFailure({ dialect, status: r.status, headers: r.headers, body: errorBody }) : undefined;
const verdict = judge();
const retryableByCondition = netErr !== undefined || (verdict !== undefined && PROVIDER_CONDITION_TRAITS[verdict.condition].retryable);
const retryable = shouldRetryHeaderVerdict(r) ?? retryableByCondition;
const connectReading = netErr !== undefined ? readConnectFailure(netErr) : undefined;
const deterministicConnect = connectReading?.verdict === "deterministic";
const fastLane = deterministicConnect && deterministicLaneActive;
const laneMaxRetries = fastLane ? deterministicConnectBudget : maxRetries;
const errClass = netErr !== undefined
? deterministicConnect
? "connect_refused"
: "transport"
: r?.status === 429
? "rate_limit"
: r !== undefined && (r.status >= 500 || r.status === 408 || r.status === 409)
? "server"
: "http";
if (retryable && attempt < laneMaxRetries) {
const delayMs = fastLane
? retryBackoffMs(baseDelay, attempt, r, Math.random, FAST_MAX_BACKOFF_MS)
: retryBackoffMs(baseDelay, attempt, r);
const waitHint = providerWaitHint(r);
const truncationNote = waitHint?.rawMs !== undefined
? ` (the requested wait was longer than the cap; honoring ${Math.ceil(waitHint.ms / 1000)}s instead of ${Math.ceil(waitHint.rawMs / 1000)}s)`
: "";
const refusedSubject = connectReading?.proxyRefusal !== undefined ? "proxy refused the tunnel" : "no endpoint answering";
const reconnectDetail = errClass !== "connect_refused" ? TRANSPORT_RETRY_DETAIL : fastLane ? `${refusedSubject}, retrying briefly` : `${refusedSubject}, retrying`;
emitBrainTelemetry({ kind: "retry", attempt: attempt + 1, phase: "connect", errClass, nextDelayMs: delayMs });
announcedRetry = true;
discardResponseBody(r);
cc.abort();
cc.dispose();
const retryStatus = namesTheFailure(r?.status) ? r.status : undefined;
const phase = RETRY_PHASE_OF_ERR_CLASS[errClass];
await sleepAnnouncingRetry(delayMs, signal, (remainingMs) => ({
phase,
detail: phase === "rate_limited"
? `rate limited, backing off${truncationNote}`
: phase === "reconnecting"
? reconnectDetail
: `transient error, retrying${truncationNote}`,
retryInSec: Math.ceil(remainingMs / 1000),
retryInMs: remainingMs,
attempt: attempt + 1,
maxRetries: laneMaxRetries,
errClass,
...(retryStatus !== undefined ? { errorStatus: retryStatus } : {}),
...(verdict !== undefined ? { providerCondition: verdict.condition, providerConditionSource: verdict.source } : {}),
}));
attempt++;
continue;
}
if (netErr) {
const chain = describeNetworkError(netErr);
const proxyRefusal = connectReading?.proxyRefusal;
throw new BrainError("network", proxyRefusal === undefined ? chain : `${proxyTunnelRefusedSentence(proxyRefusal.status)} — ${chain}`, undefined, { apiError: true });
}
if (r !== undefined && errorBody === undefined)
errorBody = await readErrorBody(r, signal);
const terminalVerdict = judge();
const detail = errorBody?.text ?? "";
const status = r?.status ?? 0;
const itl = terminalVerdict?.inputTooLong ?? "no";
const providerCode = refusedTheRequest(status) ? readProviderErrorCode(errorBody) : undefined;
throw new BrainError(classifyHttp(status), `${httpLabel} HTTP ${status || "ERR"}: ${detail.slice(0, 500)}${overflowDeclineNote}`, status, {
apiError: true,
...(itl === "yes" ? { inputTooLong: true } : itl === "no" ? { inputTooLongRuledOut: true } : {}),
...(providerCode !== undefined ? { providerCode } : {}),
...(terminalVerdict !== undefined ? { verdict: { condition: terminalVerdict.condition, source: terminalVerdict.source, ...(terminalVerdict.resetAtMs !== undefined ? { resetAtMs: terminalVerdict.resetAtMs } : {}) } } : {}),
...(errorBody !== undefined && !errorBody.endedNormally ? { errorBodyIncomplete: true } : {}),
});
}
finally {
cc.abort();
cc.dispose();
}
}
const partial = emptyAssistant(model);
attemptPartial = partial;
if (!startEmitted) {
startEmitted = true;
out.push({ type: "start", partial: { ...partial } });
}
const reader = res.body.getReader();
const decoder = new TextDecoder();
let buffer = "";
let firstTokenSeen = false;
let firedFirstTokenDeadlineMs;
let ftTimer;
let firstTokenWait = { stop: () => undefined, expire: () => undefined };
if (firstTokenTimeoutMs && firstTokenTimeoutMs > 0) {
firstTokenWait = armWaitDisclosure(firstTokenTimeoutMs);
ftTimer = setTimeout(() => {
firedFirstTokenDeadlineMs = firstTokenTimeoutMs;
firstTokenWait.expire();
void reader.cancel().catch(() => undefined);
}, firstTokenTimeoutMs);
}
let firedIdleDeadlineMs;
let idleTimer;
const bumpIdle = () => {
if (!idleTimeoutMs || idleTimeoutMs <= 0)
return;
firedIdleDeadlineMs = undefined;
if (idleTimer)
clearTimeout(idleTimer);
idleTimer = setTimeout(() => {
firedIdleDeadlineMs = idleTimeoutMs;
void reader.cancel().catch(() => undefined);
}, idleTimeoutMs);
};
const sawContentToken = () => {
bumpIdle();
if (firstTokenSeen)
return;
firstTokenSeen = true;
firstTokenArrived = true;
firedFirstTokenDeadlineMs = undefined;
if (ftTimer) {
clearTimeout(ftTimer);
ftTimer = undefined;
}
firstTokenWait.stop();
};
cleanup = () => {
if (ftTimer) {
clearTimeout(ftTimer);
ftTimer = undefined;
}
firstTokenWait.stop();
if (idleTimer) {
clearTimeout(idleTimer);
idleTimer = undefined;
}
connect?.dispose();
void reader.cancel().catch(() => undefined);
};
let midstreamWarning;
const parserOut = Object.create(out);
parserOut.push = (ev) => {
if (midstreamWarning !== undefined && (ev.type === "done" || ev.type === "error")) {
const msg = ev.type === "done" ? ev.message : ev.error;
msg.partialFinalized = true;
msg.errorMessage = msg.errorMessage ? `${msg.errorMessage}\n${midstreamWarning}` : midstreamWarning;
}
out.push(ev);
};
const parser = makeParser({
out: parserOut,
partial,
sawContentToken,
cancel: () => void reader.cancel().catch(() => undefined),
dialect,
keepOutput: (output) => {
attemptOutput = output;
},
});
const INBAND_OVERLOAD_WARNING = "API overloaded mid-response. The response above may be incomplete.";
const onLineOverloadTierA = (line) => {
try {
parser.onLine(line);
return false;
}
catch (e) {
if (brainErrorCodeOf(e) !== "server" || !parser.snapshot().hasSubstantiveText)
throw e;
return true;
}
};
let failure;
readLoop: for (;;) {
let readRes;
try {
readRes = await reader.read();
}
catch (e) {
if (isAbortError(e) || signal?.aborted)
throw e;
failure = {
kind: "connection",
err: new BrainError("network", `mid-stream read failed: ${describeNetworkError(e)}`, undefined, { apiError: true }),
};
break readLoop;
}
const { done, value } = readRes;
if (done)
break;
buffer += decoder.decode(value, { stream: true });
let nl;
let start = 0;
while ((nl = buffer.indexOf("\n", start)) !== -1) {
const line = buffer.slice(start, nl);
start = nl + 1;
if (onLineOverloadTierA(line)) {
midstreamWarning = INBAND_OVERLOAD_WARNING;
parser.finalize({ truncated: true });
return;
}
}
if (start > 0)
buffer = buffer.slice(start);
}
if (ftTimer) {
clearTimeout(ftTimer);
ftTimer = undefined;
}
firstTokenWait.stop();
if (idleTimer) {
clearTimeout(idleTimer);
idleTimer = undefined;
}
if (!failure && firedFirstTokenDeadlineMs !== undefined) {
failure = {
kind: "stalled",
watchdog: "first-token",
knob: "firstTokenTimeoutMs",
timeoutMs: firedFirstTokenDeadlineMs,
err: new BrainError("network", `first-token timeout after ${firedFirstTokenDeadlineMs}ms`, undefined, { apiError: true }),
};
}
if (!failure && firedIdleDeadlineMs !== undefined) {
failure = {
kind: "stalled",
watchdog: "stream idle",
knob: "idleTimeoutMs",
timeoutMs: firedIdleDeadlineMs,
err: new BrainError("network", `stream idle timeout after ${firedIdleDeadlineMs}ms`, undefined, { apiError: true }),
};
}
if (!failure) {
buffer += decoder.decode();
if (buffer.trim() && onLineOverloadTierA(buffer))
midstreamWarning = INBAND_OVERLOAD_WARNING;
parser.finalize(midstreamWarning !== undefined ? { truncated: true } : undefined);
return;
}
const snap = parser.snapshot();
if (snap.terminalFrameSeen) {
parser.finalize();
return;
}
if (snap.hasSubstantiveText || snap.hasCompletedToolCall) {
midstreamWarning =
failure.kind === "stalled"
? "Response stalled mid-stream. The response above may be incomplete."
: STREAM_TORN_WARNING;
parser.finalize({ truncated: true });
return;
}
if (snap.hasCompletedToolCall)
throw failure.err;
cleanup?.();
cleanup = undefined;
const announcement = retryAnnouncement(failure);
if (snap.hasOnlyThinking) {
if (thinkingRetries >= THINKING_RETRY_BUDGET)
throw failure.err;
thinkingRetries++;
parser.sealForRetry();
const delayMs = 100 * thinkingRetries;
emitBrainTelemetry({ kind: "retry", attempt: thinkingRetries, phase: "midstream", errClass: announcement.errClass, nextDelayMs: delayMs });
announcedRetry = true;
await sleepAnnouncingRetry(delayMs, signal, (remainingMs) => ({
phase: RETRY_PHASE_OF_ERR_CLASS[announcement.errClass],
detail: announcement.detail,
retryInSec: Math.ceil(remainingMs / 1000),
retryInMs: remainingMs,
attempt: thinkingRetries,
maxRetries: THINKING_RETRY_BUDGET,
errClass: announcement.errClass,
}));
continue;
}
if (attempt >= maxRetries)
throw failure.err;
const delayMs = retryBackoffMs(baseDelay, attempt);
attempt++;
emitBrainTelemetry({ kind: "retry", attempt, phase: "midstream", errClass: announcement.errClass, nextDelayMs: delayMs });
announcedRetry = true;
await sleepAnnouncingRetry(delayMs, signal, (remainingMs) => ({
phase: RETRY_PHASE_OF_ERR_CLASS[announcement.errClass],
detail: announcement.detail,
retryInSec: Math.ceil(remainingMs / 1000),
retryInMs: remainingMs,
attempt,
maxRetries,
errClass: announcement.errClass,
}));
}
}
}