@sema-agent/core
Version:
Stateless, task-oriented AI agent core
810 lines (809 loc) • 37.3 kB
JavaScript
import { createAssistantMessageEventStream, } from "../internal/llm.js";
import { STREAM_TORN_WARNING } from "./stream-shared.js";
import { FLOOR_OUTPUT_TOKENS, parseContextOverflow, planOutputCapAdjustment } from "./context-overflow.js";
import { abortLikeName, BrainError, brainErrorCodeOf, classifyConnectFailure, classifyFetchThrow, classifyHttp, describeNetworkError, isProviderEchoShape, namesTheFailure, readProviderErrorCode, readProviderRequestId, refusedTheRequest } from "./errors.js";
import { stripUrlCredentials } from "./route-adjudicator.js";
import { classifyInputTooLong } from "./input-too-long.js";
import { FAST_MAX_BACKOFF_MS, providerWaitHint, retryBackoffMs } from "./retry.js";
import { bindBrainStatusEmitter, emitBrainStatus, emitBrainTelemetry } from "./status-sink.js";
import { createAttemptController, firstByteWindow, refuseRetiredTimeoutKnobs, resolveFirstByteRetries, resolveWatchdogMs, STALL_FIRST_BYTE_MS, STALL_FIRST_TOKEN_MS, STALL_IDLE_MS, } from "./timeout.js";
import { readMember, thrownText } from "../core/thrown-value.js";
import { redactSecrets } from "../core/untrusted-egress.js";
const DEFAULT_MAX_RETRIES = 10;
const MAX_RETRIES_ENV_CEILING = 15;
const MAX_RETRIES_ENV = "SEMA_MAX_RETRIES";
export function resolveMaxRetries(configured) {
if (configured !== undefined && Number.isFinite(configured))
return configured;
const raw = process.env[MAX_RETRIES_ENV];
if (raw !== undefined && raw.trim() !== "") {
const parsed = Number(raw);
if (Number.isFinite(parsed) && parsed >= 0)
return Math.min(Math.floor(parsed), MAX_RETRIES_ENV_CEILING);
}
return DEFAULT_MAX_RETRIES;
}
const DEFAULT_DETERMINISTIC_CONNECT_RETRIES = 2;
export function resolveDeterministicConnectRetries(configured) {
if (configured !== undefined && Number.isFinite(configured) && configured >= 0)
return Math.floor(configured);
return DEFAULT_DETERMINISTIC_CONNECT_RETRIES;
}
const SHOULD_RETRY_HEADER = "x-should-retry";
function sameHeaders(a, b) {
const aKeys = Object.keys(a);
if (aKeys.length !== Object.keys(b).length)
return false;
return aKeys.every((k) => Object.prototype.hasOwnProperty.call(b, k) && a[k] === b[k]);
}
function canonicalJson(value) {
if (Array.isArray(value))
return `[${value.map(canonicalJson).join(",")}]`;
if (typeof value === "object" && value !== null) {
const rec = value;
return `{${Object.keys(rec)
.sort()
.map((k) => `${JSON.stringify(k)}:${canonicalJson(rec[k])}`)
.join(",")}}`;
}
return JSON.stringify(value) ?? "null";
}
function leafHolder(root, path) {
const parts = path.split(".");
const leaf = parts.pop();
if (leaf === undefined)
return undefined;
let cur = root;
for (const part of parts) {
if (typeof cur !== "object" || cur === null || Array.isArray(cur))
return undefined;
cur = cur[part];
}
if (typeof cur !== "object" || cur === null || Array.isArray(cur))
return undefined;
return { holder: cur, leaf };
}
export function sameRequestModuloCap(original, rebuilt, imposedCap) {
if (rebuilt.url !== original.url || !sameHeaders(rebuilt.headers, original.headers))
return false;
if (rebuilt.outputCapTokens !== imposedCap)
return false;
if (rebuilt.body === original.body)
return true;
const parse = (raw) => {
let parsed;
try {
parsed = JSON.parse(raw);
}
catch {
return undefined;
}
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed))
return undefined;
return parsed;
};
const a = parse(original.body);
const b = parse(rebuilt.body);
if (a === undefined || b === undefined)
return false;
for (const path of new Set([...(original.capDerivedPaths ?? []), ...(rebuilt.capDerivedPaths ?? [])])) {
const from = leafHolder(a, path);
const to = leafHolder(b, path);
const before = from === undefined ? undefined : from.holder[from.leaf];
const after = to === undefined ? undefined : to.holder[to.leaf];
if (before !== undefined) {
if (typeof before !== "number" || typeof after !== "number")
return false;
if (after > before)
return false;
}
if (from !== undefined)
delete from.holder[from.leaf];
if (to !== undefined)
delete to.holder[to.leaf];
}
return canonicalJson(a) === canonicalJson(b);
}
const ERROR_BODY_BYTE_CAP = 64 * 1024;
const ERROR_BODY_READ_TIMEOUT_MS = 5_000;
async function readErrorBody(res, signal) {
const body = res.body;
if (body === null)
return { text: "", endedNormally: true };
let reader;
try {
reader = body.getReader();
}
catch {
return { text: "", endedNormally: false };
}
const decoder = new TextDecoder();
let stopped = false;
const stop = () => {
stopped = true;
void reader.cancel().catch(() => undefined);
};
let timer;
let out = "";
let endedNormally = false;
try {
timer = setTimeout(stop, ERROR_BODY_READ_TIMEOUT_MS);
signal?.addEventListener("abort", stop, { once: true });
if (signal?.aborted === true)
stop();
let bytes = 0;
for (;;) {
const chunk = await reader.read();
if (chunk.done) {
endedNormally = !stopped;
break;
}
const value = chunk.value;
if (value === undefined)
continue;
const room = ERROR_BODY_BYTE_CAP - bytes;
const usable = value.byteLength <= room ? value : value.subarray(0, room);
bytes += usable.byteLength;
out += decoder.decode(usable, { stream: true });
if (bytes >= ERROR_BODY_BYTE_CAP) {
stop();
break;
}
}
out += decoder.decode();
}
catch {
endedNormally = false;
}
finally {
if (timer !== undefined)
clearTimeout(timer);
signal?.removeEventListener("abort", stop);
stop();
}
return { text: out, endedNormally };
}
function discardResponseBody(res) {
const body = res?.body;
if (body === undefined || body === null || body.locked)
return;
try {
void body.cancel().catch(() => undefined);
}
catch {
}
}
export function shouldRetryHeaderVerdict(res) {
if (res === undefined)
return undefined;
let raw = null;
try {
const headers = res.headers;
if (headers === undefined || headers === null || typeof headers.get !== "function")
return undefined;
raw = res.headers.get(SHOULD_RETRY_HEADER);
}
catch {
return undefined;
}
if (raw === "true")
return true;
if (raw === "false")
return false;
return undefined;
}
const RETRY_STATUS_SLICE_MS = 30_000;
export const WAITING_FIRST_TOKEN_AFTER_MS = 30_000;
export const WAITING_FIRST_TOKEN_EVERY_MS = 30_000;
const THINKING_RETRY_BUDGET = 2;
const EMPTY_USAGE = {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};
function emptyAssistant(model) {
return {
role: "assistant",
content: [],
api: model.api,
provider: model.provider,
model: model.id,
stopReason: "stop",
usage: { ...EMPTY_USAGE, cost: { ...EMPTY_USAGE.cost } },
timestamp: Date.now(),
};
}
function isAbortError(err) {
return abortLikeName(err) !== undefined;
}
function sleep(ms, signal) {
return new Promise((resolve) => {
if (signal?.aborted)
return resolve();
let t;
const onAbort = () => {
clearTimeout(t);
resolve();
};
t = setTimeout(() => {
signal?.removeEventListener("abort", onAbort);
resolve();
}, ms);
signal?.addEventListener("abort", onAbort, { once: true });
});
}
async function sleepAnnouncingRetry(totalMs, signal, frame) {
let remaining = Math.max(0, totalMs);
for (;;) {
if (signal?.aborted)
return;
emitBrainStatus(frame(remaining));
if (remaining <= 0)
return;
const slice = Math.min(remaining, RETRY_STATUS_SLICE_MS);
await sleep(slice, signal);
remaining -= slice;
if (remaining <= 0)
return;
}
}
function retryAnnouncement(failure) {
if (failure.kind !== "stalled")
return { errClass: "transport", detail: "connection lost, reconnecting" };
const knob = failure.allowanceMs !== undefined && failure.allowanceMs > 0
? `${failure.knob} ${failure.timeoutMs - failure.allowanceMs} + ${failure.allowanceMs} upload allowance`
: failure.knob;
return { errClass: "stall", detail: `${failure.watchdog} watchdog fired after ${failure.timeoutMs}ms (${knob}), retrying` };
}
function firstByteTimeoutMessage(windowsMs) {
const [first, ...retries] = windowsMs;
const waited = retries.length === 0 ? `within ${first ?? 0}ms` : `within ${first ?? 0}ms, then ${retries.map((w) => `${w}ms`).join(", then ")} on the retry`;
return `first-byte timeout: no response headers ${waited} (firstByteTimeoutMs) — if a proxy or gateway holds responses until they complete, raise firstByteTimeoutMs to wait longer`;
}
export function runStreamingBrain(args) {
const { model, doFetch, signal, config, httpLabel, buildRequest, makeParser, stallTimeouts } = args;
const callRetryCap = args.maxRetries !== undefined && Number.isFinite(args.maxRetries) && args.maxRetries >= 0 ? Math.floor(args.maxRetries) : Infinity;
const out = createAssistantMessageEventStream();
let cleanup;
let announcedRetry = false;
let announcedWait = false;
let terminalWaitDetail = "first token arrived";
let firstTokenArrived = false;
let terminalRetryPhase = "recovered";
let terminalRetryDetail = "recovered after retrying";
let requestIdSeen;
let attemptPartial;
void run()
.catch((err) => {
const aborted = signal?.aborted === true || isAbortError(err);
terminalRetryPhase = "gave_up";
terminalRetryDetail = aborted ? "cancelled while retrying" : "retries exhausted";
terminalWaitDetail = firstTokenArrived
? aborted
? "cancelled after the first token arrived"
: "the call failed after the first token arrived"
: aborted
? "cancelled while waiting for the first token"
: "the first-token wait ended in failure";
const errorMsg = emptyAssistant(model);
errorMsg.stopReason = aborted ? "aborted" : "error";
errorMsg.errorMessage = thrownText(err);
if (!aborted && brainErrorCodeOf(err) !== undefined) {
if (readMember(err, "inputTooLong") === true)
errorMsg.errorKind = "input_too_long";
else if (readMember(err, "inputTooLongRuledOut") === true)
errorMsg.inputTooLongRuledOut = true;
if (readMember(err, "apiError") === true) {
errorMsg.isApiErrorMessage = true;
const status = readMember(err, "status");
const statusNum = typeof status === "number" ? status : undefined;
if (namesTheFailure(statusNum))
errorMsg.apiErrorStatus = statusNum;
if (requestIdSeen !== undefined)
errorMsg.requestId = requestIdSeen;
const providerCode = readMember(err, "providerCode");
if (refusedTheRequest(statusNum) && typeof providerCode === "string" && isProviderEchoShape(providerCode))
errorMsg.apiErrorCode = providerCode;
}
}
errorMsg.usageMissing = true;
const seenResponseModel = attemptPartial?.responseModel;
if (seenResponseModel !== undefined)
errorMsg.responseModel = seenResponseModel;
out.push({ type: "error", reason: aborted ? "aborted" : "error", error: errorMsg });
})
.finally(() => {
cleanup?.();
if (!announcedRetry && !announcedWait)
return;
try {
emitBrainStatus({ phase: terminalRetryPhase, detail: announcedRetry ? terminalRetryDetail : terminalWaitDetail });
}
catch {
}
});
return out;
async function run() {
let req = buildRequest();
req.onCommitted?.();
let maxOutputTokensOverride;
const maxRetries = Math.min(resolveMaxRetries(config.maxRetries), callRetryCap);
const baseDelay = config.retryDelayMs ?? 500;
const deterministicConnectBudget = Math.min(resolveDeterministicConnectRetries(config.deterministicConnectRetries), maxRetries);
const deterministicLaneActive = deterministicConnectBudget < maxRetries;
refuseRetiredTimeoutKnobs(stallTimeouts, "advisory", "StreamOptions.stallTimeouts");
const firstByteTimeoutMs = resolveWatchdogMs(config.firstByteTimeoutMs, stallTimeouts?.firstByteMs, STALL_FIRST_BYTE_MS, "firstByteTimeoutMs");
const firstTokenTimeoutMs = resolveWatchdogMs(config.firstTokenTimeoutMs, stallTimeouts?.firstTokenMs, STALL_FIRST_TOKEN_MS, "firstTokenTimeoutMs");
const idleTimeoutMs = resolveWatchdogMs(config.idleTimeoutMs, stallTimeouts?.idleMs, STALL_IDLE_MS, "idleTimeoutMs");
const firstByteRetryBudget = resolveFirstByteRetries(config.firstByteRetries);
const armWaitDisclosure = (ceilingMs) => {
if (!(ceilingMs >= WAITING_FIRST_TOKEN_AFTER_MS))
return { stop: () => undefined, expire: () => undefined };
const startedAt = Date.now();
const emitWaitFrame = bindBrainStatusEmitter();
let timer;
let stopped = false;
let disclosed = false;
const disclose = () => {
disclosed = true;
announcedWait = true;
emitWaitFrame({ phase: "waiting_first_token", detail: "waiting for the first token", elapsedMs: Date.now() - startedAt, timeoutMs: ceilingMs });
};
const schedule = (delayMs) => {
timer = setTimeout(() => {
timer = undefined;
if (stopped || Date.now() - startedAt >= ceilingMs)
return;
disclose();
schedule(WAITING_FIRST_TOKEN_EVERY_MS);
}, delayMs);
};
schedule(WAITING_FIRST_TOKEN_AFTER_MS);
const stop = () => {
stopped = true;
if (timer)
clearTimeout(timer);
timer = undefined;
};
return {
stop,
expire: () => {
if (!stopped && !disclosed)
disclose();
stop();
},
};
};
let attempt = 0;
let thinkingRetries = 0;
let firstByteStalls = 0;
const firstByteWindowsWaited = [];
let startEmitted = false;
for (;;) {
let res;
let connect;
for (;;) {
if (signal?.aborted) {
const e = new Error("aborted");
e.name = "AbortError";
throw e;
}
const window = firstByteWindow(firstByteTimeoutMs, Buffer.byteLength(req.body), firstByteStalls);
const headerWait = armWaitDisclosure(window.windowMs);
const cc = createAttemptController(window.windowMs, signal);
let r;
let netErr;
let firstByteStall;
requestIdSeen = undefined;
attemptPartial = undefined;
const stalledAtFirstByte = () => {
firstByteWindowsWaited.push(window.windowMs);
return {
kind: "stalled",
watchdog: "first-byte",
knob: "firstByteTimeoutMs",
timeoutMs: window.windowMs,
allowanceMs: window.allowanceMs,
err: new BrainError("network", firstByteTimeoutMessage(firstByteWindowsWaited), undefined, { apiError: true }),
};
};
try {
r = await doFetch(req.url, { method: "POST", headers: req.headers, body: req.body, signal: cc.signal });
requestIdSeen = readProviderRequestId(r);
if (cc.timedOut()) {
discardResponseBody(r);
r = undefined;
requestIdSeen = undefined;
firstByteStall = stalledAtFirstByte();
}
}
catch (e) {
if (cc.timedOut()) {
firstByteStall = stalledAtFirstByte();
}
else if (isAbortError(e) || signal?.aborted) {
cc.dispose();
throw e;
}
else if (classifyFetchThrow(e) === "request_unbuildable") {
cc.dispose();
throw new BrainError("invalid_request", `${httpLabel} request could not be built: ${redactSecrets(stripUrlCredentials(thrownText(e)))}`);
}
else {
netErr = e;
}
}
finally {
if (firstByteStall !== undefined)
headerWait.expire();
else
headerWait.stop();
}
if (r && r.ok && r.body) {
cc.clearTimer();
connect = cc;
cleanup = () => connect?.dispose();
res = r;
break;
}
cc.clearTimer();
try {
if (firstByteStall !== undefined) {
if (firstByteStalls >= firstByteRetryBudget)
throw firstByteStall.err;
firstByteStalls++;
const announcement = retryAnnouncement(firstByteStall);
const delayMs = retryBackoffMs(baseDelay, firstByteStalls - 1);
emitBrainTelemetry({ kind: "retry", attempt: firstByteStalls, phase: "connect", errClass: announcement.errClass, nextDelayMs: delayMs });
announcedRetry = true;
cc.abort();
cc.dispose();
await sleepAnnouncingRetry(delayMs, signal, (remainingMs) => ({
phase: "reconnecting",
detail: announcement.detail,
retryInSec: Math.ceil(remainingMs / 1000),
retryInMs: remainingMs,
attempt: firstByteStalls,
maxRetries: firstByteRetryBudget,
errClass: announcement.errClass,
}));
continue;
}
let overflowBody;
let overflowDeclineNote = "";
if (r !== undefined && r.status === 400) {
const read = await readErrorBody(r, signal);
overflowBody = read.text;
const report = read.endedNormally ? parseContextOverflow(r.status, read.text) : undefined;
if (report !== undefined) {
const plan = planOutputCapAdjustment(report, [maxOutputTokensOverride, req.outputCapTokens]);
if (plan.outcome === "adjust" && attempt < maxRetries) {
let rebuilt;
try {
rebuilt = buildRequest({ maxOutputTokens: plan.maxOutputTokens });
}
catch {
rebuilt = undefined;
}
let acceptable = false;
try {
acceptable = rebuilt !== undefined && sameRequestModuloCap(req, rebuilt, plan.maxOutputTokens);
}
catch {
acceptable = false;
}
if (acceptable && rebuilt !== undefined) {
const previousCap = plan.current;
maxOutputTokensOverride = plan.maxOutputTokens;
req = rebuilt;
req.onCommitted?.();
emitBrainTelemetry({ kind: "retry", attempt: attempt + 1, phase: "connect", errClass: "output_cap", nextDelayMs: 0 });
announcedRetry = true;
emitBrainStatus({
phase: "retrying",
detail: `output limit lowered from ${previousCap} to ${plan.maxOutputTokens} tokens, retrying`,
attempt: attempt + 1,
maxRetries,
errClass: "output_cap",
...(namesTheFailure(r?.status) ? { errorStatus: r.status } : {}),
});
attempt++;
continue;
}
overflowDeclineNote =
rebuilt === undefined
? ` — output-cap recovery declined: the request could not be rebuilt with a smaller cap`
: ` — output-cap recovery declined: rebuilding the request changed more than the output cap (its destination, headers, or some other body field moved), so this would no longer be the same request with a smaller cap`;
}
else {
overflowDeclineNote =
plan.outcome === "below_floor"
? ` — output-cap recovery declined: only ${plan.available} tokens are available under the context limit, below the ${FLOOR_OUTPUT_TOKENS}-token floor; the input itself must shrink`
: plan.outcome === "no_progress"
? ` — output-cap recovery declined: the recomputed cap (${plan.available}) is not below the ${plan.current} already in effect, so re-sending cannot help`
: ` — output-cap recovery declined: the retry budget (${maxRetries}) is exhausted`;
}
}
}
const retryableByStatus = netErr !== undefined ||
(r !== undefined && (r.status >= 500 || r.status === 429 || r.status === 408 || r.status === 409));
const retryable = shouldRetryHeaderVerdict(r) ?? retryableByStatus;
const deterministicConnect = netErr !== undefined && classifyConnectFailure(netErr) === "deterministic";
const fastLane = deterministicConnect && deterministicLaneActive;
const laneMaxRetries = fastLane ? deterministicConnectBudget : maxRetries;
const errClass = netErr !== undefined
? deterministicConnect
? "connect_refused"
: "transport"
: r?.status === 429
? "rate_limit"
: r !== undefined && (r.status >= 500 || r.status === 408 || r.status === 409)
? "server"
: "http";
if (retryable && attempt < laneMaxRetries) {
const delayMs = fastLane
? retryBackoffMs(baseDelay, attempt, r, Math.random, FAST_MAX_BACKOFF_MS)
: retryBackoffMs(baseDelay, attempt, r);
const waitHint = providerWaitHint(r);
const truncationNote = waitHint?.rawMs !== undefined
? ` (the requested wait was longer than the cap; honoring ${Math.ceil(waitHint.ms / 1000)}s instead of ${Math.ceil(waitHint.rawMs / 1000)}s)`
: "";
const statusPhase = r?.status === 429 ? "rate_limited" : netErr !== undefined ? "reconnecting" : "retrying";
const reconnectDetail = errClass !== "connect_refused"
? "connection lost, reconnecting"
: fastLane
? "no endpoint answering, retrying briefly"
: "no endpoint answering, retrying";
emitBrainTelemetry({ kind: "retry", attempt: attempt + 1, phase: "connect", errClass, nextDelayMs: delayMs });
announcedRetry = true;
discardResponseBody(r);
cc.abort();
cc.dispose();
const retryStatus = namesTheFailure(r?.status) ? r.status : undefined;
await sleepAnnouncingRetry(delayMs, signal, (remainingMs) => ({
phase: statusPhase,
detail: statusPhase === "rate_limited"
? `rate limited, backing off${truncationNote}`
: statusPhase === "reconnecting"
? reconnectDetail
: `transient error, retrying${truncationNote}`,
retryInSec: Math.ceil(remainingMs / 1000),
retryInMs: remainingMs,
attempt: attempt + 1,
maxRetries: laneMaxRetries,
errClass,
...(retryStatus !== undefined ? { errorStatus: retryStatus } : {}),
}));
attempt++;
continue;
}
if (netErr)
throw new BrainError("network", describeNetworkError(netErr), undefined, { apiError: true });
const detail = overflowBody ?? (r ? (await readErrorBody(r, signal)).text : "");
const status = r?.status ?? 0;
const itl = classifyInputTooLong(status, detail);
const providerCode = refusedTheRequest(status) ? readProviderErrorCode(detail) : undefined;
throw new BrainError(classifyHttp(status), `${httpLabel} HTTP ${status || "ERR"}: ${detail.slice(0, 500)}${overflowDeclineNote}`, status, {
apiError: true,
...(itl === "yes" ? { inputTooLong: true } : itl === "no" ? { inputTooLongRuledOut: true } : {}),
...(providerCode !== undefined ? { providerCode } : {}),
});
}
finally {
cc.abort();
cc.dispose();
}
}
const partial = emptyAssistant(model);
attemptPartial = partial;
if (!startEmitted) {
startEmitted = true;
out.push({ type: "start", partial: { ...partial } });
}
const reader = res.body.getReader();
const decoder = new TextDecoder();
let buffer = "";
let firstTokenSeen = false;
let firedFirstTokenDeadlineMs;
let ftTimer;
let firstTokenWait = { stop: () => undefined, expire: () => undefined };
if (firstTokenTimeoutMs && firstTokenTimeoutMs > 0) {
firstTokenWait = armWaitDisclosure(firstTokenTimeoutMs);
ftTimer = setTimeout(() => {
firedFirstTokenDeadlineMs = firstTokenTimeoutMs;
firstTokenWait.expire();
void reader.cancel().catch(() => undefined);
}, firstTokenTimeoutMs);
}
let firedIdleDeadlineMs;
let idleTimer;
const bumpIdle = () => {
if (!idleTimeoutMs || idleTimeoutMs <= 0)
return;
firedIdleDeadlineMs = undefined;
if (idleTimer)
clearTimeout(idleTimer);
idleTimer = setTimeout(() => {
firedIdleDeadlineMs = idleTimeoutMs;
void reader.cancel().catch(() => undefined);
}, idleTimeoutMs);
};
const sawContentToken = () => {
bumpIdle();
if (firstTokenSeen)
return;
firstTokenSeen = true;
firstTokenArrived = true;
firedFirstTokenDeadlineMs = undefined;
if (ftTimer) {
clearTimeout(ftTimer);
ftTimer = undefined;
}
firstTokenWait.stop();
};
cleanup = () => {
if (ftTimer) {
clearTimeout(ftTimer);
ftTimer = undefined;
}
firstTokenWait.stop();
if (idleTimer) {
clearTimeout(idleTimer);
idleTimer = undefined;
}
connect?.dispose();
void reader.cancel().catch(() => undefined);
};
let midstreamWarning;
const parserOut = Object.create(out);
parserOut.push = (ev) => {
if (midstreamWarning !== undefined && (ev.type === "done" || ev.type === "error")) {
const msg = ev.type === "done" ? ev.message : ev.error;
msg.partialFinalized = true;
msg.errorMessage = msg.errorMessage ? `${msg.errorMessage}\n${midstreamWarning}` : midstreamWarning;
}
out.push(ev);
};
const parser = makeParser({
out: parserOut,
partial,
sawContentToken,
cancel: () => void reader.cancel().catch(() => undefined),
});
const INBAND_OVERLOAD_WARNING = "API overloaded mid-response. The response above may be incomplete.";
const onLineOverloadTierA = (line) => {
try {
parser.onLine(line);
return false;
}
catch (e) {
if (brainErrorCodeOf(e) !== "server" || !parser.snapshot().hasSubstantiveText)
throw e;
return true;
}
};
let failure;
readLoop: for (;;) {
let readRes;
try {
readRes = await reader.read();
}
catch (e) {
if (isAbortError(e) || signal?.aborted)
throw e;
failure = {
kind: "connection",
err: new BrainError("network", `mid-stream read failed: ${describeNetworkError(e)}`, undefined, { apiError: true }),
};
break readLoop;
}
const { done, value } = readRes;
if (done)
break;
buffer += decoder.decode(value, { stream: true });
let nl;
let start = 0;
while ((nl = buffer.indexOf("\n", start)) !== -1) {
const line = buffer.slice(start, nl);
start = nl + 1;
if (onLineOverloadTierA(line)) {
midstreamWarning = INBAND_OVERLOAD_WARNING;
parser.finalize({ truncated: true });
return;
}
}
if (start > 0)
buffer = buffer.slice(start);
}
if (ftTimer) {
clearTimeout(ftTimer);
ftTimer = undefined;
}
firstTokenWait.stop();
if (idleTimer) {
clearTimeout(idleTimer);
idleTimer = undefined;
}
if (!failure && firedFirstTokenDeadlineMs !== undefined) {
failure = {
kind: "stalled",
watchdog: "first-token",
knob: "firstTokenTimeoutMs",
timeoutMs: firedFirstTokenDeadlineMs,
err: new BrainError("network", `first-token timeout after ${firedFirstTokenDeadlineMs}ms`, undefined, { apiError: true }),
};
}
if (!failure && firedIdleDeadlineMs !== undefined) {
failure = {
kind: "stalled",
watchdog: "stream idle",
knob: "idleTimeoutMs",
timeoutMs: firedIdleDeadlineMs,
err: new BrainError("network", `stream idle timeout after ${firedIdleDeadlineMs}ms`, undefined, { apiError: true }),
};
}
if (!failure) {
buffer += decoder.decode();
if (buffer.trim() && onLineOverloadTierA(buffer))
midstreamWarning = INBAND_OVERLOAD_WARNING;
parser.finalize(midstreamWarning !== undefined ? { truncated: true } : undefined);
return;
}
const snap = parser.snapshot();
if (snap.terminalFrameSeen) {
parser.finalize();
return;
}
if (snap.hasSubstantiveText || snap.hasCompletedToolCall) {
midstreamWarning =
failure.kind === "stalled"
? "Response stalled mid-stream. The response above may be incomplete."
: STREAM_TORN_WARNING;
parser.finalize({ truncated: true });
return;
}
if (snap.hasCompletedToolCall)
throw failure.err;
cleanup?.();
cleanup = undefined;
const announcement = retryAnnouncement(failure);
if (snap.hasOnlyThinking) {
if (thinkingRetries >= THINKING_RETRY_BUDGET)
throw failure.err;
thinkingRetries++;
parser.sealForRetry();
const delayMs = 100 * thinkingRetries;
emitBrainTelemetry({ kind: "retry", attempt: thinkingRetries, phase: "midstream", errClass: announcement.errClass, nextDelayMs: delayMs });
announcedRetry = true;
await sleepAnnouncingRetry(delayMs, signal, (remainingMs) => ({
phase: "reconnecting",
detail: announcement.detail,
retryInSec: Math.ceil(remainingMs / 1000),
retryInMs: remainingMs,
attempt: thinkingRetries,
maxRetries: THINKING_RETRY_BUDGET,
errClass: announcement.errClass,
}));
continue;
}
if (attempt >= maxRetries)
throw failure.err;
const delayMs = retryBackoffMs(baseDelay, attempt);
attempt++;
emitBrainTelemetry({ kind: "retry", attempt, phase: "midstream", errClass: announcement.errClass, nextDelayMs: delayMs });
announcedRetry = true;
await sleepAnnouncingRetry(delayMs, signal, (remainingMs) => ({
phase: "reconnecting",
detail: announcement.detail,
retryInSec: Math.ceil(remainingMs / 1000),
retryInMs: remainingMs,
attempt,
maxRetries,
errClass: announcement.errClass,
}));
}
}
}