manifest
Version:
Self-hosted Manifest LLM router with embedded server, SQLite database, and dashboard
202 lines • 8.91 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.buildMetaHeaders = buildMetaHeaders;
exports.handleProviderError = handleProviderError;
exports.recordFallbackFailures = recordFallbackFailures;
exports.handleStreamResponse = handleStreamResponse;
exports.handleNonStreamResponse = handleNonStreamResponse;
exports.recordSuccess = recordSuccess;
const common_1 = require("@nestjs/common");
const stream_writer_1 = require("./stream-writer");
const proxy_error_sanitizer_1 = require("./proxy-error-sanitizer");
const logger = new common_1.Logger('ProxyResponseHandler');
function buildMetaHeaders(meta) {
const headers = {
'X-Manifest-Tier': meta.tier,
'X-Manifest-Model': meta.model,
'X-Manifest-Provider': meta.provider,
'X-Manifest-Confidence': String(meta.confidence),
'X-Manifest-Reason': meta.reason,
};
if (meta.fallbackFromModel) {
headers['X-Manifest-Fallback-From'] = meta.fallbackFromModel;
headers['X-Manifest-Fallback-Index'] = String(meta.fallbackIndex ?? 0);
}
return headers;
}
function setHeaders(res, headers) {
for (const [k, v] of Object.entries(headers))
res.setHeader(k, v);
}
async function handleProviderError(res, ctx, meta, metaHeaders, errorStatus, errorBody, failedFallbacks, recorder, traceId) {
if (failedFallbacks && failedFallbacks.length > 0 && !meta.fallbackFromModel) {
await handleFallbackExhausted(res, ctx, meta, metaHeaders, errorStatus, errorBody, failedFallbacks, recorder, traceId);
return;
}
recorder
.recordProviderError(ctx, errorStatus, errorBody, {
model: meta.model,
tier: meta.tier,
traceId,
fallbackFromModel: meta.fallbackFromModel,
fallbackIndex: meta.fallbackIndex,
authType: meta.auth_type,
})
.catch((e) => logger.warn(`Failed to record provider error: ${e}`));
logger.warn(`Upstream error ${errorStatus}: provider=${meta.provider} model=${meta.model} tier=${meta.tier} body=${errorBody.slice(0, 500)}`);
res.status(errorStatus);
setHeaders(res, metaHeaders);
res.json({
error: {
message: (0, proxy_error_sanitizer_1.sanitizeProviderError)(errorStatus, errorBody, process.env.NODE_ENV),
type: 'upstream_error',
status: errorStatus,
},
});
}
function handleFallbackExhausted(res, ctx, meta, metaHeaders, errorStatus, errorBody, failedFallbacks, recorder, traceId) {
const baseTime = Date.now();
recorder
.recordFailedFallbacks(ctx, meta.tier, meta.model, failedFallbacks, {
traceId,
baseTimeMs: baseTime,
markHandled: true,
lastAsError: true,
authType: meta.auth_type,
})
.catch((e) => logger.warn(`Failed to record fallback errors: ${e}`));
const primaryTs = new Date(baseTime + (failedFallbacks.length + 1) * 100).toISOString();
recorder
.recordPrimaryFailure(ctx, meta.tier, meta.model, errorBody, primaryTs, meta.auth_type)
.catch((e) => logger.warn(`Failed to record primary failure: ${e}`));
logger.warn(`Fallback chain exhausted: ${errorBody.slice(0, 200)}`);
res.status(errorStatus);
setHeaders(res, metaHeaders);
res.setHeader('X-Manifest-Fallback-Exhausted', 'true');
res.json({
error: {
message: (0, proxy_error_sanitizer_1.sanitizeProviderError)(errorStatus, errorBody, process.env.NODE_ENV),
type: 'fallback_exhausted',
status: errorStatus,
primary_model: meta.model,
primary_provider: meta.provider,
attempted_fallbacks: failedFallbacks.map((f) => ({
model: f.model,
provider: f.provider,
status: f.status,
})),
},
});
}
function recordFallbackFailures(ctx, meta, failedFallbacks, recorder) {
if (!meta.fallbackFromModel)
return undefined;
const fallbackBaseTime = Date.now();
const failures = failedFallbacks ?? [];
recorder
.recordPrimaryFailure(ctx, meta.tier, meta.fallbackFromModel, meta.primaryErrorBody ?? `Provider returned HTTP ${meta.primaryErrorStatus ?? 500}`, new Date(fallbackBaseTime).toISOString(), meta.auth_type)
.catch((e) => logger.warn(`Failed to record primary failure: ${e}`));
if (failures.length > 0) {
recorder
.recordFailedFallbacks(ctx, meta.tier, meta.fallbackFromModel, failures, {
baseTimeMs: fallbackBaseTime,
markHandled: true,
authType: meta.auth_type,
})
.catch((e) => logger.warn(`Failed to record fallback errors: ${e}`));
}
return new Date(fallbackBaseTime + (failures.length + 1) * 100).toISOString();
}
async function handleStreamResponse(res, forward, meta, metaHeaders, providerClient, signatureCache, sessionKey) {
(0, stream_writer_1.initSseHeaders)(res, metaHeaders);
if (forward.isGoogle) {
return (0, stream_writer_1.pipeStream)(forward.response.body, res, (chunk) => {
const result = providerClient.convertGoogleStreamChunk(chunk, meta.model);
if (signatureCache && sessionKey && result) {
const sigRe = /"thought_signature"\s*:\s*"([^"]+)"/g;
const idRe = /"id"\s*:\s*"([^"]+)"/g;
const ids = [...result.matchAll(idRe)].map((m) => m[1]);
const sigs = [...result.matchAll(sigRe)].map((m) => m[1]);
for (let i = 0; i < Math.min(ids.length, sigs.length); i++) {
signatureCache.store(sessionKey, ids[i], sigs[i]);
}
}
return result;
});
}
if (forward.isAnthropic) {
return (0, stream_writer_1.pipeStream)(forward.response.body, res, providerClient.createAnthropicStreamTransformer(meta.model));
}
if (forward.isChatGpt) {
return (0, stream_writer_1.pipeStream)(forward.response.body, res, (chunk) => providerClient.convertChatGptStreamChunk(chunk, meta.model));
}
return (0, stream_writer_1.pipeStream)(forward.response.body, res);
}
async function handleNonStreamResponse(res, forward, meta, metaHeaders, providerClient, signatureCache, sessionKey) {
let responseBody;
if (forward.isGoogle) {
const googleData = (await forward.response.json());
responseBody = providerClient.convertGoogleResponse(googleData, meta.model);
if (signatureCache && sessionKey) {
const sigs = responseBody?._extractedSignatures;
if (sigs) {
for (const s of sigs)
signatureCache.store(sessionKey, s.toolCallId, s.signature);
delete responseBody._extractedSignatures;
}
}
}
else if (forward.isAnthropic) {
const anthropicData = (await forward.response.json());
responseBody = providerClient.convertAnthropicResponse(anthropicData, meta.model);
}
else if (forward.isChatGpt) {
const sseText = await forward.response.text();
responseBody = providerClient.collectChatGptSseResponse(sseText, meta.model);
}
else {
responseBody = await forward.response.json();
}
const body = responseBody;
const usage = body?.usage;
let streamUsage = null;
if (usage && typeof usage.prompt_tokens === 'number') {
streamUsage = {
prompt_tokens: usage.prompt_tokens,
completion_tokens: usage.completion_tokens ?? 0,
cache_read_tokens: usage.cache_read_tokens,
cache_creation_tokens: usage.cache_creation_tokens,
};
}
res.status(200);
setHeaders(res, metaHeaders);
res.json(responseBody);
return streamUsage;
}
function recordSuccess(ctx, meta, streamUsage, fallbackSuccessTs, recorder, traceId, sessionKey, startTime) {
if (meta.fallbackFromModel && fallbackSuccessTs) {
recorder
.recordFallbackSuccess(ctx, meta.model, meta.tier, {
traceId,
fallbackFromModel: meta.fallbackFromModel,
fallbackIndex: meta.fallbackIndex ?? 0,
timestamp: fallbackSuccessTs,
authType: meta.auth_type,
usage: streamUsage ?? undefined,
})
.catch((e) => logger.warn(`Failed to record fallback success: ${e}`));
}
else {
const usage = streamUsage ?? { prompt_tokens: 0, completion_tokens: 0 };
const durationMs = startTime ? Date.now() - startTime : undefined;
recorder
.recordSuccessMessage(ctx, meta.model, meta.tier, meta.reason, usage, {
traceId,
authType: meta.auth_type,
sessionKey,
durationMs,
})
.catch((e) => logger.warn(`Failed to record success message: ${e}`));
}
}
//# sourceMappingURL=proxy-response-handler.js.map