manifest
Version:
Self-hosted Manifest LLM router with embedded server, SQLite database, and dashboard
174 lines • 8.32 kB
JavaScript
;
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
var ProxyController_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ProxyController = void 0;
const common_1 = require("@nestjs/common");
const throttler_1 = require("@nestjs/throttler");
const public_decorator_1 = require("../../common/decorators/public.decorator");
const agent_key_auth_guard_1 = require("../../otlp/guards/agent-key-auth.guard");
const proxy_service_1 = require("./proxy.service");
const proxy_rate_limiter_1 = require("./proxy-rate-limiter");
const provider_client_1 = require("./provider-client");
const proxy_message_recorder_1 = require("./proxy-message-recorder");
const thought_signature_cache_1 = require("./thought-signature-cache");
const proxy_response_handler_1 = require("./proxy-response-handler");
const proxy_exception_filter_1 = require("./proxy-exception.filter");
const proxy_friendly_response_1 = require("./proxy-friendly-response");
const MAX_SEEN_USERS = 10_000;
const SEEN_USER_TTL_MS = 24 * 60 * 60 * 1000;
let ProxyController = ProxyController_1 = class ProxyController {
proxyService;
rateLimiter;
providerClient;
recorder;
signatureCache;
logger = new common_1.Logger(ProxyController_1.name);
seenUsers = new Map();
constructor(proxyService, rateLimiter, providerClient, recorder, signatureCache) {
this.proxyService = proxyService;
this.rateLimiter = rateLimiter;
this.providerClient = providerClient;
this.recorder = recorder;
this.signatureCache = signatureCache;
}
async chatCompletions(req, res) {
const { userId } = req.ingestionContext;
const body = req.body;
const sessionKey = req.headers['x-session-key'] || 'default';
const traceId = this.extractTraceId(req);
const isStream = body.stream === true;
let headersSent = false;
let slotAcquired = false;
const clientAbort = new AbortController();
res.once('close', () => clientAbort.abort());
const startTime = Date.now();
try {
this.rateLimiter.checkLimit(userId);
this.rateLimiter.acquireSlot(userId);
slotAcquired = true;
const { forward, meta, failedFallbacks } = await this.proxyService.proxyRequest({
agentId: req.ingestionContext.agentId,
userId,
body,
sessionKey,
tenantId: req.ingestionContext.tenantId,
agentName: req.ingestionContext.agentName,
signal: clientAbort.signal,
});
this.trackFirstProxyRequest(userId);
const metaHeaders = (0, proxy_response_handler_1.buildMetaHeaders)(meta);
const providerResponse = forward.response;
if (!providerResponse.ok) {
const errorBody = await providerResponse.text();
await (0, proxy_response_handler_1.handleProviderError)(res, req.ingestionContext, meta, metaHeaders, providerResponse.status, errorBody, failedFallbacks, this.recorder, traceId);
return;
}
const fallbackSuccessTs = (0, proxy_response_handler_1.recordFallbackFailures)(req.ingestionContext, meta, failedFallbacks, this.recorder);
let streamUsage = null;
if (isStream && providerResponse.body) {
headersSent = true;
streamUsage = await (0, proxy_response_handler_1.handleStreamResponse)(res, forward, meta, metaHeaders, this.providerClient, this.signatureCache, sessionKey);
}
else {
streamUsage = await (0, proxy_response_handler_1.handleNonStreamResponse)(res, forward, meta, metaHeaders, this.providerClient, this.signatureCache, sessionKey);
}
(0, proxy_response_handler_1.recordSuccess)(req.ingestionContext, meta, streamUsage, fallbackSuccessTs, this.recorder, traceId, sessionKey, startTime);
}
catch (err) {
if (clientAbort.signal.aborted) {
if (!res.writableEnded)
res.end();
return;
}
const message = err instanceof Error ? err.message : String(err);
const status = err instanceof common_1.HttpException ? err.getStatus() : 500;
this.logger.error(`Proxy error: ${message}`);
this.recorder
.recordProviderError(req.ingestionContext, status, message, { traceId })
.catch((e) => this.logger.warn(`Failed to record provider error: ${e}`));
if (headersSent) {
if (!res.writableEnded)
res.end();
return;
}
if (status === 429) {
const response = err instanceof common_1.HttpException ? err.getResponse() : message;
res
.status(429)
.json(typeof response === 'string'
? { error: { message: response, type: 'proxy_error' } }
: response);
return;
}
const isStream = req.body?.stream === true;
const clientMessage = status >= 500 ? '[🦚 Manifest] Something broke on our end. Try again shortly.' : message;
(0, proxy_friendly_response_1.sendFriendlyResponse)(res, clientMessage, isStream);
}
finally {
if (slotAcquired)
this.rateLimiter.releaseSlot(userId);
}
}
extractTraceId(req) {
const header = req.headers['traceparent'];
if (!header)
return undefined;
const parts = header.split('-');
return parts.length >= 2 ? parts[1] : undefined;
}
trackFirstProxyRequest(userId) {
const now = Date.now();
if (this.seenUsers.has(userId))
return;
this.evictExpiredUsers(now);
if (this.seenUsers.size >= MAX_SEEN_USERS) {
const oldest = this.seenUsers.keys().next().value;
this.seenUsers.delete(oldest);
}
this.seenUsers.set(userId, now);
}
evictExpiredUsers(now) {
for (const [key, timestamp] of this.seenUsers) {
if (now - timestamp > SEEN_USER_TTL_MS) {
this.seenUsers.delete(key);
}
else {
break;
}
}
}
};
exports.ProxyController = ProxyController;
__decorate([
(0, common_1.Post)('chat/completions'),
__param(0, (0, common_1.Req)()),
__param(1, (0, common_1.Res)()),
__metadata("design:type", Function),
__metadata("design:paramtypes", [Object, Object]),
__metadata("design:returntype", Promise)
], ProxyController.prototype, "chatCompletions", null);
exports.ProxyController = ProxyController = ProxyController_1 = __decorate([
(0, common_1.Controller)('v1'),
(0, public_decorator_1.Public)(),
(0, common_1.UseGuards)(agent_key_auth_guard_1.AgentKeyAuthGuard),
(0, common_1.UseFilters)(proxy_exception_filter_1.ProxyExceptionFilter),
(0, throttler_1.SkipThrottle)(),
__metadata("design:paramtypes", [proxy_service_1.ProxyService,
proxy_rate_limiter_1.ProxyRateLimiter,
provider_client_1.ProviderClient,
proxy_message_recorder_1.ProxyMessageRecorder,
thought_signature_cache_1.ThoughtSignatureCache])
], ProxyController);
//# sourceMappingURL=proxy.controller.js.map