manifest
Version:
Self-hosted Manifest LLM router with embedded server, SQLite database, and dashboard
264 lines • 12 kB
JavaScript
;
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ProxyMessageRecorder = void 0;
const common_1 = require("@nestjs/common");
const typeorm_1 = require("@nestjs/typeorm");
const typeorm_2 = require("typeorm");
const uuid_1 = require("uuid");
const agent_message_entity_1 = require("../../entities/agent-message.entity");
const model_pricing_cache_service_1 = require("../../model-prices/model-pricing-cache.service");
const ingest_event_bus_service_1 = require("../../common/services/ingest-event-bus.service");
const proxy_message_dedup_1 = require("./proxy-message-dedup");
const cost_calculator_1 = require("../../common/utils/cost-calculator");
let ProxyMessageRecorder = class ProxyMessageRecorder {
messageRepo;
pricingCache;
dedup;
eventBus;
rateLimitCooldown = new Map();
RATE_LIMIT_COOLDOWN_MS = 60_000;
MAX_COOLDOWN_ENTRIES = 1_000;
cooldownCleanupTimer;
constructor(messageRepo, pricingCache, dedup, eventBus) {
this.messageRepo = messageRepo;
this.pricingCache = pricingCache;
this.dedup = dedup;
this.eventBus = eventBus;
this.cooldownCleanupTimer = setInterval(() => this.evictExpiredCooldowns(), 60_000);
if (typeof this.cooldownCleanupTimer === 'object' && 'unref' in this.cooldownCleanupTimer) {
this.cooldownCleanupTimer.unref();
}
}
onModuleDestroy() {
clearInterval(this.cooldownCleanupTimer);
}
async recordProviderError(ctx, httpStatus, errorMessage, opts) {
const { model, tier, traceId, fallbackFromModel, fallbackIndex, authType } = opts ?? {};
if (httpStatus === 429) {
const key = `${ctx.tenantId}:${ctx.agentId}`;
const now = Date.now();
const lastRecorded = this.rateLimitCooldown.get(key) ?? 0;
if (now - lastRecorded < this.RATE_LIMIT_COOLDOWN_MS)
return;
this.rateLimitCooldown.set(key, now);
if (this.rateLimitCooldown.size > this.MAX_COOLDOWN_ENTRIES) {
for (const [k, v] of this.rateLimitCooldown) {
if (now - v >= this.RATE_LIMIT_COOLDOWN_MS)
this.rateLimitCooldown.delete(k);
}
}
}
const messageStatus = httpStatus === 429 ? 'rate_limited' : 'error';
await this.messageRepo.insert({
id: (0, uuid_1.v4)(),
tenant_id: ctx.tenantId,
agent_id: ctx.agentId,
trace_id: traceId ?? null,
timestamp: new Date().toISOString(),
status: messageStatus,
error_message: errorMessage.slice(0, 2000),
error_http_status: httpStatus,
agent_name: ctx.agentName,
model: model ?? null,
routing_tier: tier ?? null,
input_tokens: 0,
output_tokens: 0,
cache_read_tokens: 0,
cache_creation_tokens: 0,
fallback_from_model: fallbackFromModel ?? null,
fallback_index: fallbackIndex ?? null,
auth_type: authType ?? null,
user_id: ctx.userId,
});
this.eventBus.emit(ctx.userId);
}
async recordFailedFallbacks(ctx, tier, primaryModel, failures, opts) {
const { traceId, baseTimeMs, markHandled = false, lastAsError = false, authType } = opts ?? {};
for (let i = 0; i < failures.length; i++) {
const f = failures[i];
const ts = baseTimeMs
? new Date(baseTimeMs + (failures.length - i) * 100).toISOString()
: new Date().toISOString();
const isLast = i === failures.length - 1;
const useHandledStatus = markHandled && !(lastAsError && isLast);
const status = useHandledStatus
? 'fallback_error'
: f.status === 429
? 'rate_limited'
: 'error';
await this.messageRepo.insert({
id: (0, uuid_1.v4)(),
tenant_id: ctx.tenantId,
agent_id: ctx.agentId,
trace_id: traceId ?? null,
timestamp: ts,
status,
error_message: f.errorBody.slice(0, 2000),
error_http_status: f.status,
agent_name: ctx.agentName,
model: f.model,
routing_tier: tier,
input_tokens: 0,
output_tokens: 0,
cache_read_tokens: 0,
cache_creation_tokens: 0,
fallback_from_model: primaryModel,
fallback_index: f.fallbackIndex,
auth_type: authType ?? null,
user_id: ctx.userId,
});
}
this.eventBus.emit(ctx.userId);
}
async recordPrimaryFailure(ctx, tier, model, errorBody, timestamp, authType) {
await this.messageRepo.insert({
id: (0, uuid_1.v4)(),
tenant_id: ctx.tenantId,
agent_id: ctx.agentId,
trace_id: null,
timestamp,
status: 'fallback_error',
error_message: errorBody.slice(0, 2000),
agent_name: ctx.agentName,
model,
routing_tier: tier,
input_tokens: 0,
output_tokens: 0,
cache_read_tokens: 0,
cache_creation_tokens: 0,
fallback_from_model: null,
fallback_index: null,
auth_type: authType ?? null,
user_id: ctx.userId,
});
this.eventBus.emit(ctx.userId);
}
async recordFallbackSuccess(ctx, model, tier, opts) {
const { traceId, fallbackFromModel, fallbackIndex, timestamp, authType, usage } = opts ?? {};
const inputTokens = usage?.prompt_tokens ?? 0;
const outputTokens = usage?.completion_tokens ?? 0;
const costUsd = (0, cost_calculator_1.computeTokenCost)({
inputTokens,
outputTokens,
model,
pricing: usage ? this.pricingCache.getByModel(model) : undefined,
isSubscription: authType === 'subscription',
});
await this.messageRepo.insert({
id: (0, uuid_1.v4)(),
tenant_id: ctx.tenantId,
agent_id: ctx.agentId,
trace_id: traceId ?? null,
timestamp: timestamp ?? new Date().toISOString(),
status: 'ok',
agent_name: ctx.agentName,
model,
routing_tier: tier,
input_tokens: inputTokens,
output_tokens: outputTokens,
cache_read_tokens: usage?.cache_read_tokens ?? 0,
cache_creation_tokens: usage?.cache_creation_tokens ?? 0,
cost_usd: costUsd,
auth_type: authType ?? null,
fallback_from_model: fallbackFromModel ?? null,
fallback_index: fallbackIndex ?? null,
user_id: ctx.userId,
});
this.eventBus.emit(ctx.userId);
}
async recordSuccessMessage(ctx, model, tier, reason, usage, opts) {
const { traceId, authType, sessionKey, durationMs } = opts ?? {};
const costUsd = (0, cost_calculator_1.computeTokenCost)({
inputTokens: usage.prompt_tokens,
outputTokens: usage.completion_tokens,
model,
pricing: this.pricingCache.getByModel(model),
isSubscription: authType === 'subscription',
});
const normalizedSessionKey = this.dedup.normalizeSessionKey(sessionKey);
let wrote = false;
await this.dedup.withSuccessWriteLock(this.dedup.getSuccessWriteLockKey(ctx, model, traceId, normalizedSessionKey), async () => {
await this.dedup.withAgentMessageTransaction(this.messageRepo, ctx, async (messageRepo) => {
const existing = await this.dedup.findExistingSuccessMessage(messageRepo, ctx, model, usage, traceId, normalizedSessionKey);
if (existing) {
const hasRecordedTokens = (existing.input_tokens ?? 0) > 0 || (existing.output_tokens ?? 0) > 0;
if (hasRecordedTokens)
return;
const updatePayload = {
model,
routing_tier: tier,
routing_reason: reason,
input_tokens: usage.prompt_tokens,
output_tokens: usage.completion_tokens,
cache_read_tokens: usage.cache_read_tokens ?? 0,
cache_creation_tokens: usage.cache_creation_tokens ?? 0,
cost_usd: costUsd,
auth_type: authType ?? null,
user_id: ctx.userId,
duration_ms: durationMs ?? null,
};
if (normalizedSessionKey)
updatePayload.session_key = normalizedSessionKey;
await messageRepo.update({ id: existing.id }, updatePayload);
wrote = true;
return;
}
await messageRepo.insert({
id: (0, uuid_1.v4)(),
tenant_id: ctx.tenantId,
agent_id: ctx.agentId,
trace_id: traceId ?? null,
session_key: normalizedSessionKey,
timestamp: new Date().toISOString(),
status: 'ok',
agent_name: ctx.agentName,
model,
routing_tier: tier,
routing_reason: reason,
input_tokens: usage.prompt_tokens,
output_tokens: usage.completion_tokens,
cache_read_tokens: usage.cache_read_tokens ?? 0,
cache_creation_tokens: usage.cache_creation_tokens ?? 0,
cost_usd: costUsd,
auth_type: authType ?? null,
fallback_from_model: null,
fallback_index: null,
user_id: ctx.userId,
duration_ms: durationMs ?? null,
});
wrote = true;
});
});
if (wrote)
this.eventBus.emit(ctx.userId);
}
evictExpiredCooldowns() {
const now = Date.now();
for (const [k, v] of this.rateLimitCooldown) {
if (now - v >= this.RATE_LIMIT_COOLDOWN_MS)
this.rateLimitCooldown.delete(k);
}
}
};
exports.ProxyMessageRecorder = ProxyMessageRecorder;
exports.ProxyMessageRecorder = ProxyMessageRecorder = __decorate([
(0, common_1.Injectable)(),
__param(0, (0, typeorm_1.InjectRepository)(agent_message_entity_1.AgentMessage)),
__metadata("design:paramtypes", [typeorm_2.Repository,
model_pricing_cache_service_1.ModelPricingCacheService,
proxy_message_dedup_1.ProxyMessageDedup,
ingest_event_bus_service_1.IngestEventBusService])
], ProxyMessageRecorder);
//# sourceMappingURL=proxy-message-recorder.js.map