manifest
Version:
Self-hosted Manifest LLM router with embedded server, SQLite database, and dashboard
227 lines • 12.3 kB
JavaScript
;
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var ProxyService_1;
Object.defineProperty(exports, "__esModule", { value: true });
exports.ProxyService = void 0;
const common_1 = require("@nestjs/common");
const config_1 = require("@nestjs/config");
const resolve_service_1 = require("../resolve/resolve.service");
const provider_key_service_1 = require("../routing-core/provider-key.service");
const tier_service_1 = require("../routing-core/tier.service");
const openai_oauth_service_1 = require("../oauth/openai-oauth.service");
const minimax_oauth_service_1 = require("../oauth/minimax-oauth.service");
const session_momentum_service_1 = require("./session-momentum.service");
const limit_check_service_1 = require("../../notifications/services/limit-check.service");
const fallback_status_codes_1 = require("./fallback-status-codes");
const proxy_fallback_service_1 = require("./proxy-fallback.service");
const thought_signature_cache_1 = require("./thought-signature-cache");
const proxy_friendly_response_1 = require("./proxy-friendly-response");
const SCORING_EXCLUDED_ROLES = new Set(['system', 'developer']);
const SCORING_RECENT_MESSAGES = 10;
let ProxyService = ProxyService_1 = class ProxyService {
resolveService;
providerKeyService;
tierService;
openaiOauth;
minimaxOauth;
momentum;
limitCheck;
fallbackService;
config;
signatureCache;
logger = new common_1.Logger(ProxyService_1.name);
constructor(resolveService, providerKeyService, tierService, openaiOauth, minimaxOauth, momentum, limitCheck, fallbackService, config, signatureCache) {
this.resolveService = resolveService;
this.providerKeyService = providerKeyService;
this.tierService = tierService;
this.openaiOauth = openaiOauth;
this.minimaxOauth = minimaxOauth;
this.momentum = momentum;
this.limitCheck = limitCheck;
this.fallbackService = fallbackService;
this.config = config;
this.signatureCache = signatureCache;
}
async proxyRequest(opts) {
const { agentId, userId, body, sessionKey, tenantId, agentName, signal } = opts;
const messages = body.messages;
if (!messages || !Array.isArray(messages) || messages.length === 0) {
throw new common_1.BadRequestException('messages array is required');
}
sanitizeNullContent(messages);
const limitMessage = await this.enforceLimits(tenantId, agentName);
if (limitMessage) {
return (0, proxy_friendly_response_1.buildFriendlyResponse)(limitMessage, body.stream === true, 'limit_exceeded');
}
const scoringMessages = this.filterScoringMessages(messages);
const scoringTools = Array.isArray(body.tools) ? body.tools : undefined;
const isHeartbeat = this.detectHeartbeat(scoringMessages);
const recentTiers = this.momentum.getRecentTiers(sessionKey);
const resolved = isHeartbeat
? await this.resolveService.resolveForTier(agentId, 'simple')
: await this.resolveService.resolve(agentId, scoringMessages, scoringTools, body.tool_choice, body.max_tokens, recentTiers);
if (!resolved.model || !resolved.provider) {
this.logger.warn(`No model available for agent=${agentId}: ` +
`tier=${resolved.tier} model=${resolved.model} provider=${resolved.provider} ` +
`confidence=${resolved.confidence} reason=${resolved.reason}`);
return this.buildNoProviderResult(body.stream === true, agentName);
}
let apiKey = await this.providerKeyService.getProviderApiKey(agentId, resolved.provider, resolved.auth_type);
if (apiKey === null) {
const dashboardUrl = (0, proxy_friendly_response_1.getDashboardUrl)(this.config, agentName);
const content = `[🦚 Manifest] No API key set for ${resolved.provider} yet. Add one here: ${dashboardUrl}`;
return (0, proxy_friendly_response_1.buildFriendlyResponse)(content, body.stream === true, 'no_provider_key');
}
const resolvedCredentials = await (0, proxy_fallback_service_1.resolveApiKey)(resolved.provider, apiKey, resolved.auth_type, agentId, userId, this.openaiOauth, this.minimaxOauth);
const providerRegion = await this.providerKeyService.getProviderRegion(agentId, resolved.provider, resolved.auth_type);
const primaryModel = (0, proxy_fallback_service_1.normalizeProviderModel)(resolved.provider, resolved.model);
this.logger.log(`Proxy: tier=${resolved.tier} model=${primaryModel} provider=${resolved.provider} auth_type=${resolved.auth_type} confidence=${resolved.confidence}`);
const stream = body.stream === true;
const signatureLookup = (toolCallId) => this.signatureCache.retrieve(sessionKey, toolCallId);
const forward = await this.fallbackService.tryForwardToProvider({
provider: resolved.provider,
apiKey: resolvedCredentials.apiKey,
model: primaryModel,
body,
stream,
sessionKey,
signal,
authType: resolved.auth_type,
resourceUrl: resolvedCredentials.resourceUrl,
providerRegion,
signatureLookup,
});
if (!forward.response.ok && (0, fallback_status_codes_1.shouldTriggerFallback)(forward.response.status)) {
const tiers = await this.tierService.getTiers(agentId);
const assignment = tiers.find((t) => t.tier === resolved.tier);
const fallbackModels = assignment?.fallback_models;
if (fallbackModels && fallbackModels.length > 0) {
const primaryStatus = forward.response.status;
const primaryErrorBody = await forward.response.text();
const { success, failures } = await this.fallbackService.tryFallbacks(agentId, userId, fallbackModels, body, stream, sessionKey, primaryModel, signal, resolved.provider ?? undefined, resolved.auth_type, signatureLookup);
if (success) {
this.momentum.recordTier(sessionKey, resolved.tier);
return {
forward: success.forward,
meta: {
tier: resolved.tier,
model: success.model,
provider: success.provider,
confidence: resolved.confidence,
reason: resolved.reason,
auth_type: resolved.auth_type,
fallbackFromModel: primaryModel,
fallbackIndex: success.fallbackIndex,
primaryErrorStatus: primaryStatus,
primaryErrorBody: primaryErrorBody,
},
failedFallbacks: failures,
};
}
const safeHeaders = new Headers(forward.response.headers);
safeHeaders.delete('content-encoding');
safeHeaders.delete('content-length');
safeHeaders.delete('transfer-encoding');
const rebuilt = new Response(primaryErrorBody, {
status: primaryStatus,
headers: safeHeaders,
});
this.momentum.recordTier(sessionKey, resolved.tier);
return {
forward: {
response: rebuilt,
isGoogle: forward.isGoogle,
isAnthropic: forward.isAnthropic,
isChatGpt: forward.isChatGpt,
},
meta: {
tier: resolved.tier,
model: primaryModel,
provider: resolved.provider,
confidence: resolved.confidence,
reason: resolved.reason,
auth_type: resolved.auth_type,
},
failedFallbacks: failures,
};
}
}
this.momentum.recordTier(sessionKey, resolved.tier);
return {
forward,
meta: {
tier: resolved.tier,
model: primaryModel,
provider: resolved.provider,
confidence: resolved.confidence,
reason: resolved.reason,
auth_type: resolved.auth_type,
},
};
}
async enforceLimits(tenantId, agentName) {
if (!tenantId || !agentName)
return null;
const exceeded = await this.limitCheck.checkLimits(tenantId, agentName);
if (!exceeded)
return null;
const fmt = exceeded.metricType === 'cost'
? `$${Number(exceeded.actual).toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`
: Number(exceeded.actual).toLocaleString(undefined, { maximumFractionDigits: 0 });
const threshFmt = exceeded.metricType === 'cost'
? `$${Number(exceeded.threshold).toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`
: Number(exceeded.threshold).toLocaleString(undefined, { maximumFractionDigits: 0 });
const dashboardUrl = (0, proxy_friendly_response_1.getDashboardUrl)(this.config, agentName);
return `[🦚 Manifest] Usage limit hit: ${exceeded.metricType} is at ${fmt} (limit: ${threshFmt}/${exceeded.period}). You can adjust it here: ${dashboardUrl}`;
}
filterScoringMessages(messages) {
return messages
.filter((m) => !SCORING_EXCLUDED_ROLES.has(m.role))
.slice(-SCORING_RECENT_MESSAGES);
}
detectHeartbeat(scoringMessages) {
const lastUser = [...scoringMessages].reverse().find((m) => m.role === 'user');
if (!lastUser)
return false;
if (typeof lastUser.content === 'string')
return lastUser.content.includes('HEARTBEAT_OK');
if (Array.isArray(lastUser.content)) {
return lastUser.content.some((p) => p.type === 'text' && typeof p.text === 'string' && p.text.includes('HEARTBEAT_OK'));
}
return false;
}
buildNoProviderResult(stream, agentName) {
const dashboardUrl = (0, proxy_friendly_response_1.getDashboardUrl)(this.config, agentName);
const content = `[🦚 Manifest] Manifest is connected successfully. To start routing requests, connect a model provider: ${dashboardUrl}`;
return (0, proxy_friendly_response_1.buildFriendlyResponse)(content, stream, 'no_provider');
}
};
exports.ProxyService = ProxyService;
exports.ProxyService = ProxyService = ProxyService_1 = __decorate([
(0, common_1.Injectable)(),
__metadata("design:paramtypes", [resolve_service_1.ResolveService,
provider_key_service_1.ProviderKeyService,
tier_service_1.TierService,
openai_oauth_service_1.OpenaiOauthService,
minimax_oauth_service_1.MinimaxOauthService,
session_momentum_service_1.SessionMomentumService,
limit_check_service_1.LimitCheckService,
proxy_fallback_service_1.ProxyFallbackService,
config_1.ConfigService,
thought_signature_cache_1.ThoughtSignatureCache])
], ProxyService);
function sanitizeNullContent(messages) {
for (const msg of messages) {
if (msg && typeof msg === 'object' && msg.content === null)
msg.content = '';
}
}
//# sourceMappingURL=proxy.service.js.map