@tehreet/conduit
Version:
LLM API gateway with intelligent routing, robust process management, and health monitoring
795 lines • 29.9 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.router = exports.createCommonRoutingRules = exports.registerCustomEvaluator = exports.getAllRoutingRules = exports.removeRoutingRule = exports.registerRoutingRule = exports.getAdvancedRoutingEngine = exports.initializePlugins = exports.getUseModel = exports.AdvancedRoutingEngine = void 0;
const tiktoken_1 = require("tiktoken");
const log_1 = require("./log");
const context_extractor_1 = require("./context-extractor");
const plugins_1 = require("../plugins");
const enc = (0, tiktoken_1.get_encoding)('cl100k_base');
/**
* Enhanced routing engine with advanced rule evaluation
*/
class AdvancedRoutingEngine {
constructor() {
this.rules = [];
this.customEvaluators = new Map();
this.registerDefaultEvaluators();
}
/**
* Register a custom routing rule
*/
registerRule(rule) {
// Remove existing rule with same ID
this.rules = this.rules.filter(r => r.id !== rule.id);
// Add new rule and sort by priority
this.rules.push(rule);
this.rules.sort((a, b) => b.priority - a.priority);
}
/**
* Remove a routing rule
*/
removeRule(ruleId) {
this.rules = this.rules.filter(r => r.id !== ruleId);
}
/**
* Get all routing rules
*/
getRules() {
return [...this.rules];
}
/**
* Register custom condition evaluator
*/
registerCustomEvaluator(name, evaluator) {
this.customEvaluators.set(name, evaluator);
}
/**
* Evaluate routing rules and make routing decision
*/
async evaluate(context) {
// Enhance context with content analysis
const enhancedContext = await this.enhanceContext(context);
// Evaluate rules in priority order
for (const rule of this.rules) {
if (!rule.enabled)
continue;
if (await this.evaluateRule(rule, enhancedContext)) {
return {
selectedModel: rule.model,
confidence: rule.confidence || 0.8,
reason: `Matched rule: ${rule.name}`,
rule,
fallbackModel: rule.fallback,
metadata: rule.metadata
};
}
}
// No rules matched, use fallback
const fallbackModel = context.config.Router?.default || 'claude-3-5-sonnet-20241022';
return {
selectedModel: fallbackModel,
confidence: 0.5,
reason: 'No rules matched, using fallback',
fallbackModel: undefined
};
}
/**
* Evaluate a single routing rule
*/
async evaluateRule(rule, context) {
try {
// All conditions must be true for rule to match
for (const condition of rule.conditions) {
const result = await this.evaluateCondition(condition, context);
if (!result) {
return false;
}
}
return true;
}
catch (error) {
(0, log_1.log)(`Error evaluating rule ${rule.name}:`, error);
return false;
}
}
/**
* Evaluate a single condition
*/
async evaluateCondition(condition, context) {
let result = false;
switch (condition.type) {
case 'token_count':
result = this.evaluateTokenCountCondition(condition, context);
break;
case 'model_request':
result = this.evaluateModelRequestCondition(condition, context);
break;
case 'flags':
result = this.evaluateFlagsCondition(condition, context);
break;
case 'content':
result = this.evaluateContentCondition(condition, context);
break;
case 'context':
result = this.evaluateContextCondition(condition, context);
break;
case 'time':
result = this.evaluateTimeCondition(condition, context);
break;
case 'custom':
result = this.evaluateCustomCondition(condition, context);
break;
default:
(0, log_1.log)(`Unknown condition type: ${condition.type}`);
return false;
}
return condition.negate ? !result : result;
}
/**
* Evaluate token count condition
*/
evaluateTokenCountCondition(condition, context) {
const tokenCount = context.tokenCount;
const threshold = Number(condition.value);
switch (condition.operator) {
case '<': return tokenCount < threshold;
case '<=': return tokenCount <= threshold;
case '>': return tokenCount > threshold;
case '>=': return tokenCount >= threshold;
case '=': return tokenCount === threshold;
case '!=': return tokenCount !== threshold;
default: return false;
}
}
/**
* Evaluate model request condition
*/
evaluateModelRequestCondition(condition, context) {
const requestedModel = context.request.body?.model || '';
const value = String(condition.value);
switch (condition.operator) {
case '=': return requestedModel === value;
case '!=': return requestedModel !== value;
case 'includes': return requestedModel.includes(value);
case 'excludes': return !requestedModel.includes(value);
case 'matches': return new RegExp(value).test(requestedModel);
default: return false;
}
}
/**
* Evaluate flags condition
*/
evaluateFlagsCondition(condition, context) {
const body = context.request.body || {};
const field = condition.field || 'thinking';
const value = condition.value;
switch (condition.operator) {
case '=': return body[field] === value;
case '!=': return body[field] !== value;
case 'includes': return String(body[field]).includes(String(value));
case 'excludes': return !String(body[field]).includes(String(value));
default: return !!body[field];
}
}
/**
* Evaluate content condition
*/
evaluateContentCondition(condition, context) {
const analysis = context.contentAnalysis;
if (!analysis)
return false;
const field = condition.field || 'complexity';
const value = condition.value;
switch (field) {
case 'complexity':
return this.compareValues(analysis.complexity, condition.operator, value);
case 'language':
return this.compareValues(analysis.language, condition.operator, value);
case 'topics':
return this.evaluateArrayCondition(analysis.topics, condition.operator, value);
case 'codeBlocks':
return this.compareValues(analysis.codeBlocks, condition.operator, value);
default:
return false;
}
}
/**
* Evaluate context condition (Synapse context)
*/
evaluateContextCondition(condition, context) {
const synapseContext = context.synapseContext;
const field = condition.field || 'projectId';
const value = condition.value;
const contextValue = synapseContext[field];
return this.compareValues(contextValue, condition.operator, value);
}
/**
* Evaluate time condition
*/
evaluateTimeCondition(condition, context) {
const now = context.timestamp;
const field = condition.field || 'hour';
const value = condition.value;
switch (field) {
case 'hour':
return this.compareValues(now.getHours(), condition.operator, value);
case 'day':
return this.compareValues(now.getDay(), condition.operator, value);
case 'date':
return this.compareValues(now.getDate(), condition.operator, value);
case 'month':
return this.compareValues(now.getMonth(), condition.operator, value);
default:
return false;
}
}
/**
* Evaluate custom condition
*/
evaluateCustomCondition(condition, context) {
const evaluator = this.customEvaluators.get(condition.field || '');
if (!evaluator) {
(0, log_1.log)(`Custom evaluator not found: ${condition.field}`);
return false;
}
return evaluator(condition, context);
}
/**
* Compare values based on operator
*/
compareValues(actual, operator, expected) {
switch (operator) {
case '=': return actual === expected;
case '!=': return actual !== expected;
case '<': return actual < expected;
case '<=': return actual <= expected;
case '>': return actual > expected;
case '>=': return actual >= expected;
case 'includes': return String(actual).includes(String(expected));
case 'excludes': return !String(actual).includes(String(expected));
case 'matches': return new RegExp(String(expected)).test(String(actual));
default: return false;
}
}
/**
* Evaluate array condition
*/
evaluateArrayCondition(array, operator, value) {
switch (operator) {
case 'includes': return array.includes(value);
case 'excludes': return !array.includes(value);
case '=': return array.length === value;
case '!=': return array.length !== value;
case '<': return array.length < value;
case '<=': return array.length <= value;
case '>': return array.length > value;
case '>=': return array.length >= value;
default: return false;
}
}
/**
* Enhance context with content analysis
*/
async enhanceContext(context) {
const messages = context.request.body?.messages || [];
const contentAnalysis = await this.analyzeContent(messages);
return {
...context,
contentAnalysis
};
}
/**
* Analyze content for routing decisions
*/
async analyzeContent(messages) {
let allContent = '';
let codeBlocks = 0;
// Extract all text content
for (const message of messages) {
if (typeof message.content === 'string') {
allContent += message.content + '\n';
}
else if (Array.isArray(message.content)) {
for (const part of message.content) {
if (part.type === 'text') {
allContent += part.text + '\n';
}
}
}
}
// Count code blocks
codeBlocks = (allContent.match(/```/g) || []).length / 2;
// Analyze complexity (simple heuristic)
const complexity = this.calculateComplexity(allContent);
// Detect language (simple heuristic)
const language = this.detectLanguage(allContent);
// Extract topics (simple keyword extraction)
const topics = this.extractTopics(allContent);
return {
complexity,
language,
topics,
codeBlocks
};
}
/**
* Calculate content complexity
*/
calculateComplexity(content) {
const factors = {
length: Math.min(content.length / 10000, 1), // 0-1 based on length
sentences: Math.min((content.match(/[.!?]/g) || []).length / 100, 1),
technical: (content.match(/\b(algorithm|implementation|architecture|database|function|class|method)\b/gi) || []).length / 20,
code: (content.match(/```|`[^`]+`/g) || []).length / 10
};
return Math.min((factors.length * 0.2 + factors.sentences * 0.2 + factors.technical * 0.3 + factors.code * 0.3), 1);
}
/**
* Detect content language
*/
detectLanguage(content) {
const patterns = {
python: /\b(def|import|from|class|if __name__|print\()/gi,
javascript: /\b(function|const|let|var|=>|console\.log)/gi,
java: /\b(public|private|class|static|void|System\.out)/gi,
cpp: /\b(#include|using namespace|std::|cout|cin)/gi,
html: /<[^>]+>/gi,
sql: /\b(SELECT|FROM|WHERE|INSERT|UPDATE|DELETE)\b/gi
};
for (const [lang, pattern] of Object.entries(patterns)) {
if (pattern.test(content)) {
return lang;
}
}
return 'text';
}
/**
* Extract topics from content
*/
extractTopics(content) {
const topics = [];
const topicPatterns = {
coding: /\b(code|coding|programming|development|software)\b/gi,
data: /\b(data|database|sql|analytics|statistics)\b/gi,
web: /\b(web|html|css|javascript|frontend|backend)\b/gi,
ai: /\b(ai|artificial intelligence|machine learning|neural|model)\b/gi,
design: /\b(design|ui|ux|interface|user experience)\b/gi
};
for (const [topic, pattern] of Object.entries(topicPatterns)) {
if (pattern.test(content)) {
topics.push(topic);
}
}
return topics;
}
/**
* Register default evaluators
*/
registerDefaultEvaluators() {
// Register built-in custom evaluators
this.registerCustomEvaluator('has_code', (condition, context) => {
return (context.contentAnalysis?.codeBlocks || 0) > 0;
});
this.registerCustomEvaluator('is_complex', (condition, context) => {
return (context.contentAnalysis?.complexity || 0) > 0.7;
});
this.registerCustomEvaluator('is_synapse_request', (condition, context) => {
return !!(context.synapseContext.projectId || context.synapseContext.agentId);
});
}
}
exports.AdvancedRoutingEngine = AdvancedRoutingEngine;
// Global instances
let pluginManager = null;
let routingEngine = null;
const getPluginManager = () => {
if (!pluginManager) {
pluginManager = new plugins_1.DefaultPluginManager();
}
return pluginManager;
};
const getRoutingEngine = () => {
if (!routingEngine) {
routingEngine = new AdvancedRoutingEngine();
}
return routingEngine;
};
const mapToClaudeModel = (synapseModelId) => {
// Map Synapse model IDs to Claude CLI model names
const mapping = {
'anthropic/claude-3-5-sonnet-20241022': 'claude-3-5-sonnet-20241022',
'anthropic/claude-3-5-haiku-20241022': 'claude-3-5-haiku-20241022',
'anthropic/claude-3-opus-20240229': 'claude-3-opus-20240229',
};
return (mapping[synapseModelId] || synapseModelId.split('/').pop() || synapseModelId);
};
const selectModelWithSynapseContext = (synapseContext, req, tokenCount, config) => {
// If we have Synapse context, use it for routing
if (synapseContext.agentModelConfig) {
try {
const agentConfig = JSON.parse(synapseContext.agentModelConfig);
if (!agentConfig.useProjectDefaults && agentConfig.overrideModel) {
(0, log_1.log)('Using agent override model:', agentConfig.overrideModel);
return mapToClaudeModel(agentConfig.overrideModel);
}
}
catch (error) {
(0, log_1.log)('Error parsing agent model config:', error);
}
}
if (synapseContext.projectModelConfig) {
try {
const projectConfig = JSON.parse(synapseContext.projectModelConfig);
if (projectConfig.enabled && projectConfig.defaultModel) {
(0, log_1.log)('Using project default model:', projectConfig.defaultModel);
return mapToClaudeModel(projectConfig.defaultModel);
}
}
catch (error) {
(0, log_1.log)('Error parsing project model config:', error);
}
}
return null; // Fall back to standard routing
};
const evaluateRoutingRule = (rule, tokenCount, req) => {
try {
const condition = rule.condition.toLowerCase();
// Token count conditions
if (condition.includes('tokencount')) {
const match = condition.match(/tokencount\s*([<>]=?)\s*(\d+)/);
if (match) {
const operator = match[1];
const threshold = parseInt(match[2]);
switch (operator) {
case '<':
return tokenCount < threshold;
case '<=':
return tokenCount <= threshold;
case '>':
return tokenCount > threshold;
case '>=':
return tokenCount >= threshold;
}
}
}
// Flags conditions
if (condition.includes('flags.includes')) {
const match = condition.match(/flags\.includes\(['"]([^'"]+)['"]\)/);
if (match) {
const flag = match[1];
if (flag === '--thinking' && req.body?.thinking) {
return true;
}
}
}
// Model conditions
if (condition.includes('model')) {
const requestedModel = req.body?.model || '';
if (condition.includes('haiku') && requestedModel.includes('haiku')) {
return true;
}
if (condition.includes('sonnet') && requestedModel.includes('sonnet')) {
return true;
}
if (condition.includes('opus') && requestedModel.includes('opus')) {
return true;
}
}
return false;
}
catch (error) {
(0, log_1.log)(`Error evaluating routing rule '${rule.name}':`, error);
return false;
}
};
const getUseModel = async (req, tokenCount, config, env = process.env) => {
// CRITICAL: If both model and provider are explicitly provided, validate and use them
if (req.body.model && req.body.provider && !req.body.model.includes(',')) {
const providers = config.providers || config.Providers || [];
const provider = providers.find(p => p.name === req.body.provider ||
p.providerName === req.body.provider);
if (provider) {
const model = provider.models?.find((m) => m.id === req.body.model);
if (model) {
(0, log_1.log)('Using explicitly provided model/provider:', {
model: req.body.model,
provider: req.body.provider
});
return `${req.body.provider},${req.body.model}`;
}
else {
(0, log_1.log)('Model not found in provider, but using explicit model anyway:', {
model: req.body.model,
provider: req.body.provider,
availableModels: provider.models?.map((m) => m.id) || []
});
return `${req.body.provider},${req.body.model}`;
}
}
else {
(0, log_1.log)('Provider not found, but using explicit model anyway:', {
model: req.body.model,
provider: req.body.provider,
availableProviders: providers.map((p) => p.name || p.providerName)
});
return `${req.body.provider},${req.body.model}`;
}
}
// CRITICAL: If only model is explicitly provided, use it
if (req.body.model && !req.body.model.includes(',')) {
(0, log_1.log)('Using explicitly provided model:', req.body.model);
return req.body.model;
}
if (req.body.model && req.body.model.includes(',')) {
return req.body.model;
}
// Extract Synapse context for routing
const synapseContext = context_extractor_1.ContextExtractor.extractSynapseContext(env);
// Create routing context for plugins
const routingContext = {
request: req,
tokenCount,
config,
env,
synapseContext,
requestBody: req.body, // Add request body for plugins
};
// Try plugin-based routing first
const manager = getPluginManager();
const pluginModel = await manager.executeCustomRouting(routingContext);
if (pluginModel) {
(0, log_1.log)('Plugin provided custom routing:', pluginModel);
return pluginModel;
}
// Try advanced routing engine
const routingEngine = getRoutingEngine();
const evaluationContext = {
request: req,
tokenCount,
config,
env,
synapseContext,
timestamp: new Date()
};
const decision = await routingEngine.evaluate(evaluationContext);
if (decision.confidence > 0.6) {
(0, log_1.log)(`Advanced routing decision: ${decision.reason} (confidence: ${decision.confidence})`);
return decision.selectedModel;
}
// Try preset routing rules
if (config.presetConfig?.defaultRouting?.rules) {
for (const rule of config.presetConfig.defaultRouting.rules) {
if (evaluateRoutingRule(rule, tokenCount, req)) {
(0, log_1.log)(`Preset routing rule '${rule.name}' matched, using model:`, rule.model);
return rule.model;
}
}
}
// Try Synapse-based routing (fallback)
if (synapseContext.projectId || synapseContext.agentId) {
const synapseModel = selectModelWithSynapseContext(synapseContext, req, tokenCount, config);
if (synapseModel) {
return synapseModel;
}
}
// Fall back to standard Conduit routing
// if tokenCount is greater than 60K, use the long context model
if (tokenCount > 1000 * 60 && config.Router?.longContext) {
(0, log_1.log)('Using long context model due to token count:', tokenCount);
return config.Router.longContext;
}
// If the model is claude-3-5-haiku, use the background model
if (req.body.model?.startsWith('claude-3-5-haiku') &&
config.Router?.background) {
(0, log_1.log)('Using background model for ', req.body.model);
return config.Router.background;
}
// if exits thinking, use the think model
if (req.body.thinking && config.Router?.think) {
(0, log_1.log)('Using think model for ', req.body.thinking);
return config.Router.think;
}
// Use flexible fallback logic
const defaultModel = req.body.model || // Use request model as highest priority
config.Router?.default ||
config.presetConfig?.defaultRouting?.fallback ||
(config.providers?.[0]?.models?.[0]?.id) || // Use first available model
(config.Providers?.[0]?.models?.[0]?.id) || // Check alternate providers field
'gpt-3.5-turbo'; // Generic fallback, not Claude-specific
(0, log_1.log)('Using fallback model:', defaultModel);
return defaultModel;
};
exports.getUseModel = getUseModel;
// Export function to access plugin manager for initialization
const initializePlugins = async (pluginDir) => {
const manager = getPluginManager();
if (pluginDir) {
await manager.loadPlugins(pluginDir);
}
return manager;
};
exports.initializePlugins = initializePlugins;
// Export functions to access routing engine
const getAdvancedRoutingEngine = () => {
return getRoutingEngine();
};
exports.getAdvancedRoutingEngine = getAdvancedRoutingEngine;
const registerRoutingRule = (rule) => {
const engine = getRoutingEngine();
engine.registerRule(rule);
};
exports.registerRoutingRule = registerRoutingRule;
const removeRoutingRule = (ruleId) => {
const engine = getRoutingEngine();
engine.removeRule(ruleId);
};
exports.removeRoutingRule = removeRoutingRule;
const getAllRoutingRules = () => {
const engine = getRoutingEngine();
return engine.getRules();
};
exports.getAllRoutingRules = getAllRoutingRules;
const registerCustomEvaluator = (name, evaluator) => {
const engine = getRoutingEngine();
engine.registerCustomEvaluator(name, evaluator);
};
exports.registerCustomEvaluator = registerCustomEvaluator;
// Helper function to create common routing rules
const createCommonRoutingRules = () => {
return [
{
id: 'high-token-count',
name: 'High Token Count',
description: 'Use long context model for high token counts',
priority: 100,
conditions: [
{
type: 'token_count',
operator: '>',
value: 60000
}
],
model: 'claude-3-5-sonnet-20241022',
confidence: 0.9,
enabled: true
},
{
id: 'coding-request',
name: 'Coding Request',
description: 'Use code-optimized model for coding requests',
priority: 90,
conditions: [
{
type: 'content',
field: 'codeBlocks',
operator: '>',
value: 0
}
],
model: 'claude-3-5-sonnet-20241022',
confidence: 0.8,
enabled: true
},
{
id: 'thinking-request',
name: 'Thinking Request',
description: 'Use thinking model for reasoning requests',
priority: 95,
conditions: [
{
type: 'flags',
field: 'thinking',
operator: '=',
value: true
}
],
model: 'claude-3-5-sonnet-20241022',
confidence: 0.95,
enabled: true
},
{
id: 'simple-request',
name: 'Simple Request',
description: 'Use fast model for simple requests',
priority: 70,
conditions: [
{
type: 'token_count',
operator: '<',
value: 1000
},
{
type: 'content',
field: 'complexity',
operator: '<',
value: 0.3
}
],
model: 'claude-3-5-haiku-20241022',
confidence: 0.85,
enabled: true
},
{
id: 'complex-request',
name: 'Complex Request',
description: 'Use powerful model for complex requests',
priority: 80,
conditions: [
{
type: 'content',
field: 'complexity',
operator: '>',
value: 0.7
}
],
model: 'claude-3-5-sonnet-20241022',
confidence: 0.9,
enabled: true
}
];
};
exports.createCommonRoutingRules = createCommonRoutingRules;
const router = async (req, res, config) => {
const { messages, system = [], tools } = req.body;
try {
let tokenCount = 0;
if (Array.isArray(messages)) {
messages.forEach(message => {
if (typeof message.content === 'string') {
tokenCount += enc.encode(message.content).length;
}
else if (Array.isArray(message.content)) {
message.content.forEach((contentPart) => {
if (contentPart.type === 'text') {
tokenCount += enc.encode(contentPart.text).length;
}
else if (contentPart.type === 'tool_use') {
tokenCount += enc.encode(JSON.stringify(contentPart.input)).length;
}
else if (contentPart.type === 'tool_result') {
tokenCount += enc.encode(typeof contentPart.content === 'string'
? contentPart.content
: JSON.stringify(contentPart.content)).length;
}
});
}
});
}
if (typeof system === 'string') {
tokenCount += enc.encode(system).length;
}
else if (Array.isArray(system)) {
system.forEach((item) => {
if (item.type !== 'text')
return;
if (typeof item.text === 'string') {
tokenCount += enc.encode(item.text).length;
}
else if (Array.isArray(item.text)) {
item.text.forEach((textPart) => {
tokenCount += enc.encode(textPart || '').length;
});
}
});
}
if (tools) {
tools.forEach((tool) => {
if (tool.description) {
tokenCount += enc.encode(tool.name + tool.description).length;
}
if (tool.input_schema) {
tokenCount += enc.encode(JSON.stringify(tool.input_schema)).length;
}
});
}
const model = await (0, exports.getUseModel)(req, tokenCount, config, process.env);
req.body.model = model;
}
catch (error) {
(0, log_1.log)('Error in router middleware:', error.message);
req.body.model = config.Router.default;
}
return;
};
exports.router = router;
//# sourceMappingURL=router.js.map