mcp-context-engineering/src/mongodb/models/effectivenessMetrics.ts

The intelligent context optimization system for AI coding assistants. Built with Cole's PRP methodology, Context Portal knowledge graphs, and production-ready MongoDB architecture.

import { ObjectId } from 'mongodb';
import { z } from 'zod';

// Individual effectiveness measurement
export const EffectivenessMeasurementSchema = z.object({
  measurement_id: z.string(),
  context_pattern_id: z.string(),
  agent_type: z.enum(['cursor', 'windsurf', 'claude_code', 'generic']),
  
  // Implementation results
  implementation_result: z.object({
    success: z.boolean(),
    quality_score: z.number().min(0).max(10),
    time_to_completion: z.number().optional(), // minutes
    iterations_required: z.number(),
    user_satisfaction: z.number().min(0).max(10).optional()
  }),
  
  // Context effectiveness
  context_effectiveness: z.object({
    relevance_score: z.number().min(0).max(10),
    completeness_score: z.number().min(0).max(10),
    clarity_score: z.number().min(0).max(10),
    actionability_score: z.number().min(0).max(10)
  }),
  
  // Specific feedback
  feedback: z.object({
    what_worked: z.array(z.string()),
    what_failed: z.array(z.string()),
    missing_information: z.array(z.string()),
    suggestions: z.array(z.string()),
    agent_specific_notes: z.string()
  }),
  
  // Context metadata
  context_metadata: z.object({
    feature_type: z.string(),
    complexity: z.enum(['low', 'medium', 'high']),
    project_type: z.string(),
    tech_stack: z.array(z.string())
  }),
  
  timestamp: z.date()
});

// Aggregated effectiveness analytics
export const EffectivenessAnalyticsSchema = z.object({
  pattern_id: z.string(),
  
  // Overall metrics
  overall: z.object({
    total_usages: z.number(),
    success_rate: z.number().min(0).max(1),
    average_quality_score: z.number().min(0).max(10),
    average_relevance: z.number().min(0).max(10),
    trend: z.enum(['improving', 'stable', 'declining'])
  }),
  
  // Agent-specific metrics
  by_agent: z.record(z.object({
    usage_count: z.number(),
    success_rate: z.number().min(0).max(1),
    average_quality_score: z.number().min(0).max(10),
    preferred_formats: z.array(z.string()),
    common_issues: z.array(z.string()),
    optimization_suggestions: z.array(z.string())
  })),
  
  // Project type metrics
  by_project_type: z.record(z.object({
    usage_count: z.number(),
    success_rate: z.number().min(0).max(1),
    average_quality_score: z.number().min(0).max(10),
    specific_adaptations: z.array(z.string())
  })),
  
  // Complexity metrics
  by_complexity: z.record(z.object({
    usage_count: z.number(),
    success_rate: z.number().min(0).max(1),
    average_completion_time: z.number().optional(),
    common_challenges: z.array(z.string())
  })),
  
  // Learning insights
  insights: z.object({
    top_success_factors: z.array(z.string()),
    common_failure_points: z.array(z.string()),
    optimization_opportunities: z.array(z.string()),
    cross_agent_learnings: z.array(z.string())
  }),
  
  last_updated: z.date()
});

// Cross-agent learning patterns
export const CrossAgentLearningSchema = z.object({
  learning_id: z.string(),
  
  // Source of learning
  source: z.object({
    agent_type: z.string(),
    pattern_id: z.string(),
    success_factor: z.string(),
    context: z.string()
  }),
  
  // Applicability analysis
  applicability: z.object({
    tested_agents: z.array(z.string()),
    success_rates: z.record(z.number()),
    adaptation_requirements: z.record(z.array(z.string())),
    limitations: z.array(z.string())
  }),
  
  // Implementation guidance
  implementation: z.object({
    universal_principle: z.string(),
    agent_specific_adaptations: z.record(z.string()),
    validation_criteria: z.array(z.string()),
    expected_impact: z.number().min(0).max(1)
  }),
  
  // Validation results
  validation: z.object({
    tested: z.boolean(),
    validation_results: z.array(z.object({
      agent_type: z.string(),
      before_score: z.number(),
      after_score: z.number(),
      improvement: z.number(),
      confirmed: z.boolean()
    })),
    deployment_ready: z.boolean()
  }),
  
  created_at: z.date(),
  updated_at: z.date()
});

// Main effectiveness tracking schema
export const EffectivenessTrackingSchema = z.object({
  _id: z.instanceof(ObjectId).optional(),
  
  // Individual measurements
  measurements: z.array(EffectivenessMeasurementSchema),
  
  // Aggregated analytics
  analytics: EffectivenessAnalyticsSchema,
  
  // Cross-agent learning
  cross_agent_learning: z.array(CrossAgentLearningSchema),
  
  // System-wide insights
  system_insights: z.object({
    most_effective_patterns: z.array(z.string()),
    agent_strengths: z.record(z.array(z.string())),
    universal_success_factors: z.array(z.string()),
    improvement_priorities: z.array(z.string())
  }),
  
  metadata: z.object({
    last_analysis: z.date(),
    next_analysis_due: z.date(),
    analysis_version: z.number()
  })
});

// TypeScript types
export type EffectivenessMeasurement = z.infer<typeof EffectivenessMeasurementSchema>;
export type EffectivenessAnalytics = z.infer<typeof EffectivenessAnalyticsSchema>;
export type CrossAgentLearning = z.infer<typeof CrossAgentLearningSchema>;
export type EffectivenessTracking = z.infer<typeof EffectivenessTrackingSchema>;