aiwg/agentic/code/frameworks/sdlc-complete/schemas/research/lats-evaluation.yaml

Cognitive architecture for AI-augmented software development with structured memory, ensemble validation, and closed-loop correction. FAIR-aligned artifacts, 84% cost reduction via human-in-the-loop, standards adopted by 100+ organizations.

# LATS Hybrid Value Function Schema
# Based on REF-024 LATS (ICML 2024)
# Finding: 92.7% HumanEval pass@1 (state-of-the-art)
# Formula: V(s) = λ * V_LM(s) + (1-λ) * V_SC(s) where λ=0.5 optimal
# Issue: #98

$schema: "https://json-schema.org/draft/2020-12/schema"
$id: "https://aiwg.io/schemas/lats-evaluation/v1"
title: "LATS Hybrid Value Function Schema"
description: |
  Hybrid value function combining LLM scoring with self-consistency voting.
  Achieves state-of-the-art performance through balanced ensemble approach.

type: object
required:
  - evaluation_id
  - target
  - v_lm
  - v_sc
  - combined

properties:
  evaluation_id:
    type: string
    format: uuid
    description: "Unique evaluation identifier"

  timestamp:
    type: string
    format: date-time

  target:
    type: object
    required: [type, content_hash]
    properties:
      type:
        type: string
        enum:
          - code
          - document
          - architecture_decision
          - requirement
          - test_case
          - artifact
        description: "Type of artifact being evaluated"
      path:
        type: string
        description: "File path if applicable"
      content_hash:
        type: string
        description: "SHA-256 hash of content"
      context:
        type: string
        description: "Evaluation context"

  # V_LM: LLM-generated scalar score
  v_lm:
    type: object
    required: [score, prompt_template, rationale]
    description: "LLM-based value estimation"
    properties:
      score:
        type: number
        minimum: 0
        maximum: 1
        description: "LLM-generated score (0-1)"

      prompt_template:
        type: string
        enum:
          - quality_assessment
          - correctness_check
          - completeness_review
          - custom
        description: "Prompt template used for scoring"

      custom_prompt:
        type: string
        description: "Custom prompt if template is 'custom'"

      rationale:
        type: string
        minLength: 50
        description: "LLM's explanation for the score"

      aspects:
        type: array
        items:
          type: object
          properties:
            name:
              type: string
            score:
              type: number
              minimum: 0
              maximum: 1
            weight:
              type: number
              minimum: 0
              maximum: 1
        description: "Breakdown by quality aspects"

      confidence:
        type: number
        minimum: 0
        maximum: 1
        description: "LLM's confidence in score"

      model:
        type: string
        description: "Model used for evaluation"

  # V_SC: Self-consistency voting
  v_sc:
    type: object
    required: [score, rollouts, voting_method]
    description: "Self-consistency based value estimation"
    properties:
      score:
        type: number
        minimum: 0
        maximum: 1
        description: "Self-consistency score (0-1)"

      n_rollouts:
        type: integer
        minimum: 3
        default: 5
        description: "Number of rollouts (default: 5)"

      rollouts:
        type: array
        minItems: 3
        items:
          type: object
          required: [id, outcome, cluster]
          properties:
            id:
              type: integer
            outcome:
              type: string
              description: "Result of this rollout"
            outcome_hash:
              type: string
              description: "Hash for clustering"
            cluster:
              type: integer
              description: "Cluster assignment"
            execution_success:
              type: boolean
              description: "Did rollout execute successfully?"
            test_passed:
              type: boolean
              description: "Did tests pass? (for code)"
        description: "Individual rollout results"

      voting_method:
        type: string
        enum:
          - majority
          - plurality
          - weighted
          - execution_guided
        description: "How votes are counted"

      cluster_distribution:
        type: array
        items:
          type: object
          properties:
            cluster_id:
              type: integer
            count:
              type: integer
            representative:
              type: string
        description: "Distribution of outcomes across clusters"

      consensus_strength:
        type: number
        minimum: 0
        maximum: 1
        description: "How strong is the consensus?"

  # Combined score
  combined:
    type: object
    required: [score, lambda]
    description: "Combined hybrid value"
    properties:
      score:
        type: number
        minimum: 0
        maximum: 1
        description: "V(s) = λ * V_LM(s) + (1-λ) * V_SC(s)"

      lambda:
        type: number
        minimum: 0
        maximum: 1
        default: 0.5
        description: "Weighting factor (0.5 optimal per LATS paper)"

      formula:
        type: string
        default: "V(s) = λ * V_LM(s) + (1-λ) * V_SC(s)"

      interpretation:
        type: string
        enum:
          - high_confidence   # Both V_LM and V_SC agree, high scores
          - llm_favored       # V_LM high, V_SC low
          - consensus_favored # V_LM low, V_SC high
          - low_confidence    # Both low
          - divergent         # High disagreement
        description: "How to interpret the combined score"

      decision:
        type: string
        enum:
          - accept
          - refine
          - reject
          - expand_search
        description: "Recommended action based on score"

  # Tuning metadata
  tuning:
    type: object
    description: "Lambda tuning for task type"
    properties:
      task_type:
        type: string
        description: "Type of task (affects optimal lambda)"
      recommended_lambda:
        type: number
        description: "Recommended lambda for this task type"
      lambda_rationale:
        type: string
        description: "Why this lambda is recommended"

  # Search context (if part of tree search)
  search_context:
    type: object
    description: "Context when used in tree search"
    properties:
      node_id:
        type: string
      depth:
        type: integer
      parent_score:
        type: number
      siblings_evaluated:
        type: integer
      best_sibling_score:
        type: number

# Lambda tuning recommendations by task type
lambda_recommendations:
  code_generation:
    lambda: 0.4
    rationale: "Execution-based V_SC provides strong signal for code"
  documentation:
    lambda: 0.6
    rationale: "LLM judgment more reliable for prose quality"
  architecture:
    lambda: 0.5
    rationale: "Balance needed between expert judgment and consistency"
  requirements:
    lambda: 0.55
    rationale: "Slight LLM bias for assessing completeness"
  test_cases:
    lambda: 0.45
    rationale: "Slight execution bias for test quality"

# Prompt templates for V_LM
v_lm_templates:
  quality_assessment: |
    Evaluate the following {{artifact_type}} on a scale of 0-1:

    {{content}}

    Consider:
    1. Correctness: Does it achieve its intended purpose?
    2. Completeness: Are all necessary elements present?
    3. Clarity: Is it easy to understand?
    4. Best practices: Does it follow established conventions?

    Provide:
    - Overall score (0-1)
    - Aspect scores
    - Brief rationale

  correctness_check: |
    Verify the correctness of this {{artifact_type}}:

    {{content}}

    Check for:
    1. Logical errors
    2. Inconsistencies
    3. Missing edge cases
    4. Incorrect assumptions

    Score: 0 (definitely incorrect) to 1 (definitely correct)

  completeness_review: |
    Assess the completeness of this {{artifact_type}}:

    {{content}}

    Requirements:
    {{requirements}}

    Score: 0 (missing critical elements) to 1 (fully complete)

# Decision thresholds
thresholds:
  accept:
    combined_score: 0.85
    interpretation: ["high_confidence"]
  refine:
    combined_score_range: [0.5, 0.85]
    interpretation: ["llm_favored", "consensus_favored"]
  reject:
    combined_score: 0.5
    interpretation: ["low_confidence"]
  expand_search:
    condition: "divergent interpretation AND search_budget_remaining"

# Integration with AIWG agents
agent_integration:
  quality_agents:
    - "Test Engineer"
    - "Code Reviewer"
    - "Security Auditor"
  evaluation_trigger:
    - artifact_complete
    - iteration_complete
    - gate_check

# Examples
examples:
  - evaluation_id: "eval-001-example"
    target:
      type: code
      path: "src/auth/validate.ts"
      content_hash: "abc123..."
    v_lm:
      score: 0.75
      prompt_template: quality_assessment
      rationale: "Good structure, missing edge case handling for null input"
      aspects:
        - name: correctness
          score: 0.7
          weight: 0.3
        - name: completeness
          score: 0.6
          weight: 0.25
        - name: clarity
          score: 0.9
          weight: 0.2
        - name: best_practices
          score: 0.8
          weight: 0.25
      confidence: 0.8
      model: "claude-3-opus"
    v_sc:
      score: 0.8
      n_rollouts: 5
      rollouts:
        - id: 1
          outcome: "Tests pass, handles standard cases"
          cluster: 1
          execution_success: true
          test_passed: true
        - id: 2
          outcome: "Tests pass, handles standard cases"
          cluster: 1
          execution_success: true
          test_passed: true
        - id: 3
          outcome: "Tests pass, handles standard cases"
          cluster: 1
          execution_success: true
          test_passed: true
        - id: 4
          outcome: "Fails on null input"
          cluster: 2
          execution_success: true
          test_passed: false
        - id: 5
          outcome: "Tests pass, handles standard cases"
          cluster: 1
          execution_success: true
          test_passed: true
      voting_method: majority
      cluster_distribution:
        - cluster_id: 1
          count: 4
          representative: "Tests pass, handles standard cases"
        - cluster_id: 2
          count: 1
          representative: "Fails on null input"
      consensus_strength: 0.8
    combined:
      score: 0.775  # 0.5 * 0.75 + 0.5 * 0.8
      lambda: 0.5
      interpretation: high_confidence
      decision: refine

# References
references:
  research:
    - "@.aiwg/research/findings/REF-024-lats.md"
  implementation:
    - "#98"
  related:
    - "@agentic/code/frameworks/sdlc-complete/schemas/research/quality-assessment.yaml"
    - "@agentic/code/addons/ralph/schemas/actionable-feedback.yaml"