aiwg/agentic/code/frameworks/sdlc-complete/schemas/flows/quality-assurance.yaml

Cognitive architecture for AI-augmented software development with structured memory, ensemble validation, and closed-loop correction. FAIR-aligned artifacts, 84% cost reduction via human-in-the-loop, standards adopted by 100+ organizations.

# Quality Assurance Framework Schema
# Based on REF-059 LitLLM best practices
# Issues: #219 (Hallucination Detection), #220 (Retrieval-First Pattern)

$schema: "https://json-schema.org/draft/2020-12/schema"
$id: "https://aiwg.io/schemas/quality-assurance/v1"
title: "Quality Assurance Framework Schema"
description: |
  Comprehensive quality assurance framework implementing hallucination detection
  and retrieval-first workflow patterns per REF-059 LitLLM.

type: object
required:
  - version
  - hallucination_detection
  - retrieval_first

properties:
  version:
    type: string
    pattern: "^\\d+\\.\\d+\\.\\d+$"
    default: "1.0.0"

  hallucination_detection:
    $ref: "#/$defs/HallucinationDetection"

  retrieval_first:
    $ref: "#/$defs/RetrievalFirstPattern"

$defs:
  HallucinationDetection:
    type: object
    description: "Pattern-based hallucination detection configuration"
    properties:
      enabled:
        type: boolean
        default: true

      rule_categories:
        type: object
        properties:
          structural:
            type: object
            description: "Structural hallucination rules"
            properties:
              contradictions:
                type: object
                properties:
                  enabled:
                    type: boolean
                    default: true
                  patterns:
                    type: array
                    items:
                      type: string
                    default:
                      - "X is true.*X is false"
                      - "always.*never"
                      - "must.*must not"
                  severity:
                    type: string
                    default: "warning"

              tense_inconsistency:
                type: object
                properties:
                  enabled:
                    type: boolean
                    default: true
                  severity:
                    type: string
                    default: "warning"

              unsupported_claims:
                type: object
                properties:
                  enabled:
                    type: boolean
                    default: true
                  markers:
                    type: array
                    items:
                      type: string
                    default:
                      - "studies show"
                      - "research proves"
                      - "according to"
                  require_citation:
                    type: boolean
                    default: true
                  severity:
                    type: string
                    default: "warning"

          technical:
            type: object
            description: "Technical hallucination rules"
            properties:
              invented_api_methods:
                type: object
                properties:
                  enabled:
                    type: boolean
                    default: true
                  schema_sources:
                    type: array
                    items:
                      type: string
                    default:
                      - "node_modules/@types/**/*.d.ts"
                      - "agentic/code/frameworks/sdlc-complete/schemas/flows/**/*.json"
                  severity:
                    type: string
                    default: "error"

              nonexistent_files:
                type: object
                properties:
                  enabled:
                    type: boolean
                    default: true
                  pattern:
                    type: string
                    default: "@[\\w\\/\\-\\.]+\\.(?:md|ts|tsx|js|json)"
                  validate_existence:
                    type: boolean
                    default: true
                  severity:
                    type: string
                    default: "error"

              invalid_commands:
                type: object
                properties:
                  enabled:
                    type: boolean
                    default: true
                  known_commands:
                    type: array
                    items:
                      type: string
                    default:
                      - "aiwg"
                      - "npm"
                      - "git"
                      - "curl"
                  severity:
                    type: string
                    default: "error"

              nonexistent_dependencies:
                type: object
                properties:
                  enabled:
                    type: boolean
                    default: true
                  check_npm:
                    type: boolean
                    default: true
                  severity:
                    type: string
                    default: "error"

          factual:
            type: object
            description: "Factual hallucination rules"
            properties:
              outdated_versions:
                type: object
                properties:
                  enabled:
                    type: boolean
                    default: true
                  version_sources:
                    type: array
                    items:
                      type: string
                    default:
                      - "package.json"
                      - "package-lock.json"
                  severity:
                    type: string
                    default: "warning"

              incorrect_syntax:
                type: object
                properties:
                  enabled:
                    type: boolean
                    default: true
                  languages:
                    type: array
                    items:
                      type: string
                    default:
                      - "typescript"
                      - "javascript"
                      - "yaml"
                      - "json"
                  severity:
                    type: string
                    default: "error"

              misattributed_quotes:
                type: object
                properties:
                  enabled:
                    type: boolean
                    default: true
                  verify_sources:
                    type: boolean
                    default: true
                  severity:
                    type: string
                    default: "warning"

      validation_config:
        type: object
        properties:
          storage_path:
            type: string
            default: ".aiwg/quality/hallucination-rules.json"
          on_violation:
            type: object
            properties:
              block_output:
                type: boolean
                default: false
                description: "Block output on critical violations"
              report_violations:
                type: boolean
                default: true
              suggest_remediation:
                type: boolean
                default: true

  RetrievalFirstPattern:
    type: object
    description: "Retrieval-first workflow pattern for agents"
    properties:
      enabled:
        type: boolean
        default: true

      enforcement:
        type: string
        enum: [strict, advisory, disabled]
        default: "advisory"
        description: |
          strict: Block execution if retrieval not performed
          advisory: Warn but allow execution
          disabled: No enforcement

      workflow:
        type: object
        description: "Retrieval-first workflow steps"
        properties:
          step1_context_retrieval:
            type: object
            properties:
              required:
                type: boolean
                default: true
              actions:
                type: array
                items:
                  type: string
                default:
                  - "List required @-mentions"
                  - "Identify relevant files"
                  - "Determine project state to verify"

          step2_read_context:
            type: object
            properties:
              required:
                type: boolean
                default: true
              actions:
                type: array
                items:
                  type: string
                default:
                  - "Use Read tool for each identified file"
                  - "Capture relevant content"
                  - "Note any missing files"

          step3_verify_completeness:
            type: object
            properties:
              required:
                type: boolean
                default: true
              actions:
                type: array
                items:
                  type: string
                default:
                  - "Confirm all necessary information retrieved"
                  - "Identify any gaps"
                  - "Request additional files if needed"

          step4_execute_task:
            type: object
            properties:
              required:
                type: boolean
                default: true
              actions:
                type: array
                items:
                  type: string
                default:
                  - "Perform task using retrieved context"
                  - "Reference sources in output"
                  - "Validate against context before completing"

      anti_patterns:
        type: array
        items:
          type: object
          properties:
            name:
              type: string
            description:
              type: string
            detection_pattern:
              type: string
        default:
          - name: "immediate-generation"
            description: "Agent generates output without reading files"
            detection_pattern: "No Read tool calls before Write"
          - name: "assumed-knowledge"
            description: "Agent assumes file contents without verification"
            detection_pattern: "References to files not in context"
          - name: "generic-output"
            description: "Output doesn't reference specific project files"
            detection_pattern: "No @-mentions in generated content"

      agent_template:
        type: string
        default: |
          ## Execution Protocol

          **CRITICAL: Retrieval-First Workflow**

          ### Before ANY Task Execution:

          1. **List Context**: Identify all @-mentions and files needed
             - What files do you need to read?
             - What @-mentions are relevant?
             - What project state must you verify?

          2. **Read Context**: Use Read tool for each file
             - Read all identified files
             - Capture relevant content
             - Note any missing files

          3. **Verify Completeness**: Confirm you have all information
             - All necessary context retrieved?
             - Any gaps to fill?
             - Ready to proceed?

          4. **Execute Task**: Perform with retrieved context
             - Use actual code/content from files
             - Reference sources in output
             - Validate against context

          5. **Validate Output**: Check against context
             - All references valid?
             - Matches actual implementation?
             - Follows project patterns?

          ### FORBIDDEN

          - Generating output without reading files
          - Assuming file contents
          - Creating generic output without @-mentions
          - Inventing API methods or paths

# Validation rules schema
validation_rule:
  type: object
  required:
    - name
    - category
    - severity
    - check
  properties:
    name:
      type: string
    category:
      type: string
      enum: [structural, technical, factual]
    severity:
      type: string
      enum: [critical, error, warning, info]
    check:
      type: object
      properties:
        type:
          type: string
          enum: [pattern, function, schema]
        value:
          description: "Pattern regex, function name, or schema path"

# Validation result schema
validation_result:
  type: object
  properties:
    passed:
      type: boolean
    violations:
      type: array
      items:
        type: object
        properties:
          rule:
            type: string
          severity:
            type: string
          line:
            type: integer
          message:
            type: string
          suggestion:
            type: string
    summary:
      type: object
      properties:
        total_violations:
          type: integer
        critical:
          type: integer
        errors:
          type: integer
        warnings:
          type: integer

# CLI commands
cli_commands:
  validate_response:
    command: "aiwg validate-response <file>"
    description: "Validate content for hallucinations"
    options:
      - name: "--rules"
        description: "Rule categories to check"
      - name: "--strict"
        description: "Fail on any warning"
      - name: "--fix"
        description: "Auto-fix where possible"

  hallucination_check:
    command: "aiwg hallucination-check <content>"
    description: "Check content for hallucination patterns"
    options:
      - name: "--category"
        description: "Specific category (structural, technical, factual)"

  retrieval_audit:
    command: "aiwg retrieval-audit <session>"
    description: "Audit session for retrieval-first compliance"
    options:
      - name: "--report"
        description: "Generate detailed report"

# Agent protocol
agent_protocol:
  detect_hallucinations:
    description: "Check content for hallucination patterns"
    triggers:
      - agent_response_complete
      - pre_write_hook
    steps:
      - load_hallucination_rules
      - for_each_category:
          - apply_structural_rules
          - apply_technical_rules
          - apply_factual_rules
      - collect_violations
      - if_critical_violations:
          - block_output
          - report_violations
          - request_revision
      - if_warnings:
          - annotate_output
          - log_warnings
      - return_validation_result

  enforce_retrieval_first:
    description: "Enforce retrieval-first pattern"
    triggers:
      - task_start
    steps:
      - check_enforcement_level
      - if_strict:
          - require_context_listing
          - require_read_calls
          - require_verification
      - track_read_calls
      - on_write_attempt:
          - verify_context_read
          - if_no_reads:
              - block_write
              - request_retrieval
      - return_compliance_status

# Storage
storage:
  hallucination_rules: ".aiwg/quality/hallucination-rules.json"
  validation_logs: ".aiwg/logs/validation/"
  retrieval_audit: ".aiwg/logs/retrieval-audit/"

# Research targets (from REF-059 LitLLM)
research_targets:
  hallucination_detection: "Pattern-based detection of common hallucination types"
  retrieval_first: "Enforce retrieve-then-reason workflow"
  grounding: "Ground all outputs in actual project context"
  validation: "Automated quality assurance on agent outputs"

# Example validation report
example_report: |
  ================================================================================
                         QUALITY VALIDATION REPORT
  ================================================================================

  File: src/api/gitea-client.ts
  Agent: api-designer
  Timestamp: 2026-01-25T14:30:00Z

  RETRIEVAL-FIRST COMPLIANCE: ✓ PASSED
  - Context files read: 4
  - @-mentions resolved: 7
  - All references valid: Yes

  HALLUCINATION CHECK: ⚠ WARNINGS

  Violations:
  1. [WARNING] Line 45: Unsupported claim
     "Studies show this is the best approach"
     Rule: structural.unsupported_claims
     Fix: Add citation or rephrase

  2. [ERROR] Line 78: Non-existent file reference
     @src/utils/deprecated-helper.ts
     Rule: technical.nonexistent_files
     Fix: Verify file exists or remove reference

  Summary:
  - Critical: 0
  - Errors: 1
  - Warnings: 1

  Status: REVIEW REQUIRED

# References
references:
  research:
    - "@.aiwg/research/findings/REF-059-litllm.md"
  implementation:
    - "#219"
    - "#220"
  related:
    - "@.claude/rules/mention-wiring.md"
    - "@src/quality/"
    - "@agentic/code/frameworks/sdlc-complete/agents/"