@nestbox-ai/cli
Version:
The cli tools that helps developers to build agents
249 lines (242 loc) • 8.14 kB
YAML
# JSON Schema for evaluation test case files
# This schema defines the structure of YAML files used for GraphRAG evaluation
# Used by eval-runner.ts to validate test case files
$schema: "http://json-schema.org/draft-07/schema#"
title: "Evaluation Test Cases"
description: "Schema for nest-doc-processing-cli evaluation test case YAML files"
type: object
additionalProperties: false
# At least one of basic_search, local_search, or global_search must be present
anyOf:
- required: [basic_search]
- required: [local_search]
- required: [global_search]
properties:
eval_params:
type: object
description: "Optional query parameters that apply to all test cases in each search mode"
additionalProperties: false
properties:
basic_search:
type: object
description: "Parameters for all basic search queries"
additionalProperties: false
properties:
k:
type: integer
minimum: 1
maximum: 100
description: "Number of text units to retrieve"
temperature:
type: number
minimum: 0
maximum: 2
description: "LLM temperature"
max_tokens:
type: integer
minimum: 1
maximum: 16000
description: "Maximum response tokens"
local_search:
type: object
description: "Parameters for all local search queries"
additionalProperties: false
properties:
text_unit_prop:
type: number
minimum: 0
maximum: 1
description: "Text unit proportion"
community_prop:
type: number
minimum: 0
maximum: 1
description: "Community proportion"
conversation_history_max_turns:
type: integer
minimum: 0
maximum: 20
description: "Max conversation turns"
top_k_entities:
type: integer
minimum: 1
maximum: 100
description: "Top k entities"
top_k_relationships:
type: integer
minimum: 1
maximum: 100
description: "Top k relationships"
max_context_tokens:
type: integer
minimum: 1000
maximum: 128000
description: "Max context tokens"
temperature:
type: number
minimum: 0
maximum: 2
description: "LLM temperature"
max_tokens:
type: integer
minimum: 1
maximum: 16000
description: "Maximum response tokens"
global_search:
type: object
description: "Parameters for all global search queries"
additionalProperties: false
properties:
max_context_tokens:
type: integer
minimum: 1000
maximum: 128000
description: "Max context tokens"
data_max_tokens:
type: integer
minimum: 1000
maximum: 128000
description: "Max data tokens"
map_max_length:
type: integer
minimum: 100
maximum: 10000
description: "Map phase max length"
reduce_max_length:
type: integer
minimum: 100
maximum: 10000
description: "Reduce phase max length"
dynamic_search_threshold:
type: integer
minimum: 0
maximum: 10
description: "Rating threshold"
dynamic_search_keep_parent:
type: boolean
description: "Keep parent community"
dynamic_search_num_repeats:
type: integer
minimum: 0
maximum: 10
description: "Rating repeats"
dynamic_search_use_summary:
type: boolean
description: "Use community summary"
dynamic_search_max_level:
type: integer
minimum: 0
maximum: 10
description: "Max hierarchy level"
temperature:
type: number
minimum: 0
maximum: 2
description: "LLM temperature"
max_tokens:
type: integer
minimum: 1
maximum: 16000
description: "Maximum response tokens"
basic_search:
type: array
description: "Test cases for simple vector similarity search queries"
minItems: 1
items:
type: object
required:
- question
- expected_answer
- bad_answer
additionalProperties: false
properties:
question:
type: string
description: "The question to ask GraphRAG (simple factual queries)"
minLength: 1
examples:
- "What is mentioned about parking?"
- "Are pets allowed?"
- "What is the security deposit?"
expected_answer:
type: string
description: "The expected correct answer (for similarity comparison)"
minLength: 1
examples:
- "The property includes one assigned parking space in the garage."
- "Small pets under 20 lbs are allowed with a $300 deposit."
bad_answer:
type: string
description: "A bad/wrong answer (for negative similarity comparison)"
minLength: 1
examples:
- "No parking information is available."
- "The document does not mention pets."
local_search:
type: array
description: "Test cases for entity-focused local search queries"
minItems: 1
items:
type: object
required:
- question
- expected_answer
- bad_answer
additionalProperties: false
properties:
question:
type: string
description: "The question to ask GraphRAG"
minLength: 1
examples:
- "What is the monthly rent amount?"
- "What are the payment terms?"
- "Who is the landlord?"
expected_answer:
type: string
description: "The expected correct answer (for similarity comparison)"
minLength: 1
examples:
- "The monthly rent is $2,500, payable on the first of each month."
- "Rent is due on the 1st of each month, with a 5-day grace period."
bad_answer:
type: string
description: "A bad/wrong answer (for negative similarity comparison)"
minLength: 1
examples:
- "I don't know what the rent is."
- "Payment is required monthly."
global_search:
type: array
description: "Test cases for community-based global search queries (summaries)"
minItems: 1
items:
type: object
required:
- question
- expected_answer
- bad_answer
additionalProperties: false
properties:
question:
type: string
description: "The question to ask GraphRAG (typically asking for summaries)"
minLength: 1
examples:
- "Summarize all tenant obligations in the document."
- "What are the key themes and entities discussed in this document?"
- "What are all the financial terms mentioned?"
expected_answer:
type: string
description: "The expected correct answer (for similarity comparison)"
minLength: 1
examples:
- "Tenants must pay rent on time, maintain the premises, report damages within 24 hours, and follow all building rules."
- "The document discusses rental terms, tenant and landlord obligations, payment schedules, and termination conditions."
bad_answer:
type: string
description: "A bad/wrong answer (for negative similarity comparison)"
minLength: 1
examples:
- "The tenant has some responsibilities."
- "This is a legal document."