@lobehub/chat/packages/prompts/promptfoo/translate/eval.yaml

Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.

description: Test translation accuracy between different languages

providers:
  - openai:chat:gpt-5-mini
  - openai:chat:claude-3-5-haiku-latest
  - openai:chat:gemini-flash-latest
  - openai:chat:deepseek-chat

prompts:
  - file://promptfoo/translate/prompt.ts

tests:
  - vars:
      content: 'Hello, how are you?'
      from: 'en-US'
      to: 'zh-CN'
    assert:
      - type: contains-any
        value: ['你好', '您好']
      - type: not-contains
        value: 'Hello'

  - vars:
      content: '你好，你怎么样？'
      from: 'zh-CN'
      to: 'en-US'
    assert:
      - type: contains-any
        value: ['Hello', 'Hi', 'how are you', 'How are you']
      - type: not-contains
        value: '你好'

  - vars:
      content: 'Je suis content de vous rencontrer'
      from: 'fr-FR'
      to: 'en-US'
    assert:
      - type: contains-any
        value: ['pleased', 'happy', 'glad', 'meet', 'meeting']
      - type: not-contains
        value: 'Je suis'

  - vars:
      content: 'The weather is beautiful today'
      from: 'en-US'
      to: 'es-ES'
    assert:
      - type: contains-any
        value: ['tiempo', 'clima', 'hermoso', 'bonito', 'hoy', 'día', 'precioso']
      - type: not-contains
        value: 'weather'

  - vars:
      content: 'I love programming with TypeScript'
      from: 'en-US'
      to: 'ja-JP'
    assert:
      - type: contains
        value: 'TypeScript' # Technical terms often remain unchanged
      - type: not-contains
        value: 'I love'

  - vars:
      content: 'Machine learning is revolutionizing technology'
      from: 'en-US'
      to: 'de-DE'
    assert:
      - type: contains-any
        value: ['Technologie', 'revolutioniert', 'maschinelles', 'Lernen']
      - type: not-contains
        value: 'Machine learning'

  - vars:
      content: 'API_KEY_12345'
      from: 'en-US'
      to: 'zh-CN'
    assert:
      - type: contains
        value: 'API_KEY_12345'