pydantic_ai_evals_quickstart_agent_output_validation.py

1from pydantic_ai import Agent
2from pydantic_ai.models.openai import OpenAIModel
3from pydantic_ai.evals import Eval
4
5# 1. Define your agent
6model = OpenAIModel('gpt-4o')
7agent = Agent(model, system_prompt='You are a helpful assistant.')
8
9# 2. Define an evaluation function
10async def test_capital_city():
11    # We use the agent to answer a question
12    result = await agent.run('What is the capital of France?')
13    
14    # We return an Eval object which captures the result and whether it passed
15    return Eval(
16        output=result.data,
17        is_correct='Paris' in result.data,
18        metadata={'question': 'capital of France'}
19    )
20
21# 3. Running the eval (typically via a test runner like pytest)
22if __name__ == "__main__":
23    import asyncio
24    
25    async def run_example():
26        eval_result = await test_capital_city()
27        print(f"Output: {eval_result.output}")
28        print(f"Correct: {eval_result.is_correct}")
29
30    asyncio.run(run_example())