Evaluation Strategies for AI Agents

{
  "question": "Do you have printer paper in stock?",
  "reference_answer": "Should check database and provide current inventory",
  "metadata": {
    "category": "inventory_check",
    "requires_tools": ["query_database"]
  }
}

{
  "question": "What's your return policy?",
  "reference_answer": "Should search knowledge base for return policy",
  "metadata": {
    "category": "policy_question",
    "requires_tools": ["search_knowledge_base"]
  }
}

import asyncio
from pathlib import Path
from langsmith import aevaluate
from agent_v1 import chat, load_knowledge_base

async def chat_wrapper(inputs: dict) -> dict:
    """Wrapper to adapt dataset inputs to chat function signature."""
    question = inputs.get("question", "")
    result = await chat(question)
    return {"answer": result["output"], "messages": result["messages"]}

async def main():
    # Load any required resources (like knowledge base)
    await load_knowledge_base(kb_dir="./knowledge_base")
    
    # Run evaluation
    results = await aevaluate(
        chat_wrapper,
        data="officeflow-dataset",  # Dataset name in LangSmith
        experiment_prefix="agent-v1"
    )
    
    print(f"Evaluation complete! Results: {results}")

if __name__ == "__main__":
    asyncio.run(main())

import re

SCHEMA_PATTERNS = [
    r"PRAGMA\s+table_info",
    r"SELECT\s+.*FROM\s+sqlite_master",
    r"PRAGMA\s+database_list",
]

def _is_schema_query(sql: str) -> bool:
    """Return True if the SQL is a schema-inspection query."""
    for pattern in SCHEMA_PATTERNS:
        if re.search(pattern, sql, re.IGNORECASE):
            return True
    return False

def schema_before_query(run, example) -> dict:
    """Score 1 if agent checks DB schema before querying data, 0 otherwise."""
    # Extract tool calls from the run
    tool_calls = _extract_tool_calls(run)
    db_calls = [tc for tc in tool_calls if tc["name"] == "query_database"]
    
    if not db_calls:
        return {"score": 1, "comment": "No database calls - N/A"}
    
    # Check if schema query appears before first data query
    seen_schema_check = False
    for tc in db_calls:
        sql = tc.get("arguments", "")
        if _is_schema_query(sql):
            seen_schema_check = True
        else:
            # First data query - was schema checked first?
            if not seen_schema_check:
                return {
                    "score": 0,
                    "comment": f"Agent queried data without checking schema first."
                }
            break
    
    return {"score": 1, "comment": "Agent checked schema before querying"}

from openai import OpenAI

client = OpenAI()

CORRECTNESS_PROMPT = """You are evaluating an AI customer support agent's response.

User Question: {question}

Agent Response: {response}

Reference Context: {reference}

Is the agent's response correct and helpful? Consider:
- Does it answer the question accurately?
- Does it use appropriate tools if needed?
- Is the information factually correct?

Respond with a score from 0 to 1:
- 1.0: Completely correct and helpful
- 0.5: Partially correct but missing important details
- 0.0: Incorrect or unhelpful

Output only the numeric score."""

def evaluate_correctness(run, example) -> dict:
    """Use GPT to evaluate response correctness."""
    question = example.inputs.get("question", "")
    response = run.outputs.get("answer", "")
    reference = example.outputs.get("reference_answer", "N/A")
    
    result = client.chat.completions.create(
        model="gpt-5-nano",
        messages=[
            {"role": "system", "content": "You are an evaluator. Respond with only a number."},
            {"role": "user", "content": CORRECTNESS_PROMPT.format(
                question=question,
                response=response,
                reference=reference
            )}
        ]
    )
    
    score = float(result.choices[0].message.content.strip())
    
    return {
        "score": score,
        "comment": f"LLM judge scored {score} for correctness"
    }

from openai import OpenAI
from langsmith import evaluate

client = OpenAI()

CONCISENESS_PROMPT = """You are evaluating two responses to the same customer question.
Determine which response is MORE CONCISE while still providing all crucial information.

**Question:** {question}

**Response A:**
{response_a}

**Response B:**
{response_b}

Output your verdict as a single number:
1 if Response A is more concise while preserving crucial information
2 if Response B is more concise while preserving crucial information
0 if they are roughly equal"""

def conciseness_evaluator(inputs: dict, outputs: list[dict]) -> list[int]:
    """Compare two responses for conciseness."""
    response = client.chat.completions.create(
        model="gpt-5-nano",
        messages=[
            {"role": "system", "content": "You are a conciseness evaluator. Respond with only: 0, 1, or 2."},
            {"role": "user", "content": CONCISENESS_PROMPT.format(
                question=inputs["question"],
                response_a=outputs[0].get("answer", "N/A"),
                response_b=outputs[1].get("answer", "N/A"),
            )}
        ],
    )
    
    preference = int(response.choices[0].message.content.strip())
    
    if preference == 1:
        return [1, 0]  # A wins
    elif preference == 2:
        return [0, 1]  # B wins
    else:
        return [0, 0]  # Tie

# Run pairwise evaluation
evaluate(
    ("agent-v4-experiment", "agent-v5-experiment"),
    evaluators=[conciseness_evaluator],
    randomize_order=True,  # Prevent position bias
)

from langsmith import aevaluate

results = await aevaluate(
    chat_wrapper,
    data="officeflow-dataset",
    evaluators=[
        schema_before_query,      # Code-based: tool usage correctness
        evaluate_correctness,      # LLM-judge: response quality
        measure_latency,           # Code-based: performance
        check_hallucination,       # LLM-judge: factual accuracy
    ],
    experiment_prefix="agent-v2"
)

1. Run experiment on current agent version
2. Analyze results - identify failures
3. Hypothesize improvements (better prompt, different tool, etc.)
4. Implement changes
5. Run new experiment
6. Compare with previous version
7. Repeat until metrics meet your targets