Agent Evaluation & Benchmarking

import json
from your_agent import run_agent

def evaluate_agent(agent, test_cases, grader):
    """Run agent on test cases and grade results."""
    results = []

    for tc in test_cases:
        # Step 1: Run the agent
        try:
            output = run_agent(tc["input"])
        except Exception as e:
            results.append({"task": tc["id"], "success": False, "error": str(e)})
            continue

        # Step 2: Grade the output
        grade = grader(output, tc["expected"])
        results.append({
            "task": tc["id"],
            "success": grade["pass"],
            "score": grade.get("score", 0),
            "steps": output.get("steps", 0),
            "cost": output.get("cost", 0),
            "trajectory": grade.get("reasoning", "")
        })

    # Step 3: Aggregate
    passed = sum(1 for r in results if r["success"])
    return {
        "total": len(results),
        "passed": passed,
        "success_rate": passed / len(results),
        "avg_steps": sum(r["steps"] for r in results) / len(results),
        "avg_cost": sum(r["cost"] for r in results) / len(results),
        "details": results
    }

def exact_match_grader(output, expected):
    """For tasks with deterministic output (classification, extraction)."""
    return {
        "pass": output["answer"].strip().lower() == expected.strip().lower(),
        "score": 1.0 if output["answer"].strip().lower() == expected.strip().lower() else 0.0
    }

def llm_judge_grader(output, expected, model="gpt-4o"):
    """For open-ended tasks (writing, reasoning, analysis).
    Use a different model than the agent to avoid self-evaluation bias."""
    prompt = f"""Grade this agent output against the expected answer.
Expected: {expected}
Agent output: {output["answer"]}

Rate on: accuracy (0-4), completeness (0-3), conciseness (0-3).
Return JSON: {{"accuracy": int, "completeness": int, "conciseness": int, "pass": bool}}"""

    response = openai.chat.completions.create(
        model=model,
        messages=[{"role": "user", "content": prompt}],
        response_format={"type": "json_object"}
    )
    grade = json.loads(response.choices[0].message.content)
    total = sum(grade[k] for k in ["accuracy", "completeness", "conciseness"])
    grade["pass"] = total >= 7  # Threshold for passing
    grade["score"] = total / 10.0
    return grade

def tool_accuracy_grader(output, expected_calls):
    """For tasks where tool call correctness matters more than final output."""
    actual_calls = [(tc["name"], tc["args"]) for tc in output.get("tool_calls", [])]
    expected = [(ec["name"], ec["args"]) for ec in expected_calls]

    correct = sum(1 for ac, ec in zip(actual_calls, expected)
                  if ac[0] == ec[0] and all(ac[1].get(k) == ec[1].get(k)
                  for k in ec[1]))
    return {
        "pass": correct == len(expected_calls),
        "score": correct / len(expected_calls) if expected_calls else 0
    }

Task: "What's the population of Tokyo and what % of Japan is that?"

Bad trajectory (3/10):               Good trajectory (8/10):
1. Search "Tokyo population"        1. Search "Tokyo population 2024"
2. Search "Japan area" ← wrong     2. Search "Japan population 2024"
3. Search "Japan population"        3. Calculate 37M / 124M = 29.8%
4. Calculate (wrong numbers)        4. Verify with second source
5. Search "Tokyo population 2024"   5. Return answer with sources
6. Calculate (now correct)
7. Return answer

def score_trajectory(trajectory, optimal_steps):
    """Score a trajectory on efficiency, correctness, and safety."""
    score = 10.0

    # Efficiency penalty: each step beyond optimal
    extra_steps = len(trajectory["steps"]) - optimal_steps
    if extra_steps > 0:
        score -= min(extra_steps * 0.5, 3.0)

    # Error penalty: each failed tool call or correction
    errors = sum(1 for step in trajectory["steps"] if step.get("error") or step.get("correction"))
    score -= min(errors * 1.0, 4.0)

    # Safety bonus: no harmful actions
    if not trajectory.get("safety_violations"):
        score += 1.0

    return max(0, score)

1. Run agent on test suite → collect results
2. Identify failure patterns (which tasks fail? why?)
3. Fix the agent (system prompt, tools, model, memory)
4. Re-run test suite → compare to baseline
5. If improvement → deploy. If not → go to step 3.