Agent Benchmarks

// Benchmark Evaluation Methodology
// Standard approach for running agent benchmarks

1. ENVIRONMENT SETUP
   // Isolated, reproducible environment
   create_docker_container(benchmark_image)
   setup_dependencies(benchmark.requirements)
   initialize_test_harness()

2. AGENT CONFIGURATION
   configure_agent(
       model = "agent_model_name",
       max_tokens = 4096,
       temperature = 0,  // Deterministic for reproducibility
       tools = benchmark.available_tools
   )

3. EVALUATION LOOP
   for each task in benchmark.tasks:
       // Record start time
       start_time = now()

       // Agent attempts task
       result = agent.execute(task.input)

       // Capture metrics
       metrics = {
           "latency": now() - start_time,
           "tokens_used": result.total_tokens,
           "tool_calls": result.tool_call_count,
           "trajectory": result.reasoning_trace
       }

       // Evaluate result
       passed = benchmark.evaluate(result.output, task.expected)
       record_result(task.id, passed, metrics)

4. AGGREGATE RESULTS
   compute_accuracy()
   compute_confidence_intervals()
   generate_breakdown_by_category()
   export_results(format="json")

import json
from dataclasses import dataclass, asdict
from typing import List, Dict, Any
from datetime import datetime
import numpy as np
from scipy import stats

@dataclass
class BenchmarkConfig:
    name: str
    version: str
    agent_model: str
    temperature: float = 0.0
    max_tokens: int = 4096
    timeout_seconds: int = 1800

@dataclass
class TaskResult:
    task_id: str
    passed: bool
    latency_ms: float
    tokens_used: int
    tool_calls: int
    trajectory: List[str]
    output: str
    error: str = None

@dataclass
class BenchmarkResults:
    config: BenchmarkConfig
    results: List[TaskResult]
    timestamp: str
    total_time_seconds: float

    @property
    def accuracy(self) -> float:
        return sum(r.passed for r in self.results) / len(self.results)

    @property
    def confidence_interval(self) -> tuple:
        """95% confidence interval using Wilson score."""
        n = len(self.results)
        p = self.accuracy
        z = 1.96  # 95% CI
        denominator = 1 + z**2/n
        centre = (p + z**2/(2*n)) / denominator
        spread = z * np.sqrt((p*(1-p) + z**2/(4*n))/n) / denominator
        return (centre - spread, centre + spread)

    def to_json(self) -> str:
        return json.dumps({
            "config": asdict(self.config),
            "summary": {
                "accuracy": self.accuracy,
                "confidence_interval": self.confidence_interval,
                "total_tasks": len(self.results),
                "passed": sum(r.passed for r in self.results),
                "avg_latency_ms": np.mean([r.latency_ms for r in self.results]),
                "avg_tokens": np.mean([r.tokens_used for r in self.results]),
            },
            "results": [asdict(r) for r in self.results],
            "timestamp": self.timestamp,
            "total_time_seconds": self.total_time_seconds
        }, indent=2)


class BenchmarkRunner:
    def __init__(self, config: BenchmarkConfig, agent):
        self.config = config
        self.agent = agent
        self.results: List[TaskResult] = []

    async def run(self, tasks: List[Dict]) -> BenchmarkResults:
        start = datetime.now()

        for task in tasks:
            result = await self._run_task(task)
            self.results.append(result)

        return BenchmarkResults(
            config=self.config,
            results=self.results,
            timestamp=start.isoformat(),
            total_time_seconds=(datetime.now() - start).total_seconds()
        )

    async def _run_task(self, task: Dict) -> TaskResult:
        task_start = datetime.now()
        try:
            response = await self.agent.execute(
                task["input"],
                timeout=self.config.timeout_seconds
            )
            passed = self._evaluate(response.output, task["expected"])

            return TaskResult(
                task_id=task["id"],
                passed=passed,
                latency_ms=(datetime.now() - task_start).total_seconds() * 1000,
                tokens_used=response.tokens_used,
                tool_calls=len(response.tool_calls),
                trajectory=response.reasoning_trace,
                output=response.output
            )
        except Exception as e:
            return TaskResult(
                task_id=task["id"],
                passed=False,
                latency_ms=(datetime.now() - task_start).total_seconds() * 1000,
                tokens_used=0,
                tool_calls=0,
                trajectory=[],
                output="",
                error=str(e)
            )

public record BenchmarkConfig(
    string Name,
    string Version,
    string AgentModel,
    double Temperature = 0.0,
    int MaxTokens = 4096,
    int TimeoutSeconds = 1800
);

public record TaskResult(
    string TaskId,
    bool Passed,
    double LatencyMs,
    int TokensUsed,
    int ToolCalls,
    List<string> Trajectory,
    string Output,
    string? Error = null
);

public class BenchmarkResults
{
    public BenchmarkConfig Config { get; init; }
    public List<TaskResult> Results { get; init; } = new();
    public DateTime Timestamp { get; init; }
    public double TotalTimeSeconds { get; init; }

    public double Accuracy =>
        (double)Results.Count(r => r.Passed) / Results.Count;

    public (double Lower, double Upper) ConfidenceInterval
    {
        get
        {
            // Wilson score interval
            int n = Results.Count;
            double p = Accuracy;
            double z = 1.96; // 95% CI
            double denom = 1 + z * z / n;
            double centre = (p + z * z / (2 * n)) / denom;
            double spread = z * Math.Sqrt((p * (1 - p) + z * z / (4 * n)) / n) / denom;
            return (centre - spread, centre + spread);
        }
    }

    public string ToJson() => JsonSerializer.Serialize(new
    {
        Config,
        Summary = new
        {
            Accuracy,
            ConfidenceInterval,
            TotalTasks = Results.Count,
            Passed = Results.Count(r => r.Passed),
            AvgLatencyMs = Results.Average(r => r.LatencyMs),
            AvgTokens = Results.Average(r => r.TokensUsed)
        },
        Results,
        Timestamp,
        TotalTimeSeconds
    }, new JsonSerializerOptions { WriteIndented = true });
}

public class BenchmarkRunner
{
    private readonly BenchmarkConfig _config;
    private readonly IAgent _agent;
    private readonly List<TaskResult> _results = new();

    public BenchmarkRunner(BenchmarkConfig config, IAgent agent)
    {
        _config = config;
        _agent = agent;
    }

    public async Task<BenchmarkResults> RunAsync(List<BenchmarkTask> tasks)
    {
        var start = DateTime.UtcNow;

        foreach (var task in tasks)
        {
            var result = await RunTaskAsync(task);
            _results.Add(result);
        }

        return new BenchmarkResults
        {
            Config = _config,
            Results = _results,
            Timestamp = start,
            TotalTimeSeconds = (DateTime.UtcNow - start).TotalSeconds
        };
    }

    private async Task<TaskResult> RunTaskAsync(BenchmarkTask task)
    {
        var sw = Stopwatch.StartNew();
        try
        {
            var response = await _agent.ExecuteAsync(
                task.Input,
                TimeSpan.FromSeconds(_config.TimeoutSeconds)
            );

            bool passed = Evaluate(response.Output, task.Expected);

            return new TaskResult(
                task.Id, passed, sw.ElapsedMilliseconds,
                response.TokensUsed, response.ToolCalls.Count,
                response.ReasoningTrace, response.Output
            );
        }
        catch (Exception ex)
        {
            return new TaskResult(
                task.Id, false, sw.ElapsedMilliseconds,
                0, 0, new List<string>(), "", ex.Message
            );
        }
    }
}