{
  "tracker": "ai-agent-benchmarks",
  "title": "AI Agent Benchmark Observatory",
  "standfirst": "Independent tracking of agent capabilities, costs and evaluation methodologies.",
  "checked_at": "2026-11-15T10:00:00Z",
  "license": "CC BY 4.0, attribution: link to https://kernelpost.com/research/ai-agent-benchmarks/",
  "benchmarks": [
    {
      "id": "gdpval",
      "name": "GDPval",
      "maintainer": "OpenAI",
      "launched": "2025-09-25",
      "focus": "Professional knowledge-work tasks",
      "environment": "One-shot work assignments with reference files; deliverables are documents, slides, spreadsheets, diagrams and media",
      "tasks": "1,320 tasks across 44 occupations in 9 industries; 220-task gold set is public",
      "metric": "Win-or-tie rate against an expert's own deliverable",
      "grading": "Blind pairwise comparison by occupational experts (avg. 14 years' experience); an experimental automated grader exists but does not replace experts",
      "harness": "Set by the evaluator",
      "cost_reporting": "Launch post estimates models are ~100x faster and cheaper than experts, counting inference time and API billing only",
      "status": "No public leaderboard from OpenAI; third parties run the open gold set",
      "caveat": "One-shot only: no drafts, client feedback or ambiguity about what the job is.",
      "sources": [
        {
          "label": "OpenAI launch post",
          "url": "https://openai.com/index/gdpval/"
        },
        {
          "label": "Paper (arXiv 2510.04374)",
          "url": "https://arxiv.org/abs/2510.04374"
        }
      ],
      "snapshot": {
        "board": "OpenAI launch results (gold set)",
        "url": "https://openai.com/index/gdpval/",
        "version": "v1",
        "unit": "text",
        "note": "Launch-time result only. Of the seven models tested, Claude Opus 4.1 scored best, rated as good as or better than the expert deliverable in just under half of the 220 gold tasks.",
        "rows": []
      }
    },
    {
      "id": "gdpval-aa",
      "name": "GDPval-AA",
      "maintainer": "Artificial Analysis",
      "launched": null,
      "focus": "Professional knowledge-work tasks (GDPval gold set)",
      "environment": "Agentic loop with shell access and web browsing (Stirrup harness)",
      "tasks": "220 GDPval gold-set tasks",
      "metric": "Elo from blind pairwise comparisons, anchored to DeepSeek V4.1 Flash (max) at 1600",
      "grading": "Pairwise model-vs-model judgement",
      "harness": "Fixed by Artificial Analysis (Stirrup)",
      "cost_reporting": "Cost per task, output tokens and turns per task",
      "status": "Live, but labelled Legacy on Artificial Analysis",
      "caveat": "Elo is relative to the other models on the board, not to human experts, so a rising score need not mean better-than-human work.",
      "sources": [
        {
          "label": "Artificial Analysis GDPval-AA v2.1",
          "url": "https://artificialanalysis.ai/evaluations/gdpval-aa"
        }
      ],
      "snapshot": {
        "board": "GDPval-AA v2.1 leaderboard",
        "url": "https://artificialanalysis.ai/evaluations/gdpval-aa",
        "version": "v2.1",
        "unit": "Elo",
        "note": "The top three sit inside each other's confidence intervals: treat them as tied.",
        "rows": [
          {
            "rank": 1,
            "model": "Claude Opus 5.5",
            "setting": "Max, Default Fallback",
            "agent": "Stirrup",
            "score": 1866,
            "ci": "±26",
            "cost": null,
            "date": "2026-09"
          },
          {
            "rank": 2,
            "model": "Kimi K3.1 Pro",
            "setting": null,
            "agent": "Stirrup",
            "score": 1861,
            "ci": "±28",
            "cost": null,
            "date": "2026-10"
          },
          {
            "rank": 3,
            "model": "Muse Wonder Full",
            "setting": "Max",
            "agent": "Stirrup",
            "score": 1857,
            "ci": "±25",
            "cost": null,
            "date": "2026-10"
          }
        ]
      }
    },
    {
      "id": "swe-bench-verified",
      "name": "SWE-bench Verified",
      "maintainer": "SWE-bench team (Princeton/Stanford), with OpenAI",
      "launched": "2024-08",
      "focus": "Resolving real GitHub issues",
      "environment": "Repository-level coding in Docker; patch judged by the repo's tests",
      "tasks": "500 human-filtered issues from 12 Python repositories",
      "metric": "% resolved",
      "grading": "Automated: fail-to-pass and pass-to-pass unit tests",
      "harness": "Official view defaults to bash-only mini-SWE-agent, so every model gets the same scaffold",
      "cost_reporting": "Average $ per instance on the bash-only board",
      "status": "Live; newest official bash-only entry dated 2026-02-26",
      "caveat": "Saturating and long public, so contamination is a live concern. Vendor-reported scores for newer models use their own scaffolds and do not appear on the official board.",
      "sources": [
        {
          "label": "Official leaderboard",
          "url": "https://www.swebench.com/"
        },
        {
          "label": "Original paper (arXiv 2310.06770)",
          "url": "https://arxiv.org/abs/2310.06770"
        }
      ],
      "snapshot": {
        "board": "Official Verified leaderboard, bash-only (mini-SWE-agent)",
        "url": "https://www.swebench.com/",
        "version": "Verified, bash-only",
        "unit": "% resolved",
        "note": "The official board has not added a run since February 2026, so it trails the frontier. That gap is itself the finding: newer headline numbers are self-reported.",
        "rows": [
          {
            "rank": 1,
            "model": "Claude 4.5 Opus",
            "setting": "high",
            "agent": "mini-SWE-agent 2.0.0",
            "score": 76.8,
            "ci": null,
            "cost": "$0.75/instance",
            "date": "2026-02-17"
          },
          {
            "rank": 2,
            "model": "Gemini 3 Flash",
            "setting": "high",
            "agent": "mini-SWE-agent 2.0.0",
            "score": 75.8,
            "ci": null,
            "cost": "$0.36/instance",
            "date": "2026-02-17"
          },
          {
            "rank": 3,
            "model": "MiniMax M2.5",
            "setting": "high",
            "agent": "mini-SWE-agent 2.0.0",
            "score": 75.8,
            "ci": null,
            "cost": "$0.07/instance",
            "date": "2026-02-17"
          }
        ]
      }
    },
    {
      "id": "swe-bench-pro",
      "name": "SWE-Bench Pro",
      "maintainer": "Scale AI",
      "launched": "2025-09",
      "focus": "Long-horizon software engineering",
      "environment": "Repository-level coding in professional codebases (consumer apps, B2B services, dev tools)",
      "tasks": "1,865 tasks across 41 repositories: 731 public, 276 private, 858 held out",
      "metric": "Resolve rate",
      "grading": "Automated: fail-to-pass tests plus no regressions in pass-to-pass tests",
      "harness": "SWE-Agent at launch; recent entries use mini-swe-agent (marked *). Uncapped cost, 250-turn limit",
      "cost_reporting": "Not reported per run",
      "status": "Live",
      "caveat": "Copyleft and private repositories are used to reduce training contamination. Reference fixes average 107 lines across 4.1 files, far larger than SWE-bench Verified.",
      "sources": [
        {
          "label": "Scale Labs leaderboard and methodology",
          "url": "https://labs.scale.com/leaderboard/swe_bench_pro"
        }
      ],
      "snapshot": {
        "board": "SWE-Bench Pro public set",
        "url": "https://labs.scale.com/leaderboard/swe_bench_pro",
        "version": "Public set",
        "unit": "% resolved",
        "note": "* run with the mini-swe-agent harness.",
        "rows": [
          {
            "rank": 1,
            "model": "Muse Spark 1.1*",
            "setting": null,
            "agent": "mini-swe-agent",
            "score": 61.5,
            "ci": "±3.10",
            "cost": null,
            "date": null
          },
          {
            "rank": 2,
            "model": "gpt-5.4*",
            "setting": "xHigh",
            "agent": "mini-swe-agent",
            "score": 59.1,
            "ci": "±3.56",
            "cost": null,
            "date": null
          },
          {
            "rank": 3,
            "model": "Muse Spark*",
            "setting": null,
            "agent": "mini-swe-agent",
            "score": 55.0,
            "ci": "±3.60",
            "cost": null,
            "date": null
          }
        ]
      }
    },
    {
      "id": "terminal-bench",
      "name": "Terminal-Bench",
      "maintainer": "Stanford, Harbor and Laude Institute",
      "launched": "2025-05-19",
      "focus": "Hard work in a terminal",
      "environment": "Containerised shell tasks: software, ML, science, operations, security, hardware and media",
      "tasks": "4.0: 66 tasks (2.1: 89 tasks)",
      "metric": "Resolution rate ± 95% CI",
      "grading": "Automated: per-task verifier tests",
      "harness": "Submitter's agent (Claude Code, Codex, Antigravity CLI, mini-swe-agent and others)",
      "cost_reporting": "Total tokens and total dollar cost per full leaderboard run",
      "status": "Live; version 4.1 leaderboard",
      "caveat": "Agent and model are scored together, so the same model can rank differently in another harness. Each major version is a new task set: 2.0 and 4.x scores are not comparable.",
      "sources": [
        {
          "label": "Leaderboard",
          "url": "https://www.tbench.ai/leaderboard"
        },
        {
          "label": "Version history",
          "url": "https://www.tbench.ai/benchmarks"
        }
      ],
      "snapshot": {
        "board": "Terminal-Bench 4.1 leaderboard",
        "url": "https://www.tbench.ai/leaderboard",
        "version": "4.1",
        "unit": "% resolved",
        "note": "Ranks 1 and 2 tie on score while their run costs differ by 3.5x. Cost is for the whole leaderboard run, not per task.",
        "rows": [
          {
            "rank": 1,
            "model": "Opus 5.5",
            "setting": "max",
            "agent": "Claude Code",
            "score": 65.8,
            "ci": "±3.1",
            "cost": "$4.7k/run",
            "date": "2026-09-22"
          },
          {
            "rank": 1,
            "model": "Fable 5.2",
            "setting": "max",
            "agent": "Claude Code",
            "score": 65.8,
            "ci": "±2.8",
            "cost": "$16.5k/run",
            "date": "2026-11-09"
          },
          {
            "rank": 3,
            "model": "Gemini 4 Carbon",
            "setting": "high",
            "agent": "Antigravity CLI",
            "score": 64.5,
            "ci": "±3.2",
            "cost": "$1.4k/run",
            "date": "2026-10-28"
          }
        ]
      }
    },
    {
      "id": "osworld",
      "name": "OSWorld 2.0",
      "maintainer": "XLANG Lab, University of Hong Kong, with partners",
      "launched": "2026-06-26",
      "focus": "Long-horizon computer use",
      "environment": "Real desktop environment and 31 self-hosted websites; GUI and tool control",
      "tasks": "108 long-horizon workflows; median human time about 1.6 hours, 69.6% take over an hour",
      "metric": "Binary task completion at 500 steps, plus a partial (checkpoint) score",
      "grading": "Automated checkpoints (27.25 per task on average) plus separate safety reports",
      "harness": "Set per evaluated agent",
      "cost_reporting": "Not reported on the launch page",
      "status": "Live; successor to OSWorld and OSWorld-Verified",
      "caveat": "Tasks average over 250 agent steps, roughly ten times OSWorld 1.0, so OSWorld 1.0 and Verified scores do not carry over.",
      "sources": [
        {
          "label": "OSWorld 2.0 site",
          "url": "https://osworld-v2.xlang.ai/"
        },
        {
          "label": "OSWorld 1.0 / Verified",
          "url": "https://os-world.github.io/"
        }
      ],
      "snapshot": {
        "board": "OSWorld 2.0 launch results",
        "url": "https://osworld-v2.xlang.ai/",
        "version": "2.0",
        "unit": "% completed",
        "note": "Best result reported at launch. Partial score: 54.8%.",
        "rows": [
          {
            "rank": 1,
            "model": "Claude Opus 4.8",
            "setting": "max thinking, batched tool calls",
            "agent": null,
            "score": 20.6,
            "ci": null,
            "cost": null,
            "date": "2026-06"
          }
        ]
      }
    },
    {
      "id": "remote-labor-index",
      "name": "Remote Labor Index (RLI)",
      "maintainer": "Scale AI and Center for AI Safety",
      "launched": "2025-10",
      "focus": "Paid freelance projects, end to end",
      "environment": "CLI (OpenHands) or computer-use environment, with image, speech and video generation tools; $30 generation budget per project",
      "tasks": "240 Upwork projects across 23 domains (230 private for scoring); mean human time 28.9 hours, $143,991 total value",
      "metric": "Automation rate: share of projects a reasonable client would accept",
      "grading": "Manual expert review against the human deliverable; 94.4% inter-annotator agreement",
      "harness": "Standardised by Scale; best of the two environments is reported",
      "cost_reporting": "Dollars earned and cost savings are defined metrics; the leaderboard shows automation rate",
      "status": "Live",
      "caveat": "No confidence interval is published for automation rate. Excludes client interaction, physical work and long-term outcomes.",
      "sources": [
        {
          "label": "Scale Labs leaderboard",
          "url": "https://labs.scale.com/leaderboard/rli"
        },
        {
          "label": "Paper",
          "url": "https://scale.com/research/rli"
        }
      ],
      "snapshot": {
        "board": "RLI leaderboard",
        "url": "https://labs.scale.com/leaderboard/rli",
        "version": "v1",
        "unit": "% automated",
        "note": "The leader nearly doubles second place. At launch the best agent automated 2.5% of projects.",
        "rows": [
          {
            "rank": 1,
            "model": "Fable 5.2",
            "setting": null,
            "agent": null,
            "score": 41.25,
            "ci": null,
            "cost": null,
            "date": null
          },
          {
            "rank": 2,
            "model": "Opus 5.5",
            "setting": null,
            "agent": null,
            "score": 21.25,
            "ci": null,
            "cost": null,
            "date": null
          },
          {
            "rank": 3,
            "model": "GPT 6 Astra",
            "setting": null,
            "agent": null,
            "score": 20.83,
            "ci": null,
            "cost": null,
            "date": null
          }
        ]
      }
    },
    {
      "id": "metr-time-horizons",
      "name": "METR time horizons",
      "maintainer": "METR",
      "launched": "2025-03",
      "focus": "Length of task an agent can finish",
      "environment": "Self-contained software, ML and cybersecurity tasks (RE-Bench, HCAST and shorter novel tasks)",
      "tasks": "Over a hundred tasks, each timed against skilled human contractors",
      "metric": "50% and 80% time horizon: human task length at which the agent is predicted to succeed that often",
      "grading": "Automated scoring; flagged runs reviewed by humans for reward hacking",
      "harness": "Chosen per model during elicitation (ReAct, Triframe, Claude Code, Codex and others); 6 runs per task",
      "cost_reporting": "Not reported",
      "status": "No longer actively updated (since 2026-09-08)",
      "caveat": "METR says measurements above 16 hours are unreliable with its task suite, and that coverage of models was never complete.",
      "sources": [
        {
          "label": "Time horizons page",
          "url": "https://metr.org/time-horizons/"
        },
        {
          "label": "Paper (arXiv 2503.14499)",
          "url": "https://arxiv.org/abs/2503.14499"
        }
      ],
      "snapshot": {
        "board": "METR time horizons page",
        "url": "https://metr.org/time-horizons/",
        "version": "Time Horizon 1.1",
        "unit": "text",
        "note": "The last model added was Claude Mythos Preview (early), on 8 May 2026, when METR also warned that measurements above 16 hours are unreliable. The page has been frozen since 8 September 2026.",
        "rows": []
      }
    }
  ],
  "comparison": {
    "title": "GDPval vs SWE-bench",
    "columns": [
      "GDPval",
      "SWE-bench Verified"
    ],
    "rows": [
      [
        "Primary focus",
        "Professional tasks",
        "Software issues"
      ],
      [
        "Environment",
        "Work assignments with reference files across 44 occupations",
        "Repository-level coding"
      ],
      [
        "Output",
        "Documents, slides, spreadsheets, media",
        "A code patch"
      ],
      [
        "Judged by",
        "Expert humans, blind, against an expert's deliverable",
        "The repository's unit tests"
      ],
      [
        "Score type",
        "Win-or-tie rate (or Elo in GDPval-AA)",
        "% of issues resolved"
      ],
      [
        "Score comparable?",
        "No: different tasks and scoring methods",
        "No: different tasks and scoring methods"
      ]
    ]
  },
  "history": [
    {
      "date": "2024-08",
      "benchmark": "SWE-bench Verified",
      "change": "Released with OpenAI as a 500-issue human-filtered subset of SWE-bench.",
      "url": "https://openai.com/index/introducing-swe-bench-verified/"
    },
    {
      "date": "2025-03",
      "benchmark": "METR time horizons",
      "change": "Time-horizon method introduced in METR's paper.",
      "url": "https://arxiv.org/abs/2503.14499"
    },
    {
      "date": "2025-05-19",
      "benchmark": "Terminal-Bench",
      "change": "Terminal-Bench 1.0 released.",
      "url": "https://www.tbench.ai/benchmarks"
    },
    {
      "date": "2025-07-28",
      "benchmark": "OSWorld",
      "change": "OSWorld-Verified: fixed community-reported tasks and re-ran results.",
      "url": "https://os-world.github.io/"
    },
    {
      "date": "2025-09-25",
      "benchmark": "GDPval",
      "change": "Released: 1,320 tasks, 220-task public gold set.",
      "url": "https://openai.com/index/gdpval/"
    },
    {
      "date": "2025-11-07",
      "benchmark": "Terminal-Bench",
      "change": "Terminal-Bench 2.0 released.",
      "url": "https://www.tbench.ai/benchmarks"
    },
    {
      "date": "2026-02-17",
      "benchmark": "SWE-bench Verified",
      "change": "A batch of runs on mini-SWE-agent 2.0.0 lands on the official bash-only board.",
      "url": "https://www.swebench.com/"
    },
    {
      "date": "2026-03-03",
      "benchmark": "METR time horizons",
      "change": "Corrected a regularization mistake that affected published measurements.",
      "url": "https://metr.org/time-horizons/"
    },
    {
      "date": "2026-05-06",
      "benchmark": "Terminal-Bench",
      "change": "2.1: verified refresh of 2.0 with 89 tasks and environment fixes.",
      "url": "https://www.tbench.ai/benchmarks"
    },
    {
      "date": "2026-05-08",
      "benchmark": "METR time horizons",
      "change": "Added notice that measurements above 16 hours are unreliable.",
      "url": "https://metr.org/time-horizons/"
    },
    {
      "date": "2026-06-26",
      "benchmark": "OSWorld",
      "change": "OSWorld 2.0 released: 108 long-horizon tasks; earlier scores no longer comparable.",
      "url": "https://osworld-v2.xlang.ai/"
    },
    {
      "date": "2026-07-30",
      "benchmark": "Terminal-Bench",
      "change": "Terminal-Bench 3.0 released.",
      "url": "https://www.tbench.ai/benchmarks"
    },
    {
      "date": "2026-08-28",
      "benchmark": "Terminal-Bench",
      "change": "Terminal-Bench 4.0 released: 66 harder tasks with recalibrated time and compute allowances.",
      "url": "https://www.tbench.ai/benchmarks"
    },
    {
      "date": "2026-09-08",
      "benchmark": "METR time horizons",
      "change": "Page marked as no longer actively updated.",
      "url": "https://metr.org/time-horizons/"
    }
  ],
  "changelog": [
    {
      "date": "2026-11-15",
      "entry": "First published: eight benchmarks, leaderboard snapshots captured 15 November 2026."
    }
  ]
}
