{
 "schema_version": 1,
 "updated": "2026-11-08",
 "links": {
  "validation": "https://github.com/rogermas05/agent-recap/blob/main/research/VALIDATION.md",
  "repo": "https://github.com/rogermas05/agent-recap",
  "explorer_audit": "https://rogermas05.github.io/agent-recap/explorer.html#audit-title",
  "cli": "npx github:rogermas05/agent-recap audit"
 },
 "source": {
  "dataset": "JetBrains-Research/SWE-TaskFlow-SWE-Bench-Pro-Trajectories",
  "dataset_url": "https://huggingface.co/datasets/JetBrains-Research/SWE-TaskFlow-SWE-Bench-Pro-Trajectories",
  "benchmark": "SWE-Bench Pro",
  "harness": "mini-swe-agent",
  "condition": "single_prompt",
  "runs_per_model": "3 runs of 700 tasks",
  "citation": "Slinko, Golubev and Titov (2026). Coding-Agent Benchmarks Should Match Their Users' Task Flows.",
  "use": "Released for research on coding agents. Used here for research only."
 },
 "claims_audit": {
  "status": "validated",
  "dataset_label": "SWE-TaskFlow · SWE-Bench Pro · mini-swe-agent · single_prompt",
  "rows": [
   {
    "model": "GPT-5.6 luna",
    "group": "GPT-5.6",
    "runs": 2100,
    "runs_with_claim": 422,
    "claims": 432,
    "verified": 347,
    "contradicted": 0,
    "disclosed": 17,
    "unclear": 67,
    "unverified": 1
   },
   {
    "model": "GPT-5.6 terra",
    "group": "GPT-5.6",
    "runs": 2100,
    "runs_with_claim": 550,
    "claims": 644,
    "verified": 518,
    "contradicted": 2,
    "disclosed": 31,
    "unclear": 88,
    "unverified": 5
   },
   {
    "model": "GPT-5.6 mini",
    "group": "GPT-5.6",
    "runs": 2100,
    "runs_with_claim": 211,
    "claims": 239,
    "verified": 181,
    "contradicted": 3,
    "disclosed": 0,
    "unclear": 34,
    "unverified": 21
   },
   {
    "model": "Gemini 3.7 Flash",
    "group": "Gemini 3.7 Flash",
    "runs": 2100,
    "runs_with_claim": 1042,
    "claims": 1648,
    "verified": 1338,
    "contradicted": 33,
    "disclosed": 85,
    "unclear": 126,
    "unverified": 66
   }
  ]
 },
 "silent_after_failure": {
  "id": "swe-taskflow",
  "label": "SWE-TaskFlow on SWE-Bench Pro",
  "harness": "mini-swe-agent",
  "format": "OpenAI Responses transcripts",
  "status": "validated",
  "status_note": "",
  "url": "https://github.com/rogermas05/agent-recap/blob/main/research/VALIDATION.md",
  "rows": [
   {
    "model": "GPT-5.6 terra",
    "last_run_failed": 227,
    "mentioned": 220,
    "claimed_success": 1,
    "silent": 6,
    "confirmed_silent": 2,
    "real_failures_estimate": 226,
    "floor_range_pct": [
     0.9,
     1.2
    ],
    "attribution": {
     "likely_agent_change": 2
    }
   },
   {
    "model": "GPT-5.6 luna",
    "last_run_failed": 247,
    "mentioned": 220,
    "claimed_success": 1,
    "silent": 26,
    "confirmed_silent": 12,
    "real_failures_estimate": 232,
    "floor_range_pct": [
     5.2,
     7.0
    ],
    "attribution": {
     "likely_agent_change": 10,
     "likely_preexisting": 2
    }
   },
   {
    "model": "Gemini 3.7 Flash",
    "last_run_failed": 348,
    "mentioned": 331,
    "claimed_success": 3,
    "silent": 14,
    "confirmed_silent": 12,
    "real_failures_estimate": 147,
    "floor_range_pct": [
     4.9,
     17.0
    ],
    "attribution": {
     "likely_preexisting": 5,
     "likely_agent_change": 4,
     "unknown": 3
    }
   },
   {
    "model": "GPT-5.6 mini",
    "last_run_failed": 59,
    "mentioned": 31,
    "claimed_success": 0,
    "silent": 28,
    "confirmed_silent": 14,
    "real_failures_estimate": 36,
    "floor_range_pct": [
     32.7,
     52.4
    ],
    "attribution": {
     "likely_agent_change": 13,
     "unknown": 1
    }
   }
  ]
 },
 "datasets": [
  {
   "id": "swe-rebench-openhands",
   "label": "SWE-rebench",
   "harness": "OpenHands",
   "format": "OpenHands event trajectories",
   "status": "validated",
   "url": "https://huggingface.co/datasets/nebius/SWE-rebench-openhands-trajectories",
   "rows": [
    {
     "model": "Qwen3-Coder",
     "last_run_failed": 548,
     "mentioned": 534,
     "claimed_success": 2,
     "silent": 12,
     "confirmed_silent": 1,
     "real_failures_estimate": 385,
     "floor_range_pct": [
      0.2,
      0.4
     ],
     "attribution": {
      "unknown": 1
     }
    }
   ]
  },
  {
   "id": "glm-claude-code",
   "label": "SWE-smith",
   "harness": "Claude Code driven by GLM-5.3",
   "format": "Claude Code session format, rebuilt from API captures",
   "status": "preliminary",
   "status_note": "Automated counts only, not reviewed. The check flagged no silent runs, so there was nothing to confirm.",
   "url": "https://huggingface.co/datasets/liangzhidanta/claude-code-glm53-swesmith-trajectories",
   "rows": [
    {
     "model": "GLM-5.3",
     "last_run_failed": 62,
     "mentioned": 61,
     "claimed_success": 1,
     "silent": 0
    }
   ]
  }
 ],
 "review": {
  "reviewers": "Two AI models (Claude Sonnet first pass, Claude Opus blind second pass). No human reviewed these labels.",
  "double_reviewed": 70,
  "binary_agreement": 0.8,
  "cohen_kappa": 0.52,
  "blind_controls": 12,
  "controls_called_silent": 0
 }
}