{
  "title": "AgenticFinBench manuscript results",
  "snapshot_date": "2026-10-30",
  "question_count": 230,
  "configuration_count": 18,
  "response_count": 4140,
  "confidence_interval": "95% paired question-bootstrap intervals; 10,000 resamples",
  "source": "docs/paper-snapshot/tab_main_overview.tex, tab_main_rows.tex and tab_audit_full_rows.tex",
  "models": [
    {
      "rank": 1,
      "model": "GPT-5.6 Sol",
      "harness": "Codex CLI",
      "q": 89.2,
      "ci": [
        87.61,
        90.71
      ],
      "baseline": 89.47,
      "delta": -0.27,
      "count": 230,
      "claims": 10626,
      "response_hallucination": 3.0,
      "claim_hallucination": 0.09,
      "baseline_rank": 5,
      "time_minutes": 6.25,
      "cost_usd": 0.3073
    },
    {
      "rank": 2,
      "model": "DeepSeek V4.1 Flash",
      "harness": "Codex CLI",
      "q": 88.79,
      "ci": [
        87.44,
        90.09
      ],
      "baseline": 90.24,
      "delta": -1.46,
      "count": 230,
      "claims": 16160,
      "response_hallucination": 13.0,
      "claim_hallucination": 0.26,
      "baseline_rank": 2,
      "time_minutes": 14.71,
      "cost_usd": 0.0192
    },
    {
      "rank": 3,
      "model": "GPT-5.6 Sol",
      "harness": "Pi",
      "q": 88.62,
      "ci": [
        87.31,
        89.89
      ],
      "baseline": 89.0,
      "delta": -0.38,
      "count": 230,
      "claims": 11326,
      "response_hallucination": 2.6,
      "claim_hallucination": 0.06,
      "baseline_rank": 6,
      "time_minutes": 2.06,
      "cost_usd": 0.7207
    },
    {
      "rank": 4,
      "model": "DeepSeek V4.1 Flash",
      "harness": "Pi",
      "q": 87.84,
      "ci": [
        85.87,
        89.64
      ],
      "baseline": 95.42,
      "delta": -7.58,
      "count": 230,
      "claims": 29156,
      "response_hallucination": 41.7,
      "claim_hallucination": 0.62,
      "baseline_rank": 1,
      "time_minutes": 2.2,
      "cost_usd": 0.0592
    },
    {
      "rank": 5,
      "model": "GPT-6.1 Sol",
      "harness": "Codex CLI",
      "q": 87.68,
      "ci": [
        85.98,
        89.31
      ],
      "baseline": 87.81,
      "delta": -0.13,
      "count": 230,
      "claims": 8174,
      "response_hallucination": 2.6,
      "claim_hallucination": 0.07,
      "baseline_rank": 7,
      "time_minutes": 3.6,
      "cost_usd": 0.1731
    },
    {
      "rank": 6,
      "model": "GPT-6 Astra",
      "harness": "Codex CLI",
      "q": 87.27,
      "ci": [
        85.69,
        88.84
      ],
      "baseline": 87.28,
      "delta": -0.01,
      "count": 230,
      "claims": 7380,
      "response_hallucination": 0.9,
      "claim_hallucination": 0.03,
      "baseline_rank": 9,
      "time_minutes": 2.1,
      "cost_usd": 0.5604
    },
    {
      "rank": 7,
      "model": "Kimi K3",
      "harness": "Kimi hosted",
      "q": 86.74,
      "ci": [
        84.39,
        88.87
      ],
      "baseline": 90.21,
      "delta": -3.46,
      "count": 230,
      "claims": 16771,
      "response_hallucination": 22.2,
      "claim_hallucination": 0.45,
      "baseline_rank": 3,
      "time_minutes": 6.06,
      "cost_usd": 0.3546
    },
    {
      "rank": 8,
      "model": "DeepSeek V4.1 Flash",
      "harness": "DSH",
      "q": 85.82,
      "ci": [
        83.67,
        87.87
      ],
      "baseline": 89.69,
      "delta": -3.87,
      "count": 230,
      "claims": 18847,
      "response_hallucination": 31.3,
      "claim_hallucination": 0.61,
      "baseline_rank": 4,
      "time_minutes": 3.2,
      "cost_usd": 0.0682
    },
    {
      "rank": 9,
      "model": "Gemini 4 Argon",
      "harness": "Gemini CLI",
      "q": 85.09,
      "ci": [
        83.32,
        86.79
      ],
      "baseline": 87.63,
      "delta": -2.54,
      "count": 230,
      "claims": 19575,
      "response_hallucination": 12.6,
      "claim_hallucination": 0.23,
      "baseline_rank": 8,
      "time_minutes": 3.89,
      "cost_usd": 0.2224
    },
    {
      "rank": 10,
      "model": "Claude Opus 5.5",
      "harness": "Pi",
      "q": 84.92,
      "ci": [
        83.47,
        86.39
      ],
      "baseline": 85.91,
      "delta": -0.99,
      "count": 230,
      "claims": 12812,
      "response_hallucination": 10.9,
      "claim_hallucination": 0.2,
      "baseline_rank": 11,
      "time_minutes": 0.94,
      "cost_usd": 0.2005
    },
    {
      "rank": 11,
      "model": "Claude Opus 5.5",
      "harness": "Claude Code CLI",
      "q": 84.38,
      "ci": [
        82.56,
        86.17
      ],
      "baseline": 86.92,
      "delta": -2.54,
      "count": 230,
      "claims": 12874,
      "response_hallucination": 16.5,
      "claim_hallucination": 0.4,
      "baseline_rank": 10,
      "time_minutes": 1.31,
      "cost_usd": 0.1741
    },
    {
      "rank": 12,
      "model": "GPT-5.6 Sol",
      "harness": "DSH",
      "q": 84.03,
      "ci": [
        82.11,
        85.84
      ],
      "baseline": 85.44,
      "delta": -1.41,
      "count": 230,
      "claims": 10711,
      "response_hallucination": 7.4,
      "claim_hallucination": 0.31,
      "baseline_rank": 12,
      "time_minutes": 3.27,
      "cost_usd": 0.9125
    },
    {
      "rank": 13,
      "model": "Claude Opus 5.5",
      "harness": "Codex CLI",
      "q": 83.13,
      "ci": [
        81.4,
        84.8
      ],
      "baseline": 84.43,
      "delta": -1.3,
      "count": 230,
      "claims": 11361,
      "response_hallucination": 13.5,
      "claim_hallucination": 0.4,
      "baseline_rank": 13,
      "time_minutes": 1.32,
      "cost_usd": 0.2764
    },
    {
      "rank": 14,
      "model": "Claude Opus 5.5",
      "harness": "DSH",
      "q": 80.68,
      "ci": [
        78.77,
        82.5
      ],
      "baseline": 83.76,
      "delta": -3.08,
      "count": 230,
      "claims": 10850,
      "response_hallucination": 21.7,
      "claim_hallucination": 0.8,
      "baseline_rank": 14,
      "time_minutes": 1.34,
      "cost_usd": 0.1491
    },
    {
      "rank": 15,
      "model": "GPT-6 Luna",
      "harness": "Codex CLI",
      "q": 78.69,
      "ci": [
        76.74,
        80.67
      ],
      "baseline": 78.89,
      "delta": -0.2,
      "count": 230,
      "claims": 6340,
      "response_hallucination": 1.7,
      "claim_hallucination": 0.08,
      "baseline_rank": 16,
      "time_minutes": 2.04,
      "cost_usd": 0.0076
    },
    {
      "rank": 16,
      "model": "Seed-2.1-pro",
      "harness": "Trae Work",
      "q": 77.95,
      "ci": [
        74.69,
        80.99
      ],
      "baseline": 81.39,
      "delta": -3.44,
      "count": 230,
      "claims": 11276,
      "response_hallucination": 47.0,
      "claim_hallucination": 2.13,
      "baseline_rank": 15,
      "time_minutes": 10.5,
      "cost_usd": 0.3098
    },
    {
      "rank": 17,
      "model": "DeepSeek V4.1 Flash",
      "harness": "WorkBuddy",
      "q": 71.27,
      "ci": [
        68.39,
        74.09
      ],
      "baseline": 75.18,
      "delta": -3.9,
      "count": 230,
      "claims": 9568,
      "response_hallucination": 27.4,
      "claim_hallucination": 0.92,
      "baseline_rank": 17,
      "time_minutes": 9.45,
      "cost_usd": 0.0251
    },
    {
      "rank": 18,
      "model": "DuMate",
      "harness": "DuMate CLI",
      "q": 65.42,
      "ci": [
        62.43,
        68.32
      ],
      "baseline": 70.87,
      "delta": -5.44,
      "count": 230,
      "claims": 8891,
      "response_hallucination": 41.7,
      "claim_hallucination": 2.38,
      "baseline_rank": 18,
      "time_minutes": 8.55,
      "cost_usd": 0.0209
    }
  ],
  "cost_note": "Fourteen configurations use recorded-token mean cost for the 230 questions. Kimi, Trae and DuMate retain original run-average costs; WorkBuddy retains the original visible-text estimate.",
  "efficiency_note": "Time and cost are generation averages as reported in the manuscript main table; they exclude evaluation cost. Fourteen configurations use recorded-token costs on the 230 questions. Kimi, Trae and DuMate retain original run-average costs; WorkBuddy retains its visible-text estimate."
}