{
  "benchmark": "eval.tax AI Accounting Arena",
  "url": "https://eval.tax/",
  "release": {
    "date": "2026-09-08",
    "pack": "v1.2",
    "harness": "1.2.3",
    "run_window": {
      "start": "2026-09-02",
      "end": "2026-09-06",
      "timezone": "UTC"
    },
    "passes": 3,
    "rows_total": 512,
    "rows_scored": 490,
    "requests_per_pass": 16,
    "participants": 30,
    "organizations": 13,
    "previous_release": "https://eval.tax/results/2026-06-29.json"
  },
  "scores_note": "Scores are eval.tax run scores on the shared statement pack, not vendor benchmark claims. Track scores are medians of three seeded passes.",
  "columns": {
    "overall": "Weighted composite of the six track scores, 0-100. Weights are frozen per pack version.",
    "categorization": "Exact category match against the 14-label gold set, 490 scored rows, 0-100.",
    "vendor": "F1 over normalized merchant entities, 490 scored rows, 0-100.",
    "context": "Context-weighted accuracy on 118 rows that need client policy or vendor history, 0-100.",
    "coa": "Exact account match against the 48-account target chart of accounts, 490 scored rows, 0-100.",
    "splits": "Structured exact match on 64 mixed-use rows, 0-100.",
    "tax": "Tax treatment correctness on 96 tax-sensitive rows, 0-100.",
    "reliability": "Percent of requests that returned schema-valid output on the first attempt (48 requests per participant).",
    "latency": "Median wall-clock seconds per 32-row request.",
    "cost": "List-price USD per 1,000 input rows at the release date's pricing, retries included."
  },
  "tracks": [
    {
      "key": "categorization",
      "name": "Transaction Categorization",
      "metric": "Exact category match",
      "url": "https://eval.tax/tracks/transaction-categorization/",
      "scored_rows": 490,
      "leader": "Jupid Agent",
      "top_score": 95.4
    },
    {
      "key": "vendor",
      "name": "Vendor Normalization",
      "metric": "Vendor match F1",
      "url": "https://eval.tax/tracks/vendor-normalization/",
      "scored_rows": 490,
      "leader": "Jupid Agent",
      "top_score": 94
    },
    {
      "key": "context",
      "name": "Client Context Fit",
      "metric": "Context-weighted accuracy",
      "url": "https://eval.tax/tracks/client-context-fit/",
      "scored_rows": 118,
      "leader": "Jupid Agent",
      "top_score": 96.2
    },
    {
      "key": "coa",
      "name": "COA Mapping",
      "metric": "COA exact match",
      "url": "https://eval.tax/tracks/coa-mapping/",
      "scored_rows": 490,
      "leader": "Jupid Agent",
      "top_score": 94.7
    },
    {
      "key": "splits",
      "name": "Split Detection",
      "metric": "Structured split exact match",
      "url": "https://eval.tax/tracks/split-detection/",
      "scored_rows": 64,
      "leader": "GPT-6 Astra Preview",
      "top_score": 95
    },
    {
      "key": "tax",
      "name": "Tax Sensitivity",
      "metric": "Tax treatment correctness",
      "url": "https://eval.tax/tracks/tax-sensitivity/",
      "scored_rows": 96,
      "leader": "Jupid Agent",
      "top_score": 94
    }
  ],
  "participants": [
    {
      "id": "jupid-agent",
      "rank": 1,
      "name": "Jupid Agent",
      "org": "Jupid",
      "type": "Agent",
      "access": "Hosted agent",
      "overall": 94.6,
      "categorization": 95.4,
      "vendor": 94,
      "context": 96.2,
      "coa": 94.7,
      "splits": 93,
      "tax": 94,
      "reliability": 92,
      "latency_s": 4.5,
      "cost_index": 7.6
    },
    {
      "id": "claude-mythos-5.1",
      "rank": 2,
      "name": "Claude Mythos 5.1",
      "org": "Anthropic",
      "type": "Model",
      "access": "Partner access",
      "overall": 94.4,
      "categorization": 94.9,
      "vendor": 93.9,
      "context": 94,
      "coa": 94.1,
      "splits": 94.6,
      "tax": 93.8,
      "reliability": 90,
      "latency_s": 6.2,
      "cost_index": 10.9
    },
    {
      "id": "claude-fable-5.1",
      "rank": 3,
      "name": "Claude Fable 5.1",
      "org": "Anthropic",
      "type": "Model",
      "access": "Public API",
      "overall": 94.2,
      "categorization": 94.7,
      "vendor": 93.8,
      "context": 93.8,
      "coa": 93.9,
      "splits": 94.4,
      "tax": 93.6,
      "reliability": 90,
      "latency_s": 5.9,
      "cost_index": 10.6
    },
    {
      "id": "gpt-6-astra-preview",
      "rank": 4,
      "name": "GPT-6 Astra Preview",
      "org": "OpenAI",
      "type": "Model",
      "access": "Limited preview",
      "overall": 94.1,
      "categorization": 94.5,
      "vendor": 93.7,
      "context": 93.4,
      "coa": 93.7,
      "splits": 95,
      "tax": 93.4,
      "reliability": 88,
      "latency_s": 6.6,
      "cost_index": 13.5
    },
    {
      "id": "gpt-5.6-sol-pro",
      "rank": 5,
      "name": "GPT-5.6 Sol Pro",
      "org": "OpenAI",
      "type": "Model",
      "access": "Public API",
      "overall": 93.8,
      "categorization": 94.2,
      "vendor": 93.6,
      "context": 93,
      "coa": 93.4,
      "splits": 94.5,
      "tax": 93.1,
      "reliability": 88,
      "latency_s": 7,
      "cost_index": 14.6
    },
    {
      "id": "claude-opus-5",
      "rank": 6,
      "name": "Claude Opus 5",
      "org": "Anthropic",
      "type": "Model",
      "access": "Public API",
      "overall": 93.6,
      "categorization": 94,
      "vendor": 93.5,
      "context": 93.2,
      "coa": 93.3,
      "splits": 93.8,
      "tax": 93,
      "reliability": 89,
      "latency_s": 5.4,
      "cost_index": 9.4
    },
    {
      "id": "gpt-5.6-sol",
      "rank": 7,
      "name": "GPT-5.6 Sol",
      "org": "OpenAI",
      "type": "Model",
      "access": "Public API",
      "overall": 93.3,
      "categorization": 93.7,
      "vendor": 93.4,
      "context": 92.4,
      "coa": 92.9,
      "splits": 94,
      "tax": 92.5,
      "reliability": 87,
      "latency_s": 5.8,
      "cost_index": 12
    },
    {
      "id": "grok-4.6",
      "rank": 8,
      "name": "Grok 4.6",
      "org": "xAI",
      "type": "Model",
      "access": "Public API",
      "overall": 92.9,
      "categorization": 93.4,
      "vendor": 93.3,
      "context": 91.9,
      "coa": 92.5,
      "splits": 93.4,
      "tax": 92.1,
      "reliability": 85,
      "latency_s": 4.9,
      "cost_index": 7.4
    },
    {
      "id": "claude-sonnet-5",
      "rank": 9,
      "name": "Claude Sonnet 5",
      "org": "Anthropic",
      "type": "Model",
      "access": "Public API",
      "overall": 92.5,
      "categorization": 93,
      "vendor": 93.1,
      "context": 91.8,
      "coa": 92.1,
      "splits": 92.8,
      "tax": 91.7,
      "reliability": 86,
      "latency_s": 3.6,
      "cost_index": 4.6
    },
    {
      "id": "claude-opus-4.8",
      "rank": 10,
      "name": "Claude Opus 4.8",
      "org": "Anthropic",
      "type": "Model",
      "access": "Public API",
      "overall": 92.2,
      "categorization": 92.6,
      "vendor": 93,
      "context": 91.2,
      "coa": 91.6,
      "splits": 92.4,
      "tax": 91.2,
      "reliability": 85,
      "latency_s": 5.2,
      "cost_index": 9.5
    },
    {
      "id": "deepseek-v4-pro",
      "rank": 11,
      "name": "DeepSeek V4-Pro",
      "org": "DeepSeek",
      "type": "Model",
      "access": "Open weights · vendor API",
      "overall": 92,
      "categorization": 92.5,
      "vendor": 92.9,
      "context": 90.8,
      "coa": 91.5,
      "splits": 92.3,
      "tax": 91,
      "reliability": 84,
      "latency_s": 4.8,
      "cost_index": 6.2
    },
    {
      "id": "gemini-3.1-pro-preview",
      "rank": 12,
      "name": "Gemini 3.1 Pro Preview",
      "org": "Google",
      "type": "Model",
      "access": "Public preview",
      "overall": 91.6,
      "categorization": 92,
      "vendor": 92.5,
      "context": 90.4,
      "coa": 91.2,
      "splits": 92,
      "tax": 90.7,
      "reliability": 83,
      "latency_s": 5,
      "cost_index": 8.5
    },
    {
      "id": "gemini-3.8-flash",
      "rank": 13,
      "name": "Gemini 3.8 Flash",
      "org": "Google",
      "type": "Model",
      "access": "Public API",
      "overall": 91.4,
      "categorization": 91.8,
      "vendor": 92.4,
      "context": 90.2,
      "coa": 90.9,
      "splits": 91.8,
      "tax": 90.5,
      "reliability": 83,
      "latency_s": 2.6,
      "cost_index": 3.2
    },
    {
      "id": "qwen3.8-max",
      "rank": 14,
      "name": "Qwen3.8-Max",
      "org": "Alibaba",
      "type": "Model",
      "access": "Vendor API",
      "overall": 91,
      "categorization": 91.5,
      "vendor": 92.1,
      "context": 89.8,
      "coa": 90.6,
      "splits": 91.3,
      "tax": 90.1,
      "reliability": 82,
      "latency_s": 4.3,
      "cost_index": 5.8
    },
    {
      "id": "kimi-k3",
      "rank": 15,
      "name": "Kimi K3",
      "org": "Moonshot AI",
      "type": "Model",
      "access": "Open weights · vendor API",
      "overall": 90.6,
      "categorization": 91,
      "vendor": 91.7,
      "context": 89.4,
      "coa": 90.2,
      "splits": 90.9,
      "tax": 89.7,
      "reliability": 81,
      "latency_s": 4.4,
      "cost_index": 5.4
    },
    {
      "id": "grok-4.5",
      "rank": 16,
      "name": "Grok 4.5",
      "org": "xAI",
      "type": "Model",
      "access": "Public API",
      "overall": 90.3,
      "categorization": 90.8,
      "vendor": 91.4,
      "context": 89,
      "coa": 89.9,
      "splits": 90.7,
      "tax": 89.4,
      "reliability": 80,
      "latency_s": 4.6,
      "cost_index": 6.4
    },
    {
      "id": "gpt-5.6-terra",
      "rank": 17,
      "name": "GPT-5.6 Terra",
      "org": "OpenAI",
      "type": "Model",
      "access": "Public API",
      "overall": 90,
      "categorization": 90.4,
      "vendor": 91,
      "context": 88.7,
      "coa": 89.5,
      "splits": 90.6,
      "tax": 89.1,
      "reliability": 80,
      "latency_s": 3.8,
      "cost_index": 6.6
    },
    {
      "id": "gemini-3.7-flash",
      "rank": 18,
      "name": "Gemini 3.7 Flash",
      "org": "Google",
      "type": "Model",
      "access": "Public API",
      "overall": 89.6,
      "categorization": 90,
      "vendor": 90.7,
      "context": 88.3,
      "coa": 89.1,
      "splits": 89.9,
      "tax": 88.6,
      "reliability": 79,
      "latency_s": 2.5,
      "cost_index": 3
    },
    {
      "id": "muse-spark",
      "rank": 19,
      "name": "Muse Spark",
      "org": "Meta",
      "type": "Model",
      "access": "Public API",
      "overall": 89.2,
      "categorization": 89.6,
      "vendor": 90.3,
      "context": 87.9,
      "coa": 88.7,
      "splits": 89.5,
      "tax": 88.2,
      "reliability": 78,
      "latency_s": 3.9,
      "cost_index": 5
    },
    {
      "id": "qwen3.8-flash",
      "rank": 20,
      "name": "Qwen3.8-Flash",
      "org": "Alibaba",
      "type": "Model",
      "access": "Open weights · vendor API",
      "overall": 88.8,
      "categorization": 89.2,
      "vendor": 89.9,
      "context": 87.5,
      "coa": 88.3,
      "splits": 89.1,
      "tax": 87.8,
      "reliability": 77,
      "latency_s": 3,
      "cost_index": 3.4
    },
    {
      "id": "deepseek-v4-flash",
      "rank": 21,
      "name": "DeepSeek V4-Flash",
      "org": "DeepSeek",
      "type": "Model",
      "access": "Open weights · vendor API",
      "overall": 88.4,
      "categorization": 88.9,
      "vendor": 89.6,
      "context": 87,
      "coa": 87.9,
      "splits": 88.7,
      "tax": 87.4,
      "reliability": 76,
      "latency_s": 3,
      "cost_index": 3.3
    },
    {
      "id": "gpt-5.6-luna",
      "rank": 22,
      "name": "GPT-5.6 Luna",
      "org": "OpenAI",
      "type": "Model",
      "access": "Public API",
      "overall": 88,
      "categorization": 88.4,
      "vendor": 89.1,
      "context": 86.6,
      "coa": 87.5,
      "splits": 88.4,
      "tax": 87,
      "reliability": 76,
      "latency_s": 1.8,
      "cost_index": 2.2
    },
    {
      "id": "mistral-medium-3.5",
      "rank": 23,
      "name": "Mistral Medium 3.5",
      "org": "Mistral",
      "type": "Model",
      "access": "Public API",
      "overall": 86.2,
      "categorization": 86.6,
      "vendor": 87.3,
      "context": 84.8,
      "coa": 85.7,
      "splits": 86.5,
      "tax": 85.2,
      "reliability": 73,
      "latency_s": 3.5,
      "cost_index": 4.2
    },
    {
      "id": "gemini-3.5-flash-lite",
      "rank": 24,
      "name": "Gemini 3.5 Flash-Lite",
      "org": "Google",
      "type": "Model",
      "access": "Public API",
      "overall": 85.8,
      "categorization": 86.2,
      "vendor": 86.9,
      "context": 84.4,
      "coa": 85.3,
      "splits": 86.1,
      "tax": 84.8,
      "reliability": 72,
      "latency_s": 1.6,
      "cost_index": 1.7
    },
    {
      "id": "mistral-large-3",
      "rank": 25,
      "name": "Mistral Large 3",
      "org": "Mistral",
      "type": "Model",
      "access": "Open weights · vendor API",
      "overall": 85.3,
      "categorization": 85.7,
      "vendor": 86.4,
      "context": 83.9,
      "coa": 84.8,
      "splits": 85.6,
      "tax": 84.3,
      "reliability": 71,
      "latency_s": 4,
      "cost_index": 4.6
    },
    {
      "id": "cohere-command-a-plus",
      "rank": 26,
      "name": "Cohere Command A+",
      "org": "Cohere",
      "type": "Model",
      "access": "Open weights · vendor API",
      "overall": 84.5,
      "categorization": 84.9,
      "vendor": 85.6,
      "context": 83.1,
      "coa": 84,
      "splits": 84.8,
      "tax": 83.5,
      "reliability": 70,
      "latency_s": 3.6,
      "cost_index": 4
    },
    {
      "id": "amazon-nova-2-pro",
      "rank": 27,
      "name": "Amazon Nova 2 Pro",
      "org": "Amazon",
      "type": "Model",
      "access": "Public API",
      "overall": 84,
      "categorization": 84.4,
      "vendor": 85,
      "context": 82.6,
      "coa": 83.5,
      "splits": 84.3,
      "tax": 83,
      "reliability": 69,
      "latency_s": 3.3,
      "cost_index": 3.6
    },
    {
      "id": "claude-haiku-4.5",
      "rank": 28,
      "name": "Claude Haiku 4.5",
      "org": "Anthropic",
      "type": "Model",
      "access": "Public API",
      "overall": 83.4,
      "categorization": 83.8,
      "vendor": 84.5,
      "context": 82,
      "coa": 82.9,
      "splits": 83.7,
      "tax": 82.4,
      "reliability": 68,
      "latency_s": 2.1,
      "cost_index": 2.5
    },
    {
      "id": "perplexity-sonar-reasoning-pro",
      "rank": 29,
      "name": "Perplexity Sonar Reasoning Pro",
      "org": "Perplexity",
      "type": "Model",
      "access": "Public API",
      "overall": 83,
      "categorization": 83.4,
      "vendor": 84,
      "context": 81.6,
      "coa": 82.5,
      "splits": 83.2,
      "tax": 82,
      "reliability": 67,
      "latency_s": 4.4,
      "cost_index": 4
    },
    {
      "id": "mistral-small-4",
      "rank": 30,
      "name": "Mistral Small 4",
      "org": "Mistral",
      "type": "Model",
      "access": "Open weights · vendor API",
      "overall": 82.6,
      "categorization": 83,
      "vendor": 83.6,
      "context": 81.2,
      "coa": 82.1,
      "splits": 82.9,
      "tax": 81.7,
      "reliability": 66,
      "latency_s": 2.6,
      "cost_index": 2.7
    }
  ]
}