{
 "arms": [
  {
   "accepted": 74,
   "cost_usd": 0.014904,
   "dir": "haiku",
   "duration_s": 18.5,
   "id": "haiku-shift",
   "input_tokens": 79530,
   "invalid_json": 1,
   "latency_ms": {
    "p50": 991.0,
    "p95": 2075.2
   },
   "model": "claude-haiku-5-5",
   "n_scored": 100,
   "n_skipped": 0,
   "n_tasks": 100,
   "output_tokens": 13902,
   "per_domain": {
    "business_decision": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "data_analysis": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "governance": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "memory_retrieval": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "planning": {
     "accepted": 7,
     "invalid": 1,
     "n": 10,
     "rate": 0.7
    },
    "recovery": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "science_calculation": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "security_triage": {
     "accepted": 2,
     "invalid": 0,
     "n": 10,
     "rate": 0.2
    },
    "software_repair": {
     "accepted": 5,
     "invalid": 0,
     "n": 10,
     "rate": 0.5
    },
    "tool_orchestration": {
     "accepted": 0,
     "invalid": 0,
     "n": 10,
     "rate": 0.0
    }
   },
   "rate": 0.74,
   "suite_seed": 9502,
   "wilson95": [
    0.6463,
    0.816
   ]
  },
  {
   "accepted": 81,
   "cost_usd": 0.26027,
   "dir": "sonnet",
   "duration_s": 31.3,
   "id": "sonnet-shift",
   "input_tokens": 79530,
   "invalid_json": 2,
   "latency_ms": {
    "p50": 1550.4,
    "p95": 4167.6
   },
   "model": "claude-sonnet-5-5",
   "n_scored": 100,
   "n_skipped": 0,
   "n_tasks": 100,
   "output_tokens": 10121,
   "per_domain": {
    "business_decision": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "data_analysis": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "governance": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "memory_retrieval": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "planning": {
     "accepted": 6,
     "invalid": 2,
     "n": 10,
     "rate": 0.6
    },
    "recovery": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "science_calculation": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "security_triage": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "software_repair": {
     "accepted": 5,
     "invalid": 0,
     "n": 10,
     "rate": 0.5
    },
    "tool_orchestration": {
     "accepted": 0,
     "invalid": 0,
     "n": 10,
     "rate": 0.0
    }
   },
   "rate": 0.81,
   "suite_seed": 9502,
   "wilson95": [
    0.7222,
    0.8749
   ]
  },
  {
   "accepted": 54,
   "cost_usd": 0.75112,
   "dir": "fable",
   "duration_s": 32.3,
   "id": "fable-shift",
   "input_tokens": 51560,
   "invalid_json": 4,
   "latency_ms": {
    "p50": 1051.9,
    "p95": 5549.0
   },
   "model": "claude-fable-5-2",
   "n_scored": 65,
   "n_skipped": 35,
   "n_tasks": 100,
   "output_tokens": 8466,
   "per_domain": {
    "business_decision": {
     "accepted": 7,
     "invalid": 0,
     "n": 7,
     "rate": 1.0
    },
    "data_analysis": {
     "accepted": 7,
     "invalid": 0,
     "n": 7,
     "rate": 1.0
    },
    "governance": {
     "accepted": 6,
     "invalid": 0,
     "n": 6,
     "rate": 1.0
    },
    "memory_retrieval": {
     "accepted": 6,
     "invalid": 0,
     "n": 6,
     "rate": 1.0
    },
    "planning": {
     "accepted": 6,
     "invalid": 0,
     "n": 6,
     "rate": 1.0
    },
    "recovery": {
     "accepted": 6,
     "invalid": 0,
     "n": 6,
     "rate": 1.0
    },
    "science_calculation": {
     "accepted": 7,
     "invalid": 0,
     "n": 7,
     "rate": 1.0
    },
    "security_triage": {
     "accepted": 7,
     "invalid": 0,
     "n": 7,
     "rate": 1.0
    },
    "software_repair": {
     "accepted": 2,
     "invalid": 4,
     "n": 7,
     "rate": 0.2857
    },
    "tool_orchestration": {
     "accepted": 0,
     "invalid": 0,
     "n": 6,
     "rate": 0.0
    }
   },
   "rate": 0.8308,
   "suite_seed": 9502,
   "wilson95": [
    0.7218,
    0.9028
   ]
  },
  {
   "accepted": 20,
   "cost_usd": 0.28588,
   "dir": "fable-replication",
   "duration_s": 17.5,
   "id": "fable-shift-replication",
   "input_tokens": 18350,
   "invalid_json": 1,
   "latency_ms": {
    "p50": 1155.3,
    "p95": 4769.9
   },
   "model": "claude-fable-5-2",
   "n_scored": 23,
   "n_skipped": 77,
   "n_tasks": 100,
   "output_tokens": 3477,
   "per_domain": {
    "business_decision": {
     "accepted": 2,
     "invalid": 0,
     "n": 2,
     "rate": 1.0
    },
    "data_analysis": {
     "accepted": 3,
     "invalid": 0,
     "n": 3,
     "rate": 1.0
    },
    "governance": {
     "accepted": 2,
     "invalid": 0,
     "n": 2,
     "rate": 1.0
    },
    "memory_retrieval": {
     "accepted": 2,
     "invalid": 0,
     "n": 2,
     "rate": 1.0
    },
    "planning": {
     "accepted": 2,
     "invalid": 0,
     "n": 2,
     "rate": 1.0
    },
    "recovery": {
     "accepted": 2,
     "invalid": 0,
     "n": 2,
     "rate": 1.0
    },
    "science_calculation": {
     "accepted": 2,
     "invalid": 0,
     "n": 2,
     "rate": 1.0
    },
    "security_triage": {
     "accepted": 3,
     "invalid": 0,
     "n": 3,
     "rate": 1.0
    },
    "software_repair": {
     "accepted": 2,
     "invalid": 1,
     "n": 3,
     "rate": 0.6667
    },
    "tool_orchestration": {
     "accepted": 0,
     "invalid": 0,
     "n": 2,
     "rate": 0.0
    }
   },
   "rate": 0.8696,
   "suite_seed": 9503,
   "wilson95": [
    0.6787,
    0.9546
   ]
  }
 ],
 "finding": "The hand-written library scores 5/100. Raw models score much higher (Haiku 74/100, Sonnet 81/100, Fable 54/65 scored before its cap, replication 20/23). The 5/100 is therefore substantially a library-coverage gap, not a general intelligence gap. The probe is also adversarial and partly underspecified: the software target is not inferable from the public tests, and all models scored 0 on tool_orchestration.",
 "generated_at": "2026-11-18T20:31:09.465149+00:00",
 "honest_limits": [
  "Fable arm was stopped by the budget cap at 65/100; replication at 23/100. Rates are on scored tasks only.",
  "The probe is deliberately adversarial to the hand-written library; it is not a fair AGI benchmark.",
  "Software task hidden target (x % 3) is not inferable from public tests; failure there is expected.",
  "All models scored 0/10 on tool_orchestration; that indicates a probe/prompt problem, not a general model failure.",
  "Cost overshoot of 3.2 cents occurred because concurrent requests passed the budget check together.",
  "Constructed tasks; no tools, no memory, one attempt; no external reviewer.",
  "Not AGI, not VAGI, not production, not a certification."
 ],
 "library_reference": {
  "accepted": 5,
  "n": 100
 },
 "overshoot_usd": 0.031616,
 "remaining_after_usd": -0.031616,
 "remaining_before_usd": 1.280558,
 "schema": "cain42.vagi_suite.shift_model_summary.v1",
 "suite": "VAGI Behavior Suite v1 \u2014 distribution shift",
 "total_cost_usd": 1.312174,
 "what_this_is": "Paid raw-model arms on the 100-task distribution-shift probe, with the same intended policy card. This is a gap map, not a capability headline."
}