{
 "arms": {
  "fable-shift": {
   "accepted": 54,
   "cost_usd": 0.75112,
   "model": "claude-fable-5-2",
   "n": 65,
   "per_domain": {
    "business_decision": {
     "accepted": 7,
     "invalid": 0,
     "n": 7,
     "rate": 1.0
    },
    "data_analysis": {
     "accepted": 7,
     "invalid": 0,
     "n": 7,
     "rate": 1.0
    },
    "governance": {
     "accepted": 6,
     "invalid": 0,
     "n": 6,
     "rate": 1.0
    },
    "memory_retrieval": {
     "accepted": 6,
     "invalid": 0,
     "n": 6,
     "rate": 1.0
    },
    "planning": {
     "accepted": 6,
     "invalid": 0,
     "n": 6,
     "rate": 1.0
    },
    "recovery": {
     "accepted": 6,
     "invalid": 0,
     "n": 6,
     "rate": 1.0
    },
    "science_calculation": {
     "accepted": 7,
     "invalid": 0,
     "n": 7,
     "rate": 1.0
    },
    "security_triage": {
     "accepted": 7,
     "invalid": 0,
     "n": 7,
     "rate": 1.0
    },
    "software_repair": {
     "accepted": 2,
     "invalid": 4,
     "n": 7,
     "rate": 0.2857
    },
    "tool_orchestration": {
     "accepted": 0,
     "invalid": 0,
     "n": 6,
     "rate": 0.0
    }
   },
   "rate": 0.8308,
   "suite_seed": 9502,
   "wilson95": [
    0.7218,
    0.9028
   ]
  },
  "fable-shift-replication": {
   "accepted": 20,
   "cost_usd": 0.28588,
   "model": "claude-fable-5-2",
   "n": 23,
   "per_domain": {
    "business_decision": {
     "accepted": 2,
     "invalid": 0,
     "n": 2,
     "rate": 1.0
    },
    "data_analysis": {
     "accepted": 3,
     "invalid": 0,
     "n": 3,
     "rate": 1.0
    },
    "governance": {
     "accepted": 2,
     "invalid": 0,
     "n": 2,
     "rate": 1.0
    },
    "memory_retrieval": {
     "accepted": 2,
     "invalid": 0,
     "n": 2,
     "rate": 1.0
    },
    "planning": {
     "accepted": 2,
     "invalid": 0,
     "n": 2,
     "rate": 1.0
    },
    "recovery": {
     "accepted": 2,
     "invalid": 0,
     "n": 2,
     "rate": 1.0
    },
    "science_calculation": {
     "accepted": 2,
     "invalid": 0,
     "n": 2,
     "rate": 1.0
    },
    "security_triage": {
     "accepted": 3,
     "invalid": 0,
     "n": 3,
     "rate": 1.0
    },
    "software_repair": {
     "accepted": 2,
     "invalid": 1,
     "n": 3,
     "rate": 0.6667
    },
    "tool_orchestration": {
     "accepted": 0,
     "invalid": 0,
     "n": 2,
     "rate": 0.0
    }
   },
   "rate": 0.8696,
   "suite_seed": 9503,
   "wilson95": [
    0.6787,
    0.9546
   ]
  },
  "haiku-shift": {
   "accepted": 74,
   "cost_usd": 0.014904,
   "model": "claude-haiku-5-5",
   "n": 100,
   "per_domain": {
    "business_decision": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "data_analysis": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "governance": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "memory_retrieval": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "planning": {
     "accepted": 7,
     "invalid": 1,
     "n": 10,
     "rate": 0.7
    },
    "recovery": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "science_calculation": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "security_triage": {
     "accepted": 2,
     "invalid": 0,
     "n": 10,
     "rate": 0.2
    },
    "software_repair": {
     "accepted": 5,
     "invalid": 0,
     "n": 10,
     "rate": 0.5
    },
    "tool_orchestration": {
     "accepted": 0,
     "invalid": 0,
     "n": 10,
     "rate": 0.0
    }
   },
   "rate": 0.74,
   "suite_seed": 9502,
   "wilson95": [
    0.6463,
    0.816
   ]
  },
  "sonnet-shift": {
   "accepted": 81,
   "cost_usd": 0.26027,
   "model": "claude-sonnet-5-5",
   "n": 100,
   "per_domain": {
    "business_decision": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "data_analysis": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "governance": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "memory_retrieval": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "planning": {
     "accepted": 6,
     "invalid": 2,
     "n": 10,
     "rate": 0.6
    },
    "recovery": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "science_calculation": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "security_triage": {
     "accepted": 10,
     "invalid": 0,
     "n": 10,
     "rate": 1.0
    },
    "software_repair": {
     "accepted": 5,
     "invalid": 0,
     "n": 10,
     "rate": 0.5
    },
    "tool_orchestration": {
     "accepted": 0,
     "invalid": 0,
     "n": 10,
     "rate": 0.0
    }
   },
   "rate": 0.81,
   "suite_seed": 9502,
   "wilson95": [
    0.7222,
    0.8749
   ]
  }
 },
 "finding": "The hand-written library scores 5/100. Raw models score much higher (Haiku 74/100, Sonnet 81/100, Fable 54/65 scored before its cap, replication 20/23). The 5/100 is therefore substantially a library-coverage gap, not a general intelligence gap. The probe is also adversarial and partly underspecified: the software target is not inferable from the public tests, and all models scored 0 on tool_orchestration.",
 "library": {
  "accepted": 5,
  "n": 100
 },
 "schema": "cain42.vagi_suite.shift_model_benchmark.v1"
}