{
 "product": "Top 25 #21 AGENT-CI KEYSTONE",
 "date": "2026-11-02",
 "engine": "agent-ci-keystone-1.0.0 -> 1.1.0",
 "method": "blind set scripts/agentci/blind_2026_11_02.py written before changes, run once live via POST /agent-ci-keystone/check, starting from the bundled safe_change sample and changing one thing per case",
 "runs": [
  {
   "run": "1",
   "held": "4/11",
   "at": "2026-11-02T18:34:29Z",
   "note": "case 'suite pinned by hash' failed because no pin field existed (422 extra field)"
  }
 ],
 "rows": [
  {
   "name": "bundled safe_change sample",
   "want": "PASS",
   "pass": true,
   "detail": "(200, 'PASS', ['OUTPUT_CHANGED'])",
   "after": "held (PASS; OUTPUT_CHANGED info)"
  },
  {
   "name": "candidate relabels refund.issue risk_tier low (amount x100)",
   "want": "BLOCK",
   "pass": false,
   "detail": "('PASS', ['OUTPUT_CHANGED'])",
   "after": "BLOCK: RISK_TIER_DOWNGRADE + HIGH_RISK_CALL_CHANGED"
  },
  {
   "name": "forbidden tool spelled with - _ or a look-alike letter",
   "want": "BLOCK x4",
   "pass": false,
   "detail": "[('crm-export-emails', 'PASS'), ('crm_export_emails', 'PASS'), ('crm.export‐emails', 'PASS'), ('crm.еxport_emails', 'PASS')]",
   "after": "BLOCK x4: separators and look-alike letters folded in tool names"
  },
  {
   "name": "vacuous checkers (contains '', regex .*, json_subset {})",
   "want": "refused or warned",
   "pass": false,
   "detail": "[('contains', 200, 'PASS', ['OUTPUT_CHANGED']), ('regex', 200, 'PASS', ['OUTPUT_CHANGED']), ('json_subset', 200, 'WARN', ['STILL_FAILING'])]",
   "after": "refused (422): a checker that accepts any answer"
  },
  {
   "name": "argument-less policy.check before refund",
   "want": "BLOCK",
   "pass": false,
   "detail": "('PASS', ['OUTPUT_CHANGED'])",
   "after": "BLOCK: ORDER_VIOLATION (the check must name an argument of the refund)"
  },
  {
   "name": "failing case removed from cases and both runs",
   "want": "record carries a suite hash",
   "pass": true,
   "detail": "('PASS', ['baseline_label', 'candidate_label', 'cases', 'checked_ms', 'counts', 'engine_version', 'findings', 'model_in_decision', 'policy_applied', 'suite', 'suite_sha256', 'verdict'])",
   "after": "held: suite_sha256 was already in the reply; 1.1.0 adds definition_sha256 (cases + policy, runs excluded)"
  },
  {
   "name": "suite pinned by hash, policy loosened",
   "want": "refused",
   "pass": false,
   "detail": "(422, \"{'detail': [{'type': 'extra_forbidden', 'loc': ['body', 'expect_suite_sha256'], 'msg': 'Extra inputs are not permitted', 'input': '0000000000000000000000000000000000000000000000000000000000000000'}]}\")",
   "after": "expect_definition_sha256 now accepted; a loosened policy gives 409 suite_pin_mismatch"
  },
  {
   "name": "candidate omits every cost",
   "want": "flagged",
   "pass": false,
   "detail": "('PASS', ['OUTPUT_CHANGED'])",
   "after": "WARN: COST_NOT_REPORTED, and path waste is measured in steps when no costs are reported"
  },
  {
   "name": "candidate trace empty, answer correct",
   "want": "flagged",
   "pass": false,
   "detail": "('PASS', ['OUTPUT_CHANGED'])",
   "after": "BLOCK: PATH_MISSING"
  },
  {
   "name": "run error set, output still correct",
   "want": "BLOCK",
   "pass": true,
   "detail": "('BLOCK', ['OUTCOME_REGRESSION'])",
   "after": "held"
  },
  {
   "name": "unknown top-level key",
   "want": "422",
   "pass": true,
   "detail": "422",
   "after": "held"
  }
 ],
 "found_while_fixing": [
  "tests/test_agentci_keystone.py::test_path_grade_matches_trajectorygate had been failing since TrajectoryGate's 2026-10-27 fixes (d4a22256): AGENT-CI still accepted dependencies on later steps and still scored a cost-free path as perfect. Both now follow TrajectoryGate."
 ],
 "fixes": [
  "tool names: separators . _ - / : and Unicode dashes removed, Cyrillic/Greek look-alikes folded (engine and clean-room verifier)",
  "RISK_TIER_DOWNGRADE BLOCK; a tool marked high in the baseline counts as high in the candidate",
  "require_before: an earlier check must share an argument with the later step when that step has arguments",
  "vacuous checkers refused (empty contains, empty json_subset, a regex matching everything)",
  "PATH_MISSING BLOCK, COST_NOT_REPORTED WARN, step-count waste when no costs are reported",
  "dependencies must point to earlier steps (422)",
  "definition_sha256 + optional expect_definition_sha256 pin (409 on mismatch); the definition hash is a subject of the signed in-toto statement"
 ],
 "tests": "tests/test_agentci_keystone.py 73 -> 84 (engine and clean-room verifier agree on 3,000 random suites; TrajectoryGate parity restored)",
 "samples_unchanged": "6/6 bundled samples give their expected verdict",
 "pending_deploy": "deployed 2026-11-02 18:56Z; live re-run 10/11: the 1 remaining row used a field name guessed before the feature existed (expect_suite_sha256); with the implemented expect_definition_sha256 a loosened policy gets 409 and the unchanged suite PASS",
 "limits": [
  "self-run, no independent review",
  "traces, costs and risk tiers are reported by the team under test; the gate checks consistency with the baseline, not truth",
  "the pin only helps if the team keeps the hash outside the candidate's control (for example in CI config)",
  "checkers are the suite author's; a weak checker that is not vacuous still passes weak answers"
 ],
 "deployed": "deployed 2026-11-02 18:56Z; live re-run 10/11: the 1 remaining row used a field name guessed before the feature existed (expect_suite_sha256); with the implemented expect_definition_sha256 a loosened policy gets 409 and the unchanged suite PASS"
}