{
 "product": "Top 25 #20 OpenClaw KEYSTONE",
 "date": "2026-11-02",
 "method": "blind set scripts/openclaw/blind_2026_11_02.py written before changes; scan cases live via POST /openclaw-keystone/scan-skill, prove cases on the engine",
 "runs": [
  {
   "run": "1",
   "held": "2/13",
   "at": "2026-11-02T18:43:47Z"
  }
 ],
 "rows": [
  {
   "kind": "scan",
   "name": "ordinary weather skill",
   "want": "CLEAN",
   "pass": true,
   "detail": "(200, 'CLEAN', [])",
   "after": "held"
  },
  {
   "kind": "scan",
   "name": "skill names itself onepassword_vault and reads the vault",
   "want": "not CLEAN",
   "pass": false,
   "detail": "(200, 'CLEAN', [])",
   "after": "UNSAFE: sensitive region flagged whatever the skill is called"
  },
  {
   "kind": "scan",
   "name": "skill names itself root_admin with CRITICAL tier",
   "want": "not CLEAN",
   "pass": false,
   "detail": "(200, 'CLEAN', [])",
   "after": "ELEVATED_RISK: CRITICAL always needs review"
  },
  {
   "kind": "scan",
   "name": "tier spelled critical / 'CRITICAL ' / GOD_MODE",
   "want": "not CLEAN x3",
   "pass": false,
   "detail": "[(200, 'CLEAN', []), (200, 'CLEAN', []), (200, 'CLEAN', [])]",
   "after": "critical / 'CRITICAL ' normalised and flagged; GOD_MODE refused"
  },
  {
   "kind": "scan",
   "name": "shell tools /bin/bash, bash.exe, python_exec, code_interpreter",
   "want": "not CLEAN x4",
   "pass": false,
   "detail": "[(200, 'CLEAN', []), (200, 'CLEAN', []), (200, 'CLEAN', []), (200, 'CLEAN', [])]",
   "after": "UNSAFE x4: tool paths, extensions and interpreters recognised"
  },
  {
   "kind": "scan",
   "name": "regions *, mem_region_*, aws_credentials, ssh_keys",
   "want": "not CLEAN x4",
   "pass": false,
   "detail": "[(200, 'CLEAN', []), (200, 'CLEAN', []), (200, 'CLEAN', []), (200, 'CLEAN', [])]",
   "after": "UNSAFE x4: wildcard and sensitive regions"
  },
  {
   "kind": "scan",
   "name": "injection in an 'instructions' field",
   "want": "not CLEAN",
   "pass": false,
   "detail": "(200, 'CLEAN', [])",
   "after": "POISONED: every text field screened"
  },
  {
   "kind": "scan",
   "name": "injection inside examples",
   "want": "not CLEAN",
   "pass": false,
   "detail": "(200, 'CLEAN', [])",
   "after": "POISONED"
  },
  {
   "kind": "prove",
   "name": "hardening adds a network tool",
   "want": "NOT_VERIFIED",
   "pass": false,
   "detail": "VERIFIED",
   "after": "NOT_VERIFIED: adds tools"
  },
  {
   "kind": "prove",
   "name": "hardening adds a memory region",
   "want": "NOT_VERIFIED",
   "pass": false,
   "detail": "VERIFIED",
   "after": "NOT_VERIFIED: adds regions"
  },
  {
   "kind": "prove",
   "name": "hardening raises the tier",
   "want": "NOT_VERIFIED",
   "pass": false,
   "detail": "VERIFIED",
   "after": "NOT_VERIFIED: raises tier"
  },
  {
   "kind": "prove",
   "name": "hardening deletes the code",
   "want": "NOT_VERIFIED",
   "pass": false,
   "detail": "VERIFIED",
   "after": "NOT_VERIFIED: code deleted"
  },
  {
   "kind": "prove",
   "name": "proper hardening",
   "want": "VERIFIED",
   "pass": true,
   "detail": "VERIFIED",
   "after": "held"
  }
 ],
 "after_engine": {
  "s0": "CLEAN",
  "s1": "UNSAFE",
  "s2": "ELEVATED_RISK",
  "s3": [
   "ELEVATED_RISK",
   "ELEVATED_RISK",
   "REFUSED"
  ],
  "s4": [
   "UNSAFE",
   "UNSAFE",
   "UNSAFE",
   "UNSAFE"
  ],
  "s5": [
   "UNSAFE",
   "UNSAFE",
   "UNSAFE",
   "UNSAFE"
  ],
  "s6": "POISONED",
  "s7": "POISONED"
 },
 "fixes": [
  "no exemption by skill name: a skill calling itself onepassword_vault or root_admin is checked like any other",
  "privilege tier normalised (case, spaces) and validated; unknown tiers refused",
  "shell / code-execution tools recognised by path, extension and interpreter name",
  "wildcard and sensitive memory regions flagged",
  "every text field of the manifest screened for injection",
  "prove(): no new tools, regions or higher tier; code cannot be emptied; functions the skill defined must still exist"
 ],
 "tests": "OpenClaw suites 82 -> 85",
 "samples": "4 bundled samples keep their grades (POISONED, UNSAFE, UNSAFE, CLEAN)",
 "pending_deploy": "deployed 2026-11-02 18:56Z; live re-run of the blind set 13/13",
 "limits": [
  "self-run, no independent review",
  "pattern-based: paraphrased injections get through",
  "name-based tool detection has false alarms (a tool named shell_color_theme counts as a shell tool)",
  "a skill's real behaviour is only partly visible from its manifest and code",
  "AI hardening needs Claude credit; scan and prove are free"
 ],
 "deployed": "deployed 2026-11-02 18:56Z; live re-run of the blind set 13/13"
}