#!/usr/bin/env python3
"""Re-run every KEYSTONE record yourself and compare with what was signed.

Standalone: imports no CAIN code. Needs Python 3.9+ and pytest.

    curl -s -o bundle.zip https://cainstudio.online/keystone/bundle && unzip -o bundle.zip -d ks
    curl -s -o reference_tests.json https://cainstudio.online/keystone-reference-tests.json
    curl -s -o rerun_keystone.py https://cainstudio.online/keystone-rerun.py
    python3 rerun_keystone.py ks reference_tests.json

For each record it writes original_code and fixed_code to two temporary folders, puts the model's test (and the
published reference test, when the record's sample has one) next to each, runs pytest in both, and compares every
test outcome with the outcome stored in the signed record. It then applies the CURRENT verdict rule
(since 2026-10-23): VERIFIED_FIX needs an independent reference test that fails on the original and passes on
the fix, a model test that does the same, and a detector grade of REAL after the fix.

WARNING: this executes AI-written code from the records. Run it in a throwaway container or VM.
Exit code 0 = every re-run matched its record; 1 = at least one mismatch; 2 = bad input.
Records that were VERIFIED_FIX under an older rule but fail the current rule are reported, not counted as
mismatches (the record is historic evidence and is never rewritten).
"""
import json
import os
import re
import subprocess
import sys
import tempfile
from pathlib import Path

RESULT_RE = re.compile(r"^(PASSED|FAILED|ERROR)\s+(\S+?)::(\S+)", re.M)


def run_pytest(folder: Path) -> dict:
    """Return {(file, test): 'passed'|'not_passed'} for every test pytest reports."""
    env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "PYTHONDONTWRITEBYTECODE": "1"}
    try:
        p = subprocess.run([sys.executable, "-m", "pytest", "-q", "-rA", "-p", "no:cacheprovider", "."],
                           cwd=folder, capture_output=True, text=True, timeout=60, env=env)
        out = p.stdout + p.stderr
    except subprocess.TimeoutExpired:
        return {"__timeout__": "not_passed"}
    res = {}
    for status, f, t in RESULT_RE.findall(out):
        res[(Path(f).name, t.split("[")[0])] = "passed" if status == "PASSED" else "not_passed"
    if not res and "no tests ran" not in out:
        res["__collect__"] = "not_passed"   # import error etc.: nothing passed
    return res


def recorded(block) -> dict:
    if not block:
        return {}
    return {(r.get("file"), r.get("test")): ("passed" if r.get("outcome") == "passed" else "not_passed")
            for r in block.get("results", [])}


def all_pass(results: dict, file: str) -> bool:
    mine = {k: v for k, v in results.items() if isinstance(k, tuple) and k[0] == file}
    return bool(mine) and all(v == "passed" for v in mine.values())


def some_fail(results: dict, file: str) -> bool:
    mine = {k: v for k, v in results.items() if isinstance(k, tuple) and k[0] == file}
    return (not mine) or any(v != "passed" for v in mine.values()) or "__collect__" in results or "__timeout__" in results


def main() -> int:
    if len(sys.argv) != 3:
        print(__doc__)
        return 2
    folder, refs_path = Path(sys.argv[1]), Path(sys.argv[2])
    try:
        refs = json.loads(refs_path.read_text())["samples"]
    except Exception as e:  # noqa: BLE001
        print("cannot read reference tests: %s" % e)
        return 2
    records = sorted(folder.glob("record-*.json"))
    if not records:
        print("no record-*.json files in %s" % folder)
        return 2
    mismatches, rows = 0, []
    for rp in records:
        r = json.loads(rp.read_text())
        seq, verdict = r.get("sequence"), r.get("verdict")
        if not r.get("original_code") or not r.get("fixed_code") or not r.get("claude_test_code"):
            rows.append((seq, verdict, "no code in record (nothing to re-run)", "-"))
            continue
        ref = refs.get(r.get("sample_id") or "")
        got = {}
        for side, code in (("before", r["original_code"]), ("after", r["fixed_code"])):
            with tempfile.TemporaryDirectory() as tmp:
                t = Path(tmp)
                (t / r["path"]).write_text(code)
                (t / "test_claude.py").write_text(r["claude_test_code"])
                if ref:
                    (t / "test_reference.py").write_text(ref["reference_test"])
                got[side] = run_pytest(t)
        notes = []
        for who, file in (("claude", "test_claude.py"), ("reference", "test_reference.py")):
            block = (r.get("tests") or {}).get(who)
            if not block:
                continue
            for side in ("before", "after"):
                want = recorded(block.get(side))
                have = {k: v for k, v in got[side].items() if isinstance(k, tuple) and k[0] == file}
                if want != have:
                    mismatches += 1
                    notes.append("MISMATCH %s/%s: signed %s, re-run %s" % (who, side, sorted(want.items()), sorted(have.items())))
        b, a = got["before"], got["after"]
        current_ok = (bool(ref) and some_fail(b, "test_reference.py") and all_pass(a, "test_reference.py")
                      and some_fail(b, "test_claude.py") and all_pass(a, "test_claude.py")
                      and (r.get("falsework_after") or {}).get("grade") == "REAL")
        rule = "meets current rule" if current_ok else "does NOT meet current rule"
        if verdict == "VERIFIED_FIX" and not current_ok:
            why = "no reference test" if not ref else "a check failed on re-run"
            notes.append("signed VERIFIED_FIX under an older rule; today it would be NOT_VERIFIED (%s)" % why)
        rows.append((seq, verdict, "; ".join(notes) or "re-run matches the signed record", rule))
    print("seq  signed verdict   current rule                 result")
    for seq, v, note, rule in rows:
        print("%-4s %-16s %-28s %s" % (seq, v, rule, note))
    print("\n%d record(s), %d test-outcome mismatch(es)" % (len(rows), mismatches))
    return 1 if mismatches else 0


if __name__ == "__main__":
    sys.exit(main())