#!/usr/bin/env python3
"""Offline verifier for the VAGI distribution-shift model baseline.

Usage: python3 -I verify_offline.py <bundle-root>

Regenerates the shift tasks for seeds 9502/9503, re-scores every recorded answer with
the published shift checker, recomputes counts and cost, and checks that the prompt
view contains no `expect`, `hidden`, `fixes` or `bad` field.

It imports the bundled suite source, not CAIN. It is an offline recomputation with
the published checker, not an independent reimplementation.
"""
from __future__ import annotations

import hashlib
import json
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent
sys.path.insert(0, str(ROOT))

from vagi_suite import model_arm, model_shift  # noqa: E402
from vagi_suite.shift import check_shift, make_shift_tasks  # noqa: E402

CHECKS: list[tuple[str, bool, str]] = []


def check(name: str, ok: bool, detail: str = "") -> None:
    CHECKS.append((name, bool(ok), detail))


def jl(p: Path) -> list[dict]:
    return [json.loads(x) for x in p.read_text().splitlines() if x.strip()]


def main() -> int:
    root = Path(sys.argv[1] if len(sys.argv) > 1 else ".")
    s = json.loads((root / "SUMMARY.json").read_text())
    tasks = {9502: {t["task_id"]: t for t in make_shift_tasks(9502)},
             9503: {t["task_id"]: t for t in make_shift_tasks(9503)}}
    check("both shift seeds have 100 tasks", all(len(v) == 100 for v in tasks.values()))
    total = 0.0
    for arm in s["arms"]:
        sub = root / arm["dir"]
        res = json.loads((sub / "results.json").read_text())
        rows = jl(sub / "rows.jsonl")
        check(f"{arm['id']}: rows present", len(rows) == arm["n_tasks"], str(len(rows)))
        check(f"{arm['id']}: model and seed recorded", res["model"] == arm["model"] and res["suite_seed"] == arm["suite_seed"])
        blob = (sub / "model_tasks.json").read_text()
        for field in ("expect", "hidden", '"fixes"', '"bad"', "note_critical"):
            check(f"{arm['id']}: prompt view has no {field}", field not in blob)
        bad = 0
        for r in rows:
            if r.get("skipped"):
                continue
            t = tasks[arm["suite_seed"]][r["task_id"]]
            ans = r.get("answer")
            if "INVALID" in str(r.get("reason")):
                ok = False
            elif t["domain"] == "software_repair":
                ok = isinstance(ans, str) and bool(ans.strip()) and not model_arm.static_screen(ans) \
                    and bool(check_shift(t, ans))
            elif t["domain"] == "planning":
                ok = isinstance(ans, dict) and isinstance(ans.get("order"), list) \
                    and isinstance(ans.get("makespan"), (int, float)) and not isinstance(ans.get("makespan"), bool) \
                    and bool(check_shift(t, (ans["order"], int(ans["makespan"]))))
            else:
                ok = bool(check_shift(t, ans))
            if ok != bool(r["accepted"]):
                bad += 1
        check(f"{arm['id']}: every row re-scores identically", bad == 0, f"mismatches={bad}")
        done = [r for r in rows if not r.get("skipped")]
        k = sum(1 for r in done if r["accepted"])
        check(f"{arm['id']}: accepted count", k == res["accepted"], f"{k}/{len(done)}")
        check(f"{arm['id']}: partial arm recorded", res["n_scored"] == len(done))
        for d, v in res["per_domain"].items():
            kk = sum(1 for r in done if r["domain"] == d and r["accepted"])
            check(f"{arm['id']}: {d}", kk == v["accepted"], f"{kk} vs {v['accepted']}")
        pin, pout = model_arm.PRICES[res["model"]]
        cost = sum(int(r.get("input_tokens") or 0) / 1e6 * pin + int(r.get("output_tokens") or 0) / 1e6 * pout
                   for r in done)
        check(f"{arm['id']}: cost arithmetic", abs(cost - res["cost"]["total_usd"]) <= 1e-6,
              f"{cost:.6f} vs {res['cost']['total_usd']}")
        total += cost
    check("total spend matches summary", abs(total - s["total_cost_usd"]) <= 1e-6,
          f"{total:.6f} vs {s['total_cost_usd']}")
    check("overshoot is recorded and non-negative", s["overshoot_usd"] >= 0)
    passed = sum(1 for _, ok, _ in CHECKS if ok)
    for name, ok, detail in CHECKS:
        print(("PASS " if ok else "FAIL ") + name + ((" :: " + detail) if detail and not ok else ""))
    print(f"\n{passed}/{len(CHECKS)} checks passed")
    print("NOTE: counts are recomputed from recorded rows; the symbolic tasks are regenerated, not re-searched.")
    return 0 if passed == len(CHECKS) else 1


if __name__ == "__main__":
    raise SystemExit(main())
