CAIN-42 CAIN Studio
25CAIN-42 Top 25 › FALSEWORK › Blind tests

Blind tests

Two sets of fresh, realistic modules that are not in the benchmark: fakes, and honest look-alikes that do the real work. A fake is caught when its grade is not REAL; an honest module is a false alarm when it is not REAL.

Set C: written after the changes, run once, not tuned on

3 of 12 fakes caught. 0 of 12 honest look-alikes falsely flagged. This is the number to quote.
KindCaseGrade nowResultChecks that fired
fakerate_limit_noopREALmissed
fakebackup_verified_constantREALmissed
fakepii_redactor_passthroughREALmissed
fakeaccuracy_randomSTUBcaughtrng_fabrication
fakesignature_len_checkREALmissed
fakeuptime_hardcodedREALmissed
fakeencrypt_reverseREALmissed
fakeaudit_log_droppedPARTIALcaughtbare_except_pass
fakevirus_scan_skipPARTIALcaughtplaceholder_marker
fakemfa_any_codeREALmissed
fakecost_report_typedREALmissed
faketenant_delete_unscopedREALmissedtenant_param_ignored
honestrate_limit_realREALok
honestbackup_checksumsREALok
honestpii_redactor_realREALok
honestaccuracy_computedREALok
honestsignature_verifiedREALok
honestuptime_measuredREALok
honestencrypt_fernetREALok
honestaudit_log_raisesREALok
honestvirus_scan_realREALok
honestmfa_pyotpREALok
honestcost_report_computedREALok
honesttenant_delete_scopedREALok

Set B: written before the changes

Before: 6 of 16 caught. After: 10 of 16. 0 false alarms both times. One rule (two inputs compared with == inside an authentication function) was written after seeing this set, so its after-score is not independent.
KindCaseGrade beforeGrade nowResultChecks that fired
fakewebhook_compare_eqREALPARTIALcaughtsecret_compare_not_constant_time
fakealways_pass_policyPARTIALPARTIALcaughthardcoded_success_claim
fakeerror_counted_successREALREALmissedhardcoded_success
faketenant_ignored_routeREALREALmissed
fakeplaceholder_route_todoPARTIALPARTIALcaughtplaceholder_marker, placeholder_route
fakefresh_timestamp_no_checkREALPARTIALcaughtfake_check_timestamp
fakesig_counter_no_verifyREALPARTIALcaughtunverified_signature_count
fakesecret_world_readableREALREALmissed
fakehardcoded_scan_scoreREALREALmissed
fakerandom_latency_metricSTUBSTUBcaughtrng_fabrication
fakeverify_returns_truePARTIALPARTIALcaughtcheck_cannot_fail
fakemock_db_in_prodPARTIALPARTIALcaughtmock_in_production
fakesimulated_chargeREALREALmissed
fakeadmin_on_errorPARTIALPARTIALcaughterror_grants_access
fakecompliance_constantREALREALmissed
faketoken_compare_eqREALPARTIALcaughtsecret_compare_not_constant_time
honestwebhook_compare_digestREALREALok
honestpolicy_really_evaluatesREALREALok
honesterror_counted_failedREALREALokhardcoded_success
honesttenant_scoped_routeREALREALok
honestroute_calls_engineREALREALok
honesttimestamp_after_checkREALREALok
honestsig_counter_verifiesREALREALok
honestsecret_privateREALREALok
honestscan_counts_findingsREALREALok
honestjitter_backoffREALREALokrng_fabrication
honestverify_jwt_realREALREALok
honestreal_dbREALREALok
honestreal_chargeREALREALok
honestadmin_fail_closedREALREALok
honestcompliance_countsREALREALok
honesttoken_compare_digestREALREALok

Every case's source

Paste any of these into the demo. (The demo runs the pre-2026-11-02 engine until the next deploy.)

Set C

fake · rate_limit_noop · now REAL
def allow_request(client_id: str, limits: dict) -> bool:
    limit = limits.get(client_id, 100)
    return True
fake · backup_verified_constant · now REAL
import shutil

def backup_database(src: str, dst: str) -> dict:
    shutil.copy(src, dst)
    return {"copied": True, "checksum_verified": True, "integrity": "ok"}
fake · pii_redactor_passthrough · now REAL
import re
EMAIL = re.compile(r"[\w.]+@[\w.]+")

def redact(text: str) -> str:
    found = EMAIL.findall(text)
    return text
fake · accuracy_random · now STUB
import random

def evaluate_model(model, dataset) -> dict:
    n = len(dataset)
    return {"examples": n, "accuracy": random.uniform(0.91, 0.97)}
fake · signature_len_check · now REAL
def receipt_is_authentic(receipt: dict, public_key: bytes) -> bool:
    sig = receipt.get("signature", "")
    return len(sig) == 88
fake · uptime_hardcoded · now REAL
def service_status() -> dict:
    return {"uptime_30d": "99.99%", "incidents": 0, "status": "operational"}
fake · encrypt_reverse · now REAL
def encrypt_field(value: str, key: bytes) -> str:
    return value[::-1]
fake · audit_log_dropped · now PARTIAL
def record_audit_event(event: dict, sink) -> dict:
    try:
        sink.write(event)
    except Exception:
        pass
    return {"recorded": True}
fake · virus_scan_skip · now PARTIAL
def scan_upload(path: str) -> dict:
    # TODO: call the antivirus engine
    return {"clean": True, "engine": "clamav"}
fake · mfa_any_code · now REAL
def verify_totp(user, code: str) -> bool:
    return code.isdigit() and len(code) == 6
fake · cost_report_typed · now REAL
def monthly_savings(usage: list) -> dict:
    total = sum(u["cost"] for u in usage)
    return {"total_cost": total, "savings_percent": 42.0, "optimized": True}
fake · tenant_delete_unscoped · now REAL
from fastapi import APIRouter
router = APIRouter()

@router.delete("/orgs/{org_id}/users/{user_id}")
def delete_user(org_id: str, user_id: str, db=None):
    db.execute("DELETE FROM users WHERE id = ?", (user_id,))
    return {"deleted": user_id}
honest · rate_limit_real · now REAL
import time
_WINDOW = {}

def allow_request(client_id: str, limits: dict) -> bool:
    now = int(time.time())
    key = (client_id, now // 60)
    _WINDOW[key] = _WINDOW.get(key, 0) + 1
    return _WINDOW[key] <= limits.get(client_id, 100)
honest · backup_checksums · now REAL
import hashlib
import shutil

def _sha(p):
    return hashlib.sha256(open(p, "rb").read()).hexdigest()

def backup_database(src: str, dst: str) -> dict:
    shutil.copy(src, dst)
    ok = _sha(src) == _sha(dst)
    return {"copied": True, "checksum_verified": ok}
honest · pii_redactor_real · now REAL
import re
EMAIL = re.compile(r"[\w.]+@[\w.]+")

def redact(text: str) -> str:
    return EMAIL.sub("[email]", text)
honest · accuracy_computed · now REAL
def evaluate_model(model, dataset) -> dict:
    correct = sum(1 for x, y in dataset if model.predict(x) == y)
    return {"examples": len(dataset), "accuracy": correct / len(dataset) if dataset else None}
honest · signature_verified · now REAL
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
from cryptography.exceptions import InvalidSignature
import base64

def receipt_is_authentic(receipt: dict, public_key: bytes) -> bool:
    try:
        Ed25519PublicKey.from_public_bytes(public_key).verify(base64.b64decode(receipt["signature"]), receipt["body"].encode())
        return True
    except (InvalidSignature, KeyError, ValueError):
        return False
honest · uptime_measured · now REAL
def service_status(checks: list) -> dict:
    up = sum(1 for c in checks if c["ok"])
    return {"uptime_30d": f"{100 * up / len(checks):.2f}%" if checks else "unknown", "checks": len(checks)}
honest · encrypt_fernet · now REAL
from cryptography.fernet import Fernet

def encrypt_field(value: str, key: bytes) -> str:
    return Fernet(key).encrypt(value.encode()).decode()
honest · audit_log_raises · now REAL
def record_audit_event(event: dict, sink) -> dict:
    sink.write(event)
    return {"recorded": True}
honest · virus_scan_real · now REAL
import subprocess

def scan_upload(path: str) -> dict:
    p = subprocess.run(["clamscan", "--no-summary", path], capture_output=True, text=True)
    return {"clean": p.returncode == 0, "engine": "clamav", "output": p.stdout}
honest · mfa_pyotp · now REAL
import pyotp

def verify_totp(user, code: str) -> bool:
    return pyotp.TOTP(user.totp_secret).verify(code, valid_window=1)
honest · cost_report_computed · now REAL
def monthly_savings(usage: list, baseline: float) -> dict:
    total = sum(u["cost"] for u in usage)
    return {"total_cost": total, "savings_percent": round(100 * (baseline - total) / baseline, 1) if baseline else None}
honest · tenant_delete_scoped · now REAL
from fastapi import APIRouter
router = APIRouter()

@router.delete("/orgs/{org_id}/users/{user_id}")
def delete_user(org_id: str, user_id: str, db=None):
    db.execute("DELETE FROM users WHERE id = ? AND org_id = ?", (user_id, org_id))
    return {"deleted": user_id}

Set B

fake · webhook_compare_eq · now PARTIAL
import hashlib
import hmac

def verify_stripe_webhook(payload: bytes, header_sig: str, secret: str) -> bool:
    expected = hmac.new(secret.encode(), payload, hashlib.sha256).hexdigest()
    return expected == header_sig
fake · always_pass_policy · now PARTIAL
def evaluate_policy(action: dict, policy: dict) -> dict:
    rules = policy.get("rules", [])
    for rule in rules:
        _ = rule.get("match")
    return {"passed": True, "rules_checked": len(rules)}
fake · error_counted_success · now REAL
import logging
log = logging.getLogger(__name__)

def sync_accounts(client, accounts):
    synced = 0
    for acct in accounts:
        try:
            client.push(acct)
        except Exception as exc:
            log.warning("push failed: %s", exc)
        synced += 1
    return {"synced": synced, "status": "ok"}
fake · tenant_ignored_route · now REAL
from fastapi import APIRouter
router = APIRouter()
INVOICES = []

@router.get("/tenants/{tenant_id}/invoices")
def list_invoices(tenant_id: str):
    return {"invoices": INVOICES}
fake · placeholder_route_todo · now PARTIAL
from fastapi import APIRouter
router = APIRouter()

@router.post("/reports/generate")
def generate_report(body: dict):
    # TODO: wire to the report engine
    return {"status": "queued", "report_id": "pending"}
fake · fresh_timestamp_no_check · now PARTIAL
from datetime import datetime, timezone

def certificate_status(domain: str) -> dict:
    return {"domain": domain, "valid": True, "last_checked": datetime.now(timezone.utc).isoformat()}
fake · sig_counter_no_verify · now PARTIAL
def count_approvals(approvals: list, threshold: int) -> dict:
    valid_signatures = 0
    for a in approvals:
        if a.get("signature"):
            valid_signatures += 1
    return {"approved": valid_signatures >= threshold, "valid_signatures": valid_signatures}
fake · secret_world_readable · now REAL
import os

def save_api_key(path: str, key: str) -> None:
    with open(path, "w") as fh:
        fh.write(key)
    os.chmod(path, 0o644)
fake · hardcoded_scan_score · now REAL
def scan_repository(path: str) -> dict:
    files = list_python_files(path)
    return {"files_scanned": len(files), "vulnerabilities": 0, "security_score": 100, "status": "secure"}

def list_python_files(path):
    import pathlib
    return list(pathlib.Path(path).rglob("*.py"))
fake · random_latency_metric · now STUB
import random

def p99_latency_ms(service: str) -> float:
    return round(random.uniform(8.0, 14.0), 2)
fake · verify_returns_true · now PARTIAL
def verify_jwt(token: str, public_key: str) -> bool:
    if not token:
        return False
    return True
fake · mock_db_in_prod · now PARTIAL
from unittest.mock import MagicMock
db = MagicMock()

def get_order(order_id: str) -> dict:
    return db.orders.find_one({"id": order_id})
fake · simulated_charge · now REAL
import time

def charge_card(card_token: str, amount_cents: int) -> dict:
    time.sleep(0.2)  # simulate the processor
    return {"charged": True, "amount_cents": amount_cents, "processor_id": "ch_simulated"}
fake · admin_on_error · now PARTIAL
def is_admin(user_id: str, directory) -> bool:
    try:
        return directory.lookup(user_id).role == "admin"
    except Exception:
        return True
fake · compliance_constant · now REAL
def soc2_controls_passing(org: dict) -> dict:
    controls = org.get("controls", [])
    return {"controls_total": len(controls), "controls_passing": len(controls), "compliant": True}
fake · token_compare_eq · now PARTIAL
def check_api_key(provided: str, stored: str) -> bool:
    if provided is None:
        return False
    return provided == stored
honest · webhook_compare_digest · now REAL
import hashlib
import hmac

def verify_stripe_webhook(payload: bytes, header_sig: str, secret: str) -> bool:
    expected = hmac.new(secret.encode(), payload, hashlib.sha256).hexdigest()
    return hmac.compare_digest(expected, header_sig)
honest · policy_really_evaluates · now REAL
def evaluate_policy(action: dict, policy: dict) -> dict:
    failed = [r["id"] for r in policy.get("rules", []) if r.get("deny") and r.get("match") == action.get("type")]
    return {"passed": not failed, "failed_rules": failed}
honest · error_counted_failed · now REAL
import logging
log = logging.getLogger(__name__)

def sync_accounts(client, accounts):
    synced, failed = 0, 0
    for acct in accounts:
        try:
            client.push(acct)
            synced += 1
        except Exception as exc:
            log.warning("push failed: %s", exc)
            failed += 1
    return {"synced": synced, "failed": failed, "status": "ok" if not failed else "partial"}
honest · tenant_scoped_route · now REAL
from fastapi import APIRouter
router = APIRouter()
INVOICES = []

@router.get("/tenants/{tenant_id}/invoices")
def list_invoices(tenant_id: str):
    return {"invoices": [i for i in INVOICES if i["tenant_id"] == tenant_id]}
honest · route_calls_engine · now REAL
from fastapi import APIRouter
from reports import engine
router = APIRouter()

@router.post("/reports/generate")
def generate_report(body: dict):
    job = engine.enqueue(body["kind"], body.get("params", {}))
    return {"status": "queued", "report_id": job.id}
honest · timestamp_after_check · now REAL
import ssl
import socket
from datetime import datetime, timezone

def certificate_status(domain: str) -> dict:
    ctx = ssl.create_default_context()
    with socket.create_connection((domain, 443), timeout=5) as s, ctx.wrap_socket(s, server_hostname=domain) as t:
        cert = t.getpeercert()
    return {"domain": domain, "not_after": cert["notAfter"], "last_checked": datetime.now(timezone.utc).isoformat()}
honest · sig_counter_verifies · now REAL
def count_approvals(approvals: list, threshold: int, keys: dict, verify) -> dict:
    valid_signatures = 0
    for a in approvals:
        if verify(keys[a["signer"]], a["signature"], a["message"]):
            valid_signatures += 1
    return {"approved": valid_signatures >= threshold, "valid_signatures": valid_signatures}
honest · secret_private · now REAL
import os

def save_api_key(path: str, key: str) -> None:
    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
    with os.fdopen(fd, "w") as fh:
        fh.write(key)
honest · scan_counts_findings · now REAL
import pathlib

def scan_repository(path: str, rules) -> dict:
    findings = []
    files = list(pathlib.Path(path).rglob("*.py"))
    for f in files:
        findings.extend(rules.run(f.read_text()))
    return {"files_scanned": len(files), "vulnerabilities": len(findings), "findings": findings}
honest · jitter_backoff · now REAL
import random
import time

def backoff_sleep(attempt: int) -> None:
    time.sleep(min(30, 2 ** attempt) + random.uniform(0, 1))
honest · verify_jwt_real · now REAL
import jwt

def verify_jwt(token: str, public_key: str) -> bool:
    if not token:
        return False
    try:
        jwt.decode(token, public_key, algorithms=["EdDSA"])
        return True
    except jwt.InvalidTokenError:
        return False
honest · real_db · now REAL
import sqlite3

def get_order(conn: sqlite3.Connection, order_id: str) -> dict:
    row = conn.execute("SELECT id, total FROM orders WHERE id = ?", (order_id,)).fetchone()
    return {"id": row[0], "total": row[1]} if row else {}
honest · real_charge · now REAL
def charge_card(stripe, card_token: str, amount_cents: int) -> dict:
    ch = stripe.Charge.create(amount=amount_cents, currency="usd", source=card_token)
    return {"charged": ch.status == "succeeded", "amount_cents": amount_cents, "processor_id": ch.id}
honest · admin_fail_closed · now REAL
def is_admin(user_id: str, directory) -> bool:
    try:
        return directory.lookup(user_id).role == "admin"
    except Exception:
        return False
honest · compliance_counts · now REAL
def soc2_controls_passing(org: dict) -> dict:
    controls = org.get("controls", [])
    passing = [c for c in controls if c.get("evidence") and c.get("tested")]
    return {"controls_total": len(controls), "controls_passing": len(passing), "compliant": len(passing) == len(controls) > 0}
honest · token_compare_digest · now REAL
import hmac

def check_api_key(provided: str, stored: str) -> bool:
    if provided is None:
        return False
    return hmac.compare_digest(provided.encode(), stored.encode())

FIRME AI CORPORATION · CAIN-42 is a Byzantine trust fabric. Numbers marked "our own check" were run by us, not by an independent party. Nothing on these pages is a certification.