{
 "hypotheses": {
  "H1_router_beats_best_fixed_path": {
   "a_correct": 19,
   "b_correct": 22,
   "best_fixed": "m15_direct",
   "bootstrap_95": [
    -0.1042,
    0.25
   ],
   "diff": 0.0625,
   "mcnemar": {
    "a_only": 8,
    "b_only": 11,
    "discordant": 19,
    "p_two_sided": 0.647606,
    "significant_at_0.05": false
   },
   "n": 48,
   "supported": false
  },
  "H2_some_multi_agent_arm_beats_single": {
   "arms_equal_or_better": [
    "c02_parallel_vote",
    "c03_pairwise_debate",
    "c04_critic_reviser",
    "c05_planner_executor",
    "c06_hierarchical",
    "c08_evidence_first",
    "c09_parallel_synthesis"
   ],
   "arms_worse_than_single": [
    "c07_specialist_committee"
   ],
   "best_delta": 0.1042,
   "best_multi_agent_arm": "c08_evidence_first",
   "significant_arms": [],
   "supported": false
  },
  "H3_direction_replicates": {
   "a_correct": 8,
   "b_correct": 10,
   "best_fixed": "m15_direct",
   "bootstrap_95": [
    -0.2083,
    0.3333
   ],
   "diff": 0.0833,
   "mcnemar": {
    "a_only": 5,
    "b_only": 7,
    "discordant": 12,
    "p_two_sided": 0.774414,
    "significant_at_0.05": false
   },
   "n": 24,
   "same_direction_as_main": true,
   "supported": true
  }
 },
 "inference_accounting": {
  "by_model": {
   "qwen2.5-0.5b-instruct": 933,
   "qwen2.5-1.5b-instruct": 181,
   "smollm2-135m-instruct": 40
  },
  "by_split": {
   "replication": 74,
   "test": 906,
   "train": 174
  },
  "cloud_calls": 0,
  "cost_usd": 0.0,
  "execution_class": "LOCAL_MODEL_INFERENCE",
  "host_time_priced": false,
  "input_tokens": 134710,
  "model_calls": 1154,
  "output_tokens": 18422,
  "sum_latency_s": 4438.5,
  "tokens": 153132
 },
 "model": "local CPU: Qwen2.5-0.5B / 1.5B Instruct, SmolLM2-135M Instruct",
 "model_registry": {
  "blocked_providers": {
   "anthropic": "BLOCKED_BILLING (no credit purchased; billing not enabled)",
   "gemini": "BLOCKED (no credential)",
   "huggingface_inference_api": "BLOCKED (no credential)",
   "llama_cpp": "BLOCKED (binary not installed on this host)",
   "ollama": "AVAILABLE_NOT_USED (production catalog container cain-cat-ollama, capped at 2 CPU / 4 GiB; qwen2.5-coder:3b Q4_K_M measured at 10-40 s per call in 4 probe calls; not loaded with benchmark traffic without owner approval)",
   "openai_compatible_remote": "BLOCKED (no credential)",
   "vllm": "BLOCKED (no GPU; not installed)"
  },
  "fallback_policy": "a path whose model is unavailable is refused with MODEL_BLOCKED_NOT_AVAILABLE; the router falls back to the next measured path for that family; a blocked model is never replaced by a mock",
  "measured": {
   "qwen2.5-0.5b-instruct": {
    "architecture": "decoder-only transformer",
    "calls": 933,
    "context_window": 32768,
    "cost_usd": 0.0,
    "execution_class": "LOCAL_MODEL_INFERENCE",
    "hardware": "CPU, 4 threads, shared host",
    "hf_id": "Qwen/Qwen2.5-0.5B-Instruct",
    "license": "Apache-2.0",
    "mean_latency_ms_per_call": 2882.1,
    "model_file_sha256": "fdf756fa7fcbe7404d5c60e26bff1a0c8b8aa1f72ced49e7dd0210fe288fb7fe",
    "parameters": "494M",
    "quantization": "none (fp32 safetensors)",
    "revision_pin": "model_file_sha256",
    "structured_output_compliance_direct": 0.0,
    "tokens": 124320,
    "tool_expression_validity_train": 0.7037,
    "train_direct_accuracy": 0.25,
    "train_direct_by_family": {
     "code_trace": {
      "correct": 0,
      "tasks": 5
     },
     "label_code_review": {
      "correct": 3,
      "tasks": 5
     },
     "label_logs": {
      "correct": 5,
      "tasks": 5
     },
     "label_notes": {
      "correct": 1,
      "tasks": 3
     },
     "schedule": {
      "correct": 1,
      "tasks": 5
     },
     "table": {
      "correct": 0,
      "tasks": 4
     },
     "word_math": {
      "correct": 0,
      "tasks": 13
     }
    }
   },
   "qwen2.5-1.5b-instruct": {
    "architecture": "decoder-only transformer",
    "calls": 181,
    "context_window": 32768,
    "cost_usd": 0.0,
    "execution_class": "LOCAL_MODEL_INFERENCE",
    "hardware": "CPU, 4 threads, shared host",
    "hf_id": "Qwen/Qwen2.5-1.5B-Instruct",
    "license": "Apache-2.0",
    "mean_latency_ms_per_call": 9365.0,
    "model_file_sha256": "dd924a11b4c220f385b51ffa522daea7c9f3d850e31b162bb5661df483c6d3ee",
    "parameters": "1.54B",
    "quantization": "none (fp32 safetensors)",
    "revision_pin": "model_file_sha256",
    "structured_output_compliance_direct": 0.0446,
    "tokens": 23590,
    "tool_expression_validity_train": 0.6667,
    "train_direct_accuracy": 0.325,
    "train_direct_by_family": {
     "code_trace": {
      "correct": 2,
      "tasks": 5
     },
     "label_code_review": {
      "correct": 4,
      "tasks": 5
     },
     "label_logs": {
      "correct": 5,
      "tasks": 5
     },
     "label_notes": {
      "correct": 0,
      "tasks": 3
     },
     "schedule": {
      "correct": 0,
      "tasks": 5
     },
     "table": {
      "correct": 0,
      "tasks": 4
     },
     "word_math": {
      "correct": 2,
      "tasks": 13
     }
    }
   },
   "smollm2-135m-instruct": {
    "architecture": "decoder-only transformer",
    "calls": 40,
    "context_window": 8192,
    "cost_usd": 0.0,
    "execution_class": "LOCAL_MODEL_INFERENCE",
    "hardware": "CPU, 4 threads, shared host",
    "hf_id": "HuggingFaceTB/SmolLM2-135M-Instruct",
    "license": "Apache-2.0",
    "mean_latency_ms_per_call": 1360.4,
    "model_file_sha256": "5af571cbf074e6d21a03528d2330792e532ca608f24ac70a143f6b369968ab8c",
    "parameters": "135M",
    "quantization": "none (fp32 safetensors)",
    "revision_pin": "model_file_sha256",
    "structured_output_compliance_direct": 0.0,
    "tokens": 5222,
    "tool_expression_validity_train": null,
    "train_direct_accuracy": 0.025,
    "train_direct_by_family": {
     "code_trace": {
      "correct": 0,
      "tasks": 5
     },
     "label_code_review": {
      "correct": 1,
      "tasks": 5
     },
     "label_logs": {
      "correct": 0,
      "tasks": 5
     },
     "label_notes": {
      "correct": 0,
      "tasks": 3
     },
     "schedule": {
      "correct": 0,
      "tasks": 5
     },
     "table": {
      "correct": 0,
      "tasks": 4
     },
     "word_math": {
      "correct": 0,
      "tasks": 13
     }
    }
   }
  },
  "not_evaluated": {
   "DeepSeek family": "NOT_EVALUATED (weights not on this host; not downloaded: CPU-only shared host budget)",
   "Gemma (gated)": "NOT_EVALUATED (weights not on this host; not downloaded: CPU-only shared host budget)",
   "Llama family": "NOT_EVALUATED (weights not on this host; not downloaded: CPU-only shared host budget)",
   "Mistral family": "NOT_EVALUATED (weights not on this host; not downloaded: CPU-only shared host budget)",
   "Phi family": "NOT_EVALUATED (weights not on this host; not downloaded: CPU-only shared host budget)",
   "Qwen >= 3B": "NOT_EVALUATED (weights not on this host; not downloaded: CPU-only shared host budget)"
  }
 },
 "policy_sha256": "b98b26485b018bb6b3c2a9f3225f310e8f3d93e5dbb4fbd5d341d4e7264172fb",
 "schema": "cain2027.e80.collective_benchmark.v1",
 "specialist_profiles": {
  "measured": 10,
  "not_evaluated": 10,
  "note": "All specialists share model weights; they differ by identity, instructions, tool grants and sampling. A claimed expertise is not evidence of expertise.",
  "profiles": {
   "agent-adversary": {
    "accuracy": 0.2917,
    "claimed_domain": "adversarial evaluation",
    "correct": 14,
    "measured_on": "all families as an independent solver (temperature 0.7)",
    "status": "MEASURED_AS_SOLVER_ONLY",
    "tasks": 48,
    "wilson_95": [
     0.1824,
     0.4318
    ]
   },
   "agent-cyber": {
    "accuracy": 0.5,
    "claimed_domain": "cybersecurity",
    "correct": 3,
    "measured_on": "cybersecurity",
    "status": "MEASURED",
    "tasks": 6,
    "wilson_95": [
     0.1876,
     0.8124
    ]
   },
   "agent-data": {
    "accuracy": 0.0,
    "claimed_domain": "data analysis",
    "correct": 0,
    "measured_on": "data_analysis",
    "status": "MEASURED",
    "tasks": 6,
    "wilson_95": [
     0.0,
     0.3903
    ]
   },
   "agent-distributed": {
    "claimed_domain": "distributed systems",
    "reason": "no benchmark family for this expertise in E80",
    "status": "NOT_EVALUATED"
   },
   "agent-docs": {
    "claimed_domain": "technical documentation",
    "reason": "no benchmark family for this expertise in E80",
    "status": "NOT_EVALUATED"
   },
   "agent-finance": {
    "claimed_domain": "financial analysis",
    "reason": "no benchmark family for this expertise in E80",
    "status": "NOT_EVALUATED"
   },
   "agent-healthcare": {
    "claimed_domain": "healthcare research",
    "reason": "no benchmark family for this expertise in E80",
    "status": "NOT_EVALUATED"
   },
   "agent-infrastructure": {
    "accuracy": 1.0,
    "claimed_domain": "infrastructure engineering",
    "correct": 6,
    "measured_on": "infrastructure",
    "status": "MEASURED",
    "tasks": 6,
    "wilson_95": [
     0.6097,
     1.0
    ]
   },
   "agent-legal": {
    "claimed_domain": "legal research",
    "reason": "no benchmark family for this expertise in E80",
    "status": "NOT_EVALUATED"
   },
   "agent-mathematics": {
    "accuracy": 0.0,
    "claimed_domain": "mathematical reasoning",
    "correct": 0,
    "measured_on": "mathematics",
    "status": "MEASURED",
    "tasks": 6,
    "wilson_95": [
     0.0,
     0.3903
    ]
   },
   "agent-memory": {
    "accuracy": 0.1667,
    "claimed_domain": "memory and knowledge engineering",
    "correct": 1,
    "measured_on": "knowledge_synthesis",
    "status": "MEASURED",
    "tasks": 6,
    "wilson_95": [
     0.0301,
     0.5635
    ]
   },
   "agent-ml": {
    "claimed_domain": "machine learning engineering",
    "reason": "no benchmark family for this expertise in E80",
    "status": "NOT_EVALUATED"
   },
   "agent-modeleval": {
    "claimed_domain": "model evaluation",
    "reason": "no benchmark family for this expertise in E80",
    "status": "NOT_EVALUATED"
   },
   "agent-planner": {
    "claimed_domain": "mission planning",
    "reason": "no benchmark family for this expertise in E80",
    "status": "NOT_EVALUATED"
   },
   "agent-resource": {
    "claimed_domain": "resource optimization",
    "reason": "no benchmark family for this expertise in E80",
    "status": "NOT_EVALUATED"
   },
   "agent-robotics": {
    "claimed_domain": "robotics and physical AI research",
    "reason": "no benchmark family for this expertise in E80",
    "status": "NOT_EVALUATED"
   },
   "agent-scientific": {
    "accuracy": 0.1667,
    "claimed_domain": "scientific research",
    "correct": 1,
    "measured_on": "scientific_research",
    "status": "MEASURED",
    "tasks": 6,
    "wilson_95": [
     0.0301,
     0.5635
    ]
   },
   "agent-software": {
    "accuracy": 0.0,
    "claimed_domain": "software engineering",
    "correct": 0,
    "measured_on": "software_engineering",
    "status": "MEASURED",
    "tasks": 6,
    "wilson_95": [
     0.0,
     0.3903
    ]
   },
   "agent-strategic": {
    "accuracy": 0.0,
    "claimed_domain": "strategic reasoning",
    "correct": 0,
    "measured_on": "planning",
    "status": "MEASURED",
    "tasks": 6,
    "wilson_95": [
     0.0,
     0.3903
    ]
   },
   "agent-verifier": {
    "accuracy": 0.25,
    "claimed_domain": "independent verification",
    "correct": 12,
    "measured_on": "all families as an independent solver (temperature 0.7)",
    "status": "MEASURED_AS_SOLVER_ONLY",
    "tasks": 48,
    "wilson_95": [
     0.1492,
     0.3878
    ]
   }
  }
 },
 "splits": {
  "replication": {
   "adaptive_graph_vs_router": {
    "a_correct": 10,
    "b_correct": 10,
    "bootstrap_95": [
     0.0,
     0.0
    ],
    "diff": 0.0,
    "mcnemar": {
     "a_only": 0,
     "b_only": 0,
     "discordant": 0,
     "p_two_sided": 1.0,
     "significant_at_0.05": false
    },
    "n": 24
   },
   "arms": {
    "c10_adaptive_graph": {
     "accuracy": 0.4167,
     "applicable": 24,
     "by_family": {
      "code_trace": {
       "correct": 3,
       "tasks": 3
      },
      "label_code_review": {
       "correct": 0,
       "tasks": 3
      },
      "label_logs": {
       "correct": 3,
       "tasks": 3
      },
      "label_notes": {
       "correct": 2,
       "tasks": 5
      },
      "schedule": {
       "correct": 0,
       "tasks": 3
      },
      "table": {
       "correct": 0,
       "tasks": 1
      },
      "word_math": {
       "correct": 2,
       "tasks": 6
      }
     },
     "correct": 10,
     "latency_s": 152.5,
     "latency_s_per_correct": 15.25,
     "model_calls": 30,
     "tasks": 24,
     "tokens": 4244,
     "tokens_per_correct": 424.4,
     "tool_calls": 18,
     "unparsed": 2,
     "wilson_95": [
      0.2447,
      0.6117
     ]
    },
    "det_exec": {
     "accuracy": 0.125,
     "applicable": 3,
     "by_family": {
      "code_trace": {
       "correct": 3,
       "tasks": 3
      },
      "label_code_review": {
       "correct": 0,
       "tasks": 3
      },
      "label_logs": {
       "correct": 0,
       "tasks": 3
      },
      "label_notes": {
       "correct": 0,
       "tasks": 5
      },
      "schedule": {
       "correct": 0,
       "tasks": 3
      },
      "table": {
       "correct": 0,
       "tasks": 1
      },
      "word_math": {
       "correct": 0,
       "tasks": 6
      }
     },
     "correct": 3,
     "latency_s": 0.0,
     "latency_s_per_correct": 0.0,
     "model_calls": 0,
     "tasks": 24,
     "tokens": 0,
     "tokens_per_correct": 0.0,
     "tool_calls": 3,
     "unparsed": 0,
     "wilson_95": [
      0.0434,
      0.31
     ]
    },
    "fixed_pot05_else_direct05": {
     "accuracy": 0.3333,
     "applicable": 24,
     "by_family": {
      "code_trace": {
       "correct": 1,
       "tasks": 3
      },
      "label_code_review": {
       "correct": 0,
       "tasks": 3
      },
      "label_logs": {
       "correct": 3,
       "tasks": 3
      },
      "label_notes": {
       "correct": 2,
       "tasks": 5
      },
      "schedule": {
       "correct": 0,
       "tasks": 3
      },
      "table": {
       "correct": 0,
       "tasks": 1
      },
      "word_math": {
       "correct": 2,
       "tasks": 6
      }
     },
     "correct": 8,
     "latency_s": 91.8,
     "latency_s_per_correct": 11.47,
     "model_calls": 24,
     "tasks": 24,
     "tokens": 3366,
     "tokens_per_correct": 420.8,
     "tool_calls": 13,
     "unparsed": 3,
     "wilson_95": [
      0.1797,
      0.5329
     ]
    },
    "fixed_pot15_else_direct15": {
     "accuracy": 0.4583,
     "applicable": 24,
     "by_family": {
      "code_trace": {
       "correct": 3,
       "tasks": 3
      },
      "label_code_review": {
       "correct": 2,
       "tasks": 3
      },
      "label_logs": {
       "correct": 3,
       "tasks": 3
      },
      "label_notes": {
       "correct": 1,
       "tasks": 5
      },
      "schedule": {
       "correct": 0,
       "tasks": 3
      },
      "table": {
       "correct": 0,
       "tasks": 1
      },
      "word_math": {
       "correct": 2,
       "tasks": 6
      }
     },
     "correct": 11,
     "latency_s": 267.2,
     "latency_s_per_correct": 24.29,
     "model_calls": 24,
     "tasks": 24,
     "tokens": 3421,
     "tokens_per_correct": 311.0,
     "tool_calls": 13,
     "unparsed": 3,
     "wilson_95": [
      0.2789,
      0.6493
     ]
    },
    "m05_direct": {
     "accuracy": 0.25,
     "applicable": 24,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 3
      },
      "label_code_review": {
       "correct": 0,
       "tasks": 3
      },
      "label_logs": {
       "correct": 3,
       "tasks": 3
      },
      "label_notes": {
       "correct": 2,
       "tasks": 5
      },
      "schedule": {
       "correct": 0,
       "tasks": 3
      },
      "table": {
       "correct": 0,
       "tasks": 1
      },
      "word_math": {
       "correct": 1,
       "tasks": 6
      }
     },
     "correct": 6,
     "latency_s": 50.1,
     "latency_s_per_correct": 8.34,
     "model_calls": 24,
     "tasks": 24,
     "tokens": 2712,
     "tokens_per_correct": 452.0,
     "tool_calls": 0,
     "unparsed": 0,
     "wilson_95": [
      0.12,
      0.449
     ]
    },
    "m05_pot": {
     "accuracy": 0.125,
     "applicable": 13,
     "by_family": {
      "code_trace": {
       "correct": 1,
       "tasks": 3
      },
      "label_code_review": {
       "correct": 0,
       "tasks": 3
      },
      "label_logs": {
       "correct": 0,
       "tasks": 3
      },
      "label_notes": {
       "correct": 0,
       "tasks": 5
      },
      "schedule": {
       "correct": 0,
       "tasks": 3
      },
      "table": {
       "correct": 0,
       "tasks": 1
      },
      "word_math": {
       "correct": 2,
       "tasks": 6
      }
     },
     "correct": 3,
     "latency_s": 67.9,
     "latency_s_per_correct": 22.64,
     "model_calls": 13,
     "tasks": 24,
     "tokens": 1925,
     "tokens_per_correct": 641.7,
     "tool_calls": 13,
     "unparsed": 3,
     "wilson_95": [
      0.0434,
      0.31
     ]
    },
    "m15_direct": {
     "accuracy": 0.3333,
     "applicable": 24,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 3
      },
      "label_code_review": {
       "correct": 2,
       "tasks": 3
      },
      "label_logs": {
       "correct": 3,
       "tasks": 3
      },
      "label_notes": {
       "correct": 1,
       "tasks": 5
      },
      "schedule": {
       "correct": 0,
       "tasks": 3
      },
      "table": {
       "correct": 1,
       "tasks": 1
      },
      "word_math": {
       "correct": 1,
       "tasks": 6
      }
     },
     "correct": 8,
     "latency_s": 204.5,
     "latency_s_per_correct": 25.56,
     "model_calls": 24,
     "tasks": 24,
     "tokens": 2850,
     "tokens_per_correct": 356.2,
     "tool_calls": 0,
     "unparsed": 0,
     "wilson_95": [
      0.1797,
      0.5329
     ]
    },
    "m15_pot": {
     "accuracy": 0.2083,
     "applicable": 13,
     "by_family": {
      "code_trace": {
       "correct": 3,
       "tasks": 3
      },
      "label_code_review": {
       "correct": 0,
       "tasks": 3
      },
      "label_logs": {
       "correct": 0,
       "tasks": 3
      },
      "label_notes": {
       "correct": 0,
       "tasks": 5
      },
      "schedule": {
       "correct": 0,
       "tasks": 3
      },
      "table": {
       "correct": 0,
       "tasks": 1
      },
      "word_math": {
       "correct": 2,
       "tasks": 6
      }
     },
     "correct": 5,
     "latency_s": 184.0,
     "latency_s_per_correct": 36.8,
     "model_calls": 13,
     "tasks": 24,
     "tokens": 1956,
     "tokens_per_correct": 391.2,
     "tool_calls": 13,
     "unparsed": 3,
     "wilson_95": [
      0.0924,
      0.4047
     ]
    },
    "oracle_any_path": {
     "accuracy": 0.625,
     "applicable": 24,
     "by_family": {
      "code_trace": {
       "correct": 3,
       "tasks": 3
      },
      "label_code_review": {
       "correct": 2,
       "tasks": 3
      },
      "label_logs": {
       "correct": 3,
       "tasks": 3
      },
      "label_notes": {
       "correct": 3,
       "tasks": 5
      },
      "schedule": {
       "correct": 0,
       "tasks": 3
      },
      "table": {
       "correct": 1,
       "tasks": 1
      },
      "word_math": {
       "correct": 3,
       "tasks": 6
      }
     },
     "correct": 15,
     "latency_s": 68.5,
     "latency_s_per_correct": 4.57,
     "model_calls": 21,
     "tasks": 24,
     "tokens": 2553,
     "tokens_per_correct": 170.2,
     "tool_calls": 5,
     "unparsed": 0,
     "wilson_95": [
      0.4271,
      0.7884
     ]
    },
    "router": {
     "accuracy": 0.4167,
     "applicable": 24,
     "by_family": {
      "code_trace": {
       "correct": 3,
       "tasks": 3
      },
      "label_code_review": {
       "correct": 0,
       "tasks": 3
      },
      "label_logs": {
       "correct": 3,
       "tasks": 3
      },
      "label_notes": {
       "correct": 2,
       "tasks": 5
      },
      "schedule": {
       "correct": 0,
       "tasks": 3
      },
      "table": {
       "correct": 0,
       "tasks": 1
      },
      "word_math": {
       "correct": 2,
       "tasks": 6
      }
     },
     "correct": 10,
     "latency_s": 63.8,
     "latency_s_per_correct": 6.38,
     "model_calls": 21,
     "tasks": 24,
     "tokens": 2750,
     "tokens_per_correct": 275.0,
     "tool_calls": 9,
     "unparsed": 2,
     "wilson_95": [
      0.2447,
      0.6117
     ]
    }
   },
   "router_vs_best_fixed_on_train": {
    "a_correct": 8,
    "b_correct": 10,
    "best_fixed": "m15_direct",
    "bootstrap_95": [
     -0.2083,
     0.3333
    ],
    "diff": 0.0833,
    "mcnemar": {
     "a_only": 5,
     "b_only": 7,
     "discordant": 12,
     "p_two_sided": 0.774414,
     "significant_at_0.05": false
    },
    "n": 24
   },
   "router_vs_exploratory_baselines": {
    "fixed_pot05_else_direct05": {
     "a_correct": 8,
     "b_correct": 10,
     "bootstrap_95": [
      0.0,
      0.2083
     ],
     "diff": 0.0833,
     "mcnemar": {
      "a_only": 0,
      "b_only": 2,
      "discordant": 2,
      "p_two_sided": 0.5,
      "significant_at_0.05": false
     },
     "n": 24
    },
    "fixed_pot15_else_direct15": {
     "a_correct": 11,
     "b_correct": 10,
     "bootstrap_95": [
      -0.2083,
      0.125
     ],
     "diff": -0.0417,
     "mcnemar": {
      "a_only": 3,
      "b_only": 2,
      "discordant": 5,
      "p_two_sided": 1.0,
      "significant_at_0.05": false
     },
     "n": 24
    },
    "m05_direct": {
     "a_correct": 6,
     "b_correct": 10,
     "bootstrap_95": [
      -0.0417,
      0.375
     ],
     "diff": 0.1667,
     "mcnemar": {
      "a_only": 1,
      "b_only": 5,
      "discordant": 6,
      "p_two_sided": 0.21875,
      "significant_at_0.05": false
     },
     "n": 24
    },
    "m15_direct": {
     "a_correct": 8,
     "b_correct": 10,
     "bootstrap_95": [
      -0.2083,
      0.3333
     ],
     "diff": 0.0833,
     "mcnemar": {
      "a_only": 5,
      "b_only": 7,
      "discordant": 12,
      "p_two_sided": 0.774414,
      "significant_at_0.05": false
     },
     "n": 24
    }
   },
   "routing": {
    "by_family": {
     "code_trace": {
      "best_correct": 3,
      "best_in_hindsight": "det_exec",
      "chosen": "det_exec",
      "chosen_correct": 3,
      "routing_decision_optimal": true,
      "tasks": 3
     },
     "label_code_review": {
      "best_correct": 2,
      "best_in_hindsight": "m15_direct",
      "chosen": "m05_direct",
      "chosen_correct": 0,
      "routing_decision_optimal": false,
      "tasks": 3
     },
     "label_logs": {
      "best_correct": 3,
      "best_in_hindsight": "m05_direct",
      "chosen": "m05_direct",
      "chosen_correct": 3,
      "routing_decision_optimal": true,
      "tasks": 3
     },
     "label_notes": {
      "best_correct": 2,
      "best_in_hindsight": "m05_direct",
      "chosen": "m05_direct",
      "chosen_correct": 2,
      "routing_decision_optimal": true,
      "tasks": 5
     },
     "schedule": {
      "best_correct": 0,
      "best_in_hindsight": "det_exec",
      "chosen": "m05_direct",
      "chosen_correct": 0,
      "routing_decision_optimal": true,
      "tasks": 3
     },
     "table": {
      "best_correct": 1,
      "best_in_hindsight": "m15_direct",
      "chosen": "m05_direct",
      "chosen_correct": 0,
      "routing_decision_optimal": false,
      "tasks": 1
     },
     "word_math": {
      "best_correct": 2,
      "best_in_hindsight": "m05_pot",
      "chosen": "m05_pot",
      "chosen_correct": 2,
      "routing_decision_optimal": true,
      "tasks": 6
     }
    },
    "families": 7,
    "families_where_decision_was_optimal_in_hindsight": 5,
    "oracle_accuracy": 0.625,
    "regret_vs_oracle": 0.2083,
    "selected_paths": {
     "det_exec": 3,
     "m05_direct": 15,
     "m05_pot": 6
    }
   },
   "tasks": 24
  },
  "test": {
   "adaptive_graph_vs_router": {
    "a_correct": 22,
    "b_correct": 22,
    "bootstrap_95": [
     0.0,
     0.0
    ],
    "diff": 0.0,
    "mcnemar": {
     "a_only": 0,
     "b_only": 0,
     "discordant": 0,
     "p_two_sided": 1.0,
     "significant_at_0.05": false
    },
    "n": 48
   },
   "arms": {
    "c01_single": {
     "accuracy": 0.2292,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 3,
       "tasks": 6
      },
      "label_logs": {
       "correct": 6,
       "tasks": 6
      },
      "label_notes": {
       "correct": 2,
       "tasks": 7
      },
      "schedule": {
       "correct": 0,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 0,
       "tasks": 14
      }
     },
     "correct": 11,
     "latency_s": 91.0,
     "latency_s_per_correct": 8.28,
     "model_calls": 48,
     "tasks": 48,
     "tokens": 5282,
     "tokens_per_correct": 480.2,
     "tool_calls": 0,
     "unparsed": 0,
     "wilson_95": [
      0.1331,
      0.3654
     ]
    },
    "c02_parallel_vote": {
     "accuracy": 0.2292,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 3,
       "tasks": 6
      },
      "label_logs": {
       "correct": 6,
       "tasks": 6
      },
      "label_notes": {
       "correct": 1,
       "tasks": 7
      },
      "schedule": {
       "correct": 0,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 1,
       "tasks": 14
      }
     },
     "correct": 11,
     "latency_s": 348.6,
     "latency_s_per_correct": 31.69,
     "model_calls": 144,
     "tasks": 48,
     "tokens": 16122,
     "tokens_per_correct": 1465.6,
     "tool_calls": 0,
     "unparsed": 0,
     "wilson_95": [
      0.1331,
      0.3654
     ]
    },
    "c03_pairwise_debate": {
     "accuracy": 0.2292,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 3,
       "tasks": 6
      },
      "label_logs": {
       "correct": 6,
       "tasks": 6
      },
      "label_notes": {
       "correct": 2,
       "tasks": 7
      },
      "schedule": {
       "correct": 0,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 0,
       "tasks": 14
      }
     },
     "correct": 11,
     "latency_s": 291.5,
     "latency_s_per_correct": 26.5,
     "model_calls": 139,
     "tasks": 48,
     "tokens": 15564,
     "tokens_per_correct": 1414.9,
     "tool_calls": 0,
     "unparsed": 2,
     "wilson_95": [
      0.1331,
      0.3654
     ]
    },
    "c04_critic_reviser": {
     "accuracy": 0.2292,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 3,
       "tasks": 6
      },
      "label_logs": {
       "correct": 6,
       "tasks": 6
      },
      "label_notes": {
       "correct": 2,
       "tasks": 7
      },
      "schedule": {
       "correct": 0,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 0,
       "tasks": 14
      }
     },
     "correct": 11,
     "latency_s": 197.5,
     "latency_s_per_correct": 17.96,
     "model_calls": 96,
     "tasks": 48,
     "tokens": 11512,
     "tokens_per_correct": 1046.5,
     "tool_calls": 0,
     "unparsed": 0,
     "wilson_95": [
      0.1331,
      0.3654
     ]
    },
    "c05_planner_executor": {
     "accuracy": 0.2708,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 2,
       "tasks": 6
      },
      "label_logs": {
       "correct": 6,
       "tasks": 6
      },
      "label_notes": {
       "correct": 2,
       "tasks": 7
      },
      "schedule": {
       "correct": 0,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 3,
       "tasks": 14
      }
     },
     "correct": 13,
     "latency_s": 354.3,
     "latency_s_per_correct": 27.25,
     "model_calls": 96,
     "tasks": 48,
     "tokens": 14904,
     "tokens_per_correct": 1146.5,
     "tool_calls": 0,
     "unparsed": 1,
     "wilson_95": [
      0.1657,
      0.41
     ]
    },
    "c06_hierarchical": {
     "accuracy": 0.3125,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 4,
       "tasks": 6
      },
      "label_logs": {
       "correct": 6,
       "tasks": 6
      },
      "label_notes": {
       "correct": 1,
       "tasks": 7
      },
      "schedule": {
       "correct": 1,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 3,
       "tasks": 14
      }
     },
     "correct": 15,
     "latency_s": 600.1,
     "latency_s_per_correct": 40.01,
     "model_calls": 181,
     "tasks": 48,
     "tokens": 27529,
     "tokens_per_correct": 1835.3,
     "tool_calls": 0,
     "unparsed": 1,
     "wilson_95": [
      0.1995,
      0.4533
     ]
    },
    "c07_specialist_committee": {
     "accuracy": 0.2083,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 3,
       "tasks": 6
      },
      "label_logs": {
       "correct": 6,
       "tasks": 6
      },
      "label_notes": {
       "correct": 1,
       "tasks": 7
      },
      "schedule": {
       "correct": 0,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 0,
       "tasks": 14
      }
     },
     "correct": 10,
     "latency_s": 298.5,
     "latency_s_per_correct": 29.85,
     "model_calls": 144,
     "tasks": 48,
     "tokens": 16538,
     "tokens_per_correct": 1653.8,
     "tool_calls": 0,
     "unparsed": 0,
     "wilson_95": [
      0.1173,
      0.3426
     ]
    },
    "c08_evidence_first": {
     "accuracy": 0.3333,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 1,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 5,
       "tasks": 6
      },
      "label_logs": {
       "correct": 6,
       "tasks": 6
      },
      "label_notes": {
       "correct": 2,
       "tasks": 7
      },
      "schedule": {
       "correct": 1,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 1,
       "tasks": 14
      }
     },
     "correct": 16,
     "latency_s": 280.2,
     "latency_s_per_correct": 17.51,
     "model_calls": 96,
     "tasks": 48,
     "tokens": 13525,
     "tokens_per_correct": 845.3,
     "tool_calls": 0,
     "unparsed": 0,
     "wilson_95": [
      0.2168,
      0.4746
     ]
    },
    "c09_parallel_synthesis": {
     "accuracy": 0.2708,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 3,
       "tasks": 6
      },
      "label_logs": {
       "correct": 6,
       "tasks": 6
      },
      "label_notes": {
       "correct": 2,
       "tasks": 7
      },
      "schedule": {
       "correct": 1,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 1,
       "tasks": 14
      }
     },
     "correct": 13,
     "latency_s": 493.2,
     "latency_s_per_correct": 37.94,
     "model_calls": 192,
     "tasks": 48,
     "tokens": 22942,
     "tokens_per_correct": 1764.8,
     "tool_calls": 0,
     "unparsed": 0,
     "wilson_95": [
      0.1657,
      0.41
     ]
    },
    "c10_adaptive_graph": {
     "accuracy": 0.4583,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 6,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 3,
       "tasks": 6
      },
      "label_logs": {
       "correct": 6,
       "tasks": 6
      },
      "label_notes": {
       "correct": 2,
       "tasks": 7
      },
      "schedule": {
       "correct": 0,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 5,
       "tasks": 14
      }
     },
     "correct": 22,
     "latency_s": 301.3,
     "latency_s_per_correct": 13.7,
     "model_calls": 60,
     "tasks": 48,
     "tokens": 8742,
     "tokens_per_correct": 397.4,
     "tool_calls": 38,
     "unparsed": 4,
     "wilson_95": [
      0.3258,
      0.5971
     ]
    },
    "det_exec": {
     "accuracy": 0.125,
     "applicable": 6,
     "by_family": {
      "code_trace": {
       "correct": 6,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 0,
       "tasks": 6
      },
      "label_logs": {
       "correct": 0,
       "tasks": 6
      },
      "label_notes": {
       "correct": 0,
       "tasks": 7
      },
      "schedule": {
       "correct": 0,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 0,
       "tasks": 14
      }
     },
     "correct": 6,
     "latency_s": 0.0,
     "latency_s_per_correct": 0.0,
     "model_calls": 0,
     "tasks": 48,
     "tokens": 0,
     "tokens_per_correct": 0.0,
     "tool_calls": 6,
     "unparsed": 0,
     "wilson_95": [
      0.0586,
      0.247
     ]
    },
    "fixed_pot05_else_direct05": {
     "accuracy": 0.3333,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 3,
       "tasks": 6
      },
      "label_logs": {
       "correct": 6,
       "tasks": 6
      },
      "label_notes": {
       "correct": 2,
       "tasks": 7
      },
      "schedule": {
       "correct": 0,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 5,
       "tasks": 14
      }
     },
     "correct": 16,
     "latency_s": 170.3,
     "latency_s_per_correct": 10.64,
     "model_calls": 48,
     "tasks": 48,
     "tokens": 6599,
     "tokens_per_correct": 412.4,
     "tool_calls": 29,
     "unparsed": 7,
     "wilson_95": [
      0.2168,
      0.4746
     ]
    },
    "fixed_pot15_else_direct15": {
     "accuracy": 0.5625,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 5,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 5,
       "tasks": 6
      },
      "label_logs": {
       "correct": 5,
       "tasks": 6
      },
      "label_notes": {
       "correct": 4,
       "tasks": 7
      },
      "schedule": {
       "correct": 0,
       "tasks": 6
      },
      "table": {
       "correct": 1,
       "tasks": 3
      },
      "word_math": {
       "correct": 7,
       "tasks": 14
      }
     },
     "correct": 27,
     "latency_s": 466.0,
     "latency_s_per_correct": 17.26,
     "model_calls": 48,
     "tasks": 48,
     "tokens": 6733,
     "tokens_per_correct": 249.4,
     "tool_calls": 29,
     "unparsed": 6,
     "wilson_95": [
      0.4227,
      0.693
     ]
    },
    "m05_direct": {
     "accuracy": 0.2292,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 3,
       "tasks": 6
      },
      "label_logs": {
       "correct": 6,
       "tasks": 6
      },
      "label_notes": {
       "correct": 2,
       "tasks": 7
      },
      "schedule": {
       "correct": 0,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 0,
       "tasks": 14
      }
     },
     "correct": 11,
     "latency_s": 91.0,
     "latency_s_per_correct": 8.28,
     "model_calls": 48,
     "tasks": 48,
     "tokens": 5282,
     "tokens_per_correct": 480.2,
     "tool_calls": 0,
     "unparsed": 0,
     "wilson_95": [
      0.1331,
      0.3654
     ]
    },
    "m05_pot": {
     "accuracy": 0.1042,
     "applicable": 29,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 0,
       "tasks": 6
      },
      "label_logs": {
       "correct": 0,
       "tasks": 6
      },
      "label_notes": {
       "correct": 0,
       "tasks": 7
      },
      "schedule": {
       "correct": 0,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 5,
       "tasks": 14
      }
     },
     "correct": 5,
     "latency_s": 132.5,
     "latency_s_per_correct": 26.51,
     "model_calls": 29,
     "tasks": 48,
     "tokens": 4294,
     "tokens_per_correct": 858.8,
     "tool_calls": 29,
     "unparsed": 7,
     "wilson_95": [
      0.0453,
      0.2217
     ]
    },
    "m15_direct": {
     "accuracy": 0.3958,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 5,
       "tasks": 6
      },
      "label_logs": {
       "correct": 5,
       "tasks": 6
      },
      "label_notes": {
       "correct": 4,
       "tasks": 7
      },
      "schedule": {
       "correct": 1,
       "tasks": 6
      },
      "table": {
       "correct": 1,
       "tasks": 3
      },
      "word_math": {
       "correct": 3,
       "tasks": 14
      }
     },
     "correct": 19,
     "latency_s": 325.0,
     "latency_s_per_correct": 17.11,
     "model_calls": 48,
     "tasks": 48,
     "tokens": 5558,
     "tokens_per_correct": 292.5,
     "tool_calls": 0,
     "unparsed": 0,
     "wilson_95": [
      0.2702,
      0.5369
     ]
    },
    "m15_pot": {
     "accuracy": 0.2708,
     "applicable": 29,
     "by_family": {
      "code_trace": {
       "correct": 5,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 0,
       "tasks": 6
      },
      "label_logs": {
       "correct": 0,
       "tasks": 6
      },
      "label_notes": {
       "correct": 0,
       "tasks": 7
      },
      "schedule": {
       "correct": 0,
       "tasks": 6
      },
      "table": {
       "correct": 1,
       "tasks": 3
      },
      "word_math": {
       "correct": 7,
       "tasks": 14
      }
     },
     "correct": 13,
     "latency_s": 330.2,
     "latency_s_per_correct": 25.4,
     "model_calls": 29,
     "tasks": 48,
     "tokens": 4318,
     "tokens_per_correct": 332.2,
     "tool_calls": 29,
     "unparsed": 6,
     "wilson_95": [
      0.1657,
      0.41
     ]
    },
    "oracle_any_path": {
     "accuracy": 0.6667,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 6,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 5,
       "tasks": 6
      },
      "label_logs": {
       "correct": 6,
       "tasks": 6
      },
      "label_notes": {
       "correct": 6,
       "tasks": 7
      },
      "schedule": {
       "correct": 1,
       "tasks": 6
      },
      "table": {
       "correct": 1,
       "tasks": 3
      },
      "word_math": {
       "correct": 7,
       "tasks": 14
      }
     },
     "correct": 32,
     "latency_s": 138.2,
     "latency_s_per_correct": 4.32,
     "model_calls": 42,
     "tasks": 48,
     "tokens": 5111,
     "tokens_per_correct": 159.7,
     "tool_calls": 13,
     "unparsed": 0,
     "wilson_95": [
      0.5254,
      0.7832
     ]
    },
    "router": {
     "accuracy": 0.4583,
     "applicable": 48,
     "by_family": {
      "code_trace": {
       "correct": 6,
       "tasks": 6
      },
      "label_code_review": {
       "correct": 3,
       "tasks": 6
      },
      "label_logs": {
       "correct": 6,
       "tasks": 6
      },
      "label_notes": {
       "correct": 2,
       "tasks": 7
      },
      "schedule": {
       "correct": 0,
       "tasks": 6
      },
      "table": {
       "correct": 0,
       "tasks": 3
      },
      "word_math": {
       "correct": 5,
       "tasks": 14
      }
     },
     "correct": 22,
     "latency_s": 120.3,
     "latency_s_per_correct": 5.47,
     "model_calls": 42,
     "tasks": 48,
     "tokens": 5475,
     "tokens_per_correct": 248.9,
     "tool_calls": 20,
     "unparsed": 4,
     "wilson_95": [
      0.3258,
      0.5971
     ]
    }
   },
   "calibration_vote_agreement": {
    "bins": {
     "1_of_3_agree": {
      "accuracy": 0.0,
      "correct": 0,
      "tasks": 13
     },
     "2_of_3_agree": {
      "accuracy": 0.0588,
      "correct": 1,
      "tasks": 17
     },
     "3_of_3_agree": {
      "accuracy": 0.5556,
      "correct": 10,
      "tasks": 18
     }
    },
    "ece": 0.4722,
    "note": "confidence = fraction of voters agreeing with the majority"
   },
   "communication_overhead_tokens": {
    "c01_single": 0,
    "c02_parallel_vote": 10840,
    "c03_pairwise_debate": 10282,
    "c04_critic_reviser": 6230,
    "c05_planner_executor": 9622,
    "c06_hierarchical": 22247,
    "c07_specialist_committee": 11256,
    "c08_evidence_first": 8243,
    "c09_parallel_synthesis": 17660,
    "c10_adaptive_graph": 3460
   },
   "coordination_holm_adjusted_p": {
    "c02_parallel_vote": 1.0,
    "c03_pairwise_debate": 1.0,
    "c04_critic_reviser": 1.0,
    "c05_planner_executor": 1.0,
    "c06_hierarchical": 1.0,
    "c07_specialist_committee": 1.0,
    "c08_evidence_first": 1.0,
    "c09_parallel_synthesis": 1.0
   },
   "coordination_vs_single": {
    "c02_parallel_vote": {
     "a_correct": 11,
     "b_correct": 11,
     "bootstrap_95": [
      -0.0625,
      0.0625
     ],
     "diff": 0.0,
     "mcnemar": {
      "a_only": 1,
      "b_only": 1,
      "discordant": 2,
      "p_two_sided": 1.0,
      "significant_at_0.05": false
     },
     "n": 48
    },
    "c03_pairwise_debate": {
     "a_correct": 11,
     "b_correct": 11,
     "bootstrap_95": [
      0.0,
      0.0
     ],
     "diff": 0.0,
     "mcnemar": {
      "a_only": 0,
      "b_only": 0,
      "discordant": 0,
      "p_two_sided": 1.0,
      "significant_at_0.05": false
     },
     "n": 48
    },
    "c04_critic_reviser": {
     "a_correct": 11,
     "b_correct": 11,
     "bootstrap_95": [
      0.0,
      0.0
     ],
     "diff": 0.0,
     "mcnemar": {
      "a_only": 0,
      "b_only": 0,
      "discordant": 0,
      "p_two_sided": 1.0,
      "significant_at_0.05": false
     },
     "n": 48
    },
    "c05_planner_executor": {
     "a_correct": 11,
     "b_correct": 13,
     "bootstrap_95": [
      -0.0625,
      0.1458
     ],
     "diff": 0.0417,
     "mcnemar": {
      "a_only": 2,
      "b_only": 4,
      "discordant": 6,
      "p_two_sided": 0.6875,
      "significant_at_0.05": false
     },
     "n": 48
    },
    "c06_hierarchical": {
     "a_correct": 11,
     "b_correct": 15,
     "bootstrap_95": [
      -0.0417,
      0.2083
     ],
     "diff": 0.0833,
     "mcnemar": {
      "a_only": 3,
      "b_only": 7,
      "discordant": 10,
      "p_two_sided": 0.34375,
      "significant_at_0.05": false
     },
     "n": 48
    },
    "c07_specialist_committee": {
     "a_correct": 11,
     "b_correct": 10,
     "bootstrap_95": [
      -0.0625,
      0.0
     ],
     "diff": -0.0208,
     "mcnemar": {
      "a_only": 1,
      "b_only": 0,
      "discordant": 1,
      "p_two_sided": 1.0,
      "significant_at_0.05": false
     },
     "n": 48
    },
    "c08_evidence_first": {
     "a_correct": 11,
     "b_correct": 16,
     "bootstrap_95": [
      0.0,
      0.2083
     ],
     "diff": 0.1042,
     "mcnemar": {
      "a_only": 1,
      "b_only": 6,
      "discordant": 7,
      "p_two_sided": 0.125,
      "significant_at_0.05": false
     },
     "n": 48
    },
    "c09_parallel_synthesis": {
     "a_correct": 11,
     "b_correct": 13,
     "bootstrap_95": [
      0.0,
      0.1042
     ],
     "diff": 0.0417,
     "mcnemar": {
      "a_only": 0,
      "b_only": 2,
      "discordant": 2,
      "p_two_sided": 0.5,
      "significant_at_0.05": false
     },
     "n": 48
    },
    "c10_adaptive_graph": {
     "a_correct": 11,
     "b_correct": 22,
     "bootstrap_95": [
      0.125,
      0.3542
     ],
     "diff": 0.2292,
     "mcnemar": {
      "a_only": 0,
      "b_only": 11,
      "discordant": 11,
      "p_two_sided": 0.000977,
      "significant_at_0.05": true
     },
     "n": 48
    }
   },
   "error_severity": {
    "c01_single": {
     "correct": 11,
     "numeric_far": 21,
     "numeric_near_miss": 8,
     "wrong_label": 8
    },
    "c10_adaptive_graph": {
     "correct": 22,
     "no_answer": 4,
     "numeric_far": 10,
     "numeric_near_miss": 4,
     "wrong_label": 8
    },
    "m15_direct": {
     "correct": 19,
     "numeric_far": 17,
     "numeric_near_miss": 7,
     "wrong_label": 5
    },
    "router": {
     "correct": 22,
     "no_answer": 4,
     "numeric_far": 10,
     "numeric_near_miss": 4,
     "wrong_label": 8
    }
   },
   "label_prf": {
    "c01_single": {
     "classes": 16,
     "label_tasks": 19,
     "macro_f1": 0.5104,
     "macro_precision": 0.4896,
     "macro_recall": 0.5625
    },
    "c02_parallel_vote": {
     "classes": 16,
     "label_tasks": 19,
     "macro_f1": 0.4688,
     "macro_precision": 0.4583,
     "macro_recall": 0.5
    },
    "c07_specialist_committee": {
     "classes": 16,
     "label_tasks": 19,
     "macro_f1": 0.4688,
     "macro_precision": 0.4583,
     "macro_recall": 0.5
    },
    "router": {
     "classes": 16,
     "label_tasks": 19,
     "macro_f1": 0.5104,
     "macro_precision": 0.4896,
     "macro_recall": 0.5625
    }
   },
   "router_vs_best_fixed_on_train": {
    "a_correct": 19,
    "b_correct": 22,
    "best_fixed": "m15_direct",
    "bootstrap_95": [
     -0.1042,
     0.25
    ],
    "diff": 0.0625,
    "mcnemar": {
     "a_only": 8,
     "b_only": 11,
     "discordant": 19,
     "p_two_sided": 0.647606,
     "significant_at_0.05": false
    },
    "n": 48
   },
   "router_vs_exploratory_baselines": {
    "fixed_pot05_else_direct05": {
     "a_correct": 16,
     "b_correct": 22,
     "bootstrap_95": [
      0.0417,
      0.2292
     ],
     "diff": 0.125,
     "mcnemar": {
      "a_only": 0,
      "b_only": 6,
      "discordant": 6,
      "p_two_sided": 0.03125,
      "significant_at_0.05": true
     },
     "n": 48
    },
    "fixed_pot15_else_direct15": {
     "a_correct": 27,
     "b_correct": 22,
     "bootstrap_95": [
      -0.25,
      0.0417
     ],
     "diff": -0.1042,
     "mcnemar": {
      "a_only": 9,
      "b_only": 4,
      "discordant": 13,
      "p_two_sided": 0.266846,
      "significant_at_0.05": false
     },
     "n": 48
    },
    "m05_direct": {
     "a_correct": 11,
     "b_correct": 22,
     "bootstrap_95": [
      0.125,
      0.3542
     ],
     "diff": 0.2292,
     "mcnemar": {
      "a_only": 0,
      "b_only": 11,
      "discordant": 11,
      "p_two_sided": 0.000977,
      "significant_at_0.05": true
     },
     "n": 48
    },
    "m15_direct": {
     "a_correct": 19,
     "b_correct": 22,
     "bootstrap_95": [
      -0.1042,
      0.25
     ],
     "diff": 0.0625,
     "mcnemar": {
      "a_only": 8,
      "b_only": 11,
      "discordant": 19,
      "p_two_sided": 0.647606,
      "significant_at_0.05": false
     },
     "n": 48
    }
   },
   "routing": {
    "by_family": {
     "code_trace": {
      "best_correct": 6,
      "best_in_hindsight": "det_exec",
      "chosen": "det_exec",
      "chosen_correct": 6,
      "routing_decision_optimal": true,
      "tasks": 6
     },
     "label_code_review": {
      "best_correct": 5,
      "best_in_hindsight": "m15_direct",
      "chosen": "m05_direct",
      "chosen_correct": 3,
      "routing_decision_optimal": false,
      "tasks": 6
     },
     "label_logs": {
      "best_correct": 6,
      "best_in_hindsight": "m05_direct",
      "chosen": "m05_direct",
      "chosen_correct": 6,
      "routing_decision_optimal": true,
      "tasks": 6
     },
     "label_notes": {
      "best_correct": 4,
      "best_in_hindsight": "m15_direct",
      "chosen": "m05_direct",
      "chosen_correct": 2,
      "routing_decision_optimal": false,
      "tasks": 7
     },
     "schedule": {
      "best_correct": 1,
      "best_in_hindsight": "m15_direct",
      "chosen": "m05_direct",
      "chosen_correct": 0,
      "routing_decision_optimal": false,
      "tasks": 6
     },
     "table": {
      "best_correct": 1,
      "best_in_hindsight": "m15_direct",
      "chosen": "m05_direct",
      "chosen_correct": 0,
      "routing_decision_optimal": false,
      "tasks": 3
     },
     "word_math": {
      "best_correct": 7,
      "best_in_hindsight": "m15_pot",
      "chosen": "m05_pot",
      "chosen_correct": 5,
      "routing_decision_optimal": false,
      "tasks": 14
     }
    },
    "families": 7,
    "families_where_decision_was_optimal_in_hindsight": 2,
    "oracle_accuracy": 0.6667,
    "regret_vs_oracle": 0.2084,
    "selected_paths": {
     "det_exec": 6,
     "m05_direct": 28,
     "m05_pot": 14
    }
   },
   "tasks": 48,
   "value_per_participant": {
    "c01_single": {
     "accuracy": 0.2292,
     "delta_per_extra_call": null,
     "delta_vs_single": 0.0,
     "model_calls_per_task": 1.0
    },
    "c02_parallel_vote": {
     "accuracy": 0.2292,
     "delta_per_extra_call": 0.0,
     "delta_vs_single": 0.0,
     "model_calls_per_task": 3.0
    },
    "c03_pairwise_debate": {
     "accuracy": 0.2292,
     "delta_per_extra_call": 0.0,
     "delta_vs_single": 0.0,
     "model_calls_per_task": 2.9
    },
    "c04_critic_reviser": {
     "accuracy": 0.2292,
     "delta_per_extra_call": 0.0,
     "delta_vs_single": 0.0,
     "model_calls_per_task": 2.0
    },
    "c05_planner_executor": {
     "accuracy": 0.2708,
     "delta_per_extra_call": 0.0417,
     "delta_vs_single": 0.0417,
     "model_calls_per_task": 2.0
    },
    "c06_hierarchical": {
     "accuracy": 0.3125,
     "delta_per_extra_call": 0.0301,
     "delta_vs_single": 0.0833,
     "model_calls_per_task": 3.77
    },
    "c07_specialist_committee": {
     "accuracy": 0.2083,
     "delta_per_extra_call": -0.0104,
     "delta_vs_single": -0.0208,
     "model_calls_per_task": 3.0
    },
    "c08_evidence_first": {
     "accuracy": 0.3333,
     "delta_per_extra_call": 0.1042,
     "delta_vs_single": 0.1042,
     "model_calls_per_task": 2.0
    },
    "c09_parallel_synthesis": {
     "accuracy": 0.2708,
     "delta_per_extra_call": 0.0139,
     "delta_vs_single": 0.0417,
     "model_calls_per_task": 4.0
    },
    "c10_adaptive_graph": {
     "accuracy": 0.4583,
     "delta_per_extra_call": 0.9168,
     "delta_vs_single": 0.2292,
     "model_calls_per_task": 1.25
    }
   }
  },
  "train": {
   "arms": {
    "c10_adaptive_graph": {
     "accuracy": 0.5,
     "applicable": 40,
     "by_family": {
      "code_trace": {
       "correct": 5,
       "tasks": 5
      },
      "label_code_review": {
       "correct": 3,
       "tasks": 5
      },
      "label_logs": {
       "correct": 5,
       "tasks": 5
      },
      "label_notes": {
       "correct": 1,
       "tasks": 3
      },
      "schedule": {
       "correct": 2,
       "tasks": 5
      },
      "table": {
       "correct": 0,
       "tasks": 4
      },
      "word_math": {
       "correct": 4,
       "tasks": 13
      }
     },
     "correct": 20,
     "latency_s": 234.2,
     "latency_s_per_correct": 11.71,
     "model_calls": 51,
     "tasks": 40,
     "tokens": 7552,
     "tokens_per_correct": 377.6,
     "tool_calls": 34,
     "unparsed": 3,
     "wilson_95": [
      0.352,
      0.648
     ]
    },
    "det_exec": {
     "accuracy": 0.125,
     "applicable": 5,
     "by_family": {
      "code_trace": {
       "correct": 5,
       "tasks": 5
      },
      "label_code_review": {
       "correct": 0,
       "tasks": 5
      },
      "label_logs": {
       "correct": 0,
       "tasks": 5
      },
      "label_notes": {
       "correct": 0,
       "tasks": 3
      },
      "schedule": {
       "correct": 0,
       "tasks": 5
      },
      "table": {
       "correct": 0,
       "tasks": 4
      },
      "word_math": {
       "correct": 0,
       "tasks": 13
      }
     },
     "correct": 5,
     "latency_s": 0.0,
     "latency_s_per_correct": 0.0,
     "model_calls": 0,
     "tasks": 40,
     "tokens": 0,
     "tokens_per_correct": 0.0,
     "tool_calls": 5,
     "unparsed": 0,
     "wilson_95": [
      0.0546,
      0.2611
     ]
    },
    "fixed_pot05_else_direct05": {
     "accuracy": 0.375,
     "applicable": 40,
     "by_family": {
      "code_trace": {
       "correct": 2,
       "tasks": 5
      },
      "label_code_review": {
       "correct": 3,
       "tasks": 5
      },
      "label_logs": {
       "correct": 5,
       "tasks": 5
      },
      "label_notes": {
       "correct": 1,
       "tasks": 3
      },
      "schedule": {
       "correct": 0,
       "tasks": 5
      },
      "table": {
       "correct": 0,
       "tasks": 4
      },
      "word_math": {
       "correct": 4,
       "tasks": 13
      }
     },
     "correct": 15,
     "latency_s": 142.3,
     "latency_s_per_correct": 9.48,
     "model_calls": 40,
     "tasks": 40,
     "tokens": 5875,
     "tokens_per_correct": 391.7,
     "tool_calls": 27,
     "unparsed": 8,
     "wilson_95": [
      0.2422,
      0.5297
     ]
    },
    "fixed_pot15_else_direct15": {
     "accuracy": 0.475,
     "applicable": 40,
     "by_family": {
      "code_trace": {
       "correct": 3,
       "tasks": 5
      },
      "label_code_review": {
       "correct": 4,
       "tasks": 5
      },
      "label_logs": {
       "correct": 5,
       "tasks": 5
      },
      "label_notes": {
       "correct": 0,
       "tasks": 3
      },
      "schedule": {
       "correct": 2,
       "tasks": 5
      },
      "table": {
       "correct": 0,
       "tasks": 4
      },
      "word_math": {
       "correct": 5,
       "tasks": 13
      }
     },
     "correct": 19,
     "latency_s": 446.6,
     "latency_s_per_correct": 23.5,
     "model_calls": 40,
     "tasks": 40,
     "tokens": 5884,
     "tokens_per_correct": 309.7,
     "tool_calls": 27,
     "unparsed": 9,
     "wilson_95": [
      0.3294,
      0.625
     ]
    },
    "m05_direct": {
     "accuracy": 0.25,
     "applicable": 40,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 5
      },
      "label_code_review": {
       "correct": 3,
       "tasks": 5
      },
      "label_logs": {
       "correct": 5,
       "tasks": 5
      },
      "label_notes": {
       "correct": 1,
       "tasks": 3
      },
      "schedule": {
       "correct": 1,
       "tasks": 5
      },
      "table": {
       "correct": 0,
       "tasks": 4
      },
      "word_math": {
       "correct": 0,
       "tasks": 13
      }
     },
     "correct": 10,
     "latency_s": 74.3,
     "latency_s_per_correct": 7.43,
     "model_calls": 40,
     "tasks": 40,
     "tokens": 4467,
     "tokens_per_correct": 446.7,
     "tool_calls": 0,
     "unparsed": 0,
     "wilson_95": [
      0.1419,
      0.4019
     ]
    },
    "m05_pot": {
     "accuracy": 0.15,
     "applicable": 27,
     "by_family": {
      "code_trace": {
       "correct": 2,
       "tasks": 5
      },
      "label_code_review": {
       "correct": 0,
       "tasks": 5
      },
      "label_logs": {
       "correct": 0,
       "tasks": 5
      },
      "label_notes": {
       "correct": 0,
       "tasks": 3
      },
      "schedule": {
       "correct": 0,
       "tasks": 5
      },
      "table": {
       "correct": 0,
       "tasks": 4
      },
      "word_math": {
       "correct": 4,
       "tasks": 13
      }
     },
     "correct": 6,
     "latency_s": 118.7,
     "latency_s_per_correct": 19.79,
     "model_calls": 27,
     "tasks": 40,
     "tokens": 4254,
     "tokens_per_correct": 709.0,
     "tool_calls": 27,
     "unparsed": 8,
     "wilson_95": [
      0.0706,
      0.2907
     ]
    },
    "m135_direct": {
     "accuracy": 0.025,
     "applicable": 40,
     "by_family": {
      "code_trace": {
       "correct": 0,
       "tasks": 5
      },
      "label_code_review": {
       "correct": 1,
       "tasks": 5
      },
      "label_logs": {
       "correct": 0,
       "tasks": 5
      },
      "label_notes": {
       "correct": 0,
       "tasks": 3
      },
      "schedule": {
       "correct": 0,
       "tasks": 5
      },
      "table": {
       "correct": 0,
       "tasks": 4
      },
      "word_math": {
       "correct": 0,
       "tasks": 13
      }
     },
     "correct": 1,
     "latency_s": 54.4,
     "latency_s_per_correct": 54.42,
     "model_calls": 40,
     "tasks": 40,
     "tokens": 5222,
     "tokens_per_correct": 5222.0,
     "tool_calls": 0,
     "unparsed": 20,
     "wilson_95": [
      0.0044,
      0.1288
     ]
    },
    "m15_direct": {
     "accuracy": 0.325,
     "applicable": 40,
     "by_family": {
      "code_trace": {
       "correct": 2,
       "tasks": 5
      },
      "label_code_review": {
       "correct": 4,
       "tasks": 5
      },
      "label_logs": {
       "correct": 5,
       "tasks": 5
      },
      "label_notes": {
       "correct": 0,
       "tasks": 3
      },
      "schedule": {
       "correct": 0,
       "tasks": 5
      },
      "table": {
       "correct": 0,
       "tasks": 4
      },
      "word_math": {
       "correct": 2,
       "tasks": 13
      }
     },
     "correct": 13,
     "latency_s": 300.3,
     "latency_s_per_correct": 23.1,
     "model_calls": 40,
     "tasks": 40,
     "tokens": 4677,
     "tokens_per_correct": 359.8,
     "tool_calls": 0,
     "unparsed": 0,
     "wilson_95": [
      0.2008,
      0.4798
     ]
    },
    "m15_pot": {
     "accuracy": 0.25,
     "applicable": 27,
     "by_family": {
      "code_trace": {
       "correct": 3,
       "tasks": 5
      },
      "label_code_review": {
       "correct": 0,
       "tasks": 5
      },
      "label_logs": {
       "correct": 0,
       "tasks": 5
      },
      "label_notes": {
       "correct": 0,
       "tasks": 3
      },
      "schedule": {
       "correct": 2,
       "tasks": 5
      },
      "table": {
       "correct": 0,
       "tasks": 4
      },
      "word_math": {
       "correct": 5,
       "tasks": 13
      }
     },
     "correct": 10,
     "latency_s": 351.0,
     "latency_s_per_correct": 35.1,
     "model_calls": 27,
     "tasks": 40,
     "tokens": 4231,
     "tokens_per_correct": 423.1,
     "tool_calls": 27,
     "unparsed": 9,
     "wilson_95": [
      0.1419,
      0.4019
     ]
    },
    "oracle_any_path": {
     "accuracy": 0.575,
     "applicable": 40,
     "by_family": {
      "code_trace": {
       "correct": 5,
       "tasks": 5
      },
      "label_code_review": {
       "correct": 4,
       "tasks": 5
      },
      "label_logs": {
       "correct": 5,
       "tasks": 5
      },
      "label_notes": {
       "correct": 1,
       "tasks": 3
      },
      "schedule": {
       "correct": 3,
       "tasks": 5
      },
      "table": {
       "correct": 0,
       "tasks": 4
      },
      "word_math": {
       "correct": 5,
       "tasks": 13
      }
     },
     "correct": 23,
     "latency_s": 112.6,
     "latency_s_per_correct": 4.9,
     "model_calls": 35,
     "tasks": 40,
     "tokens": 4318,
     "tokens_per_correct": 187.7,
     "tool_calls": 12,
     "unparsed": 0,
     "wilson_95": [
      0.4219,
      0.7149
     ]
    },
    "router": {
     "accuracy": 0.475,
     "applicable": 40,
     "by_family": {
      "code_trace": {
       "correct": 5,
       "tasks": 5
      },
      "label_code_review": {
       "correct": 3,
       "tasks": 5
      },
      "label_logs": {
       "correct": 5,
       "tasks": 5
      },
      "label_notes": {
       "correct": 1,
       "tasks": 3
      },
      "schedule": {
       "correct": 1,
       "tasks": 5
      },
      "table": {
       "correct": 0,
       "tasks": 4
      },
      "word_math": {
       "correct": 4,
       "tasks": 13
      }
     },
     "correct": 19,
     "latency_s": 94.5,
     "latency_s_per_correct": 4.98,
     "model_calls": 35,
     "tasks": 40,
     "tokens": 4635,
     "tokens_per_correct": 243.9,
     "tool_calls": 18,
     "unparsed": 3,
     "wilson_95": [
      0.3294,
      0.625
     ]
    }
   },
   "tasks": 40
  }
 }
}