{"attack_rate": 0.0, "audits": [], "budget_tokens": 4000, "episodes": 53, "model_calls": 0, "rows": [{"arm": "M0_none", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "34c96245-93cc-4120-9c61-bb011daeca07", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 25, "novel_recall": 0.0, "qid": "a037da30-e58c-40b0-bbe0-9120fdfcb460", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 29, "novel_recall": 0.0, "qid": "e963c86d-bd4d-405e-9a28-838aac105c1b", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 32, "novel_recall": 0.0, "qid": "b9065fe0-567b-4cb2-9285-986f6703a5e4", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 30, "novel_recall": 0.0, "qid": "3aa1c6f2-544a-4915-90e1-665a62785d55", "type": "B"}, {"arm": "M0_none", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 28, "novel_recall": 0.0, "qid": "49207036-7c69-44a5-9376-e3519e1c558a", "type": "B"}, {"arm": "M0_none", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 38, "novel_recall": 0.0, "qid": "08665618-b4e5-41be-a9b2-72e6223fa475", "type": "B"}, {"arm": "M0_none", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 30, "novel_recall": 0.0, "qid": "4a12dd4c-7cf5-4d7a-91b3-90bd3f14f38a", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 21, "novel_recall": 0.0, "qid": "02f41e69-80d4-4cf3-82cb-7128889db83a", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 22, "novel_recall": 0.0, "qid": "d4b7e11f-5d06-4a95-a669-e9ebbfc9f52f", "type": "D"}, {"arm": "M0_none", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 21, "novel_recall": 0.0, "qid": "b156014e-2786-4e48-9180-928286f457bd", "type": "D"}, {"arm": "M0_none", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "77244419-477c-4bbb-afab-0051b3a224eb", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 24, "novel_recall": 0.1667, "qid": "34c96245-93cc-4120-9c61-bb011daeca07", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 25, "novel_recall": 0.16, "qid": "a037da30-e58c-40b0-bbe0-9120fdfcb460", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 29, "novel_recall": 0.2069, "qid": "e963c86d-bd4d-405e-9a28-838aac105c1b", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 32, "novel_recall": 0.1562, "qid": "b9065fe0-567b-4cb2-9285-986f6703a5e4", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 30, "novel_recall": 0.0333, "qid": "3aa1c6f2-544a-4915-90e1-665a62785d55", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 28, "novel_recall": 0.1429, "qid": "49207036-7c69-44a5-9376-e3519e1c558a", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 38, "novel_recall": 0.2105, "qid": "08665618-b4e5-41be-a9b2-72e6223fa475", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 30, "novel_recall": 0.1667, "qid": "4a12dd4c-7cf5-4d7a-91b3-90bd3f14f38a", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 21, "novel_recall": 0.1905, "qid": "02f41e69-80d4-4cf3-82cb-7128889db83a", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 22, "novel_recall": 0.1818, "qid": "d4b7e11f-5d06-4a95-a669-e9ebbfc9f52f", "type": "D"}, {"arm": "M1_naive", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 21, "novel_recall": 0.2381, "qid": "b156014e-2786-4e48-9180-928286f457bd", "type": "D"}, {"arm": "M1_naive", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "77244419-477c-4bbb-afab-0051b3a224eb", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 24, "novel_recall": 0.1667, "qid": "34c96245-93cc-4120-9c61-bb011daeca07", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 25, "novel_recall": 0.16, "qid": "a037da30-e58c-40b0-bbe0-9120fdfcb460", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 29, "novel_recall": 0.2069, "qid": "e963c86d-bd4d-405e-9a28-838aac105c1b", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 32, "novel_recall": 0.1562, "qid": "b9065fe0-567b-4cb2-9285-986f6703a5e4", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 30, "novel_recall": 0.0333, "qid": "3aa1c6f2-544a-4915-90e1-665a62785d55", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 28, "novel_recall": 0.1429, "qid": "49207036-7c69-44a5-9376-e3519e1c558a", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 38, "novel_recall": 0.2105, "qid": "08665618-b4e5-41be-a9b2-72e6223fa475", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 30, "novel_recall": 0.1667, "qid": "4a12dd4c-7cf5-4d7a-91b3-90bd3f14f38a", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 21, "novel_recall": 0.1905, "qid": "02f41e69-80d4-4cf3-82cb-7128889db83a", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 22, "novel_recall": 0.1818, "qid": "d4b7e11f-5d06-4a95-a669-e9ebbfc9f52f", "type": "D"}, {"arm": "M2_evidence", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 21, "novel_recall": 0.2381, "qid": "b156014e-2786-4e48-9180-928286f457bd", "type": "D"}, {"arm": "M2_evidence", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "77244419-477c-4bbb-afab-0051b3a224eb", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 24, "novel_recall": 0.1667, "qid": "34c96245-93cc-4120-9c61-bb011daeca07", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 25, "novel_recall": 0.16, "qid": "a037da30-e58c-40b0-bbe0-9120fdfcb460", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 29, "novel_recall": 0.2069, "qid": "e963c86d-bd4d-405e-9a28-838aac105c1b", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 32, "novel_recall": 0.1562, "qid": "b9065fe0-567b-4cb2-9285-986f6703a5e4", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 30, "novel_recall": 0.0333, "qid": "3aa1c6f2-544a-4915-90e1-665a62785d55", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 28, "novel_recall": 0.1429, "qid": "49207036-7c69-44a5-9376-e3519e1c558a", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 38, "novel_recall": 0.2105, "qid": "08665618-b4e5-41be-a9b2-72e6223fa475", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 30, "novel_recall": 0.1667, "qid": "4a12dd4c-7cf5-4d7a-91b3-90bd3f14f38a", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 21, "novel_recall": 0.1905, "qid": "02f41e69-80d4-4cf3-82cb-7128889db83a", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 22, "novel_recall": 0.1818, "qid": "d4b7e11f-5d06-4a95-a669-e9ebbfc9f52f", "type": "D"}, {"arm": "M3_causal", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 21, "novel_recall": 0.2381, "qid": "b156014e-2786-4e48-9180-928286f457bd", "type": "D"}, {"arm": "M3_causal", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "77244419-477c-4bbb-afab-0051b3a224eb", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 24, "novel_recall": 0.1667, "qid": "34c96245-93cc-4120-9c61-bb011daeca07", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 25, "novel_recall": 0.16, "qid": "a037da30-e58c-40b0-bbe0-9120fdfcb460", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 29, "novel_recall": 0.2069, "qid": "e963c86d-bd4d-405e-9a28-838aac105c1b", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 32, "novel_recall": 0.1562, "qid": "b9065fe0-567b-4cb2-9285-986f6703a5e4", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 30, "novel_recall": 0.0333, "qid": "3aa1c6f2-544a-4915-90e1-665a62785d55", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 28, "novel_recall": 0.1429, "qid": "49207036-7c69-44a5-9376-e3519e1c558a", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 38, "novel_recall": 0.2105, "qid": "08665618-b4e5-41be-a9b2-72e6223fa475", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 30, "novel_recall": 0.1667, "qid": "4a12dd4c-7cf5-4d7a-91b3-90bd3f14f38a", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 21, "novel_recall": 0.1905, "qid": "02f41e69-80d4-4cf3-82cb-7128889db83a", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 22, "novel_recall": 0.1818, "qid": "d4b7e11f-5d06-4a95-a669-e9ebbfc9f52f", "type": "D"}, {"arm": "M4_validated", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 21, "novel_recall": 0.2381, "qid": "b156014e-2786-4e48-9180-928286f457bd", "type": "D"}, {"arm": "M4_validated", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "77244419-477c-4bbb-afab-0051b3a224eb", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 24, "novel_recall": 0.1667, "qid": "34c96245-93cc-4120-9c61-bb011daeca07", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 25, "novel_recall": 0.16, "qid": "a037da30-e58c-40b0-bbe0-9120fdfcb460", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 29, "novel_recall": 0.2069, "qid": "e963c86d-bd4d-405e-9a28-838aac105c1b", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 32, "novel_recall": 0.1562, "qid": "b9065fe0-567b-4cb2-9285-986f6703a5e4", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 30, "novel_recall": 0.0333, "qid": "3aa1c6f2-544a-4915-90e1-665a62785d55", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 28, "novel_recall": 0.1429, "qid": "49207036-7c69-44a5-9376-e3519e1c558a", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 38, "novel_recall": 0.2105, "qid": "08665618-b4e5-41be-a9b2-72e6223fa475", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 30, "novel_recall": 0.1667, "qid": "4a12dd4c-7cf5-4d7a-91b3-90bd3f14f38a", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 21, "novel_recall": 0.1905, "qid": "02f41e69-80d4-4cf3-82cb-7128889db83a", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 22, "novel_recall": 0.1818, "qid": "d4b7e11f-5d06-4a95-a669-e9ebbfc9f52f", "type": "D"}, {"arm": "R_recent", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 21, "novel_recall": 0.2381, "qid": "b156014e-2786-4e48-9180-928286f457bd", "type": "D"}, {"arm": "R_recent", "domain": "Game", "episode": "3", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "77244419-477c-4bbb-afab-0051b3a224eb", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 32, "novel_recall": 0.0, "qid": "44c68e51-30ac-4361-83e0-daf29d35c657", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 25, "novel_recall": 0.0, "qid": "c5f5963a-8206-4362-acf2-f5f8e7d9e224", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "732c8532-bea7-4f4c-89e3-c3815405a92c", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 37, "novel_recall": 0.0, "qid": "341f6b04-8ee6-498f-a182-d7a0a4cfb7af", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 29, "novel_recall": 0.0, "qid": "d0569b90-1d1c-4a32-ac47-a93db2a04fa8", "type": "B"}, {"arm": "M0_none", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 28, "novel_recall": 0.0, "qid": "9e78b817-4353-44eb-8d79-d41ecba62452", "type": "B"}, {"arm": "M0_none", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "bb8acd79-0e18-4876-b696-007901ea57a2", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 25, "novel_recall": 0.0, "qid": "b1fc6fce-55b5-4279-aa62-e16906e25596", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 31, "novel_recall": 0.0, "qid": "4adf72ef-84ac-4149-ad8b-8c622f100fad", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 28, "novel_recall": 0.0, "qid": "1ca9a534-efaa-47f9-8157-652b24d51eae", "type": "D"}, {"arm": "M0_none", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 34, "novel_recall": 0.0, "qid": "e15db4e1-5c33-4cbe-b83e-4e74d8629b03", "type": "D"}, {"arm": "M0_none", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "df17ee6d-e82e-4834-b489-1a23c8533605", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 32, "novel_recall": 0.2188, "qid": "44c68e51-30ac-4361-83e0-daf29d35c657", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 25, "novel_recall": 0.24, "qid": "c5f5963a-8206-4362-acf2-f5f8e7d9e224", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 24, "novel_recall": 0.2083, "qid": "732c8532-bea7-4f4c-89e3-c3815405a92c", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 37, "novel_recall": 0.1351, "qid": "341f6b04-8ee6-498f-a182-d7a0a4cfb7af", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 29, "novel_recall": 0.2414, "qid": "d0569b90-1d1c-4a32-ac47-a93db2a04fa8", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 28, "novel_recall": 0.1429, "qid": "9e78b817-4353-44eb-8d79-d41ecba62452", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 24, "novel_recall": 0.0417, "qid": "bb8acd79-0e18-4876-b696-007901ea57a2", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 25, "novel_recall": 0.0, "qid": "b1fc6fce-55b5-4279-aa62-e16906e25596", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 31, "novel_recall": 0.0323, "qid": "4adf72ef-84ac-4149-ad8b-8c622f100fad", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 28, "novel_recall": 0.0714, "qid": "1ca9a534-efaa-47f9-8157-652b24d51eae", "type": "D"}, {"arm": "M1_naive", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 34, "novel_recall": 0.1765, "qid": "e15db4e1-5c33-4cbe-b83e-4e74d8629b03", "type": "D"}, {"arm": "M1_naive", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "df17ee6d-e82e-4834-b489-1a23c8533605", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 32, "novel_recall": 0.2188, "qid": "44c68e51-30ac-4361-83e0-daf29d35c657", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 25, "novel_recall": 0.24, "qid": "c5f5963a-8206-4362-acf2-f5f8e7d9e224", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 24, "novel_recall": 0.2083, "qid": "732c8532-bea7-4f4c-89e3-c3815405a92c", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 37, "novel_recall": 0.1351, "qid": "341f6b04-8ee6-498f-a182-d7a0a4cfb7af", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 29, "novel_recall": 0.2414, "qid": "d0569b90-1d1c-4a32-ac47-a93db2a04fa8", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 28, "novel_recall": 0.1429, "qid": "9e78b817-4353-44eb-8d79-d41ecba62452", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 24, "novel_recall": 0.0417, "qid": "bb8acd79-0e18-4876-b696-007901ea57a2", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 25, "novel_recall": 0.0, "qid": "b1fc6fce-55b5-4279-aa62-e16906e25596", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 31, "novel_recall": 0.0323, "qid": "4adf72ef-84ac-4149-ad8b-8c622f100fad", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 28, "novel_recall": 0.0714, "qid": "1ca9a534-efaa-47f9-8157-652b24d51eae", "type": "D"}, {"arm": "M2_evidence", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 34, "novel_recall": 0.1765, "qid": "e15db4e1-5c33-4cbe-b83e-4e74d8629b03", "type": "D"}, {"arm": "M2_evidence", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "df17ee6d-e82e-4834-b489-1a23c8533605", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 32, "novel_recall": 0.2188, "qid": "44c68e51-30ac-4361-83e0-daf29d35c657", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 25, "novel_recall": 0.24, "qid": "c5f5963a-8206-4362-acf2-f5f8e7d9e224", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 24, "novel_recall": 0.2083, "qid": "732c8532-bea7-4f4c-89e3-c3815405a92c", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 37, "novel_recall": 0.1351, "qid": "341f6b04-8ee6-498f-a182-d7a0a4cfb7af", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 29, "novel_recall": 0.2069, "qid": "d0569b90-1d1c-4a32-ac47-a93db2a04fa8", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 28, "novel_recall": 0.1429, "qid": "9e78b817-4353-44eb-8d79-d41ecba62452", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 24, "novel_recall": 0.0417, "qid": "bb8acd79-0e18-4876-b696-007901ea57a2", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 25, "novel_recall": 0.0, "qid": "b1fc6fce-55b5-4279-aa62-e16906e25596", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 31, "novel_recall": 0.0323, "qid": "4adf72ef-84ac-4149-ad8b-8c622f100fad", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 28, "novel_recall": 0.0714, "qid": "1ca9a534-efaa-47f9-8157-652b24d51eae", "type": "D"}, {"arm": "M3_causal", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 34, "novel_recall": 0.1765, "qid": "e15db4e1-5c33-4cbe-b83e-4e74d8629b03", "type": "D"}, {"arm": "M3_causal", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "df17ee6d-e82e-4834-b489-1a23c8533605", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 32, "novel_recall": 0.2188, "qid": "44c68e51-30ac-4361-83e0-daf29d35c657", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 25, "novel_recall": 0.24, "qid": "c5f5963a-8206-4362-acf2-f5f8e7d9e224", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 24, "novel_recall": 0.2083, "qid": "732c8532-bea7-4f4c-89e3-c3815405a92c", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 37, "novel_recall": 0.1351, "qid": "341f6b04-8ee6-498f-a182-d7a0a4cfb7af", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 29, "novel_recall": 0.2069, "qid": "d0569b90-1d1c-4a32-ac47-a93db2a04fa8", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 28, "novel_recall": 0.1429, "qid": "9e78b817-4353-44eb-8d79-d41ecba62452", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 24, "novel_recall": 0.0417, "qid": "bb8acd79-0e18-4876-b696-007901ea57a2", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 25, "novel_recall": 0.0, "qid": "b1fc6fce-55b5-4279-aa62-e16906e25596", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 31, "novel_recall": 0.0323, "qid": "4adf72ef-84ac-4149-ad8b-8c622f100fad", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 28, "novel_recall": 0.0714, "qid": "1ca9a534-efaa-47f9-8157-652b24d51eae", "type": "D"}, {"arm": "M4_validated", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 34, "novel_recall": 0.1765, "qid": "e15db4e1-5c33-4cbe-b83e-4e74d8629b03", "type": "D"}, {"arm": "M4_validated", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "df17ee6d-e82e-4834-b489-1a23c8533605", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 32, "novel_recall": 0.1875, "qid": "44c68e51-30ac-4361-83e0-daf29d35c657", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 25, "novel_recall": 0.24, "qid": "c5f5963a-8206-4362-acf2-f5f8e7d9e224", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 24, "novel_recall": 0.2083, "qid": "732c8532-bea7-4f4c-89e3-c3815405a92c", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 37, "novel_recall": 0.1351, "qid": "341f6b04-8ee6-498f-a182-d7a0a4cfb7af", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 29, "novel_recall": 0.2069, "qid": "d0569b90-1d1c-4a32-ac47-a93db2a04fa8", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 28, "novel_recall": 0.1071, "qid": "9e78b817-4353-44eb-8d79-d41ecba62452", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 24, "novel_recall": 0.0417, "qid": "bb8acd79-0e18-4876-b696-007901ea57a2", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 25, "novel_recall": 0.0, "qid": "b1fc6fce-55b5-4279-aa62-e16906e25596", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 31, "novel_recall": 0.0323, "qid": "4adf72ef-84ac-4149-ad8b-8c622f100fad", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 28, "novel_recall": 0.0357, "qid": "1ca9a534-efaa-47f9-8157-652b24d51eae", "type": "D"}, {"arm": "R_recent", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 34, "novel_recall": 0.1471, "qid": "e15db4e1-5c33-4cbe-b83e-4e74d8629b03", "type": "D"}, {"arm": "R_recent", "domain": "Game", "episode": "11", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "df17ee6d-e82e-4834-b489-1a23c8533605", "type": "B"}, {"arm": "M0_none", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 33, "novel_recall": 0.0, "qid": "5157d3c9-cde6-428c-9b5b-8aad7afa5220", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 29, "novel_recall": 0.0, "qid": "b4db534f-8d48-42dd-b28e-f236ae8df7a5", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 23, "novel_recall": 0.0, "qid": "1c73f84c-cfa9-44c3-a8d7-d60ddceb139c", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 38, "novel_recall": 0.0, "qid": "ba5581d7-f901-47ca-9ec3-2868aa52fef4", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 39, "novel_recall": 0.0, "qid": "c43ee98c-6805-484e-b306-da88b5a889a6", "type": "B"}, {"arm": "M0_none", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 35, "novel_recall": 0.0, "qid": "2ed670fd-e035-4727-9669-1738bed60456", "type": "B"}, {"arm": "M0_none", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 32, "novel_recall": 0.0, "qid": "832ecbf9-5af8-409a-8b3d-4cfd1aacc72b", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 29, "novel_recall": 0.0, "qid": "5bd578d0-bf37-40c4-8909-e38438194ebe", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 25, "novel_recall": 0.0, "qid": "09a1177d-c15a-4ed8-8978-2d419ce22575", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 26, "novel_recall": 0.0, "qid": "7134aaf4-e17a-43ae-b380-7ebbd35e5c8f", "type": "D"}, {"arm": "M0_none", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 40, "novel_recall": 0.0, "qid": "487fd35b-c6e4-4ee3-93c2-867e15fe5253", "type": "D"}, {"arm": "M0_none", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "d7180ef5-1d6a-48dc-832c-2de672c03db2", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 33, "novel_recall": 0.2121, "qid": "5157d3c9-cde6-428c-9b5b-8aad7afa5220", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 29, "novel_recall": 0.2069, "qid": "b4db534f-8d48-42dd-b28e-f236ae8df7a5", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 23, "novel_recall": 0.0435, "qid": "1c73f84c-cfa9-44c3-a8d7-d60ddceb139c", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 38, "novel_recall": 0.0789, "qid": "ba5581d7-f901-47ca-9ec3-2868aa52fef4", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 39, "novel_recall": 0.1282, "qid": "c43ee98c-6805-484e-b306-da88b5a889a6", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 35, "novel_recall": 0.0286, "qid": "2ed670fd-e035-4727-9669-1738bed60456", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 32, "novel_recall": 0.1875, "qid": "832ecbf9-5af8-409a-8b3d-4cfd1aacc72b", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 29, "novel_recall": 0.1379, "qid": "5bd578d0-bf37-40c4-8909-e38438194ebe", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 25, "novel_recall": 0.24, "qid": "09a1177d-c15a-4ed8-8978-2d419ce22575", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 26, "novel_recall": 0.2308, "qid": "7134aaf4-e17a-43ae-b380-7ebbd35e5c8f", "type": "D"}, {"arm": "M1_naive", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 40, "novel_recall": 0.15, "qid": "487fd35b-c6e4-4ee3-93c2-867e15fe5253", "type": "D"}, {"arm": "M1_naive", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 10, "novel_recall": 0.8, "qid": "d7180ef5-1d6a-48dc-832c-2de672c03db2", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 33, "novel_recall": 0.2121, "qid": "5157d3c9-cde6-428c-9b5b-8aad7afa5220", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 29, "novel_recall": 0.2069, "qid": "b4db534f-8d48-42dd-b28e-f236ae8df7a5", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 23, "novel_recall": 0.0435, "qid": "1c73f84c-cfa9-44c3-a8d7-d60ddceb139c", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 38, "novel_recall": 0.0789, "qid": "ba5581d7-f901-47ca-9ec3-2868aa52fef4", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 39, "novel_recall": 0.1282, "qid": "c43ee98c-6805-484e-b306-da88b5a889a6", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 35, "novel_recall": 0.0286, "qid": "2ed670fd-e035-4727-9669-1738bed60456", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 32, "novel_recall": 0.1875, "qid": "832ecbf9-5af8-409a-8b3d-4cfd1aacc72b", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 29, "novel_recall": 0.1379, "qid": "5bd578d0-bf37-40c4-8909-e38438194ebe", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 25, "novel_recall": 0.24, "qid": "09a1177d-c15a-4ed8-8978-2d419ce22575", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 26, "novel_recall": 0.2308, "qid": "7134aaf4-e17a-43ae-b380-7ebbd35e5c8f", "type": "D"}, {"arm": "M2_evidence", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 40, "novel_recall": 0.15, "qid": "487fd35b-c6e4-4ee3-93c2-867e15fe5253", "type": "D"}, {"arm": "M2_evidence", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 10, "novel_recall": 0.8, "qid": "d7180ef5-1d6a-48dc-832c-2de672c03db2", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 33, "novel_recall": 0.2121, "qid": "5157d3c9-cde6-428c-9b5b-8aad7afa5220", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 29, "novel_recall": 0.2069, "qid": "b4db534f-8d48-42dd-b28e-f236ae8df7a5", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 23, "novel_recall": 0.0435, "qid": "1c73f84c-cfa9-44c3-a8d7-d60ddceb139c", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 38, "novel_recall": 0.0789, "qid": "ba5581d7-f901-47ca-9ec3-2868aa52fef4", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 39, "novel_recall": 0.1282, "qid": "c43ee98c-6805-484e-b306-da88b5a889a6", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 35, "novel_recall": 0.0, "qid": "2ed670fd-e035-4727-9669-1738bed60456", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 32, "novel_recall": 0.2188, "qid": "832ecbf9-5af8-409a-8b3d-4cfd1aacc72b", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 29, "novel_recall": 0.1379, "qid": "5bd578d0-bf37-40c4-8909-e38438194ebe", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 25, "novel_recall": 0.24, "qid": "09a1177d-c15a-4ed8-8978-2d419ce22575", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 26, "novel_recall": 0.2308, "qid": "7134aaf4-e17a-43ae-b380-7ebbd35e5c8f", "type": "D"}, {"arm": "M3_causal", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 40, "novel_recall": 0.15, "qid": "487fd35b-c6e4-4ee3-93c2-867e15fe5253", "type": "D"}, {"arm": "M3_causal", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 10, "novel_recall": 0.8, "qid": "d7180ef5-1d6a-48dc-832c-2de672c03db2", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 33, "novel_recall": 0.2121, "qid": "5157d3c9-cde6-428c-9b5b-8aad7afa5220", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 29, "novel_recall": 0.2069, "qid": "b4db534f-8d48-42dd-b28e-f236ae8df7a5", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 23, "novel_recall": 0.0435, "qid": "1c73f84c-cfa9-44c3-a8d7-d60ddceb139c", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 38, "novel_recall": 0.0789, "qid": "ba5581d7-f901-47ca-9ec3-2868aa52fef4", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 39, "novel_recall": 0.1282, "qid": "c43ee98c-6805-484e-b306-da88b5a889a6", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 35, "novel_recall": 0.0, "qid": "2ed670fd-e035-4727-9669-1738bed60456", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 32, "novel_recall": 0.2188, "qid": "832ecbf9-5af8-409a-8b3d-4cfd1aacc72b", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 29, "novel_recall": 0.1379, "qid": "5bd578d0-bf37-40c4-8909-e38438194ebe", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 25, "novel_recall": 0.24, "qid": "09a1177d-c15a-4ed8-8978-2d419ce22575", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 26, "novel_recall": 0.2308, "qid": "7134aaf4-e17a-43ae-b380-7ebbd35e5c8f", "type": "D"}, {"arm": "M4_validated", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 40, "novel_recall": 0.15, "qid": "487fd35b-c6e4-4ee3-93c2-867e15fe5253", "type": "D"}, {"arm": "M4_validated", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 10, "novel_recall": 0.8, "qid": "d7180ef5-1d6a-48dc-832c-2de672c03db2", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 33, "novel_recall": 0.2121, "qid": "5157d3c9-cde6-428c-9b5b-8aad7afa5220", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 29, "novel_recall": 0.1724, "qid": "b4db534f-8d48-42dd-b28e-f236ae8df7a5", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 23, "novel_recall": 0.0435, "qid": "1c73f84c-cfa9-44c3-a8d7-d60ddceb139c", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 38, "novel_recall": 0.0789, "qid": "ba5581d7-f901-47ca-9ec3-2868aa52fef4", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 39, "novel_recall": 0.1282, "qid": "c43ee98c-6805-484e-b306-da88b5a889a6", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 35, "novel_recall": 0.0286, "qid": "2ed670fd-e035-4727-9669-1738bed60456", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 32, "novel_recall": 0.1875, "qid": "832ecbf9-5af8-409a-8b3d-4cfd1aacc72b", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 29, "novel_recall": 0.1379, "qid": "5bd578d0-bf37-40c4-8909-e38438194ebe", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 25, "novel_recall": 0.2, "qid": "09a1177d-c15a-4ed8-8978-2d419ce22575", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 26, "novel_recall": 0.2308, "qid": "7134aaf4-e17a-43ae-b380-7ebbd35e5c8f", "type": "D"}, {"arm": "R_recent", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 40, "novel_recall": 0.1, "qid": "487fd35b-c6e4-4ee3-93c2-867e15fe5253", "type": "D"}, {"arm": "R_recent", "domain": "Game", "episode": "18", "full_hit": false, "n_novel": 10, "novel_recall": 0.8, "qid": "d7180ef5-1d6a-48dc-832c-2de672c03db2", "type": "B"}, {"arm": "M0_none", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "3b192665-c28e-4d88-9b27-166bc87415d7", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 27, "novel_recall": 0.0, "qid": "95e38021-0f26-4780-b796-b0e326edf719", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 40, "novel_recall": 0.0, "qid": "ddc24b39-4283-4301-a562-d6bb478fd78e", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 20, "novel_recall": 0.0, "qid": "711d6873-a97c-4efc-b546-68245615ef60", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 33, "novel_recall": 0.0, "qid": "a96e19a0-2881-47fa-b910-f7273d081575", "type": "B"}, {"arm": "M0_none", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 43, "novel_recall": 0.0, "qid": "607218c7-eef5-4b19-be7f-46085ed3204c", "type": "B"}, {"arm": "M0_none", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 38, "novel_recall": 0.0, "qid": "e4278915-e949-49f6-a559-0e6da9250ea9", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 29, "novel_recall": 0.0, "qid": "337ca321-57cf-401b-85c9-87ec6d056ba1", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 29, "novel_recall": 0.0, "qid": "01f9818b-58a6-4343-9222-79a24768fe9f", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 51, "novel_recall": 0.0, "qid": "781550d2-f5da-4064-998b-58eaad9554c3", "type": "D"}, {"arm": "M0_none", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 33, "novel_recall": 0.0, "qid": "b856a6c6-0a12-477d-8500-1823991c1921", "type": "D"}, {"arm": "M0_none", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "7f4e071f-ecaf-4c88-8dc9-43196036d590", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 16, "novel_recall": 0.3125, "qid": "3b192665-c28e-4d88-9b27-166bc87415d7", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 27, "novel_recall": 0.2222, "qid": "95e38021-0f26-4780-b796-b0e326edf719", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 40, "novel_recall": 0.125, "qid": "ddc24b39-4283-4301-a562-d6bb478fd78e", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 20, "novel_recall": 0.2, "qid": "711d6873-a97c-4efc-b546-68245615ef60", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 33, "novel_recall": 0.1515, "qid": "a96e19a0-2881-47fa-b910-f7273d081575", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 43, "novel_recall": 0.1163, "qid": "607218c7-eef5-4b19-be7f-46085ed3204c", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 38, "novel_recall": 0.1842, "qid": "e4278915-e949-49f6-a559-0e6da9250ea9", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 29, "novel_recall": 0.1724, "qid": "337ca321-57cf-401b-85c9-87ec6d056ba1", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 29, "novel_recall": 0.1034, "qid": "01f9818b-58a6-4343-9222-79a24768fe9f", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 51, "novel_recall": 0.1765, "qid": "781550d2-f5da-4064-998b-58eaad9554c3", "type": "D"}, {"arm": "M1_naive", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 33, "novel_recall": 0.1212, "qid": "b856a6c6-0a12-477d-8500-1823991c1921", "type": "D"}, {"arm": "M1_naive", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 9, "novel_recall": 0.5556, "qid": "7f4e071f-ecaf-4c88-8dc9-43196036d590", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 16, "novel_recall": 0.3125, "qid": "3b192665-c28e-4d88-9b27-166bc87415d7", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 27, "novel_recall": 0.2222, "qid": "95e38021-0f26-4780-b796-b0e326edf719", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 40, "novel_recall": 0.125, "qid": "ddc24b39-4283-4301-a562-d6bb478fd78e", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 20, "novel_recall": 0.2, "qid": "711d6873-a97c-4efc-b546-68245615ef60", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 33, "novel_recall": 0.1515, "qid": "a96e19a0-2881-47fa-b910-f7273d081575", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 43, "novel_recall": 0.1163, "qid": "607218c7-eef5-4b19-be7f-46085ed3204c", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 38, "novel_recall": 0.1842, "qid": "e4278915-e949-49f6-a559-0e6da9250ea9", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 29, "novel_recall": 0.1724, "qid": "337ca321-57cf-401b-85c9-87ec6d056ba1", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 29, "novel_recall": 0.1034, "qid": "01f9818b-58a6-4343-9222-79a24768fe9f", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 51, "novel_recall": 0.1765, "qid": "781550d2-f5da-4064-998b-58eaad9554c3", "type": "D"}, {"arm": "M2_evidence", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 33, "novel_recall": 0.1212, "qid": "b856a6c6-0a12-477d-8500-1823991c1921", "type": "D"}, {"arm": "M2_evidence", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 9, "novel_recall": 0.5556, "qid": "7f4e071f-ecaf-4c88-8dc9-43196036d590", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 16, "novel_recall": 0.3125, "qid": "3b192665-c28e-4d88-9b27-166bc87415d7", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 27, "novel_recall": 0.2222, "qid": "95e38021-0f26-4780-b796-b0e326edf719", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 40, "novel_recall": 0.125, "qid": "ddc24b39-4283-4301-a562-d6bb478fd78e", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 20, "novel_recall": 0.2, "qid": "711d6873-a97c-4efc-b546-68245615ef60", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 33, "novel_recall": 0.1515, "qid": "a96e19a0-2881-47fa-b910-f7273d081575", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 43, "novel_recall": 0.1163, "qid": "607218c7-eef5-4b19-be7f-46085ed3204c", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 38, "novel_recall": 0.1842, "qid": "e4278915-e949-49f6-a559-0e6da9250ea9", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 29, "novel_recall": 0.1724, "qid": "337ca321-57cf-401b-85c9-87ec6d056ba1", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 29, "novel_recall": 0.1034, "qid": "01f9818b-58a6-4343-9222-79a24768fe9f", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 51, "novel_recall": 0.1765, "qid": "781550d2-f5da-4064-998b-58eaad9554c3", "type": "D"}, {"arm": "M3_causal", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 33, "novel_recall": 0.1212, "qid": "b856a6c6-0a12-477d-8500-1823991c1921", "type": "D"}, {"arm": "M3_causal", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 9, "novel_recall": 0.5556, "qid": "7f4e071f-ecaf-4c88-8dc9-43196036d590", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 16, "novel_recall": 0.3125, "qid": "3b192665-c28e-4d88-9b27-166bc87415d7", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 27, "novel_recall": 0.2222, "qid": "95e38021-0f26-4780-b796-b0e326edf719", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 40, "novel_recall": 0.125, "qid": "ddc24b39-4283-4301-a562-d6bb478fd78e", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 20, "novel_recall": 0.2, "qid": "711d6873-a97c-4efc-b546-68245615ef60", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 33, "novel_recall": 0.1515, "qid": "a96e19a0-2881-47fa-b910-f7273d081575", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 43, "novel_recall": 0.1163, "qid": "607218c7-eef5-4b19-be7f-46085ed3204c", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 38, "novel_recall": 0.1842, "qid": "e4278915-e949-49f6-a559-0e6da9250ea9", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 29, "novel_recall": 0.1724, "qid": "337ca321-57cf-401b-85c9-87ec6d056ba1", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 29, "novel_recall": 0.1034, "qid": "01f9818b-58a6-4343-9222-79a24768fe9f", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 51, "novel_recall": 0.1765, "qid": "781550d2-f5da-4064-998b-58eaad9554c3", "type": "D"}, {"arm": "M4_validated", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 33, "novel_recall": 0.1212, "qid": "b856a6c6-0a12-477d-8500-1823991c1921", "type": "D"}, {"arm": "M4_validated", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 9, "novel_recall": 0.5556, "qid": "7f4e071f-ecaf-4c88-8dc9-43196036d590", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 16, "novel_recall": 0.3125, "qid": "3b192665-c28e-4d88-9b27-166bc87415d7", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 27, "novel_recall": 0.2222, "qid": "95e38021-0f26-4780-b796-b0e326edf719", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 40, "novel_recall": 0.125, "qid": "ddc24b39-4283-4301-a562-d6bb478fd78e", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 20, "novel_recall": 0.2, "qid": "711d6873-a97c-4efc-b546-68245615ef60", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 33, "novel_recall": 0.1212, "qid": "a96e19a0-2881-47fa-b910-f7273d081575", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 43, "novel_recall": 0.1163, "qid": "607218c7-eef5-4b19-be7f-46085ed3204c", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 38, "novel_recall": 0.1842, "qid": "e4278915-e949-49f6-a559-0e6da9250ea9", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 29, "novel_recall": 0.1724, "qid": "337ca321-57cf-401b-85c9-87ec6d056ba1", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 29, "novel_recall": 0.1034, "qid": "01f9818b-58a6-4343-9222-79a24768fe9f", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 51, "novel_recall": 0.1569, "qid": "781550d2-f5da-4064-998b-58eaad9554c3", "type": "D"}, {"arm": "R_recent", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 33, "novel_recall": 0.1212, "qid": "b856a6c6-0a12-477d-8500-1823991c1921", "type": "D"}, {"arm": "R_recent", "domain": "Game", "episode": "19", "full_hit": false, "n_novel": 9, "novel_recall": 0.4444, "qid": "7f4e071f-ecaf-4c88-8dc9-43196036d590", "type": "B"}, {"arm": "M0_none", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "98022752-b414-4f71-9654-9d33dbc525a0", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 34, "novel_recall": 0.0, "qid": "71c75a56-acd0-46ff-982c-f174d2807e4d", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 33, "novel_recall": 0.0, "qid": "c8e32445-0984-40f4-9359-bbd6a8af7070", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 45, "novel_recall": 0.0, "qid": "95f51d87-eb38-48b7-b999-ad30154dd7f8", "type": "A"}, {"arm": "M0_none", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 41, "novel_recall": 0.0, "qid": "71a9a91d-a17a-47f8-b317-fa450e32f754", "type": "B"}, {"arm": "M0_none", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 25, "novel_recall": 0.0, "qid": "2e3b51db-362a-4d7f-a380-f72ec4ec5790", "type": "B"}, {"arm": "M0_none", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 32, "novel_recall": 0.0, "qid": "d3681365-fab0-4174-8879-e0a5a5c295cc", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 32, "novel_recall": 0.0, "qid": "886a2c99-86d4-4ffd-be03-fbe3f6b98ace", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 35, "novel_recall": 0.0, "qid": "f7103601-d17b-49fb-bc0b-e02bc7a48872", "type": "C"}, {"arm": "M0_none", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 35, "novel_recall": 0.0, "qid": "6f69e61e-8e0b-4013-b4df-f90323cb52a3", "type": "D"}, {"arm": "M0_none", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 48, "novel_recall": 0.0, "qid": "24d28c9b-fdbd-4e19-93b3-7a423339eaf5", "type": "D"}, {"arm": "M0_none", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "b3fb13e4-4080-4b71-b498-6a38cc166c8e", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 24, "novel_recall": 0.125, "qid": "98022752-b414-4f71-9654-9d33dbc525a0", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 34, "novel_recall": 0.2059, "qid": "71c75a56-acd0-46ff-982c-f174d2807e4d", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 33, "novel_recall": 0.0909, "qid": "c8e32445-0984-40f4-9359-bbd6a8af7070", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 45, "novel_recall": 0.0667, "qid": "95f51d87-eb38-48b7-b999-ad30154dd7f8", "type": "A"}, {"arm": "M1_naive", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 41, "novel_recall": 0.122, "qid": "71a9a91d-a17a-47f8-b317-fa450e32f754", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 25, "novel_recall": 0.12, "qid": "2e3b51db-362a-4d7f-a380-f72ec4ec5790", "type": "B"}, {"arm": "M1_naive", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 32, "novel_recall": 0.0938, "qid": "d3681365-fab0-4174-8879-e0a5a5c295cc", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 32, "novel_recall": 0.0938, "qid": "886a2c99-86d4-4ffd-be03-fbe3f6b98ace", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 35, "novel_recall": 0.0857, "qid": "f7103601-d17b-49fb-bc0b-e02bc7a48872", "type": "C"}, {"arm": "M1_naive", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 35, "novel_recall": 0.0571, "qid": "6f69e61e-8e0b-4013-b4df-f90323cb52a3", "type": "D"}, {"arm": "M1_naive", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 48, "novel_recall": 0.1458, "qid": "24d28c9b-fdbd-4e19-93b3-7a423339eaf5", "type": "D"}, {"arm": "M1_naive", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "b3fb13e4-4080-4b71-b498-6a38cc166c8e", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 24, "novel_recall": 0.125, "qid": "98022752-b414-4f71-9654-9d33dbc525a0", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 34, "novel_recall": 0.2059, "qid": "71c75a56-acd0-46ff-982c-f174d2807e4d", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 33, "novel_recall": 0.0909, "qid": "c8e32445-0984-40f4-9359-bbd6a8af7070", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 45, "novel_recall": 0.0667, "qid": "95f51d87-eb38-48b7-b999-ad30154dd7f8", "type": "A"}, {"arm": "M2_evidence", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 41, "novel_recall": 0.122, "qid": "71a9a91d-a17a-47f8-b317-fa450e32f754", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 25, "novel_recall": 0.12, "qid": "2e3b51db-362a-4d7f-a380-f72ec4ec5790", "type": "B"}, {"arm": "M2_evidence", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 32, "novel_recall": 0.0938, "qid": "d3681365-fab0-4174-8879-e0a5a5c295cc", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 32, "novel_recall": 0.0938, "qid": "886a2c99-86d4-4ffd-be03-fbe3f6b98ace", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 35, "novel_recall": 0.0857, "qid": "f7103601-d17b-49fb-bc0b-e02bc7a48872", "type": "C"}, {"arm": "M2_evidence", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 35, "novel_recall": 0.0571, "qid": "6f69e61e-8e0b-4013-b4df-f90323cb52a3", "type": "D"}, {"arm": "M2_evidence", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 48, "novel_recall": 0.1458, "qid": "24d28c9b-fdbd-4e19-93b3-7a423339eaf5", "type": "D"}, {"arm": "M2_evidence", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "b3fb13e4-4080-4b71-b498-6a38cc166c8e", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 24, "novel_recall": 0.125, "qid": "98022752-b414-4f71-9654-9d33dbc525a0", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 34, "novel_recall": 0.2059, "qid": "71c75a56-acd0-46ff-982c-f174d2807e4d", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 33, "novel_recall": 0.0909, "qid": "c8e32445-0984-40f4-9359-bbd6a8af7070", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 45, "novel_recall": 0.0667, "qid": "95f51d87-eb38-48b7-b999-ad30154dd7f8", "type": "A"}, {"arm": "M3_causal", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 41, "novel_recall": 0.122, "qid": "71a9a91d-a17a-47f8-b317-fa450e32f754", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 25, "novel_recall": 0.12, "qid": "2e3b51db-362a-4d7f-a380-f72ec4ec5790", "type": "B"}, {"arm": "M3_causal", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 32, "novel_recall": 0.0938, "qid": "d3681365-fab0-4174-8879-e0a5a5c295cc", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 32, "novel_recall": 0.0938, "qid": "886a2c99-86d4-4ffd-be03-fbe3f6b98ace", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 35, "novel_recall": 0.0857, "qid": "f7103601-d17b-49fb-bc0b-e02bc7a48872", "type": "C"}, {"arm": "M3_causal", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 35, "novel_recall": 0.0571, "qid": "6f69e61e-8e0b-4013-b4df-f90323cb52a3", "type": "D"}, {"arm": "M3_causal", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 48, "novel_recall": 0.1458, "qid": "24d28c9b-fdbd-4e19-93b3-7a423339eaf5", "type": "D"}, {"arm": "M3_causal", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "b3fb13e4-4080-4b71-b498-6a38cc166c8e", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 24, "novel_recall": 0.125, "qid": "98022752-b414-4f71-9654-9d33dbc525a0", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 34, "novel_recall": 0.2059, "qid": "71c75a56-acd0-46ff-982c-f174d2807e4d", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 33, "novel_recall": 0.0909, "qid": "c8e32445-0984-40f4-9359-bbd6a8af7070", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 45, "novel_recall": 0.0667, "qid": "95f51d87-eb38-48b7-b999-ad30154dd7f8", "type": "A"}, {"arm": "M4_validated", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 41, "novel_recall": 0.122, "qid": "71a9a91d-a17a-47f8-b317-fa450e32f754", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 25, "novel_recall": 0.12, "qid": "2e3b51db-362a-4d7f-a380-f72ec4ec5790", "type": "B"}, {"arm": "M4_validated", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 32, "novel_recall": 0.0938, "qid": "d3681365-fab0-4174-8879-e0a5a5c295cc", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 32, "novel_recall": 0.0938, "qid": "886a2c99-86d4-4ffd-be03-fbe3f6b98ace", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 35, "novel_recall": 0.0857, "qid": "f7103601-d17b-49fb-bc0b-e02bc7a48872", "type": "C"}, {"arm": "M4_validated", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 35, "novel_recall": 0.0571, "qid": "6f69e61e-8e0b-4013-b4df-f90323cb52a3", "type": "D"}, {"arm": "M4_validated", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 48, "novel_recall": 0.1458, "qid": "24d28c9b-fdbd-4e19-93b3-7a423339eaf5", "type": "D"}, {"arm": "M4_validated", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "b3fb13e4-4080-4b71-b498-6a38cc166c8e", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 24, "novel_recall": 0.125, "qid": "98022752-b414-4f71-9654-9d33dbc525a0", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 34, "novel_recall": 0.2059, "qid": "71c75a56-acd0-46ff-982c-f174d2807e4d", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 33, "novel_recall": 0.0909, "qid": "c8e32445-0984-40f4-9359-bbd6a8af7070", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 45, "novel_recall": 0.0667, "qid": "95f51d87-eb38-48b7-b999-ad30154dd7f8", "type": "A"}, {"arm": "R_recent", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 41, "novel_recall": 0.122, "qid": "71a9a91d-a17a-47f8-b317-fa450e32f754", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 25, "novel_recall": 0.12, "qid": "2e3b51db-362a-4d7f-a380-f72ec4ec5790", "type": "B"}, {"arm": "R_recent", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 32, "novel_recall": 0.0938, "qid": "d3681365-fab0-4174-8879-e0a5a5c295cc", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 32, "novel_recall": 0.0938, "qid": "886a2c99-86d4-4ffd-be03-fbe3f6b98ace", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 35, "novel_recall": 0.0857, "qid": "f7103601-d17b-49fb-bc0b-e02bc7a48872", "type": "C"}, {"arm": "R_recent", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 35, "novel_recall": 0.0571, "qid": "6f69e61e-8e0b-4013-b4df-f90323cb52a3", "type": "D"}, {"arm": "R_recent", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 48, "novel_recall": 0.1458, "qid": "24d28c9b-fdbd-4e19-93b3-7a423339eaf5", "type": "D"}, {"arm": "R_recent", "domain": "Game", "episode": "27", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "b3fb13e4-4080-4b71-b498-6a38cc166c8e", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "044630dc-2132-4e95-ae79-5069ebd7ae30", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "247b08d0-4ffa-492d-8072-0819188eb4ca", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "a1f6f4c5-638c-4845-9a98-33f0a47f4b02", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "9fc40559-8b90-4807-92f1-0679b41ac55c", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "59503989-e831-4654-b5f5-b0ef2784c4a0", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 14, "novel_recall": 0.0, "qid": "986ce5ec-a2de-4840-8f00-0635ec6c5957", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "9ebd5463-3010-4d67-b0d1-79d25eba0b7f", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "c26ba34d-bd1c-4c28-99bb-3551a808ce7a", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "5b999631-02ae-4f10-a321-1cd1dff9de85", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "474c575b-9ff7-4c41-bc52-741752e619bc", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "e0e44b68-a0ae-4e52-b264-21a0d08dbb74", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "6b1f5af8-0e77-47d3-b21e-f8b902c4f30a", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "044630dc-2132-4e95-ae79-5069ebd7ae30", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 12, "novel_recall": 0.6667, "qid": "247b08d0-4ffa-492d-8072-0819188eb4ca", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "33", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "a1f6f4c5-638c-4845-9a98-33f0a47f4b02", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "9fc40559-8b90-4807-92f1-0679b41ac55c", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "59503989-e831-4654-b5f5-b0ef2784c4a0", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 14, "novel_recall": 0.2857, "qid": "986ce5ec-a2de-4840-8f00-0635ec6c5957", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 13, "novel_recall": 0.4615, "qid": "9ebd5463-3010-4d67-b0d1-79d25eba0b7f", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "c26ba34d-bd1c-4c28-99bb-3551a808ce7a", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "5b999631-02ae-4f10-a321-1cd1dff9de85", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 17, "novel_recall": 0.2941, "qid": "474c575b-9ff7-4c41-bc52-741752e619bc", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "e0e44b68-a0ae-4e52-b264-21a0d08dbb74", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 17, "novel_recall": 0.7647, "qid": "6b1f5af8-0e77-47d3-b21e-f8b902c4f30a", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "044630dc-2132-4e95-ae79-5069ebd7ae30", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 12, "novel_recall": 0.6667, "qid": "247b08d0-4ffa-492d-8072-0819188eb4ca", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "a1f6f4c5-638c-4845-9a98-33f0a47f4b02", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "9fc40559-8b90-4807-92f1-0679b41ac55c", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "59503989-e831-4654-b5f5-b0ef2784c4a0", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 14, "novel_recall": 0.2857, "qid": "986ce5ec-a2de-4840-8f00-0635ec6c5957", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 13, "novel_recall": 0.4615, "qid": "9ebd5463-3010-4d67-b0d1-79d25eba0b7f", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "c26ba34d-bd1c-4c28-99bb-3551a808ce7a", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "5b999631-02ae-4f10-a321-1cd1dff9de85", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 17, "novel_recall": 0.2941, "qid": "474c575b-9ff7-4c41-bc52-741752e619bc", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "e0e44b68-a0ae-4e52-b264-21a0d08dbb74", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 17, "novel_recall": 0.7647, "qid": "6b1f5af8-0e77-47d3-b21e-f8b902c4f30a", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "044630dc-2132-4e95-ae79-5069ebd7ae30", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 12, "novel_recall": 0.6667, "qid": "247b08d0-4ffa-492d-8072-0819188eb4ca", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "a1f6f4c5-638c-4845-9a98-33f0a47f4b02", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "9fc40559-8b90-4807-92f1-0679b41ac55c", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "59503989-e831-4654-b5f5-b0ef2784c4a0", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 14, "novel_recall": 0.2857, "qid": "986ce5ec-a2de-4840-8f00-0635ec6c5957", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 13, "novel_recall": 0.4615, "qid": "9ebd5463-3010-4d67-b0d1-79d25eba0b7f", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "c26ba34d-bd1c-4c28-99bb-3551a808ce7a", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "5b999631-02ae-4f10-a321-1cd1dff9de85", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 17, "novel_recall": 0.2941, "qid": "474c575b-9ff7-4c41-bc52-741752e619bc", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "e0e44b68-a0ae-4e52-b264-21a0d08dbb74", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 17, "novel_recall": 0.7647, "qid": "6b1f5af8-0e77-47d3-b21e-f8b902c4f30a", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "044630dc-2132-4e95-ae79-5069ebd7ae30", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 12, "novel_recall": 0.6667, "qid": "247b08d0-4ffa-492d-8072-0819188eb4ca", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "a1f6f4c5-638c-4845-9a98-33f0a47f4b02", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "9fc40559-8b90-4807-92f1-0679b41ac55c", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "59503989-e831-4654-b5f5-b0ef2784c4a0", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 14, "novel_recall": 0.2857, "qid": "986ce5ec-a2de-4840-8f00-0635ec6c5957", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 13, "novel_recall": 0.4615, "qid": "9ebd5463-3010-4d67-b0d1-79d25eba0b7f", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "c26ba34d-bd1c-4c28-99bb-3551a808ce7a", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "5b999631-02ae-4f10-a321-1cd1dff9de85", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 17, "novel_recall": 0.2941, "qid": "474c575b-9ff7-4c41-bc52-741752e619bc", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "e0e44b68-a0ae-4e52-b264-21a0d08dbb74", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 17, "novel_recall": 0.7647, "qid": "6b1f5af8-0e77-47d3-b21e-f8b902c4f30a", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "044630dc-2132-4e95-ae79-5069ebd7ae30", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 12, "novel_recall": 0.6667, "qid": "247b08d0-4ffa-492d-8072-0819188eb4ca", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "33", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "a1f6f4c5-638c-4845-9a98-33f0a47f4b02", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "9fc40559-8b90-4807-92f1-0679b41ac55c", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "59503989-e831-4654-b5f5-b0ef2784c4a0", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 14, "novel_recall": 0.2143, "qid": "986ce5ec-a2de-4840-8f00-0635ec6c5957", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 13, "novel_recall": 0.3846, "qid": "9ebd5463-3010-4d67-b0d1-79d25eba0b7f", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "c26ba34d-bd1c-4c28-99bb-3551a808ce7a", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "5b999631-02ae-4f10-a321-1cd1dff9de85", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 17, "novel_recall": 0.3529, "qid": "474c575b-9ff7-4c41-bc52-741752e619bc", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "e0e44b68-a0ae-4e52-b264-21a0d08dbb74", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "33", "full_hit": false, "n_novel": 17, "novel_recall": 0.7059, "qid": "6b1f5af8-0e77-47d3-b21e-f8b902c4f30a", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "bcaba2e3-051c-47bc-b520-4f27d00de534", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "32487937-2976-4ace-9e7d-7d0c267015cc", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "277db06e-0528-43d1-9298-7ea9d68dc7bd", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "bd06afef-321e-48c2-bb94-3f0422802f1b", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "aeab42b8-7219-4e8a-838c-2bbe9b7dc320", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "fa55939e-189d-4f24-9356-0b927c409e52", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "1df8abaa-0582-4d9a-83b6-309df14d1621", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "4d01f6f1-41fc-48d4-adef-a63ff3a5e6bc", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "1b40d1ae-b11f-410a-99b7-5dda7127e316", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 14, "novel_recall": 0.0, "qid": "ab7e2847-a650-4d7b-83e1-b6bfdcdff11a", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "8d4b9b1b-a76d-42ff-a2d4-318826922a56", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 18, "novel_recall": 0.0, "qid": "bc92555f-b6a8-4f14-ba11-7747ce1f893f", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "bcaba2e3-051c-47bc-b520-4f27d00de534", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "32487937-2976-4ace-9e7d-7d0c267015cc", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "277db06e-0528-43d1-9298-7ea9d68dc7bd", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "bd06afef-321e-48c2-bb94-3f0422802f1b", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 15, "novel_recall": 0.3333, "qid": "aeab42b8-7219-4e8a-838c-2bbe9b7dc320", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 7, "novel_recall": 0.8571, "qid": "fa55939e-189d-4f24-9356-0b927c409e52", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 13, "novel_recall": 0.5385, "qid": "1df8abaa-0582-4d9a-83b6-309df14d1621", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "4d01f6f1-41fc-48d4-adef-a63ff3a5e6bc", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "1b40d1ae-b11f-410a-99b7-5dda7127e316", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 14, "novel_recall": 0.3571, "qid": "ab7e2847-a650-4d7b-83e1-b6bfdcdff11a", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 15, "novel_recall": 0.4, "qid": "8d4b9b1b-a76d-42ff-a2d4-318826922a56", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 18, "novel_recall": 0.7778, "qid": "bc92555f-b6a8-4f14-ba11-7747ce1f893f", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "bcaba2e3-051c-47bc-b520-4f27d00de534", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 10, "novel_recall": 0.7, "qid": "32487937-2976-4ace-9e7d-7d0c267015cc", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "277db06e-0528-43d1-9298-7ea9d68dc7bd", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "bd06afef-321e-48c2-bb94-3f0422802f1b", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 15, "novel_recall": 0.3333, "qid": "aeab42b8-7219-4e8a-838c-2bbe9b7dc320", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 7, "novel_recall": 0.8571, "qid": "fa55939e-189d-4f24-9356-0b927c409e52", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 13, "novel_recall": 0.5385, "qid": "1df8abaa-0582-4d9a-83b6-309df14d1621", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "4d01f6f1-41fc-48d4-adef-a63ff3a5e6bc", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "1b40d1ae-b11f-410a-99b7-5dda7127e316", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 14, "novel_recall": 0.3571, "qid": "ab7e2847-a650-4d7b-83e1-b6bfdcdff11a", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 15, "novel_recall": 0.4, "qid": "8d4b9b1b-a76d-42ff-a2d4-318826922a56", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 18, "novel_recall": 0.7778, "qid": "bc92555f-b6a8-4f14-ba11-7747ce1f893f", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "bcaba2e3-051c-47bc-b520-4f27d00de534", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 10, "novel_recall": 0.7, "qid": "32487937-2976-4ace-9e7d-7d0c267015cc", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "277db06e-0528-43d1-9298-7ea9d68dc7bd", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "bd06afef-321e-48c2-bb94-3f0422802f1b", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 15, "novel_recall": 0.3333, "qid": "aeab42b8-7219-4e8a-838c-2bbe9b7dc320", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 7, "novel_recall": 0.8571, "qid": "fa55939e-189d-4f24-9356-0b927c409e52", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 13, "novel_recall": 0.5385, "qid": "1df8abaa-0582-4d9a-83b6-309df14d1621", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "4d01f6f1-41fc-48d4-adef-a63ff3a5e6bc", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "1b40d1ae-b11f-410a-99b7-5dda7127e316", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 14, "novel_recall": 0.3571, "qid": "ab7e2847-a650-4d7b-83e1-b6bfdcdff11a", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 15, "novel_recall": 0.4, "qid": "8d4b9b1b-a76d-42ff-a2d4-318826922a56", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 18, "novel_recall": 0.7778, "qid": "bc92555f-b6a8-4f14-ba11-7747ce1f893f", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "bcaba2e3-051c-47bc-b520-4f27d00de534", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 10, "novel_recall": 0.7, "qid": "32487937-2976-4ace-9e7d-7d0c267015cc", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "277db06e-0528-43d1-9298-7ea9d68dc7bd", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "bd06afef-321e-48c2-bb94-3f0422802f1b", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 15, "novel_recall": 0.3333, "qid": "aeab42b8-7219-4e8a-838c-2bbe9b7dc320", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 7, "novel_recall": 0.8571, "qid": "fa55939e-189d-4f24-9356-0b927c409e52", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 13, "novel_recall": 0.5385, "qid": "1df8abaa-0582-4d9a-83b6-309df14d1621", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "4d01f6f1-41fc-48d4-adef-a63ff3a5e6bc", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "1b40d1ae-b11f-410a-99b7-5dda7127e316", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 14, "novel_recall": 0.3571, "qid": "ab7e2847-a650-4d7b-83e1-b6bfdcdff11a", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 15, "novel_recall": 0.4, "qid": "8d4b9b1b-a76d-42ff-a2d4-318826922a56", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 18, "novel_recall": 0.7778, "qid": "bc92555f-b6a8-4f14-ba11-7747ce1f893f", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "bcaba2e3-051c-47bc-b520-4f27d00de534", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "32487937-2976-4ace-9e7d-7d0c267015cc", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "277db06e-0528-43d1-9298-7ea9d68dc7bd", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "40", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "bd06afef-321e-48c2-bb94-3f0422802f1b", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 15, "novel_recall": 0.3333, "qid": "aeab42b8-7219-4e8a-838c-2bbe9b7dc320", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 7, "novel_recall": 0.7143, "qid": "fa55939e-189d-4f24-9356-0b927c409e52", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 13, "novel_recall": 0.4615, "qid": "1df8abaa-0582-4d9a-83b6-309df14d1621", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "4d01f6f1-41fc-48d4-adef-a63ff3a5e6bc", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "1b40d1ae-b11f-410a-99b7-5dda7127e316", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 14, "novel_recall": 0.3571, "qid": "ab7e2847-a650-4d7b-83e1-b6bfdcdff11a", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 15, "novel_recall": 0.8, "qid": "8d4b9b1b-a76d-42ff-a2d4-318826922a56", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "40", "full_hit": false, "n_novel": 18, "novel_recall": 0.7222, "qid": "bc92555f-b6a8-4f14-ba11-7747ce1f893f", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "b6b58345-4e5f-4855-a03e-754b209074d9", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 14, "novel_recall": 0.0, "qid": "5e63b5aa-7608-4e5e-8129-624e17cf1a8d", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "1a4c2025-4fe0-4e36-a452-5c2314d47175", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "1b2c7f9a-fa94-4e5a-927e-9b68c06d552b", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "7a932a81-3f66-441f-b79d-203c8e5c12b6", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "0178b17c-0f76-4d7b-9d99-84b110e6b399", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 14, "novel_recall": 0.0, "qid": "d40d868b-3e55-4579-827a-05efb73142d1", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "dc68b189-81c0-4abe-bead-09ae7fe37442", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "6c3774bf-a71e-46c5-8a90-11d72091fb7c", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "e805c186-b91c-431b-9d69-db9df6a72093", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 33, "novel_recall": 0.0, "qid": "84177ba2-6c07-49cb-87d0-c1f270661845", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "d4b2cbac-b1ce-4ade-b03a-9c6faa1a19fd", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "b6b58345-4e5f-4855-a03e-754b209074d9", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 14, "novel_recall": 0.7143, "qid": "5e63b5aa-7608-4e5e-8129-624e17cf1a8d", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "43", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "1a4c2025-4fe0-4e36-a452-5c2314d47175", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "1b2c7f9a-fa94-4e5a-927e-9b68c06d552b", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "7a932a81-3f66-441f-b79d-203c8e5c12b6", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "0178b17c-0f76-4d7b-9d99-84b110e6b399", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 14, "novel_recall": 0.4286, "qid": "d40d868b-3e55-4579-827a-05efb73142d1", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "dc68b189-81c0-4abe-bead-09ae7fe37442", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "6c3774bf-a71e-46c5-8a90-11d72091fb7c", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "e805c186-b91c-431b-9d69-db9df6a72093", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 33, "novel_recall": 0.6061, "qid": "84177ba2-6c07-49cb-87d0-c1f270661845", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 16, "novel_recall": 0.8125, "qid": "d4b2cbac-b1ce-4ade-b03a-9c6faa1a19fd", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "b6b58345-4e5f-4855-a03e-754b209074d9", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 14, "novel_recall": 0.7143, "qid": "5e63b5aa-7608-4e5e-8129-624e17cf1a8d", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "43", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "1a4c2025-4fe0-4e36-a452-5c2314d47175", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "1b2c7f9a-fa94-4e5a-927e-9b68c06d552b", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "7a932a81-3f66-441f-b79d-203c8e5c12b6", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "0178b17c-0f76-4d7b-9d99-84b110e6b399", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 14, "novel_recall": 0.4286, "qid": "d40d868b-3e55-4579-827a-05efb73142d1", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "dc68b189-81c0-4abe-bead-09ae7fe37442", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "6c3774bf-a71e-46c5-8a90-11d72091fb7c", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "e805c186-b91c-431b-9d69-db9df6a72093", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 33, "novel_recall": 0.6061, "qid": "84177ba2-6c07-49cb-87d0-c1f270661845", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 16, "novel_recall": 0.8125, "qid": "d4b2cbac-b1ce-4ade-b03a-9c6faa1a19fd", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "b6b58345-4e5f-4855-a03e-754b209074d9", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 14, "novel_recall": 0.6429, "qid": "5e63b5aa-7608-4e5e-8129-624e17cf1a8d", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "43", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "1a4c2025-4fe0-4e36-a452-5c2314d47175", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "1b2c7f9a-fa94-4e5a-927e-9b68c06d552b", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "7a932a81-3f66-441f-b79d-203c8e5c12b6", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "0178b17c-0f76-4d7b-9d99-84b110e6b399", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 14, "novel_recall": 0.4286, "qid": "d40d868b-3e55-4579-827a-05efb73142d1", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "dc68b189-81c0-4abe-bead-09ae7fe37442", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "6c3774bf-a71e-46c5-8a90-11d72091fb7c", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "e805c186-b91c-431b-9d69-db9df6a72093", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 33, "novel_recall": 0.4242, "qid": "84177ba2-6c07-49cb-87d0-c1f270661845", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 16, "novel_recall": 0.75, "qid": "d4b2cbac-b1ce-4ade-b03a-9c6faa1a19fd", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "b6b58345-4e5f-4855-a03e-754b209074d9", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 14, "novel_recall": 0.6429, "qid": "5e63b5aa-7608-4e5e-8129-624e17cf1a8d", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "43", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "1a4c2025-4fe0-4e36-a452-5c2314d47175", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "1b2c7f9a-fa94-4e5a-927e-9b68c06d552b", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "7a932a81-3f66-441f-b79d-203c8e5c12b6", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "0178b17c-0f76-4d7b-9d99-84b110e6b399", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 14, "novel_recall": 0.4286, "qid": "d40d868b-3e55-4579-827a-05efb73142d1", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "dc68b189-81c0-4abe-bead-09ae7fe37442", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "6c3774bf-a71e-46c5-8a90-11d72091fb7c", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "e805c186-b91c-431b-9d69-db9df6a72093", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 33, "novel_recall": 0.4242, "qid": "84177ba2-6c07-49cb-87d0-c1f270661845", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 16, "novel_recall": 0.75, "qid": "d4b2cbac-b1ce-4ade-b03a-9c6faa1a19fd", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 8, "novel_recall": 0.125, "qid": "b6b58345-4e5f-4855-a03e-754b209074d9", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 14, "novel_recall": 0.5714, "qid": "5e63b5aa-7608-4e5e-8129-624e17cf1a8d", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "43", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "1a4c2025-4fe0-4e36-a452-5c2314d47175", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "1b2c7f9a-fa94-4e5a-927e-9b68c06d552b", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "7a932a81-3f66-441f-b79d-203c8e5c12b6", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "0178b17c-0f76-4d7b-9d99-84b110e6b399", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 14, "novel_recall": 0.4286, "qid": "d40d868b-3e55-4579-827a-05efb73142d1", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 13, "novel_recall": 0.3077, "qid": "dc68b189-81c0-4abe-bead-09ae7fe37442", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "6c3774bf-a71e-46c5-8a90-11d72091fb7c", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 13, "novel_recall": 0.3077, "qid": "e805c186-b91c-431b-9d69-db9df6a72093", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 33, "novel_recall": 0.4848, "qid": "84177ba2-6c07-49cb-87d0-c1f270661845", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "43", "full_hit": false, "n_novel": 16, "novel_recall": 0.625, "qid": "d4b2cbac-b1ce-4ade-b03a-9c6faa1a19fd", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "f26241b7-06ab-4347-be5d-00c4f0258bd4", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "df0cdd62-a8d6-4d9e-9de8-40d7601f0bd0", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "79c42e0f-78f9-4fc6-b4c5-0ab23ab82f69", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "905bb60a-4f73-457b-a61f-2df0b5917bac", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "a47061d8-fb0f-4957-b1d5-6d567e005f33", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "43ef1560-d3f9-4a6a-b2f6-99ef958739f8", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "c034c9bd-5b2e-48b2-a891-a61ed09c7554", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "b346cbe6-abb3-4e3b-86aa-1a0726a0767b", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "b9ef6ea6-f08e-4bd1-877a-664a23d6d8e2", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "c538b4e5-45f2-48ad-80dd-51896a8bd31d", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "8d6ce8d6-b1e3-4d78-b601-fa3b10e47152", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 11, "novel_recall": 0.8182, "qid": "f26241b7-06ab-4347-be5d-00c4f0258bd4", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "df0cdd62-a8d6-4d9e-9de8-40d7601f0bd0", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 8, "novel_recall": 0.875, "qid": "79c42e0f-78f9-4fc6-b4c5-0ab23ab82f69", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 13, "novel_recall": 0.3846, "qid": "905bb60a-4f73-457b-a61f-2df0b5917bac", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "a47061d8-fb0f-4957-b1d5-6d567e005f33", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "44", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "43ef1560-d3f9-4a6a-b2f6-99ef958739f8", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "c034c9bd-5b2e-48b2-a891-a61ed09c7554", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "b346cbe6-abb3-4e3b-86aa-1a0726a0767b", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "b9ef6ea6-f08e-4bd1-877a-664a23d6d8e2", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "c538b4e5-45f2-48ad-80dd-51896a8bd31d", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 16, "novel_recall": 0.6875, "qid": "8d6ce8d6-b1e3-4d78-b601-fa3b10e47152", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 11, "novel_recall": 0.8182, "qid": "f26241b7-06ab-4347-be5d-00c4f0258bd4", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "df0cdd62-a8d6-4d9e-9de8-40d7601f0bd0", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 8, "novel_recall": 0.875, "qid": "79c42e0f-78f9-4fc6-b4c5-0ab23ab82f69", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 13, "novel_recall": 0.3846, "qid": "905bb60a-4f73-457b-a61f-2df0b5917bac", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "a47061d8-fb0f-4957-b1d5-6d567e005f33", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "44", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "43ef1560-d3f9-4a6a-b2f6-99ef958739f8", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "c034c9bd-5b2e-48b2-a891-a61ed09c7554", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "b346cbe6-abb3-4e3b-86aa-1a0726a0767b", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "b9ef6ea6-f08e-4bd1-877a-664a23d6d8e2", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "c538b4e5-45f2-48ad-80dd-51896a8bd31d", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 16, "novel_recall": 0.6875, "qid": "8d6ce8d6-b1e3-4d78-b601-fa3b10e47152", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 11, "novel_recall": 0.7273, "qid": "f26241b7-06ab-4347-be5d-00c4f0258bd4", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "df0cdd62-a8d6-4d9e-9de8-40d7601f0bd0", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 8, "novel_recall": 0.875, "qid": "79c42e0f-78f9-4fc6-b4c5-0ab23ab82f69", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 13, "novel_recall": 0.3846, "qid": "905bb60a-4f73-457b-a61f-2df0b5917bac", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "a47061d8-fb0f-4957-b1d5-6d567e005f33", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "44", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "43ef1560-d3f9-4a6a-b2f6-99ef958739f8", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "c034c9bd-5b2e-48b2-a891-a61ed09c7554", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "b346cbe6-abb3-4e3b-86aa-1a0726a0767b", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "b9ef6ea6-f08e-4bd1-877a-664a23d6d8e2", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "c538b4e5-45f2-48ad-80dd-51896a8bd31d", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 16, "novel_recall": 0.6875, "qid": "8d6ce8d6-b1e3-4d78-b601-fa3b10e47152", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 11, "novel_recall": 0.7273, "qid": "f26241b7-06ab-4347-be5d-00c4f0258bd4", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "df0cdd62-a8d6-4d9e-9de8-40d7601f0bd0", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 8, "novel_recall": 0.875, "qid": "79c42e0f-78f9-4fc6-b4c5-0ab23ab82f69", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 13, "novel_recall": 0.3846, "qid": "905bb60a-4f73-457b-a61f-2df0b5917bac", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "a47061d8-fb0f-4957-b1d5-6d567e005f33", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "44", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "43ef1560-d3f9-4a6a-b2f6-99ef958739f8", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "c034c9bd-5b2e-48b2-a891-a61ed09c7554", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "b346cbe6-abb3-4e3b-86aa-1a0726a0767b", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "b9ef6ea6-f08e-4bd1-877a-664a23d6d8e2", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "c538b4e5-45f2-48ad-80dd-51896a8bd31d", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 16, "novel_recall": 0.6875, "qid": "8d6ce8d6-b1e3-4d78-b601-fa3b10e47152", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "f26241b7-06ab-4347-be5d-00c4f0258bd4", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "df0cdd62-a8d6-4d9e-9de8-40d7601f0bd0", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 8, "novel_recall": 0.875, "qid": "79c42e0f-78f9-4fc6-b4c5-0ab23ab82f69", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 13, "novel_recall": 0.5385, "qid": "905bb60a-4f73-457b-a61f-2df0b5917bac", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "a47061d8-fb0f-4957-b1d5-6d567e005f33", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "44", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "43ef1560-d3f9-4a6a-b2f6-99ef958739f8", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "c034c9bd-5b2e-48b2-a891-a61ed09c7554", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "44", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "b346cbe6-abb3-4e3b-86aa-1a0726a0767b", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 10, "novel_recall": 0.7, "qid": "b9ef6ea6-f08e-4bd1-877a-664a23d6d8e2", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "c538b4e5-45f2-48ad-80dd-51896a8bd31d", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "44", "full_hit": false, "n_novel": 16, "novel_recall": 0.75, "qid": "8d6ce8d6-b1e3-4d78-b601-fa3b10e47152", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "4d99691a-f8f4-42bb-8ffe-d61d4fd1d652", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "8aeb8683-415f-4207-a80b-0ec19e3f3f18", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "576baece-441e-4d66-996c-564a1c2788c7", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "81936854-1ab7-4e04-a8cb-3549a10bd38f", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "c1bccee2-d3ff-4285-9e32-37c5190c85c5", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "b0c85971-2c8c-428c-aff8-839b35ae3792", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "18154671-4bab-436c-80d3-ff7a307a982b", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "95c9f8e1-bf89-4864-a425-2c8c02cd2273", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "576c619f-6b6f-4135-a92d-33326880f0f6", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "82b0ff55-5fe4-4a5d-8945-d32764141f76", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "2df57c8c-7866-468f-8122-e1522aa28360", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 18, "novel_recall": 0.0, "qid": "92605f6a-2c87-488f-96e0-44476b452dee", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "4d99691a-f8f4-42bb-8ffe-d61d4fd1d652", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "8aeb8683-415f-4207-a80b-0ec19e3f3f18", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "47", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "576baece-441e-4d66-996c-564a1c2788c7", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "81936854-1ab7-4e04-a8cb-3549a10bd38f", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 13, "novel_recall": 0.3846, "qid": "c1bccee2-d3ff-4285-9e32-37c5190c85c5", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 15, "novel_recall": 0.5333, "qid": "b0c85971-2c8c-428c-aff8-839b35ae3792", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "18154671-4bab-436c-80d3-ff7a307a982b", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "95c9f8e1-bf89-4864-a425-2c8c02cd2273", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "576c619f-6b6f-4135-a92d-33326880f0f6", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 13, "novel_recall": 0.3846, "qid": "82b0ff55-5fe4-4a5d-8945-d32764141f76", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 15, "novel_recall": 0.7333, "qid": "2df57c8c-7866-468f-8122-e1522aa28360", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 18, "novel_recall": 0.7222, "qid": "92605f6a-2c87-488f-96e0-44476b452dee", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "4d99691a-f8f4-42bb-8ffe-d61d4fd1d652", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "8aeb8683-415f-4207-a80b-0ec19e3f3f18", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "47", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "576baece-441e-4d66-996c-564a1c2788c7", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "81936854-1ab7-4e04-a8cb-3549a10bd38f", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 13, "novel_recall": 0.3846, "qid": "c1bccee2-d3ff-4285-9e32-37c5190c85c5", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 15, "novel_recall": 0.5333, "qid": "b0c85971-2c8c-428c-aff8-839b35ae3792", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "18154671-4bab-436c-80d3-ff7a307a982b", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "95c9f8e1-bf89-4864-a425-2c8c02cd2273", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "576c619f-6b6f-4135-a92d-33326880f0f6", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 13, "novel_recall": 0.3846, "qid": "82b0ff55-5fe4-4a5d-8945-d32764141f76", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 15, "novel_recall": 0.7333, "qid": "2df57c8c-7866-468f-8122-e1522aa28360", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 18, "novel_recall": 0.7222, "qid": "92605f6a-2c87-488f-96e0-44476b452dee", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "4d99691a-f8f4-42bb-8ffe-d61d4fd1d652", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "8aeb8683-415f-4207-a80b-0ec19e3f3f18", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "47", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "576baece-441e-4d66-996c-564a1c2788c7", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "81936854-1ab7-4e04-a8cb-3549a10bd38f", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "c1bccee2-d3ff-4285-9e32-37c5190c85c5", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 15, "novel_recall": 0.5333, "qid": "b0c85971-2c8c-428c-aff8-839b35ae3792", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "18154671-4bab-436c-80d3-ff7a307a982b", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "95c9f8e1-bf89-4864-a425-2c8c02cd2273", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "576c619f-6b6f-4135-a92d-33326880f0f6", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "82b0ff55-5fe4-4a5d-8945-d32764141f76", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 15, "novel_recall": 0.6667, "qid": "2df57c8c-7866-468f-8122-e1522aa28360", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 18, "novel_recall": 0.7222, "qid": "92605f6a-2c87-488f-96e0-44476b452dee", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "4d99691a-f8f4-42bb-8ffe-d61d4fd1d652", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "8aeb8683-415f-4207-a80b-0ec19e3f3f18", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "47", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "576baece-441e-4d66-996c-564a1c2788c7", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "81936854-1ab7-4e04-a8cb-3549a10bd38f", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "c1bccee2-d3ff-4285-9e32-37c5190c85c5", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 15, "novel_recall": 0.5333, "qid": "b0c85971-2c8c-428c-aff8-839b35ae3792", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "18154671-4bab-436c-80d3-ff7a307a982b", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "95c9f8e1-bf89-4864-a425-2c8c02cd2273", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "576c619f-6b6f-4135-a92d-33326880f0f6", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "82b0ff55-5fe4-4a5d-8945-d32764141f76", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 15, "novel_recall": 0.6667, "qid": "2df57c8c-7866-468f-8122-e1522aa28360", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 18, "novel_recall": 0.7222, "qid": "92605f6a-2c87-488f-96e0-44476b452dee", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "4d99691a-f8f4-42bb-8ffe-d61d4fd1d652", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "8aeb8683-415f-4207-a80b-0ec19e3f3f18", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "47", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "576baece-441e-4d66-996c-564a1c2788c7", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "81936854-1ab7-4e04-a8cb-3549a10bd38f", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 13, "novel_recall": 0.3077, "qid": "c1bccee2-d3ff-4285-9e32-37c5190c85c5", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 15, "novel_recall": 0.4667, "qid": "b0c85971-2c8c-428c-aff8-839b35ae3792", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "18154671-4bab-436c-80d3-ff7a307a982b", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "95c9f8e1-bf89-4864-a425-2c8c02cd2273", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "576c619f-6b6f-4135-a92d-33326880f0f6", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 13, "novel_recall": 0.3077, "qid": "82b0ff55-5fe4-4a5d-8945-d32764141f76", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 15, "novel_recall": 0.4667, "qid": "2df57c8c-7866-468f-8122-e1522aa28360", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "47", "full_hit": false, "n_novel": 18, "novel_recall": 0.6667, "qid": "92605f6a-2c87-488f-96e0-44476b452dee", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "5ccfdc22-2048-4156-af48-2614bf09ec03", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 18, "novel_recall": 0.0, "qid": "0467ce51-017c-4f70-a214-e57a5bbfdcac", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "a239d7ff-55e6-4785-8feb-3a932bf936bb", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "212be2a3-4704-4cb9-9931-c8739ec213ac", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "d347f666-b627-41be-b19d-09e2baaa52b2", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "7ac812fc-6684-4c15-99bb-b70bc204db02", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "895acbc0-2573-4ce1-a490-2241ee24853d", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "ccb29352-2bc8-4cf6-a6a0-7ec541453087", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "5f117f28-538e-4a5e-9487-6a8ed561d2fb", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "3547ad9d-eb6e-4bc5-9779-564d8165b164", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "da6a0eda-0e68-46d4-a164-b7c8d911b0e6", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "3eecf435-996c-433e-8130-9821f829a3fd", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "5ccfdc22-2048-4156-af48-2614bf09ec03", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 18, "novel_recall": 0.7222, "qid": "0467ce51-017c-4f70-a214-e57a5bbfdcac", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "48", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "a239d7ff-55e6-4785-8feb-3a932bf936bb", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "212be2a3-4704-4cb9-9931-c8739ec213ac", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "d347f666-b627-41be-b19d-09e2baaa52b2", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "7ac812fc-6684-4c15-99bb-b70bc204db02", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 10, "novel_recall": 0.5, "qid": "895acbc0-2573-4ce1-a490-2241ee24853d", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "ccb29352-2bc8-4cf6-a6a0-7ec541453087", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "5f117f28-538e-4a5e-9487-6a8ed561d2fb", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "3547ad9d-eb6e-4bc5-9779-564d8165b164", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.7692, "qid": "da6a0eda-0e68-46d4-a164-b7c8d911b0e6", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 17, "novel_recall": 0.7647, "qid": "3eecf435-996c-433e-8130-9821f829a3fd", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "5ccfdc22-2048-4156-af48-2614bf09ec03", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 18, "novel_recall": 0.7222, "qid": "0467ce51-017c-4f70-a214-e57a5bbfdcac", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "48", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "a239d7ff-55e6-4785-8feb-3a932bf936bb", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "212be2a3-4704-4cb9-9931-c8739ec213ac", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "d347f666-b627-41be-b19d-09e2baaa52b2", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "7ac812fc-6684-4c15-99bb-b70bc204db02", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 10, "novel_recall": 0.5, "qid": "895acbc0-2573-4ce1-a490-2241ee24853d", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "ccb29352-2bc8-4cf6-a6a0-7ec541453087", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "5f117f28-538e-4a5e-9487-6a8ed561d2fb", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "3547ad9d-eb6e-4bc5-9779-564d8165b164", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.7692, "qid": "da6a0eda-0e68-46d4-a164-b7c8d911b0e6", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 17, "novel_recall": 0.7647, "qid": "3eecf435-996c-433e-8130-9821f829a3fd", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "5ccfdc22-2048-4156-af48-2614bf09ec03", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 18, "novel_recall": 0.7222, "qid": "0467ce51-017c-4f70-a214-e57a5bbfdcac", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "48", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "a239d7ff-55e6-4785-8feb-3a932bf936bb", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "212be2a3-4704-4cb9-9931-c8739ec213ac", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "d347f666-b627-41be-b19d-09e2baaa52b2", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "7ac812fc-6684-4c15-99bb-b70bc204db02", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 10, "novel_recall": 0.5, "qid": "895acbc0-2573-4ce1-a490-2241ee24853d", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "ccb29352-2bc8-4cf6-a6a0-7ec541453087", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "5f117f28-538e-4a5e-9487-6a8ed561d2fb", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "3547ad9d-eb6e-4bc5-9779-564d8165b164", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.5385, "qid": "da6a0eda-0e68-46d4-a164-b7c8d911b0e6", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 17, "novel_recall": 0.7647, "qid": "3eecf435-996c-433e-8130-9821f829a3fd", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "5ccfdc22-2048-4156-af48-2614bf09ec03", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 18, "novel_recall": 0.7222, "qid": "0467ce51-017c-4f70-a214-e57a5bbfdcac", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "48", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "a239d7ff-55e6-4785-8feb-3a932bf936bb", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "212be2a3-4704-4cb9-9931-c8739ec213ac", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "d347f666-b627-41be-b19d-09e2baaa52b2", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "7ac812fc-6684-4c15-99bb-b70bc204db02", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 10, "novel_recall": 0.5, "qid": "895acbc0-2573-4ce1-a490-2241ee24853d", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "ccb29352-2bc8-4cf6-a6a0-7ec541453087", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "5f117f28-538e-4a5e-9487-6a8ed561d2fb", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "3547ad9d-eb6e-4bc5-9779-564d8165b164", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.5385, "qid": "da6a0eda-0e68-46d4-a164-b7c8d911b0e6", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 17, "novel_recall": 0.7647, "qid": "3eecf435-996c-433e-8130-9821f829a3fd", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "5ccfdc22-2048-4156-af48-2614bf09ec03", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 18, "novel_recall": 0.6111, "qid": "0467ce51-017c-4f70-a214-e57a5bbfdcac", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "48", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "a239d7ff-55e6-4785-8feb-3a932bf936bb", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "212be2a3-4704-4cb9-9931-c8739ec213ac", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "d347f666-b627-41be-b19d-09e2baaa52b2", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "7ac812fc-6684-4c15-99bb-b70bc204db02", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 10, "novel_recall": 0.5, "qid": "895acbc0-2573-4ce1-a490-2241ee24853d", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "ccb29352-2bc8-4cf6-a6a0-7ec541453087", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "5f117f28-538e-4a5e-9487-6a8ed561d2fb", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "3547ad9d-eb6e-4bc5-9779-564d8165b164", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 13, "novel_recall": 0.6154, "qid": "da6a0eda-0e68-46d4-a164-b7c8d911b0e6", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "48", "full_hit": false, "n_novel": 17, "novel_recall": 0.7647, "qid": "3eecf435-996c-433e-8130-9821f829a3fd", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "d326a924-73d4-4376-aa2a-060551c31a0f", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "5797ff13-699b-486a-88c0-d4d43154b275", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "709e7d61-db97-4b28-8cca-dc0e3e191298", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "99a0f796-df10-4a18-90e7-8f05c309fab1", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "4b2a1323-1f7a-41d4-bad6-f6db6bad3b4b", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "ef2cb9c0-d14f-4b5b-b5af-96aae0fa13e2", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "100a4c73-b49d-40a1-8f39-3f720f679462", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "d9a4667e-2577-4fe1-aebb-c4bb51daf21a", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "082a336c-5b81-49be-9791-2e81bea77fd6", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 51, "novel_recall": 0.0, "qid": "bdb415a9-3512-4a85-9d69-61215d3fc231", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "ba07b305-dc42-4e01-a571-df4e4e2620bf", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "d326a924-73d4-4376-aa2a-060551c31a0f", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 9, "novel_recall": 0.2222, "qid": "5797ff13-699b-486a-88c0-d4d43154b275", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "50", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "709e7d61-db97-4b28-8cca-dc0e3e191298", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "99a0f796-df10-4a18-90e7-8f05c309fab1", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "4b2a1323-1f7a-41d4-bad6-f6db6bad3b4b", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 11, "novel_recall": 0.3636, "qid": "ef2cb9c0-d14f-4b5b-b5af-96aae0fa13e2", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 6, "novel_recall": 0.8333, "qid": "100a4c73-b49d-40a1-8f39-3f720f679462", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 10, "novel_recall": 0.5, "qid": "d9a4667e-2577-4fe1-aebb-c4bb51daf21a", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "082a336c-5b81-49be-9791-2e81bea77fd6", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 51, "novel_recall": 0.5686, "qid": "bdb415a9-3512-4a85-9d69-61215d3fc231", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "ba07b305-dc42-4e01-a571-df4e4e2620bf", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "d326a924-73d4-4376-aa2a-060551c31a0f", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "5797ff13-699b-486a-88c0-d4d43154b275", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "50", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "709e7d61-db97-4b28-8cca-dc0e3e191298", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "99a0f796-df10-4a18-90e7-8f05c309fab1", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "4b2a1323-1f7a-41d4-bad6-f6db6bad3b4b", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 11, "novel_recall": 0.7273, "qid": "ef2cb9c0-d14f-4b5b-b5af-96aae0fa13e2", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 6, "novel_recall": 0.8333, "qid": "100a4c73-b49d-40a1-8f39-3f720f679462", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 10, "novel_recall": 0.5, "qid": "d9a4667e-2577-4fe1-aebb-c4bb51daf21a", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "082a336c-5b81-49be-9791-2e81bea77fd6", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 51, "novel_recall": 0.5686, "qid": "bdb415a9-3512-4a85-9d69-61215d3fc231", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "ba07b305-dc42-4e01-a571-df4e4e2620bf", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "d326a924-73d4-4376-aa2a-060551c31a0f", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "5797ff13-699b-486a-88c0-d4d43154b275", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "50", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "709e7d61-db97-4b28-8cca-dc0e3e191298", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "99a0f796-df10-4a18-90e7-8f05c309fab1", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 10, "novel_recall": 0.5, "qid": "4b2a1323-1f7a-41d4-bad6-f6db6bad3b4b", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 11, "novel_recall": 0.7273, "qid": "ef2cb9c0-d14f-4b5b-b5af-96aae0fa13e2", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "100a4c73-b49d-40a1-8f39-3f720f679462", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 10, "novel_recall": 0.5, "qid": "d9a4667e-2577-4fe1-aebb-c4bb51daf21a", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "082a336c-5b81-49be-9791-2e81bea77fd6", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 51, "novel_recall": 0.5294, "qid": "bdb415a9-3512-4a85-9d69-61215d3fc231", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "ba07b305-dc42-4e01-a571-df4e4e2620bf", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "d326a924-73d4-4376-aa2a-060551c31a0f", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "5797ff13-699b-486a-88c0-d4d43154b275", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "50", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "709e7d61-db97-4b28-8cca-dc0e3e191298", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "99a0f796-df10-4a18-90e7-8f05c309fab1", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 10, "novel_recall": 0.5, "qid": "4b2a1323-1f7a-41d4-bad6-f6db6bad3b4b", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 11, "novel_recall": 0.7273, "qid": "ef2cb9c0-d14f-4b5b-b5af-96aae0fa13e2", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "100a4c73-b49d-40a1-8f39-3f720f679462", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 10, "novel_recall": 0.5, "qid": "d9a4667e-2577-4fe1-aebb-c4bb51daf21a", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "082a336c-5b81-49be-9791-2e81bea77fd6", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 51, "novel_recall": 0.5294, "qid": "bdb415a9-3512-4a85-9d69-61215d3fc231", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "ba07b305-dc42-4e01-a571-df4e4e2620bf", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "d326a924-73d4-4376-aa2a-060551c31a0f", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "5797ff13-699b-486a-88c0-d4d43154b275", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "50", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "709e7d61-db97-4b28-8cca-dc0e3e191298", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 6, "novel_recall": 0.8333, "qid": "99a0f796-df10-4a18-90e7-8f05c309fab1", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "4b2a1323-1f7a-41d4-bad6-f6db6bad3b4b", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 11, "novel_recall": 0.0909, "qid": "ef2cb9c0-d14f-4b5b-b5af-96aae0fa13e2", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 6, "novel_recall": 0.8333, "qid": "100a4c73-b49d-40a1-8f39-3f720f679462", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "d9a4667e-2577-4fe1-aebb-c4bb51daf21a", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "082a336c-5b81-49be-9791-2e81bea77fd6", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 51, "novel_recall": 0.6275, "qid": "bdb415a9-3512-4a85-9d69-61215d3fc231", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "50", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "ba07b305-dc42-4e01-a571-df4e4e2620bf", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "703562a3-6c61-4b19-a6ef-57be5fbf73a6", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "83a18678-3ae0-4e48-b607-e21a1571c137", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "00ed9476-3159-4458-8119-2bc1a7bf37c2", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "b0661e97-e5a4-499b-ba11-fd3a6a67aeb2", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "9ccbe9a1-ec8f-43eb-b7df-60fcb67865e4", "type": "B"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "173cb03b-6812-4deb-915a-00f4c68e2900", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "28183c4a-dbfb-41c1-a9da-c27f71c7fb59", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "ff726ad8-617f-44a6-b945-7324d59c04eb", "type": "C"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "c8b3afe8-f718-40ad-b895-ec0c5209e848", "type": "A"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "734f4d61-6e3d-4e7e-bf70-32ab08a2b852", "type": "D"}, {"arm": "M0_none", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "b56d823d-83b8-4703-9525-273a8f1ade31", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 9, "novel_recall": 0.2222, "qid": "703562a3-6c61-4b19-a6ef-57be5fbf73a6", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 12, "novel_recall": 0.5833, "qid": "83a18678-3ae0-4e48-b607-e21a1571c137", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "00ed9476-3159-4458-8119-2bc1a7bf37c2", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 10, "novel_recall": 0.3, "qid": "b0661e97-e5a4-499b-ba11-fd3a6a67aeb2", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "9ccbe9a1-ec8f-43eb-b7df-60fcb67865e4", "type": "B"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "173cb03b-6812-4deb-915a-00f4c68e2900", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 10, "novel_recall": 0.3, "qid": "28183c4a-dbfb-41c1-a9da-c27f71c7fb59", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "54", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "ff726ad8-617f-44a6-b945-7324d59c04eb", "type": "C"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "c8b3afe8-f718-40ad-b895-ec0c5209e848", "type": "A"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 15, "novel_recall": 0.8, "qid": "734f4d61-6e3d-4e7e-bf70-32ab08a2b852", "type": "D"}, {"arm": "M1_naive", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "b56d823d-83b8-4703-9525-273a8f1ade31", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "703562a3-6c61-4b19-a6ef-57be5fbf73a6", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 12, "novel_recall": 0.5833, "qid": "83a18678-3ae0-4e48-b607-e21a1571c137", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "00ed9476-3159-4458-8119-2bc1a7bf37c2", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 10, "novel_recall": 0.3, "qid": "b0661e97-e5a4-499b-ba11-fd3a6a67aeb2", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "9ccbe9a1-ec8f-43eb-b7df-60fcb67865e4", "type": "B"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "173cb03b-6812-4deb-915a-00f4c68e2900", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 10, "novel_recall": 0.3, "qid": "28183c4a-dbfb-41c1-a9da-c27f71c7fb59", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "54", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "ff726ad8-617f-44a6-b945-7324d59c04eb", "type": "C"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "c8b3afe8-f718-40ad-b895-ec0c5209e848", "type": "A"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 15, "novel_recall": 0.8, "qid": "734f4d61-6e3d-4e7e-bf70-32ab08a2b852", "type": "D"}, {"arm": "M2_evidence", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "b56d823d-83b8-4703-9525-273a8f1ade31", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "703562a3-6c61-4b19-a6ef-57be5fbf73a6", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 12, "novel_recall": 0.5833, "qid": "83a18678-3ae0-4e48-b607-e21a1571c137", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "00ed9476-3159-4458-8119-2bc1a7bf37c2", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 10, "novel_recall": 0.3, "qid": "b0661e97-e5a4-499b-ba11-fd3a6a67aeb2", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "9ccbe9a1-ec8f-43eb-b7df-60fcb67865e4", "type": "B"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "173cb03b-6812-4deb-915a-00f4c68e2900", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 10, "novel_recall": 0.3, "qid": "28183c4a-dbfb-41c1-a9da-c27f71c7fb59", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "54", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "ff726ad8-617f-44a6-b945-7324d59c04eb", "type": "C"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "c8b3afe8-f718-40ad-b895-ec0c5209e848", "type": "A"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 15, "novel_recall": 0.7333, "qid": "734f4d61-6e3d-4e7e-bf70-32ab08a2b852", "type": "D"}, {"arm": "M3_causal", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "b56d823d-83b8-4703-9525-273a8f1ade31", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "703562a3-6c61-4b19-a6ef-57be5fbf73a6", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 12, "novel_recall": 0.5833, "qid": "83a18678-3ae0-4e48-b607-e21a1571c137", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "00ed9476-3159-4458-8119-2bc1a7bf37c2", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 10, "novel_recall": 0.3, "qid": "b0661e97-e5a4-499b-ba11-fd3a6a67aeb2", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "9ccbe9a1-ec8f-43eb-b7df-60fcb67865e4", "type": "B"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "173cb03b-6812-4deb-915a-00f4c68e2900", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 10, "novel_recall": 0.3, "qid": "28183c4a-dbfb-41c1-a9da-c27f71c7fb59", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "54", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "ff726ad8-617f-44a6-b945-7324d59c04eb", "type": "C"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "c8b3afe8-f718-40ad-b895-ec0c5209e848", "type": "A"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 15, "novel_recall": 0.7333, "qid": "734f4d61-6e3d-4e7e-bf70-32ab08a2b852", "type": "D"}, {"arm": "M4_validated", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "b56d823d-83b8-4703-9525-273a8f1ade31", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "703562a3-6c61-4b19-a6ef-57be5fbf73a6", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 12, "novel_recall": 0.5833, "qid": "83a18678-3ae0-4e48-b607-e21a1571c137", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "00ed9476-3159-4458-8119-2bc1a7bf37c2", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 10, "novel_recall": 0.2, "qid": "b0661e97-e5a4-499b-ba11-fd3a6a67aeb2", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "9ccbe9a1-ec8f-43eb-b7df-60fcb67865e4", "type": "B"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "173cb03b-6812-4deb-915a-00f4c68e2900", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 10, "novel_recall": 0.2, "qid": "28183c4a-dbfb-41c1-a9da-c27f71c7fb59", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "ff726ad8-617f-44a6-b945-7324d59c04eb", "type": "C"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "c8b3afe8-f718-40ad-b895-ec0c5209e848", "type": "A"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 15, "novel_recall": 0.6667, "qid": "734f4d61-6e3d-4e7e-bf70-32ab08a2b852", "type": "D"}, {"arm": "R_recent", "domain": "EMBODIED_AI", "episode": "54", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "b56d823d-83b8-4703-9525-273a8f1ade31", "type": "D"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "d85dd827-80d1-4fc3-addf-fcdffc06c31f", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "434c9c5c-d7b7-4b4a-b98b-44f7ceb5ae2a", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "4e0d5ab4-bd16-4c78-b5b2-e97bd26050ff", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "f4c70654-1575-4ec4-bda5-2a8517dd5059", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "cd562378-63fa-48fb-8597-11d5a0407411", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "042c5d1a-8be6-4474-9bc7-53cad9acb1be", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "5f8d45b2-7390-44bb-b275-6cf481605b2f", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e1ba25ab-b77b-46d8-828d-03a5814102fe", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "1ff90be9-80cf-4253-be3e-975d233b4079", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "92ad0236-42a3-43e1-bed6-875093e4172b", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "0b6c5930-e0c2-4ce4-b339-b0528fc6a424", "type": "D"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 23, "novel_recall": 0.0, "qid": "e57ccaef-82a6-4a46-b71c-1eecc31ba9b9", "type": "D"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "d85dd827-80d1-4fc3-addf-fcdffc06c31f", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "434c9c5c-d7b7-4b4a-b98b-44f7ceb5ae2a", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "4e0d5ab4-bd16-4c78-b5b2-e97bd26050ff", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "f4c70654-1575-4ec4-bda5-2a8517dd5059", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "cd562378-63fa-48fb-8597-11d5a0407411", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "042c5d1a-8be6-4474-9bc7-53cad9acb1be", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 10, "novel_recall": 0.9, "qid": "5f8d45b2-7390-44bb-b275-6cf481605b2f", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e1ba25ab-b77b-46d8-828d-03a5814102fe", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "1ff90be9-80cf-4253-be3e-975d233b4079", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "92ad0236-42a3-43e1-bed6-875093e4172b", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "0b6c5930-e0c2-4ce4-b339-b0528fc6a424", "type": "D"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 23, "novel_recall": 0.1739, "qid": "e57ccaef-82a6-4a46-b71c-1eecc31ba9b9", "type": "D"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "d85dd827-80d1-4fc3-addf-fcdffc06c31f", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "434c9c5c-d7b7-4b4a-b98b-44f7ceb5ae2a", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "4e0d5ab4-bd16-4c78-b5b2-e97bd26050ff", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "f4c70654-1575-4ec4-bda5-2a8517dd5059", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "cd562378-63fa-48fb-8597-11d5a0407411", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "042c5d1a-8be6-4474-9bc7-53cad9acb1be", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 10, "novel_recall": 0.9, "qid": "5f8d45b2-7390-44bb-b275-6cf481605b2f", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "e1ba25ab-b77b-46d8-828d-03a5814102fe", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "1ff90be9-80cf-4253-be3e-975d233b4079", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 10, "novel_recall": 0.3, "qid": "92ad0236-42a3-43e1-bed6-875093e4172b", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "0b6c5930-e0c2-4ce4-b339-b0528fc6a424", "type": "D"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 23, "novel_recall": 0.1739, "qid": "e57ccaef-82a6-4a46-b71c-1eecc31ba9b9", "type": "D"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "d85dd827-80d1-4fc3-addf-fcdffc06c31f", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "434c9c5c-d7b7-4b4a-b98b-44f7ceb5ae2a", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "4e0d5ab4-bd16-4c78-b5b2-e97bd26050ff", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "f4c70654-1575-4ec4-bda5-2a8517dd5059", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "cd562378-63fa-48fb-8597-11d5a0407411", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "042c5d1a-8be6-4474-9bc7-53cad9acb1be", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 10, "novel_recall": 0.9, "qid": "5f8d45b2-7390-44bb-b275-6cf481605b2f", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "e1ba25ab-b77b-46d8-828d-03a5814102fe", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "1ff90be9-80cf-4253-be3e-975d233b4079", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 10, "novel_recall": 0.3, "qid": "92ad0236-42a3-43e1-bed6-875093e4172b", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "0b6c5930-e0c2-4ce4-b339-b0528fc6a424", "type": "D"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 23, "novel_recall": 0.1739, "qid": "e57ccaef-82a6-4a46-b71c-1eecc31ba9b9", "type": "D"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "d85dd827-80d1-4fc3-addf-fcdffc06c31f", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "434c9c5c-d7b7-4b4a-b98b-44f7ceb5ae2a", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "4e0d5ab4-bd16-4c78-b5b2-e97bd26050ff", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "f4c70654-1575-4ec4-bda5-2a8517dd5059", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "cd562378-63fa-48fb-8597-11d5a0407411", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "042c5d1a-8be6-4474-9bc7-53cad9acb1be", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 10, "novel_recall": 0.9, "qid": "5f8d45b2-7390-44bb-b275-6cf481605b2f", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "e1ba25ab-b77b-46d8-828d-03a5814102fe", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "1ff90be9-80cf-4253-be3e-975d233b4079", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 10, "novel_recall": 0.3, "qid": "92ad0236-42a3-43e1-bed6-875093e4172b", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "0b6c5930-e0c2-4ce4-b339-b0528fc6a424", "type": "D"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 23, "novel_recall": 0.1739, "qid": "e57ccaef-82a6-4a46-b71c-1eecc31ba9b9", "type": "D"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "d85dd827-80d1-4fc3-addf-fcdffc06c31f", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "434c9c5c-d7b7-4b4a-b98b-44f7ceb5ae2a", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "4e0d5ab4-bd16-4c78-b5b2-e97bd26050ff", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "f4c70654-1575-4ec4-bda5-2a8517dd5059", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "cd562378-63fa-48fb-8597-11d5a0407411", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 7, "novel_recall": 0.7143, "qid": "042c5d1a-8be6-4474-9bc7-53cad9acb1be", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 10, "novel_recall": 0.9, "qid": "5f8d45b2-7390-44bb-b275-6cf481605b2f", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e1ba25ab-b77b-46d8-828d-03a5814102fe", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "1ff90be9-80cf-4253-be3e-975d233b4079", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "92ad0236-42a3-43e1-bed6-875093e4172b", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 24, "novel_recall": 0.3333, "qid": "0b6c5930-e0c2-4ce4-b339-b0528fc6a424", "type": "D"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "65", "full_hit": false, "n_novel": 23, "novel_recall": 0.7391, "qid": "e57ccaef-82a6-4a46-b71c-1eecc31ba9b9", "type": "D"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "ba4e3175-dea6-4ed3-86eb-cbf0c3b73d1d", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "582edb99-632d-450c-b539-992f060f8ea0", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "7c7e5444-4683-42d6-9b75-61384e0d2208", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "7af745de-5101-4c6f-8afe-4b550732bde5", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 14, "novel_recall": 0.0, "qid": "da0dbb58-cad9-46ba-969c-414a7352f8ad", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "523fef22-e0ae-4957-a9d1-d38904c53f1e", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "cb16728c-5f7d-4fed-b69c-d1d805f25c3e", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "43d28da0-01c4-4690-a13e-80c309ed6cbf", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "ed4d2db0-b658-42f1-9742-53e3653ca85d", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "204112cf-c5e8-49ce-9ed3-647caefc9da2", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 29, "novel_recall": 0.0, "qid": "05e340dc-191e-4db5-8e5e-44d0a181ea4c", "type": "D"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 27, "novel_recall": 0.0, "qid": "f9611fff-648c-4c71-b377-00426d21b46d", "type": "D"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 13, "novel_recall": 0.4615, "qid": "ba4e3175-dea6-4ed3-86eb-cbf0c3b73d1d", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": true, "n_novel": 9, "novel_recall": 1.0, "qid": "582edb99-632d-450c-b539-992f060f8ea0", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "7c7e5444-4683-42d6-9b75-61384e0d2208", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 17, "novel_recall": 0.8824, "qid": "7af745de-5101-4c6f-8afe-4b550732bde5", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 14, "novel_recall": 0.2857, "qid": "da0dbb58-cad9-46ba-969c-414a7352f8ad", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 11, "novel_recall": 0.1818, "qid": "523fef22-e0ae-4957-a9d1-d38904c53f1e", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "cb16728c-5f7d-4fed-b69c-d1d805f25c3e", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "43d28da0-01c4-4690-a13e-80c309ed6cbf", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "ed4d2db0-b658-42f1-9742-53e3653ca85d", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 15, "novel_recall": 0.8667, "qid": "204112cf-c5e8-49ce-9ed3-647caefc9da2", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 29, "novel_recall": 0.2414, "qid": "05e340dc-191e-4db5-8e5e-44d0a181ea4c", "type": "D"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 27, "novel_recall": 0.3704, "qid": "f9611fff-648c-4c71-b377-00426d21b46d", "type": "D"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 13, "novel_recall": 0.4615, "qid": "ba4e3175-dea6-4ed3-86eb-cbf0c3b73d1d", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": true, "n_novel": 9, "novel_recall": 1.0, "qid": "582edb99-632d-450c-b539-992f060f8ea0", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "7c7e5444-4683-42d6-9b75-61384e0d2208", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 17, "novel_recall": 0.8824, "qid": "7af745de-5101-4c6f-8afe-4b550732bde5", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 14, "novel_recall": 0.2857, "qid": "da0dbb58-cad9-46ba-969c-414a7352f8ad", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 11, "novel_recall": 0.1818, "qid": "523fef22-e0ae-4957-a9d1-d38904c53f1e", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "cb16728c-5f7d-4fed-b69c-d1d805f25c3e", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "43d28da0-01c4-4690-a13e-80c309ed6cbf", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "ed4d2db0-b658-42f1-9742-53e3653ca85d", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 15, "novel_recall": 0.8667, "qid": "204112cf-c5e8-49ce-9ed3-647caefc9da2", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 29, "novel_recall": 0.2414, "qid": "05e340dc-191e-4db5-8e5e-44d0a181ea4c", "type": "D"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 27, "novel_recall": 0.3704, "qid": "f9611fff-648c-4c71-b377-00426d21b46d", "type": "D"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 13, "novel_recall": 0.4615, "qid": "ba4e3175-dea6-4ed3-86eb-cbf0c3b73d1d", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": true, "n_novel": 9, "novel_recall": 1.0, "qid": "582edb99-632d-450c-b539-992f060f8ea0", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "7c7e5444-4683-42d6-9b75-61384e0d2208", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 17, "novel_recall": 0.8824, "qid": "7af745de-5101-4c6f-8afe-4b550732bde5", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 14, "novel_recall": 0.2857, "qid": "da0dbb58-cad9-46ba-969c-414a7352f8ad", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 11, "novel_recall": 0.1818, "qid": "523fef22-e0ae-4957-a9d1-d38904c53f1e", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "cb16728c-5f7d-4fed-b69c-d1d805f25c3e", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "43d28da0-01c4-4690-a13e-80c309ed6cbf", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "ed4d2db0-b658-42f1-9742-53e3653ca85d", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 15, "novel_recall": 0.8667, "qid": "204112cf-c5e8-49ce-9ed3-647caefc9da2", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 29, "novel_recall": 0.2414, "qid": "05e340dc-191e-4db5-8e5e-44d0a181ea4c", "type": "D"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 27, "novel_recall": 0.3704, "qid": "f9611fff-648c-4c71-b377-00426d21b46d", "type": "D"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 13, "novel_recall": 0.4615, "qid": "ba4e3175-dea6-4ed3-86eb-cbf0c3b73d1d", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": true, "n_novel": 9, "novel_recall": 1.0, "qid": "582edb99-632d-450c-b539-992f060f8ea0", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "7c7e5444-4683-42d6-9b75-61384e0d2208", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 17, "novel_recall": 0.8824, "qid": "7af745de-5101-4c6f-8afe-4b550732bde5", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 14, "novel_recall": 0.2857, "qid": "da0dbb58-cad9-46ba-969c-414a7352f8ad", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 11, "novel_recall": 0.1818, "qid": "523fef22-e0ae-4957-a9d1-d38904c53f1e", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "cb16728c-5f7d-4fed-b69c-d1d805f25c3e", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "43d28da0-01c4-4690-a13e-80c309ed6cbf", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "ed4d2db0-b658-42f1-9742-53e3653ca85d", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 15, "novel_recall": 0.8667, "qid": "204112cf-c5e8-49ce-9ed3-647caefc9da2", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 29, "novel_recall": 0.2414, "qid": "05e340dc-191e-4db5-8e5e-44d0a181ea4c", "type": "D"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 27, "novel_recall": 0.3704, "qid": "f9611fff-648c-4c71-b377-00426d21b46d", "type": "D"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "ba4e3175-dea6-4ed3-86eb-cbf0c3b73d1d", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 9, "novel_recall": 0.2222, "qid": "582edb99-632d-450c-b539-992f060f8ea0", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "7c7e5444-4683-42d6-9b75-61384e0d2208", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 17, "novel_recall": 0.5294, "qid": "7af745de-5101-4c6f-8afe-4b550732bde5", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 14, "novel_recall": 0.5, "qid": "da0dbb58-cad9-46ba-969c-414a7352f8ad", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 11, "novel_recall": 0.7273, "qid": "523fef22-e0ae-4957-a9d1-d38904c53f1e", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "cb16728c-5f7d-4fed-b69c-d1d805f25c3e", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "43d28da0-01c4-4690-a13e-80c309ed6cbf", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "ed4d2db0-b658-42f1-9742-53e3653ca85d", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 15, "novel_recall": 0.6, "qid": "204112cf-c5e8-49ce-9ed3-647caefc9da2", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 29, "novel_recall": 0.2414, "qid": "05e340dc-191e-4db5-8e5e-44d0a181ea4c", "type": "D"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "75", "full_hit": false, "n_novel": 27, "novel_recall": 0.2963, "qid": "f9611fff-648c-4c71-b377-00426d21b46d", "type": "D"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 18, "novel_recall": 0.0, "qid": "84006d93-45f2-48b5-a35a-600ada365a27", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "9dd5a707-e88d-48fc-83d7-9f5cd3e1f6b5", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 14, "novel_recall": 0.0, "qid": "fbdcbae3-b33f-455c-893e-e90e5661e17e", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "6d2cf47b-f4c5-4592-9438-69143f32ca34", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "f4b194ff-176a-463a-8255-99891a9e92d0", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "4ef167e5-5b10-4364-bbd1-aeadc412f283", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "0b630b7c-78d5-4141-ace8-e6f5cc4c2c4d", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "51a60d2d-37a6-429c-9ee8-33a494322336", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "db885146-3608-4538-bad3-0a2a3fb77082", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "5b025eaf-e85c-44ed-a1d2-a50d1c8aae0c", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "75c600c2-8816-4259-9673-343e5c016988", "type": "D"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 28, "novel_recall": 0.0, "qid": "fcdc8f29-16d1-43bf-bcec-32d050daf47c", "type": "D"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 18, "novel_recall": 0.0, "qid": "84006d93-45f2-48b5-a35a-600ada365a27", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "9dd5a707-e88d-48fc-83d7-9f5cd3e1f6b5", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 14, "novel_recall": 0.7143, "qid": "fbdcbae3-b33f-455c-893e-e90e5661e17e", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "6d2cf47b-f4c5-4592-9438-69143f32ca34", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "f4b194ff-176a-463a-8255-99891a9e92d0", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "4ef167e5-5b10-4364-bbd1-aeadc412f283", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "0b630b7c-78d5-4141-ace8-e6f5cc4c2c4d", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "51a60d2d-37a6-429c-9ee8-33a494322336", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "db885146-3608-4538-bad3-0a2a3fb77082", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "5b025eaf-e85c-44ed-a1d2-a50d1c8aae0c", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "75c600c2-8816-4259-9673-343e5c016988", "type": "D"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 28, "novel_recall": 0.0, "qid": "fcdc8f29-16d1-43bf-bcec-32d050daf47c", "type": "D"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 18, "novel_recall": 0.0556, "qid": "84006d93-45f2-48b5-a35a-600ada365a27", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "9dd5a707-e88d-48fc-83d7-9f5cd3e1f6b5", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 14, "novel_recall": 0.7143, "qid": "fbdcbae3-b33f-455c-893e-e90e5661e17e", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "6d2cf47b-f4c5-4592-9438-69143f32ca34", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 12, "novel_recall": 0.0833, "qid": "f4b194ff-176a-463a-8255-99891a9e92d0", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 16, "novel_recall": 0.0625, "qid": "4ef167e5-5b10-4364-bbd1-aeadc412f283", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "0b630b7c-78d5-4141-ace8-e6f5cc4c2c4d", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "51a60d2d-37a6-429c-9ee8-33a494322336", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "db885146-3608-4538-bad3-0a2a3fb77082", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "5b025eaf-e85c-44ed-a1d2-a50d1c8aae0c", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "75c600c2-8816-4259-9673-343e5c016988", "type": "D"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 28, "novel_recall": 0.0, "qid": "fcdc8f29-16d1-43bf-bcec-32d050daf47c", "type": "D"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 18, "novel_recall": 0.0556, "qid": "84006d93-45f2-48b5-a35a-600ada365a27", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "9dd5a707-e88d-48fc-83d7-9f5cd3e1f6b5", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 14, "novel_recall": 0.7143, "qid": "fbdcbae3-b33f-455c-893e-e90e5661e17e", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "6d2cf47b-f4c5-4592-9438-69143f32ca34", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 12, "novel_recall": 0.0833, "qid": "f4b194ff-176a-463a-8255-99891a9e92d0", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 16, "novel_recall": 0.0625, "qid": "4ef167e5-5b10-4364-bbd1-aeadc412f283", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "0b630b7c-78d5-4141-ace8-e6f5cc4c2c4d", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "51a60d2d-37a6-429c-9ee8-33a494322336", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "db885146-3608-4538-bad3-0a2a3fb77082", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "5b025eaf-e85c-44ed-a1d2-a50d1c8aae0c", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "75c600c2-8816-4259-9673-343e5c016988", "type": "D"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 28, "novel_recall": 0.0, "qid": "fcdc8f29-16d1-43bf-bcec-32d050daf47c", "type": "D"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 18, "novel_recall": 0.0556, "qid": "84006d93-45f2-48b5-a35a-600ada365a27", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "9dd5a707-e88d-48fc-83d7-9f5cd3e1f6b5", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 14, "novel_recall": 0.7143, "qid": "fbdcbae3-b33f-455c-893e-e90e5661e17e", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "6d2cf47b-f4c5-4592-9438-69143f32ca34", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 12, "novel_recall": 0.0833, "qid": "f4b194ff-176a-463a-8255-99891a9e92d0", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 16, "novel_recall": 0.0625, "qid": "4ef167e5-5b10-4364-bbd1-aeadc412f283", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "0b630b7c-78d5-4141-ace8-e6f5cc4c2c4d", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "51a60d2d-37a6-429c-9ee8-33a494322336", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "db885146-3608-4538-bad3-0a2a3fb77082", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "5b025eaf-e85c-44ed-a1d2-a50d1c8aae0c", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "75c600c2-8816-4259-9673-343e5c016988", "type": "D"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 28, "novel_recall": 0.0, "qid": "fcdc8f29-16d1-43bf-bcec-32d050daf47c", "type": "D"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 18, "novel_recall": 0.5, "qid": "84006d93-45f2-48b5-a35a-600ada365a27", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 17, "novel_recall": 0.1765, "qid": "9dd5a707-e88d-48fc-83d7-9f5cd3e1f6b5", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 14, "novel_recall": 0.5714, "qid": "fbdcbae3-b33f-455c-893e-e90e5661e17e", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 16, "novel_recall": 0.1875, "qid": "6d2cf47b-f4c5-4592-9438-69143f32ca34", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 12, "novel_recall": 0.4167, "qid": "f4b194ff-176a-463a-8255-99891a9e92d0", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 16, "novel_recall": 0.375, "qid": "4ef167e5-5b10-4364-bbd1-aeadc412f283", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 11, "novel_recall": 0.0909, "qid": "0b630b7c-78d5-4141-ace8-e6f5cc4c2c4d", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 12, "novel_recall": 0.5833, "qid": "51a60d2d-37a6-429c-9ee8-33a494322336", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "db885146-3608-4538-bad3-0a2a3fb77082", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "5b025eaf-e85c-44ed-a1d2-a50d1c8aae0c", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 17, "novel_recall": 0.1765, "qid": "75c600c2-8816-4259-9673-343e5c016988", "type": "D"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "76", "full_hit": false, "n_novel": 28, "novel_recall": 0.3214, "qid": "fcdc8f29-16d1-43bf-bcec-32d050daf47c", "type": "D"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "ee01f7d0-9ad3-424c-a19f-07c4b42cf77b", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "388848f2-44f4-489c-b38f-2d19cff34525", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "cd2537c5-0224-4966-a348-761975c65de5", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "5d4d5e49-c9e4-40e8-bf2f-fab7150127c7", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "57dc4ee4-dda6-40cb-a034-048765863b2a", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "db1a3a29-b1ff-4e8a-93c8-b2edd7a0448a", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "912fc13f-98bb-4e35-8359-137abdc7d687", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "390aad51-0700-45b3-bea8-c58fec7d2f3c", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "f6c53689-87d9-4c79-a709-ee6bc569d982", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "2d470f65-cdc4-4e3f-9972-7cea03dc49a3", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 30, "novel_recall": 0.0, "qid": "1cdb8d18-d082-402d-a490-a0110fd556ba", "type": "D"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 33, "novel_recall": 0.0, "qid": "b28ef6d1-4aee-4cc0-bcff-c397ed7b9e3e", "type": "D"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "ee01f7d0-9ad3-424c-a19f-07c4b42cf77b", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "388848f2-44f4-489c-b38f-2d19cff34525", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "cd2537c5-0224-4966-a348-761975c65de5", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "5d4d5e49-c9e4-40e8-bf2f-fab7150127c7", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "57dc4ee4-dda6-40cb-a034-048765863b2a", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 8, "novel_recall": 1.0, "qid": "db1a3a29-b1ff-4e8a-93c8-b2edd7a0448a", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 12, "novel_recall": 0.8333, "qid": "912fc13f-98bb-4e35-8359-137abdc7d687", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "390aad51-0700-45b3-bea8-c58fec7d2f3c", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "f6c53689-87d9-4c79-a709-ee6bc569d982", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "2d470f65-cdc4-4e3f-9972-7cea03dc49a3", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 30, "novel_recall": 0.0, "qid": "1cdb8d18-d082-402d-a490-a0110fd556ba", "type": "D"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 33, "novel_recall": 0.8182, "qid": "b28ef6d1-4aee-4cc0-bcff-c397ed7b9e3e", "type": "D"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "ee01f7d0-9ad3-424c-a19f-07c4b42cf77b", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "388848f2-44f4-489c-b38f-2d19cff34525", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "cd2537c5-0224-4966-a348-761975c65de5", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "5d4d5e49-c9e4-40e8-bf2f-fab7150127c7", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "57dc4ee4-dda6-40cb-a034-048765863b2a", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 8, "novel_recall": 1.0, "qid": "db1a3a29-b1ff-4e8a-93c8-b2edd7a0448a", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 12, "novel_recall": 0.8333, "qid": "912fc13f-98bb-4e35-8359-137abdc7d687", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "390aad51-0700-45b3-bea8-c58fec7d2f3c", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "f6c53689-87d9-4c79-a709-ee6bc569d982", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "2d470f65-cdc4-4e3f-9972-7cea03dc49a3", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 30, "novel_recall": 0.1, "qid": "1cdb8d18-d082-402d-a490-a0110fd556ba", "type": "D"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 33, "novel_recall": 0.8182, "qid": "b28ef6d1-4aee-4cc0-bcff-c397ed7b9e3e", "type": "D"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "ee01f7d0-9ad3-424c-a19f-07c4b42cf77b", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "388848f2-44f4-489c-b38f-2d19cff34525", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "cd2537c5-0224-4966-a348-761975c65de5", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "5d4d5e49-c9e4-40e8-bf2f-fab7150127c7", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "57dc4ee4-dda6-40cb-a034-048765863b2a", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 8, "novel_recall": 1.0, "qid": "db1a3a29-b1ff-4e8a-93c8-b2edd7a0448a", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 12, "novel_recall": 0.8333, "qid": "912fc13f-98bb-4e35-8359-137abdc7d687", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "390aad51-0700-45b3-bea8-c58fec7d2f3c", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "f6c53689-87d9-4c79-a709-ee6bc569d982", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "2d470f65-cdc4-4e3f-9972-7cea03dc49a3", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 30, "novel_recall": 0.1, "qid": "1cdb8d18-d082-402d-a490-a0110fd556ba", "type": "D"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 33, "novel_recall": 0.8182, "qid": "b28ef6d1-4aee-4cc0-bcff-c397ed7b9e3e", "type": "D"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "ee01f7d0-9ad3-424c-a19f-07c4b42cf77b", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "388848f2-44f4-489c-b38f-2d19cff34525", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "cd2537c5-0224-4966-a348-761975c65de5", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "5d4d5e49-c9e4-40e8-bf2f-fab7150127c7", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "57dc4ee4-dda6-40cb-a034-048765863b2a", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 8, "novel_recall": 1.0, "qid": "db1a3a29-b1ff-4e8a-93c8-b2edd7a0448a", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 12, "novel_recall": 0.8333, "qid": "912fc13f-98bb-4e35-8359-137abdc7d687", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "390aad51-0700-45b3-bea8-c58fec7d2f3c", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "f6c53689-87d9-4c79-a709-ee6bc569d982", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "2d470f65-cdc4-4e3f-9972-7cea03dc49a3", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 30, "novel_recall": 0.1, "qid": "1cdb8d18-d082-402d-a490-a0110fd556ba", "type": "D"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 33, "novel_recall": 0.8182, "qid": "b28ef6d1-4aee-4cc0-bcff-c397ed7b9e3e", "type": "D"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "ee01f7d0-9ad3-424c-a19f-07c4b42cf77b", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "388848f2-44f4-489c-b38f-2d19cff34525", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "cd2537c5-0224-4966-a348-761975c65de5", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "5d4d5e49-c9e4-40e8-bf2f-fab7150127c7", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "57dc4ee4-dda6-40cb-a034-048765863b2a", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "db1a3a29-b1ff-4e8a-93c8-b2edd7a0448a", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 12, "novel_recall": 0.5, "qid": "912fc13f-98bb-4e35-8359-137abdc7d687", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "390aad51-0700-45b3-bea8-c58fec7d2f3c", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "f6c53689-87d9-4c79-a709-ee6bc569d982", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "2d470f65-cdc4-4e3f-9972-7cea03dc49a3", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 30, "novel_recall": 0.1667, "qid": "1cdb8d18-d082-402d-a490-a0110fd556ba", "type": "D"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "78", "full_hit": false, "n_novel": 33, "novel_recall": 0.6364, "qid": "b28ef6d1-4aee-4cc0-bcff-c397ed7b9e3e", "type": "D"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "97009ed8-eb3f-47c5-a1a7-498ba50da513", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "caaf99d1-3125-4cfd-b697-70c6b9f7d3d1", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "3b9b7398-6a9b-47a6-89ed-9c36e9971d23", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "71bc7e71-74ac-4cf6-92d8-42783f3e58b8", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "da6ccdef-436e-4268-a943-e2c09371c490", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "716d3b19-e5df-44fd-8703-c5c19190cdaa", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "69f4c2eb-9586-4374-986e-8800e8424883", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "5b7d9e7e-84f9-4954-9aca-af1b28841212", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "7f0cd872-35a1-4f78-8fc1-210f2753f521", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "14d4d5df-9e30-4acf-b73d-bbfddc079dd3", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "44041df8-1ba1-4029-97b2-3589be17dd90", "type": "D"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "d117cd59-8c6b-45a0-9b31-27174836c52b", "type": "D"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "97009ed8-eb3f-47c5-a1a7-498ba50da513", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "caaf99d1-3125-4cfd-b697-70c6b9f7d3d1", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": true, "n_novel": 7, "novel_recall": 1.0, "qid": "3b9b7398-6a9b-47a6-89ed-9c36e9971d23", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "71bc7e71-74ac-4cf6-92d8-42783f3e58b8", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "da6ccdef-436e-4268-a943-e2c09371c490", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "716d3b19-e5df-44fd-8703-c5c19190cdaa", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "69f4c2eb-9586-4374-986e-8800e8424883", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "5b7d9e7e-84f9-4954-9aca-af1b28841212", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "7f0cd872-35a1-4f78-8fc1-210f2753f521", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "14d4d5df-9e30-4acf-b73d-bbfddc079dd3", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "44041df8-1ba1-4029-97b2-3589be17dd90", "type": "D"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 24, "novel_recall": 0.5, "qid": "d117cd59-8c6b-45a0-9b31-27174836c52b", "type": "D"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "97009ed8-eb3f-47c5-a1a7-498ba50da513", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "caaf99d1-3125-4cfd-b697-70c6b9f7d3d1", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": true, "n_novel": 7, "novel_recall": 1.0, "qid": "3b9b7398-6a9b-47a6-89ed-9c36e9971d23", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "71bc7e71-74ac-4cf6-92d8-42783f3e58b8", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "da6ccdef-436e-4268-a943-e2c09371c490", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "716d3b19-e5df-44fd-8703-c5c19190cdaa", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "69f4c2eb-9586-4374-986e-8800e8424883", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "5b7d9e7e-84f9-4954-9aca-af1b28841212", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 8, "novel_recall": 0.125, "qid": "7f0cd872-35a1-4f78-8fc1-210f2753f521", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "14d4d5df-9e30-4acf-b73d-bbfddc079dd3", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "44041df8-1ba1-4029-97b2-3589be17dd90", "type": "D"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 24, "novel_recall": 0.5, "qid": "d117cd59-8c6b-45a0-9b31-27174836c52b", "type": "D"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "97009ed8-eb3f-47c5-a1a7-498ba50da513", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "caaf99d1-3125-4cfd-b697-70c6b9f7d3d1", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": true, "n_novel": 7, "novel_recall": 1.0, "qid": "3b9b7398-6a9b-47a6-89ed-9c36e9971d23", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "71bc7e71-74ac-4cf6-92d8-42783f3e58b8", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "da6ccdef-436e-4268-a943-e2c09371c490", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "716d3b19-e5df-44fd-8703-c5c19190cdaa", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "69f4c2eb-9586-4374-986e-8800e8424883", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "5b7d9e7e-84f9-4954-9aca-af1b28841212", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 8, "novel_recall": 0.125, "qid": "7f0cd872-35a1-4f78-8fc1-210f2753f521", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "14d4d5df-9e30-4acf-b73d-bbfddc079dd3", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "44041df8-1ba1-4029-97b2-3589be17dd90", "type": "D"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 24, "novel_recall": 0.5, "qid": "d117cd59-8c6b-45a0-9b31-27174836c52b", "type": "D"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "97009ed8-eb3f-47c5-a1a7-498ba50da513", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "caaf99d1-3125-4cfd-b697-70c6b9f7d3d1", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": true, "n_novel": 7, "novel_recall": 1.0, "qid": "3b9b7398-6a9b-47a6-89ed-9c36e9971d23", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "71bc7e71-74ac-4cf6-92d8-42783f3e58b8", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "da6ccdef-436e-4268-a943-e2c09371c490", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "716d3b19-e5df-44fd-8703-c5c19190cdaa", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "69f4c2eb-9586-4374-986e-8800e8424883", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "5b7d9e7e-84f9-4954-9aca-af1b28841212", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 8, "novel_recall": 0.125, "qid": "7f0cd872-35a1-4f78-8fc1-210f2753f521", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "14d4d5df-9e30-4acf-b73d-bbfddc079dd3", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "44041df8-1ba1-4029-97b2-3589be17dd90", "type": "D"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 24, "novel_recall": 0.5, "qid": "d117cd59-8c6b-45a0-9b31-27174836c52b", "type": "D"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "97009ed8-eb3f-47c5-a1a7-498ba50da513", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "caaf99d1-3125-4cfd-b697-70c6b9f7d3d1", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "3b9b7398-6a9b-47a6-89ed-9c36e9971d23", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 7, "novel_recall": 0.1429, "qid": "71bc7e71-74ac-4cf6-92d8-42783f3e58b8", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "da6ccdef-436e-4268-a943-e2c09371c490", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "716d3b19-e5df-44fd-8703-c5c19190cdaa", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "69f4c2eb-9586-4374-986e-8800e8424883", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "5b7d9e7e-84f9-4954-9aca-af1b28841212", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "7f0cd872-35a1-4f78-8fc1-210f2753f521", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "14d4d5df-9e30-4acf-b73d-bbfddc079dd3", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 24, "novel_recall": 0.125, "qid": "44041df8-1ba1-4029-97b2-3589be17dd90", "type": "D"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "80", "full_hit": false, "n_novel": 24, "novel_recall": 0.625, "qid": "d117cd59-8c6b-45a0-9b31-27174836c52b", "type": "D"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "44dd051d-6c4c-4a82-9feb-08ee965ff83f", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "6fae8bac-9650-4529-8721-0e4df1c557cd", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "d96eca42-5768-4cb0-a1b7-6de60e176c95", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "27c0a986-76ea-4578-852b-f33c85b9155a", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "ae9dda2a-6eb2-494f-b43e-f44d59d9cee2", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "9f53b16e-2e10-45ec-ae50-b692e54cdabd", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "116df5ca-6fae-4daa-a29c-20b7201281f8", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "9cd134db-bc3a-4059-8cfd-17b707644cd3", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e35c9cbc-18fc-4a68-aca4-a0e3b37a0c1c", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e9f04cca-fdc1-4feb-b391-eecc7baffbd6", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 41, "novel_recall": 0.0, "qid": "4156043a-25ba-473d-be9f-7013b5973fa3", "type": "D"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 42, "novel_recall": 0.0, "qid": "161afa1e-049d-47fe-9292-944931c9cb1e", "type": "D"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "44dd051d-6c4c-4a82-9feb-08ee965ff83f", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "6fae8bac-9650-4529-8721-0e4df1c557cd", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "d96eca42-5768-4cb0-a1b7-6de60e176c95", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "27c0a986-76ea-4578-852b-f33c85b9155a", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "ae9dda2a-6eb2-494f-b43e-f44d59d9cee2", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "9f53b16e-2e10-45ec-ae50-b692e54cdabd", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "116df5ca-6fae-4daa-a29c-20b7201281f8", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "9cd134db-bc3a-4059-8cfd-17b707644cd3", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "e35c9cbc-18fc-4a68-aca4-a0e3b37a0c1c", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e9f04cca-fdc1-4feb-b391-eecc7baffbd6", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 41, "novel_recall": 0.4634, "qid": "4156043a-25ba-473d-be9f-7013b5973fa3", "type": "D"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 42, "novel_recall": 0.4286, "qid": "161afa1e-049d-47fe-9292-944931c9cb1e", "type": "D"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "44dd051d-6c4c-4a82-9feb-08ee965ff83f", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "6fae8bac-9650-4529-8721-0e4df1c557cd", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "d96eca42-5768-4cb0-a1b7-6de60e176c95", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "27c0a986-76ea-4578-852b-f33c85b9155a", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "ae9dda2a-6eb2-494f-b43e-f44d59d9cee2", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "9f53b16e-2e10-45ec-ae50-b692e54cdabd", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "116df5ca-6fae-4daa-a29c-20b7201281f8", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "9cd134db-bc3a-4059-8cfd-17b707644cd3", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "e35c9cbc-18fc-4a68-aca4-a0e3b37a0c1c", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e9f04cca-fdc1-4feb-b391-eecc7baffbd6", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 41, "novel_recall": 0.4634, "qid": "4156043a-25ba-473d-be9f-7013b5973fa3", "type": "D"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 42, "novel_recall": 0.4286, "qid": "161afa1e-049d-47fe-9292-944931c9cb1e", "type": "D"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "44dd051d-6c4c-4a82-9feb-08ee965ff83f", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "6fae8bac-9650-4529-8721-0e4df1c557cd", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "d96eca42-5768-4cb0-a1b7-6de60e176c95", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "27c0a986-76ea-4578-852b-f33c85b9155a", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "ae9dda2a-6eb2-494f-b43e-f44d59d9cee2", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "9f53b16e-2e10-45ec-ae50-b692e54cdabd", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "116df5ca-6fae-4daa-a29c-20b7201281f8", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "9cd134db-bc3a-4059-8cfd-17b707644cd3", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "e35c9cbc-18fc-4a68-aca4-a0e3b37a0c1c", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e9f04cca-fdc1-4feb-b391-eecc7baffbd6", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 41, "novel_recall": 0.3415, "qid": "4156043a-25ba-473d-be9f-7013b5973fa3", "type": "D"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 42, "novel_recall": 0.4286, "qid": "161afa1e-049d-47fe-9292-944931c9cb1e", "type": "D"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "44dd051d-6c4c-4a82-9feb-08ee965ff83f", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "6fae8bac-9650-4529-8721-0e4df1c557cd", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "d96eca42-5768-4cb0-a1b7-6de60e176c95", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "27c0a986-76ea-4578-852b-f33c85b9155a", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "ae9dda2a-6eb2-494f-b43e-f44d59d9cee2", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "9f53b16e-2e10-45ec-ae50-b692e54cdabd", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "116df5ca-6fae-4daa-a29c-20b7201281f8", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "9cd134db-bc3a-4059-8cfd-17b707644cd3", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "e35c9cbc-18fc-4a68-aca4-a0e3b37a0c1c", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e9f04cca-fdc1-4feb-b391-eecc7baffbd6", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 41, "novel_recall": 0.3415, "qid": "4156043a-25ba-473d-be9f-7013b5973fa3", "type": "D"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 42, "novel_recall": 0.4286, "qid": "161afa1e-049d-47fe-9292-944931c9cb1e", "type": "D"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "44dd051d-6c4c-4a82-9feb-08ee965ff83f", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "6fae8bac-9650-4529-8721-0e4df1c557cd", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "d96eca42-5768-4cb0-a1b7-6de60e176c95", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.4444, "qid": "27c0a986-76ea-4578-852b-f33c85b9155a", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "ae9dda2a-6eb2-494f-b43e-f44d59d9cee2", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "9f53b16e-2e10-45ec-ae50-b692e54cdabd", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 11, "novel_recall": 0.1818, "qid": "116df5ca-6fae-4daa-a29c-20b7201281f8", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "9cd134db-bc3a-4059-8cfd-17b707644cd3", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "e35c9cbc-18fc-4a68-aca4-a0e3b37a0c1c", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "e9f04cca-fdc1-4feb-b391-eecc7baffbd6", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 41, "novel_recall": 0.3171, "qid": "4156043a-25ba-473d-be9f-7013b5973fa3", "type": "D"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "83", "full_hit": false, "n_novel": 42, "novel_recall": 0.4286, "qid": "161afa1e-049d-47fe-9292-944931c9cb1e", "type": "D"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "25758566-d05b-4cd7-a57e-f965cab0a32f", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "642cfa07-868f-4dbe-8a34-17705def4436", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "f50f29c8-93c0-4a85-a4d8-6da78df204da", "type": "A"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "b4b03f4f-65c2-42ff-940d-fe56d55b30e8", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "90f27e6e-2ee2-41e7-bc66-a4bc4da1e71c", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "be56c938-8028-4cbc-9ad8-7c284c164ceb", "type": "B"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "65be8c31-c44c-4b8e-b36e-6aca83cbdfba", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "424984fd-5cf6-4362-949c-dc433b4fc283", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "0bb1e0df-9b5b-4f07-b15e-9f46aee918b5", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "813fb6e1-88de-4ee2-85c3-35f3fc010c1c", "type": "C"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 26, "novel_recall": 0.0, "qid": "22667cf5-0027-4c4b-88c3-a5a86662a585", "type": "D"}, {"arm": "M0_none", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 23, "novel_recall": 0.0, "qid": "d4d71cf9-433b-44f6-97ad-e87b0bf6b9fb", "type": "D"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "25758566-d05b-4cd7-a57e-f965cab0a32f", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.8182, "qid": "642cfa07-868f-4dbe-8a34-17705def4436", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "f50f29c8-93c0-4a85-a4d8-6da78df204da", "type": "A"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "b4b03f4f-65c2-42ff-940d-fe56d55b30e8", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "90f27e6e-2ee2-41e7-bc66-a4bc4da1e71c", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 12, "novel_recall": 0.9167, "qid": "be56c938-8028-4cbc-9ad8-7c284c164ceb", "type": "B"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 7, "novel_recall": 0.8571, "qid": "65be8c31-c44c-4b8e-b36e-6aca83cbdfba", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": true, "n_novel": 3, "novel_recall": 1.0, "qid": "424984fd-5cf6-4362-949c-dc433b4fc283", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "0bb1e0df-9b5b-4f07-b15e-9f46aee918b5", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.4545, "qid": "813fb6e1-88de-4ee2-85c3-35f3fc010c1c", "type": "C"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 26, "novel_recall": 0.4231, "qid": "22667cf5-0027-4c4b-88c3-a5a86662a585", "type": "D"}, {"arm": "M1_naive", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 23, "novel_recall": 0.2174, "qid": "d4d71cf9-433b-44f6-97ad-e87b0bf6b9fb", "type": "D"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "25758566-d05b-4cd7-a57e-f965cab0a32f", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.8182, "qid": "642cfa07-868f-4dbe-8a34-17705def4436", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "f50f29c8-93c0-4a85-a4d8-6da78df204da", "type": "A"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "b4b03f4f-65c2-42ff-940d-fe56d55b30e8", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "90f27e6e-2ee2-41e7-bc66-a4bc4da1e71c", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 12, "novel_recall": 0.8333, "qid": "be56c938-8028-4cbc-9ad8-7c284c164ceb", "type": "B"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 7, "novel_recall": 0.8571, "qid": "65be8c31-c44c-4b8e-b36e-6aca83cbdfba", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": true, "n_novel": 3, "novel_recall": 1.0, "qid": "424984fd-5cf6-4362-949c-dc433b4fc283", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "0bb1e0df-9b5b-4f07-b15e-9f46aee918b5", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.4545, "qid": "813fb6e1-88de-4ee2-85c3-35f3fc010c1c", "type": "C"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 26, "novel_recall": 0.4231, "qid": "22667cf5-0027-4c4b-88c3-a5a86662a585", "type": "D"}, {"arm": "M2_evidence", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 23, "novel_recall": 0.2609, "qid": "d4d71cf9-433b-44f6-97ad-e87b0bf6b9fb", "type": "D"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "25758566-d05b-4cd7-a57e-f965cab0a32f", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.8182, "qid": "642cfa07-868f-4dbe-8a34-17705def4436", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "f50f29c8-93c0-4a85-a4d8-6da78df204da", "type": "A"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "b4b03f4f-65c2-42ff-940d-fe56d55b30e8", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "90f27e6e-2ee2-41e7-bc66-a4bc4da1e71c", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 12, "novel_recall": 0.8333, "qid": "be56c938-8028-4cbc-9ad8-7c284c164ceb", "type": "B"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 7, "novel_recall": 0.8571, "qid": "65be8c31-c44c-4b8e-b36e-6aca83cbdfba", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": true, "n_novel": 3, "novel_recall": 1.0, "qid": "424984fd-5cf6-4362-949c-dc433b4fc283", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "0bb1e0df-9b5b-4f07-b15e-9f46aee918b5", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.4545, "qid": "813fb6e1-88de-4ee2-85c3-35f3fc010c1c", "type": "C"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 26, "novel_recall": 0.4231, "qid": "22667cf5-0027-4c4b-88c3-a5a86662a585", "type": "D"}, {"arm": "M3_causal", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 23, "novel_recall": 0.2609, "qid": "d4d71cf9-433b-44f6-97ad-e87b0bf6b9fb", "type": "D"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "25758566-d05b-4cd7-a57e-f965cab0a32f", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.8182, "qid": "642cfa07-868f-4dbe-8a34-17705def4436", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "f50f29c8-93c0-4a85-a4d8-6da78df204da", "type": "A"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "b4b03f4f-65c2-42ff-940d-fe56d55b30e8", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "90f27e6e-2ee2-41e7-bc66-a4bc4da1e71c", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 12, "novel_recall": 0.8333, "qid": "be56c938-8028-4cbc-9ad8-7c284c164ceb", "type": "B"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 7, "novel_recall": 0.8571, "qid": "65be8c31-c44c-4b8e-b36e-6aca83cbdfba", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": true, "n_novel": 3, "novel_recall": 1.0, "qid": "424984fd-5cf6-4362-949c-dc433b4fc283", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "0bb1e0df-9b5b-4f07-b15e-9f46aee918b5", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.4545, "qid": "813fb6e1-88de-4ee2-85c3-35f3fc010c1c", "type": "C"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 26, "novel_recall": 0.4231, "qid": "22667cf5-0027-4c4b-88c3-a5a86662a585", "type": "D"}, {"arm": "M4_validated", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 23, "novel_recall": 0.2609, "qid": "d4d71cf9-433b-44f6-97ad-e87b0bf6b9fb", "type": "D"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "25758566-d05b-4cd7-a57e-f965cab0a32f", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.3636, "qid": "642cfa07-868f-4dbe-8a34-17705def4436", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "f50f29c8-93c0-4a85-a4d8-6da78df204da", "type": "A"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "b4b03f4f-65c2-42ff-940d-fe56d55b30e8", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "90f27e6e-2ee2-41e7-bc66-a4bc4da1e71c", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 12, "novel_recall": 0.4167, "qid": "be56c938-8028-4cbc-9ad8-7c284c164ceb", "type": "B"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": true, "n_novel": 7, "novel_recall": 1.0, "qid": "65be8c31-c44c-4b8e-b36e-6aca83cbdfba", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": true, "n_novel": 3, "novel_recall": 1.0, "qid": "424984fd-5cf6-4362-949c-dc433b4fc283", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "0bb1e0df-9b5b-4f07-b15e-9f46aee918b5", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 11, "novel_recall": 0.7273, "qid": "813fb6e1-88de-4ee2-85c3-35f3fc010c1c", "type": "C"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 26, "novel_recall": 0.5, "qid": "22667cf5-0027-4c4b-88c3-a5a86662a585", "type": "D"}, {"arm": "R_recent", "domain": "OPENWORLD_QA", "episode": "86", "full_hit": false, "n_novel": 23, "novel_recall": 0.2609, "qid": "d4d71cf9-433b-44f6-97ad-e87b0bf6b9fb", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "d058bfd4-cb61-4711-92d1-7f9ccc1931c8", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 30, "novel_recall": 0.0, "qid": "ca1fbfbd-08d4-4d1b-904c-5ddb7b323609", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 93, "novel_recall": 0.0, "qid": "36ce2d6b-5773-4f8b-a5e9-d9ec103762c7", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 100, "novel_recall": 0.0, "qid": "6e62298c-4d3e-4f8f-8487-59cccb315b86", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 42, "novel_recall": 0.0, "qid": "1993e3da-65b2-41b6-bf4e-5fe34fea8e76", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 49, "novel_recall": 0.0, "qid": "dde65a38-a383-441b-bc27-0a76f4b2d016", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 121, "novel_recall": 0.0, "qid": "e5ece92f-cc52-4560-8f2c-41d7222184df", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 62, "novel_recall": 0.0, "qid": "87ed10dc-9440-495b-95a7-9561480b7b97", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 46, "novel_recall": 0.0, "qid": "8e36da51-b068-41ac-a33f-a421df41a48a", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 129, "novel_recall": 0.0, "qid": "5d03f84e-e50f-4dd2-b73e-386ef3ba0244", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 31, "novel_recall": 0.0, "qid": "6fd4606e-6075-4017-af81-99cf39530d76", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 52, "novel_recall": 0.0, "qid": "2842bd10-a466-4212-9b78-0214a2498a56", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "95", "full_hit": true, "n_novel": 17, "novel_recall": 1.0, "qid": "d058bfd4-cb61-4711-92d1-7f9ccc1931c8", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 30, "novel_recall": 0.8, "qid": "ca1fbfbd-08d4-4d1b-904c-5ddb7b323609", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 93, "novel_recall": 0.3763, "qid": "36ce2d6b-5773-4f8b-a5e9-d9ec103762c7", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 100, "novel_recall": 0.33, "qid": "6e62298c-4d3e-4f8f-8487-59cccb315b86", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 42, "novel_recall": 0.4048, "qid": "1993e3da-65b2-41b6-bf4e-5fe34fea8e76", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 49, "novel_recall": 0.2857, "qid": "dde65a38-a383-441b-bc27-0a76f4b2d016", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 121, "novel_recall": 0.2893, "qid": "e5ece92f-cc52-4560-8f2c-41d7222184df", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 62, "novel_recall": 0.2581, "qid": "87ed10dc-9440-495b-95a7-9561480b7b97", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 46, "novel_recall": 0.2826, "qid": "8e36da51-b068-41ac-a33f-a421df41a48a", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 129, "novel_recall": 0.2403, "qid": "5d03f84e-e50f-4dd2-b73e-386ef3ba0244", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 31, "novel_recall": 0.871, "qid": "6fd4606e-6075-4017-af81-99cf39530d76", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 52, "novel_recall": 0.5962, "qid": "2842bd10-a466-4212-9b78-0214a2498a56", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "95", "full_hit": true, "n_novel": 17, "novel_recall": 1.0, "qid": "d058bfd4-cb61-4711-92d1-7f9ccc1931c8", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 30, "novel_recall": 0.8, "qid": "ca1fbfbd-08d4-4d1b-904c-5ddb7b323609", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 93, "novel_recall": 0.3763, "qid": "36ce2d6b-5773-4f8b-a5e9-d9ec103762c7", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 100, "novel_recall": 0.33, "qid": "6e62298c-4d3e-4f8f-8487-59cccb315b86", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 42, "novel_recall": 0.4048, "qid": "1993e3da-65b2-41b6-bf4e-5fe34fea8e76", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 49, "novel_recall": 0.2857, "qid": "dde65a38-a383-441b-bc27-0a76f4b2d016", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 121, "novel_recall": 0.2893, "qid": "e5ece92f-cc52-4560-8f2c-41d7222184df", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 62, "novel_recall": 0.2581, "qid": "87ed10dc-9440-495b-95a7-9561480b7b97", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 46, "novel_recall": 0.2826, "qid": "8e36da51-b068-41ac-a33f-a421df41a48a", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 129, "novel_recall": 0.2403, "qid": "5d03f84e-e50f-4dd2-b73e-386ef3ba0244", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 31, "novel_recall": 0.871, "qid": "6fd4606e-6075-4017-af81-99cf39530d76", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 52, "novel_recall": 0.5962, "qid": "2842bd10-a466-4212-9b78-0214a2498a56", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 17, "novel_recall": 0.9412, "qid": "d058bfd4-cb61-4711-92d1-7f9ccc1931c8", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 30, "novel_recall": 0.8, "qid": "ca1fbfbd-08d4-4d1b-904c-5ddb7b323609", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 93, "novel_recall": 0.2688, "qid": "36ce2d6b-5773-4f8b-a5e9-d9ec103762c7", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 100, "novel_recall": 0.22, "qid": "6e62298c-4d3e-4f8f-8487-59cccb315b86", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 42, "novel_recall": 0.4048, "qid": "1993e3da-65b2-41b6-bf4e-5fe34fea8e76", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 49, "novel_recall": 0.2449, "qid": "dde65a38-a383-441b-bc27-0a76f4b2d016", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 121, "novel_recall": 0.2727, "qid": "e5ece92f-cc52-4560-8f2c-41d7222184df", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 62, "novel_recall": 0.2742, "qid": "87ed10dc-9440-495b-95a7-9561480b7b97", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 46, "novel_recall": 0.2826, "qid": "8e36da51-b068-41ac-a33f-a421df41a48a", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 129, "novel_recall": 0.2403, "qid": "5d03f84e-e50f-4dd2-b73e-386ef3ba0244", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 31, "novel_recall": 0.7097, "qid": "6fd4606e-6075-4017-af81-99cf39530d76", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 52, "novel_recall": 0.4231, "qid": "2842bd10-a466-4212-9b78-0214a2498a56", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 17, "novel_recall": 0.9412, "qid": "d058bfd4-cb61-4711-92d1-7f9ccc1931c8", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 30, "novel_recall": 0.8, "qid": "ca1fbfbd-08d4-4d1b-904c-5ddb7b323609", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 93, "novel_recall": 0.2688, "qid": "36ce2d6b-5773-4f8b-a5e9-d9ec103762c7", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 100, "novel_recall": 0.22, "qid": "6e62298c-4d3e-4f8f-8487-59cccb315b86", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 42, "novel_recall": 0.4048, "qid": "1993e3da-65b2-41b6-bf4e-5fe34fea8e76", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 49, "novel_recall": 0.2449, "qid": "dde65a38-a383-441b-bc27-0a76f4b2d016", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 121, "novel_recall": 0.2727, "qid": "e5ece92f-cc52-4560-8f2c-41d7222184df", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 62, "novel_recall": 0.2742, "qid": "87ed10dc-9440-495b-95a7-9561480b7b97", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 46, "novel_recall": 0.2826, "qid": "8e36da51-b068-41ac-a33f-a421df41a48a", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 129, "novel_recall": 0.2403, "qid": "5d03f84e-e50f-4dd2-b73e-386ef3ba0244", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 31, "novel_recall": 0.7097, "qid": "6fd4606e-6075-4017-af81-99cf39530d76", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 52, "novel_recall": 0.4231, "qid": "2842bd10-a466-4212-9b78-0214a2498a56", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "95", "full_hit": true, "n_novel": 17, "novel_recall": 1.0, "qid": "d058bfd4-cb61-4711-92d1-7f9ccc1931c8", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 30, "novel_recall": 0.8, "qid": "ca1fbfbd-08d4-4d1b-904c-5ddb7b323609", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 93, "novel_recall": 0.4194, "qid": "36ce2d6b-5773-4f8b-a5e9-d9ec103762c7", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 100, "novel_recall": 0.28, "qid": "6e62298c-4d3e-4f8f-8487-59cccb315b86", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 42, "novel_recall": 0.381, "qid": "1993e3da-65b2-41b6-bf4e-5fe34fea8e76", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 49, "novel_recall": 0.2041, "qid": "dde65a38-a383-441b-bc27-0a76f4b2d016", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 121, "novel_recall": 0.2397, "qid": "e5ece92f-cc52-4560-8f2c-41d7222184df", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 62, "novel_recall": 0.2258, "qid": "87ed10dc-9440-495b-95a7-9561480b7b97", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 46, "novel_recall": 0.3261, "qid": "8e36da51-b068-41ac-a33f-a421df41a48a", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 129, "novel_recall": 0.2248, "qid": "5d03f84e-e50f-4dd2-b73e-386ef3ba0244", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 31, "novel_recall": 0.871, "qid": "6fd4606e-6075-4017-af81-99cf39530d76", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "95", "full_hit": false, "n_novel": 52, "novel_recall": 0.5769, "qid": "2842bd10-a466-4212-9b78-0214a2498a56", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "53e435d2-2fd9-45b1-a6e7-9eac9fe56a19", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 67, "novel_recall": 0.0, "qid": "9b2aedd1-9154-4191-a9c9-ff84f050a3a6", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 79, "novel_recall": 0.0, "qid": "f3e87e5c-a4b5-4145-a72e-9ae5890b7f89", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 57, "novel_recall": 0.0, "qid": "0f8321e9-0d52-4ffe-8f05-d1997ff03aca", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 69, "novel_recall": 0.0, "qid": "26bb030b-6263-4a97-963d-e4e13ce79560", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 56, "novel_recall": 0.0, "qid": "098c2bd8-fa25-49e0-8a1c-7e76d7f567cf", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 89, "novel_recall": 0.0, "qid": "c9fa683b-4223-4cf2-ac1b-a5513086bbf0", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 89, "novel_recall": 0.0, "qid": "a76e7356-a6e3-4e24-bd0c-456d427e8420", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "21297958-47e8-4a63-a9b8-acb24a36ae61", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 69, "novel_recall": 0.0, "qid": "e9ef18f3-edce-478c-b0fe-d1f0dc332785", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 28, "novel_recall": 0.0, "qid": "38bea23e-a52d-4705-9102-bbc21e51fae7", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 54, "novel_recall": 0.0, "qid": "de051cef-d132-45e9-9d3d-50c01b09a4de", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 17, "novel_recall": 0.2353, "qid": "53e435d2-2fd9-45b1-a6e7-9eac9fe56a19", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 67, "novel_recall": 0.0597, "qid": "9b2aedd1-9154-4191-a9c9-ff84f050a3a6", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 79, "novel_recall": 0.0506, "qid": "f3e87e5c-a4b5-4145-a72e-9ae5890b7f89", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 57, "novel_recall": 0.1404, "qid": "0f8321e9-0d52-4ffe-8f05-d1997ff03aca", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 69, "novel_recall": 0.087, "qid": "26bb030b-6263-4a97-963d-e4e13ce79560", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 56, "novel_recall": 0.1071, "qid": "098c2bd8-fa25-49e0-8a1c-7e76d7f567cf", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 89, "novel_recall": 0.0787, "qid": "c9fa683b-4223-4cf2-ac1b-a5513086bbf0", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 89, "novel_recall": 0.0562, "qid": "a76e7356-a6e3-4e24-bd0c-456d427e8420", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 13, "novel_recall": 0.1538, "qid": "21297958-47e8-4a63-a9b8-acb24a36ae61", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 69, "novel_recall": 0.087, "qid": "e9ef18f3-edce-478c-b0fe-d1f0dc332785", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 28, "novel_recall": 0.2857, "qid": "38bea23e-a52d-4705-9102-bbc21e51fae7", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 54, "novel_recall": 0.1111, "qid": "de051cef-d132-45e9-9d3d-50c01b09a4de", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 17, "novel_recall": 0.2353, "qid": "53e435d2-2fd9-45b1-a6e7-9eac9fe56a19", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 67, "novel_recall": 0.0597, "qid": "9b2aedd1-9154-4191-a9c9-ff84f050a3a6", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 79, "novel_recall": 0.0506, "qid": "f3e87e5c-a4b5-4145-a72e-9ae5890b7f89", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 57, "novel_recall": 0.1404, "qid": "0f8321e9-0d52-4ffe-8f05-d1997ff03aca", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 69, "novel_recall": 0.087, "qid": "26bb030b-6263-4a97-963d-e4e13ce79560", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 56, "novel_recall": 0.1071, "qid": "098c2bd8-fa25-49e0-8a1c-7e76d7f567cf", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 89, "novel_recall": 0.0787, "qid": "c9fa683b-4223-4cf2-ac1b-a5513086bbf0", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 89, "novel_recall": 0.0562, "qid": "a76e7356-a6e3-4e24-bd0c-456d427e8420", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 13, "novel_recall": 0.1538, "qid": "21297958-47e8-4a63-a9b8-acb24a36ae61", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 69, "novel_recall": 0.087, "qid": "e9ef18f3-edce-478c-b0fe-d1f0dc332785", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 28, "novel_recall": 0.2857, "qid": "38bea23e-a52d-4705-9102-bbc21e51fae7", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 54, "novel_recall": 0.1111, "qid": "de051cef-d132-45e9-9d3d-50c01b09a4de", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 17, "novel_recall": 0.2353, "qid": "53e435d2-2fd9-45b1-a6e7-9eac9fe56a19", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 67, "novel_recall": 0.0597, "qid": "9b2aedd1-9154-4191-a9c9-ff84f050a3a6", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 79, "novel_recall": 0.0506, "qid": "f3e87e5c-a4b5-4145-a72e-9ae5890b7f89", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 57, "novel_recall": 0.1404, "qid": "0f8321e9-0d52-4ffe-8f05-d1997ff03aca", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 69, "novel_recall": 0.1014, "qid": "26bb030b-6263-4a97-963d-e4e13ce79560", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 56, "novel_recall": 0.1071, "qid": "098c2bd8-fa25-49e0-8a1c-7e76d7f567cf", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 89, "novel_recall": 0.0787, "qid": "c9fa683b-4223-4cf2-ac1b-a5513086bbf0", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 89, "novel_recall": 0.0562, "qid": "a76e7356-a6e3-4e24-bd0c-456d427e8420", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 13, "novel_recall": 0.1538, "qid": "21297958-47e8-4a63-a9b8-acb24a36ae61", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 69, "novel_recall": 0.087, "qid": "e9ef18f3-edce-478c-b0fe-d1f0dc332785", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 28, "novel_recall": 0.2857, "qid": "38bea23e-a52d-4705-9102-bbc21e51fae7", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 54, "novel_recall": 0.1111, "qid": "de051cef-d132-45e9-9d3d-50c01b09a4de", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 17, "novel_recall": 0.2353, "qid": "53e435d2-2fd9-45b1-a6e7-9eac9fe56a19", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 67, "novel_recall": 0.0597, "qid": "9b2aedd1-9154-4191-a9c9-ff84f050a3a6", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 79, "novel_recall": 0.0506, "qid": "f3e87e5c-a4b5-4145-a72e-9ae5890b7f89", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 57, "novel_recall": 0.1404, "qid": "0f8321e9-0d52-4ffe-8f05-d1997ff03aca", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 69, "novel_recall": 0.1014, "qid": "26bb030b-6263-4a97-963d-e4e13ce79560", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 56, "novel_recall": 0.1071, "qid": "098c2bd8-fa25-49e0-8a1c-7e76d7f567cf", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 89, "novel_recall": 0.0787, "qid": "c9fa683b-4223-4cf2-ac1b-a5513086bbf0", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 89, "novel_recall": 0.0562, "qid": "a76e7356-a6e3-4e24-bd0c-456d427e8420", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 13, "novel_recall": 0.1538, "qid": "21297958-47e8-4a63-a9b8-acb24a36ae61", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 69, "novel_recall": 0.087, "qid": "e9ef18f3-edce-478c-b0fe-d1f0dc332785", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 28, "novel_recall": 0.2857, "qid": "38bea23e-a52d-4705-9102-bbc21e51fae7", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 54, "novel_recall": 0.1111, "qid": "de051cef-d132-45e9-9d3d-50c01b09a4de", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 17, "novel_recall": 0.2353, "qid": "53e435d2-2fd9-45b1-a6e7-9eac9fe56a19", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 67, "novel_recall": 0.1194, "qid": "9b2aedd1-9154-4191-a9c9-ff84f050a3a6", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 79, "novel_recall": 0.0506, "qid": "f3e87e5c-a4b5-4145-a72e-9ae5890b7f89", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 57, "novel_recall": 0.1404, "qid": "0f8321e9-0d52-4ffe-8f05-d1997ff03aca", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 69, "novel_recall": 0.1449, "qid": "26bb030b-6263-4a97-963d-e4e13ce79560", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 56, "novel_recall": 0.125, "qid": "098c2bd8-fa25-49e0-8a1c-7e76d7f567cf", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 89, "novel_recall": 0.0899, "qid": "c9fa683b-4223-4cf2-ac1b-a5513086bbf0", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 89, "novel_recall": 0.0674, "qid": "a76e7356-a6e3-4e24-bd0c-456d427e8420", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 13, "novel_recall": 0.3077, "qid": "21297958-47e8-4a63-a9b8-acb24a36ae61", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 69, "novel_recall": 0.1304, "qid": "e9ef18f3-edce-478c-b0fe-d1f0dc332785", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 28, "novel_recall": 0.5, "qid": "38bea23e-a52d-4705-9102-bbc21e51fae7", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "98", "full_hit": false, "n_novel": 54, "novel_recall": 0.2037, "qid": "de051cef-d132-45e9-9d3d-50c01b09a4de", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 34, "novel_recall": 0.0, "qid": "19484533-070f-4f49-86b4-cce0caf18b32", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 33, "novel_recall": 0.0, "qid": "330720af-db5b-4f23-b5b4-6605f0530744", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 60, "novel_recall": 0.0, "qid": "c5e51ca3-edf8-4e91-b218-033a617c116c", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 82, "novel_recall": 0.0, "qid": "3d5e7763-6377-4f1d-852d-15c4c67c4887", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 55, "novel_recall": 0.0, "qid": "26328e2a-4935-42b2-8d2b-4db0ad9e71ae", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 85, "novel_recall": 0.0, "qid": "867ba7a3-8151-4148-ae11-e2afba3d2910", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 98, "novel_recall": 0.0, "qid": "ccaa1adc-db74-443f-aedb-b5b4ad21900c", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 21, "novel_recall": 0.0, "qid": "388fecf5-e3d2-4420-b559-967631d4f57f", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 27, "novel_recall": 0.0, "qid": "9f4f278a-5465-4eb5-8bec-cf59b745730c", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 94, "novel_recall": 0.0, "qid": "7d832a86-f801-42e8-963e-cf2cae0e6102", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 36, "novel_recall": 0.0, "qid": "06077f6d-d5bf-4747-ac99-4c2538b721fe", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 64, "novel_recall": 0.0, "qid": "5d6055c1-615a-47d8-83ab-b3782b6892a2", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 34, "novel_recall": 0.0294, "qid": "19484533-070f-4f49-86b4-cce0caf18b32", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 33, "novel_recall": 0.0303, "qid": "330720af-db5b-4f23-b5b4-6605f0530744", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 60, "novel_recall": 0.1333, "qid": "c5e51ca3-edf8-4e91-b218-033a617c116c", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 82, "novel_recall": 0.0732, "qid": "3d5e7763-6377-4f1d-852d-15c4c67c4887", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 55, "novel_recall": 0.1636, "qid": "26328e2a-4935-42b2-8d2b-4db0ad9e71ae", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 85, "novel_recall": 0.0706, "qid": "867ba7a3-8151-4148-ae11-e2afba3d2910", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 98, "novel_recall": 0.051, "qid": "ccaa1adc-db74-443f-aedb-b5b4ad21900c", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 21, "novel_recall": 0.3333, "qid": "388fecf5-e3d2-4420-b559-967631d4f57f", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 27, "novel_recall": 0.4444, "qid": "9f4f278a-5465-4eb5-8bec-cf59b745730c", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 94, "novel_recall": 0.1915, "qid": "7d832a86-f801-42e8-963e-cf2cae0e6102", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 36, "novel_recall": 0.5, "qid": "06077f6d-d5bf-4747-ac99-4c2538b721fe", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 64, "novel_recall": 0.1719, "qid": "5d6055c1-615a-47d8-83ab-b3782b6892a2", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 34, "novel_recall": 0.0294, "qid": "19484533-070f-4f49-86b4-cce0caf18b32", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 33, "novel_recall": 0.0303, "qid": "330720af-db5b-4f23-b5b4-6605f0530744", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 60, "novel_recall": 0.1333, "qid": "c5e51ca3-edf8-4e91-b218-033a617c116c", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 82, "novel_recall": 0.0732, "qid": "3d5e7763-6377-4f1d-852d-15c4c67c4887", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 55, "novel_recall": 0.1636, "qid": "26328e2a-4935-42b2-8d2b-4db0ad9e71ae", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 85, "novel_recall": 0.0706, "qid": "867ba7a3-8151-4148-ae11-e2afba3d2910", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 98, "novel_recall": 0.051, "qid": "ccaa1adc-db74-443f-aedb-b5b4ad21900c", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 21, "novel_recall": 0.3333, "qid": "388fecf5-e3d2-4420-b559-967631d4f57f", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 27, "novel_recall": 0.4444, "qid": "9f4f278a-5465-4eb5-8bec-cf59b745730c", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 94, "novel_recall": 0.1915, "qid": "7d832a86-f801-42e8-963e-cf2cae0e6102", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 36, "novel_recall": 0.5, "qid": "06077f6d-d5bf-4747-ac99-4c2538b721fe", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 64, "novel_recall": 0.1719, "qid": "5d6055c1-615a-47d8-83ab-b3782b6892a2", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 34, "novel_recall": 0.0294, "qid": "19484533-070f-4f49-86b4-cce0caf18b32", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 33, "novel_recall": 0.0303, "qid": "330720af-db5b-4f23-b5b4-6605f0530744", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 60, "novel_recall": 0.1333, "qid": "c5e51ca3-edf8-4e91-b218-033a617c116c", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 82, "novel_recall": 0.0732, "qid": "3d5e7763-6377-4f1d-852d-15c4c67c4887", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 55, "novel_recall": 0.1636, "qid": "26328e2a-4935-42b2-8d2b-4db0ad9e71ae", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 85, "novel_recall": 0.0706, "qid": "867ba7a3-8151-4148-ae11-e2afba3d2910", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 98, "novel_recall": 0.051, "qid": "ccaa1adc-db74-443f-aedb-b5b4ad21900c", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 21, "novel_recall": 0.381, "qid": "388fecf5-e3d2-4420-b559-967631d4f57f", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 27, "novel_recall": 0.4444, "qid": "9f4f278a-5465-4eb5-8bec-cf59b745730c", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 94, "novel_recall": 0.1915, "qid": "7d832a86-f801-42e8-963e-cf2cae0e6102", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 36, "novel_recall": 0.5, "qid": "06077f6d-d5bf-4747-ac99-4c2538b721fe", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 64, "novel_recall": 0.1719, "qid": "5d6055c1-615a-47d8-83ab-b3782b6892a2", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 34, "novel_recall": 0.0294, "qid": "19484533-070f-4f49-86b4-cce0caf18b32", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 33, "novel_recall": 0.0303, "qid": "330720af-db5b-4f23-b5b4-6605f0530744", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 60, "novel_recall": 0.1333, "qid": "c5e51ca3-edf8-4e91-b218-033a617c116c", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 82, "novel_recall": 0.0732, "qid": "3d5e7763-6377-4f1d-852d-15c4c67c4887", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 55, "novel_recall": 0.1636, "qid": "26328e2a-4935-42b2-8d2b-4db0ad9e71ae", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 85, "novel_recall": 0.0706, "qid": "867ba7a3-8151-4148-ae11-e2afba3d2910", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 98, "novel_recall": 0.051, "qid": "ccaa1adc-db74-443f-aedb-b5b4ad21900c", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 21, "novel_recall": 0.381, "qid": "388fecf5-e3d2-4420-b559-967631d4f57f", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 27, "novel_recall": 0.4444, "qid": "9f4f278a-5465-4eb5-8bec-cf59b745730c", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 94, "novel_recall": 0.1915, "qid": "7d832a86-f801-42e8-963e-cf2cae0e6102", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 36, "novel_recall": 0.5, "qid": "06077f6d-d5bf-4747-ac99-4c2538b721fe", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 64, "novel_recall": 0.1719, "qid": "5d6055c1-615a-47d8-83ab-b3782b6892a2", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 34, "novel_recall": 0.0294, "qid": "19484533-070f-4f49-86b4-cce0caf18b32", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 33, "novel_recall": 0.0303, "qid": "330720af-db5b-4f23-b5b4-6605f0530744", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 60, "novel_recall": 0.1333, "qid": "c5e51ca3-edf8-4e91-b218-033a617c116c", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 82, "novel_recall": 0.0732, "qid": "3d5e7763-6377-4f1d-852d-15c4c67c4887", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 55, "novel_recall": 0.1636, "qid": "26328e2a-4935-42b2-8d2b-4db0ad9e71ae", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 85, "novel_recall": 0.0824, "qid": "867ba7a3-8151-4148-ae11-e2afba3d2910", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 98, "novel_recall": 0.0612, "qid": "ccaa1adc-db74-443f-aedb-b5b4ad21900c", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 21, "novel_recall": 0.3333, "qid": "388fecf5-e3d2-4420-b559-967631d4f57f", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 27, "novel_recall": 0.4444, "qid": "9f4f278a-5465-4eb5-8bec-cf59b745730c", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 94, "novel_recall": 0.2021, "qid": "7d832a86-f801-42e8-963e-cf2cae0e6102", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 36, "novel_recall": 0.5556, "qid": "06077f6d-d5bf-4747-ac99-4c2538b721fe", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "99", "full_hit": false, "n_novel": 64, "novel_recall": 0.1875, "qid": "5d6055c1-615a-47d8-83ab-b3782b6892a2", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 35, "novel_recall": 0.0, "qid": "fdc6478a-b4ac-4c29-950f-4ead8095e1bd", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 83, "novel_recall": 0.0, "qid": "aca93ba1-fb75-4d71-b411-275fa8929f3a", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 36, "novel_recall": 0.0, "qid": "8a561604-a6c6-4c60-9361-c5496d4295af", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 93, "novel_recall": 0.0, "qid": "9a684383-29e4-4627-bf24-64689ca42218", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 92, "novel_recall": 0.0, "qid": "96bb0fb3-3e8b-4c7a-9860-1673b4cb8080", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 142, "novel_recall": 0.0, "qid": "36c361dd-7ce0-429e-9d29-03a12da6ab94", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 52, "novel_recall": 0.0, "qid": "7760e759-a385-4809-bc1e-89443c0d9a00", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 50, "novel_recall": 0.0, "qid": "29a5f81d-eca0-45f3-a965-321a362ae0ff", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 72, "novel_recall": 0.0, "qid": "9f63efed-c3a9-4471-9392-bc6368ca597f", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 117, "novel_recall": 0.0, "qid": "3d6dbd2b-d104-48c8-a8c8-a40bc73c56d5", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 40, "novel_recall": 0.0, "qid": "b90add19-9620-440e-8ff1-290316e9f501", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 56, "novel_recall": 0.0, "qid": "08725ff7-dae3-4def-9894-d11e03a34ffe", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 35, "novel_recall": 0.7714, "qid": "fdc6478a-b4ac-4c29-950f-4ead8095e1bd", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 83, "novel_recall": 0.3133, "qid": "aca93ba1-fb75-4d71-b411-275fa8929f3a", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 36, "novel_recall": 0.25, "qid": "8a561604-a6c6-4c60-9361-c5496d4295af", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 93, "novel_recall": 0.2473, "qid": "9a684383-29e4-4627-bf24-64689ca42218", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 92, "novel_recall": 0.2174, "qid": "96bb0fb3-3e8b-4c7a-9860-1673b4cb8080", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 142, "novel_recall": 0.1901, "qid": "36c361dd-7ce0-429e-9d29-03a12da6ab94", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 52, "novel_recall": 0.1923, "qid": "7760e759-a385-4809-bc1e-89443c0d9a00", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 50, "novel_recall": 0.4, "qid": "29a5f81d-eca0-45f3-a965-321a362ae0ff", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 72, "novel_recall": 0.3194, "qid": "9f63efed-c3a9-4471-9392-bc6368ca597f", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 117, "novel_recall": 0.1453, "qid": "3d6dbd2b-d104-48c8-a8c8-a40bc73c56d5", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 40, "novel_recall": 0.625, "qid": "b90add19-9620-440e-8ff1-290316e9f501", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 56, "novel_recall": 0.4107, "qid": "08725ff7-dae3-4def-9894-d11e03a34ffe", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 35, "novel_recall": 0.7714, "qid": "fdc6478a-b4ac-4c29-950f-4ead8095e1bd", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 83, "novel_recall": 0.3133, "qid": "aca93ba1-fb75-4d71-b411-275fa8929f3a", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 36, "novel_recall": 0.25, "qid": "8a561604-a6c6-4c60-9361-c5496d4295af", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 93, "novel_recall": 0.2473, "qid": "9a684383-29e4-4627-bf24-64689ca42218", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 92, "novel_recall": 0.2174, "qid": "96bb0fb3-3e8b-4c7a-9860-1673b4cb8080", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 142, "novel_recall": 0.1901, "qid": "36c361dd-7ce0-429e-9d29-03a12da6ab94", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 52, "novel_recall": 0.1923, "qid": "7760e759-a385-4809-bc1e-89443c0d9a00", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 50, "novel_recall": 0.4, "qid": "29a5f81d-eca0-45f3-a965-321a362ae0ff", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 72, "novel_recall": 0.3194, "qid": "9f63efed-c3a9-4471-9392-bc6368ca597f", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 117, "novel_recall": 0.1453, "qid": "3d6dbd2b-d104-48c8-a8c8-a40bc73c56d5", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 40, "novel_recall": 0.6, "qid": "b90add19-9620-440e-8ff1-290316e9f501", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 56, "novel_recall": 0.4107, "qid": "08725ff7-dae3-4def-9894-d11e03a34ffe", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 35, "novel_recall": 0.7143, "qid": "fdc6478a-b4ac-4c29-950f-4ead8095e1bd", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 83, "novel_recall": 0.3133, "qid": "aca93ba1-fb75-4d71-b411-275fa8929f3a", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 36, "novel_recall": 0.2222, "qid": "8a561604-a6c6-4c60-9361-c5496d4295af", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 93, "novel_recall": 0.2473, "qid": "9a684383-29e4-4627-bf24-64689ca42218", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 92, "novel_recall": 0.2174, "qid": "96bb0fb3-3e8b-4c7a-9860-1673b4cb8080", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 142, "novel_recall": 0.1831, "qid": "36c361dd-7ce0-429e-9d29-03a12da6ab94", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 52, "novel_recall": 0.1731, "qid": "7760e759-a385-4809-bc1e-89443c0d9a00", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 50, "novel_recall": 0.18, "qid": "29a5f81d-eca0-45f3-a965-321a362ae0ff", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 72, "novel_recall": 0.3194, "qid": "9f63efed-c3a9-4471-9392-bc6368ca597f", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 117, "novel_recall": 0.1282, "qid": "3d6dbd2b-d104-48c8-a8c8-a40bc73c56d5", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 40, "novel_recall": 0.5, "qid": "b90add19-9620-440e-8ff1-290316e9f501", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 56, "novel_recall": 0.375, "qid": "08725ff7-dae3-4def-9894-d11e03a34ffe", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 35, "novel_recall": 0.7143, "qid": "fdc6478a-b4ac-4c29-950f-4ead8095e1bd", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 83, "novel_recall": 0.3133, "qid": "aca93ba1-fb75-4d71-b411-275fa8929f3a", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 36, "novel_recall": 0.2222, "qid": "8a561604-a6c6-4c60-9361-c5496d4295af", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 93, "novel_recall": 0.2473, "qid": "9a684383-29e4-4627-bf24-64689ca42218", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 92, "novel_recall": 0.2174, "qid": "96bb0fb3-3e8b-4c7a-9860-1673b4cb8080", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 142, "novel_recall": 0.1831, "qid": "36c361dd-7ce0-429e-9d29-03a12da6ab94", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 52, "novel_recall": 0.1731, "qid": "7760e759-a385-4809-bc1e-89443c0d9a00", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 50, "novel_recall": 0.18, "qid": "29a5f81d-eca0-45f3-a965-321a362ae0ff", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 72, "novel_recall": 0.3194, "qid": "9f63efed-c3a9-4471-9392-bc6368ca597f", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 117, "novel_recall": 0.1282, "qid": "3d6dbd2b-d104-48c8-a8c8-a40bc73c56d5", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 40, "novel_recall": 0.5, "qid": "b90add19-9620-440e-8ff1-290316e9f501", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 56, "novel_recall": 0.375, "qid": "08725ff7-dae3-4def-9894-d11e03a34ffe", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 35, "novel_recall": 0.7143, "qid": "fdc6478a-b4ac-4c29-950f-4ead8095e1bd", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 83, "novel_recall": 0.2892, "qid": "aca93ba1-fb75-4d71-b411-275fa8929f3a", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 36, "novel_recall": 0.1389, "qid": "8a561604-a6c6-4c60-9361-c5496d4295af", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 93, "novel_recall": 0.172, "qid": "9a684383-29e4-4627-bf24-64689ca42218", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 92, "novel_recall": 0.1848, "qid": "96bb0fb3-3e8b-4c7a-9860-1673b4cb8080", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 142, "novel_recall": 0.162, "qid": "36c361dd-7ce0-429e-9d29-03a12da6ab94", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 52, "novel_recall": 0.1923, "qid": "7760e759-a385-4809-bc1e-89443c0d9a00", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 50, "novel_recall": 0.38, "qid": "29a5f81d-eca0-45f3-a965-321a362ae0ff", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 72, "novel_recall": 0.25, "qid": "9f63efed-c3a9-4471-9392-bc6368ca597f", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 117, "novel_recall": 0.1368, "qid": "3d6dbd2b-d104-48c8-a8c8-a40bc73c56d5", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 40, "novel_recall": 0.6, "qid": "b90add19-9620-440e-8ff1-290316e9f501", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "104", "full_hit": false, "n_novel": 56, "novel_recall": 0.4107, "qid": "08725ff7-dae3-4def-9894-d11e03a34ffe", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 56, "novel_recall": 0.0, "qid": "041d6294-bda2-4e34-85d7-b7259a97b8e7", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 61, "novel_recall": 0.0, "qid": "0e9bc26a-a5e7-4cd2-9bcd-ecf136955380", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 73, "novel_recall": 0.0, "qid": "a33d339b-c6c5-41df-92bd-c1f32ce1a8cd", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 91, "novel_recall": 0.0, "qid": "8965138b-eb3e-41bf-bab8-81afe84e7360", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 29, "novel_recall": 0.0, "qid": "edcb4447-20ad-4110-aeba-dd0e04cb95da", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 107, "novel_recall": 0.0, "qid": "5df572ff-511d-4c56-9d39-7e331ee3b80d", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 103, "novel_recall": 0.0, "qid": "151ea22f-f933-42d7-8647-a69bb189ceca", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 76, "novel_recall": 0.0, "qid": "b1a905bc-cebc-46ec-bed6-3929051cc32c", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "fa3becc3-5742-45da-b90e-956a0c6d8018", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 63, "novel_recall": 0.0, "qid": "f783f0e3-6436-4bbc-81e0-ddd69eb58e5d", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 29, "novel_recall": 0.0, "qid": "048a7d1e-e262-40a3-8e04-0a896e2de9a3", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 43, "novel_recall": 0.0, "qid": "14479ad5-3740-426f-88cf-62a503a6cf00", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 56, "novel_recall": 0.5357, "qid": "041d6294-bda2-4e34-85d7-b7259a97b8e7", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 61, "novel_recall": 0.3443, "qid": "0e9bc26a-a5e7-4cd2-9bcd-ecf136955380", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 73, "novel_recall": 0.3151, "qid": "a33d339b-c6c5-41df-92bd-c1f32ce1a8cd", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 91, "novel_recall": 0.2637, "qid": "8965138b-eb3e-41bf-bab8-81afe84e7360", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 29, "novel_recall": 0.7931, "qid": "edcb4447-20ad-4110-aeba-dd0e04cb95da", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 107, "novel_recall": 0.243, "qid": "5df572ff-511d-4c56-9d39-7e331ee3b80d", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 103, "novel_recall": 0.1456, "qid": "151ea22f-f933-42d7-8647-a69bb189ceca", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 76, "novel_recall": 0.1579, "qid": "b1a905bc-cebc-46ec-bed6-3929051cc32c", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 24, "novel_recall": 0.2917, "qid": "fa3becc3-5742-45da-b90e-956a0c6d8018", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 63, "novel_recall": 0.2857, "qid": "f783f0e3-6436-4bbc-81e0-ddd69eb58e5d", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 29, "novel_recall": 0.5862, "qid": "048a7d1e-e262-40a3-8e04-0a896e2de9a3", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 43, "novel_recall": 0.4651, "qid": "14479ad5-3740-426f-88cf-62a503a6cf00", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 56, "novel_recall": 0.5357, "qid": "041d6294-bda2-4e34-85d7-b7259a97b8e7", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 61, "novel_recall": 0.3443, "qid": "0e9bc26a-a5e7-4cd2-9bcd-ecf136955380", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 73, "novel_recall": 0.3151, "qid": "a33d339b-c6c5-41df-92bd-c1f32ce1a8cd", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 91, "novel_recall": 0.2637, "qid": "8965138b-eb3e-41bf-bab8-81afe84e7360", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 29, "novel_recall": 0.7931, "qid": "edcb4447-20ad-4110-aeba-dd0e04cb95da", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 107, "novel_recall": 0.243, "qid": "5df572ff-511d-4c56-9d39-7e331ee3b80d", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 103, "novel_recall": 0.1456, "qid": "151ea22f-f933-42d7-8647-a69bb189ceca", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 76, "novel_recall": 0.1579, "qid": "b1a905bc-cebc-46ec-bed6-3929051cc32c", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 24, "novel_recall": 0.2917, "qid": "fa3becc3-5742-45da-b90e-956a0c6d8018", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 63, "novel_recall": 0.2857, "qid": "f783f0e3-6436-4bbc-81e0-ddd69eb58e5d", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 29, "novel_recall": 0.5862, "qid": "048a7d1e-e262-40a3-8e04-0a896e2de9a3", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 43, "novel_recall": 0.4651, "qid": "14479ad5-3740-426f-88cf-62a503a6cf00", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 56, "novel_recall": 0.5357, "qid": "041d6294-bda2-4e34-85d7-b7259a97b8e7", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 61, "novel_recall": 0.3443, "qid": "0e9bc26a-a5e7-4cd2-9bcd-ecf136955380", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 73, "novel_recall": 0.3151, "qid": "a33d339b-c6c5-41df-92bd-c1f32ce1a8cd", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 91, "novel_recall": 0.2637, "qid": "8965138b-eb3e-41bf-bab8-81afe84e7360", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 29, "novel_recall": 0.7586, "qid": "edcb4447-20ad-4110-aeba-dd0e04cb95da", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 107, "novel_recall": 0.2336, "qid": "5df572ff-511d-4c56-9d39-7e331ee3b80d", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 103, "novel_recall": 0.1359, "qid": "151ea22f-f933-42d7-8647-a69bb189ceca", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 76, "novel_recall": 0.1579, "qid": "b1a905bc-cebc-46ec-bed6-3929051cc32c", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 24, "novel_recall": 0.2917, "qid": "fa3becc3-5742-45da-b90e-956a0c6d8018", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 63, "novel_recall": 0.2857, "qid": "f783f0e3-6436-4bbc-81e0-ddd69eb58e5d", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 29, "novel_recall": 0.5862, "qid": "048a7d1e-e262-40a3-8e04-0a896e2de9a3", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 43, "novel_recall": 0.4651, "qid": "14479ad5-3740-426f-88cf-62a503a6cf00", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 56, "novel_recall": 0.5357, "qid": "041d6294-bda2-4e34-85d7-b7259a97b8e7", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 61, "novel_recall": 0.3443, "qid": "0e9bc26a-a5e7-4cd2-9bcd-ecf136955380", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 73, "novel_recall": 0.3151, "qid": "a33d339b-c6c5-41df-92bd-c1f32ce1a8cd", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 91, "novel_recall": 0.2637, "qid": "8965138b-eb3e-41bf-bab8-81afe84e7360", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 29, "novel_recall": 0.7586, "qid": "edcb4447-20ad-4110-aeba-dd0e04cb95da", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 107, "novel_recall": 0.2336, "qid": "5df572ff-511d-4c56-9d39-7e331ee3b80d", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 103, "novel_recall": 0.1359, "qid": "151ea22f-f933-42d7-8647-a69bb189ceca", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 76, "novel_recall": 0.1579, "qid": "b1a905bc-cebc-46ec-bed6-3929051cc32c", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 24, "novel_recall": 0.2917, "qid": "fa3becc3-5742-45da-b90e-956a0c6d8018", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 63, "novel_recall": 0.2857, "qid": "f783f0e3-6436-4bbc-81e0-ddd69eb58e5d", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 29, "novel_recall": 0.5862, "qid": "048a7d1e-e262-40a3-8e04-0a896e2de9a3", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 43, "novel_recall": 0.4651, "qid": "14479ad5-3740-426f-88cf-62a503a6cf00", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 56, "novel_recall": 0.5357, "qid": "041d6294-bda2-4e34-85d7-b7259a97b8e7", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 61, "novel_recall": 0.3443, "qid": "0e9bc26a-a5e7-4cd2-9bcd-ecf136955380", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 73, "novel_recall": 0.3151, "qid": "a33d339b-c6c5-41df-92bd-c1f32ce1a8cd", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 91, "novel_recall": 0.2637, "qid": "8965138b-eb3e-41bf-bab8-81afe84e7360", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 29, "novel_recall": 0.7931, "qid": "edcb4447-20ad-4110-aeba-dd0e04cb95da", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 107, "novel_recall": 0.243, "qid": "5df572ff-511d-4c56-9d39-7e331ee3b80d", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 103, "novel_recall": 0.1456, "qid": "151ea22f-f933-42d7-8647-a69bb189ceca", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 76, "novel_recall": 0.1579, "qid": "b1a905bc-cebc-46ec-bed6-3929051cc32c", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 24, "novel_recall": 0.2917, "qid": "fa3becc3-5742-45da-b90e-956a0c6d8018", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 63, "novel_recall": 0.3492, "qid": "f783f0e3-6436-4bbc-81e0-ddd69eb58e5d", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 29, "novel_recall": 0.5862, "qid": "048a7d1e-e262-40a3-8e04-0a896e2de9a3", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "107", "full_hit": false, "n_novel": 43, "novel_recall": 0.4884, "qid": "14479ad5-3740-426f-88cf-62a503a6cf00", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "f37ef8ed-e1aa-43c0-9ca8-2d07ecd505a4", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 35, "novel_recall": 0.0, "qid": "d9f28e4c-bb18-42f1-a347-dc6956e6b74a", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 80, "novel_recall": 0.0, "qid": "2e9e007a-f7dc-456a-b82f-573c9412c908", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 83, "novel_recall": 0.0, "qid": "3abaf57b-3506-44f8-b88c-f0133f21982f", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 57, "novel_recall": 0.0, "qid": "3f6aeaa8-1c34-4338-abdd-9c731074cd29", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 93, "novel_recall": 0.0, "qid": "343fbae1-9bf4-4e42-b2be-f2821b9707f8", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 69, "novel_recall": 0.0, "qid": "b9f44057-f335-41bf-87cb-5efb56ef229a", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 37, "novel_recall": 0.0, "qid": "128344e8-e85c-4724-a094-3532367bb012", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 41, "novel_recall": 0.0, "qid": "c2f47a0b-0645-46b5-a7d2-d3e03cf9b8f7", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 53, "novel_recall": 0.0, "qid": "0f8ecd29-601b-4659-8db5-8ddea964cc54", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 43, "novel_recall": 0.0, "qid": "5e004275-83a5-4128-a9f3-2e3ea04aa4d7", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 68, "novel_recall": 0.0, "qid": "7c638e43-bd3b-46ec-a2fa-1aa5b7962b4a", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 16, "novel_recall": 0.8125, "qid": "f37ef8ed-e1aa-43c0-9ca8-2d07ecd505a4", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 35, "novel_recall": 0.3143, "qid": "d9f28e4c-bb18-42f1-a347-dc6956e6b74a", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 80, "novel_recall": 0.2, "qid": "2e9e007a-f7dc-456a-b82f-573c9412c908", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 83, "novel_recall": 0.2048, "qid": "3abaf57b-3506-44f8-b88c-f0133f21982f", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 57, "novel_recall": 0.2982, "qid": "3f6aeaa8-1c34-4338-abdd-9c731074cd29", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 93, "novel_recall": 0.2151, "qid": "343fbae1-9bf4-4e42-b2be-f2821b9707f8", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 69, "novel_recall": 0.2029, "qid": "b9f44057-f335-41bf-87cb-5efb56ef229a", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 37, "novel_recall": 0.3243, "qid": "128344e8-e85c-4724-a094-3532367bb012", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 41, "novel_recall": 0.122, "qid": "c2f47a0b-0645-46b5-a7d2-d3e03cf9b8f7", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 53, "novel_recall": 0.283, "qid": "0f8ecd29-601b-4659-8db5-8ddea964cc54", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 43, "novel_recall": 0.5349, "qid": "5e004275-83a5-4128-a9f3-2e3ea04aa4d7", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 68, "novel_recall": 0.3676, "qid": "7c638e43-bd3b-46ec-a2fa-1aa5b7962b4a", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 16, "novel_recall": 0.8125, "qid": "f37ef8ed-e1aa-43c0-9ca8-2d07ecd505a4", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 35, "novel_recall": 0.3143, "qid": "d9f28e4c-bb18-42f1-a347-dc6956e6b74a", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 80, "novel_recall": 0.2, "qid": "2e9e007a-f7dc-456a-b82f-573c9412c908", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 83, "novel_recall": 0.2048, "qid": "3abaf57b-3506-44f8-b88c-f0133f21982f", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 57, "novel_recall": 0.2982, "qid": "3f6aeaa8-1c34-4338-abdd-9c731074cd29", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 93, "novel_recall": 0.2151, "qid": "343fbae1-9bf4-4e42-b2be-f2821b9707f8", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 69, "novel_recall": 0.2029, "qid": "b9f44057-f335-41bf-87cb-5efb56ef229a", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 37, "novel_recall": 0.3243, "qid": "128344e8-e85c-4724-a094-3532367bb012", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 41, "novel_recall": 0.122, "qid": "c2f47a0b-0645-46b5-a7d2-d3e03cf9b8f7", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 53, "novel_recall": 0.283, "qid": "0f8ecd29-601b-4659-8db5-8ddea964cc54", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 43, "novel_recall": 0.5581, "qid": "5e004275-83a5-4128-a9f3-2e3ea04aa4d7", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 68, "novel_recall": 0.3676, "qid": "7c638e43-bd3b-46ec-a2fa-1aa5b7962b4a", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 16, "novel_recall": 0.8125, "qid": "f37ef8ed-e1aa-43c0-9ca8-2d07ecd505a4", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 35, "novel_recall": 0.2571, "qid": "d9f28e4c-bb18-42f1-a347-dc6956e6b74a", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 80, "novel_recall": 0.1375, "qid": "2e9e007a-f7dc-456a-b82f-573c9412c908", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 83, "novel_recall": 0.1807, "qid": "3abaf57b-3506-44f8-b88c-f0133f21982f", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 57, "novel_recall": 0.2632, "qid": "3f6aeaa8-1c34-4338-abdd-9c731074cd29", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 93, "novel_recall": 0.1935, "qid": "343fbae1-9bf4-4e42-b2be-f2821b9707f8", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 69, "novel_recall": 0.1739, "qid": "b9f44057-f335-41bf-87cb-5efb56ef229a", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 37, "novel_recall": 0.3243, "qid": "128344e8-e85c-4724-a094-3532367bb012", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 41, "novel_recall": 0.122, "qid": "c2f47a0b-0645-46b5-a7d2-d3e03cf9b8f7", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 53, "novel_recall": 0.2453, "qid": "0f8ecd29-601b-4659-8db5-8ddea964cc54", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 43, "novel_recall": 0.5349, "qid": "5e004275-83a5-4128-a9f3-2e3ea04aa4d7", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 68, "novel_recall": 0.2941, "qid": "7c638e43-bd3b-46ec-a2fa-1aa5b7962b4a", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 16, "novel_recall": 0.8125, "qid": "f37ef8ed-e1aa-43c0-9ca8-2d07ecd505a4", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 35, "novel_recall": 0.2571, "qid": "d9f28e4c-bb18-42f1-a347-dc6956e6b74a", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 80, "novel_recall": 0.1375, "qid": "2e9e007a-f7dc-456a-b82f-573c9412c908", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 83, "novel_recall": 0.1807, "qid": "3abaf57b-3506-44f8-b88c-f0133f21982f", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 57, "novel_recall": 0.2632, "qid": "3f6aeaa8-1c34-4338-abdd-9c731074cd29", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 93, "novel_recall": 0.1935, "qid": "343fbae1-9bf4-4e42-b2be-f2821b9707f8", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 69, "novel_recall": 0.1739, "qid": "b9f44057-f335-41bf-87cb-5efb56ef229a", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 37, "novel_recall": 0.3243, "qid": "128344e8-e85c-4724-a094-3532367bb012", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 41, "novel_recall": 0.122, "qid": "c2f47a0b-0645-46b5-a7d2-d3e03cf9b8f7", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 53, "novel_recall": 0.2453, "qid": "0f8ecd29-601b-4659-8db5-8ddea964cc54", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 43, "novel_recall": 0.5349, "qid": "5e004275-83a5-4128-a9f3-2e3ea04aa4d7", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 68, "novel_recall": 0.2941, "qid": "7c638e43-bd3b-46ec-a2fa-1aa5b7962b4a", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 16, "novel_recall": 0.8125, "qid": "f37ef8ed-e1aa-43c0-9ca8-2d07ecd505a4", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 35, "novel_recall": 0.3143, "qid": "d9f28e4c-bb18-42f1-a347-dc6956e6b74a", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 80, "novel_recall": 0.2875, "qid": "2e9e007a-f7dc-456a-b82f-573c9412c908", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 83, "novel_recall": 0.253, "qid": "3abaf57b-3506-44f8-b88c-f0133f21982f", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 57, "novel_recall": 0.2982, "qid": "3f6aeaa8-1c34-4338-abdd-9c731074cd29", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 93, "novel_recall": 0.2043, "qid": "343fbae1-9bf4-4e42-b2be-f2821b9707f8", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 69, "novel_recall": 0.2319, "qid": "b9f44057-f335-41bf-87cb-5efb56ef229a", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 37, "novel_recall": 0.3514, "qid": "128344e8-e85c-4724-a094-3532367bb012", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 41, "novel_recall": 0.2927, "qid": "c2f47a0b-0645-46b5-a7d2-d3e03cf9b8f7", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 53, "novel_recall": 0.283, "qid": "0f8ecd29-601b-4659-8db5-8ddea964cc54", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 43, "novel_recall": 0.7674, "qid": "5e004275-83a5-4128-a9f3-2e3ea04aa4d7", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "111", "full_hit": false, "n_novel": 68, "novel_recall": 0.3971, "qid": "7c638e43-bd3b-46ec-a2fa-1aa5b7962b4a", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 14, "novel_recall": 0.0, "qid": "46e6b92b-191d-4eec-bbde-a7e81c914cf8", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 22, "novel_recall": 0.0, "qid": "4ccd31da-d4d5-4a9b-a6b3-2b31c0cc330f", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 66, "novel_recall": 0.0, "qid": "59b20f0d-1ffb-4292-aebf-b3f58ecb5d23", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 107, "novel_recall": 0.0, "qid": "b2911515-3a13-4c1e-8425-bee76e0e6e0c", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 93, "novel_recall": 0.0, "qid": "8583374e-188b-4367-b71d-012233babc67", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 105, "novel_recall": 0.0, "qid": "2c81ecc5-8b05-424e-b630-f1fe8df21050", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 80, "novel_recall": 0.0, "qid": "8f3dd074-94bd-4c41-99c5-1272a9efce9f", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 36, "novel_recall": 0.0, "qid": "43fbe771-a65a-45a6-9a72-1a0e480b42dd", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 79, "novel_recall": 0.0, "qid": "b524eef1-e28e-480a-ba93-a1469d79da23", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 90, "novel_recall": 0.0, "qid": "a7aa4c28-4c00-4abc-9623-3fb8d9024fa0", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 41, "novel_recall": 0.0, "qid": "699afd61-5b10-4300-b44b-070e7d978e03", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 66, "novel_recall": 0.0, "qid": "b6b240a2-1457-4a86-8ec6-3e72eaf13b6b", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "119", "full_hit": true, "n_novel": 14, "novel_recall": 1.0, "qid": "46e6b92b-191d-4eec-bbde-a7e81c914cf8", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 22, "novel_recall": 0.5909, "qid": "4ccd31da-d4d5-4a9b-a6b3-2b31c0cc330f", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 66, "novel_recall": 0.3788, "qid": "59b20f0d-1ffb-4292-aebf-b3f58ecb5d23", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 107, "novel_recall": 0.1963, "qid": "b2911515-3a13-4c1e-8425-bee76e0e6e0c", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 93, "novel_recall": 0.1935, "qid": "8583374e-188b-4367-b71d-012233babc67", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 105, "novel_recall": 0.181, "qid": "2c81ecc5-8b05-424e-b630-f1fe8df21050", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 80, "novel_recall": 0.2, "qid": "8f3dd074-94bd-4c41-99c5-1272a9efce9f", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 36, "novel_recall": 0.2222, "qid": "43fbe771-a65a-45a6-9a72-1a0e480b42dd", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 79, "novel_recall": 0.2785, "qid": "b524eef1-e28e-480a-ba93-a1469d79da23", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 90, "novel_recall": 0.2222, "qid": "a7aa4c28-4c00-4abc-9623-3fb8d9024fa0", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 41, "novel_recall": 0.5366, "qid": "699afd61-5b10-4300-b44b-070e7d978e03", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 66, "novel_recall": 0.3788, "qid": "b6b240a2-1457-4a86-8ec6-3e72eaf13b6b", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "119", "full_hit": true, "n_novel": 14, "novel_recall": 1.0, "qid": "46e6b92b-191d-4eec-bbde-a7e81c914cf8", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 22, "novel_recall": 0.5909, "qid": "4ccd31da-d4d5-4a9b-a6b3-2b31c0cc330f", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 66, "novel_recall": 0.3788, "qid": "59b20f0d-1ffb-4292-aebf-b3f58ecb5d23", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 107, "novel_recall": 0.1963, "qid": "b2911515-3a13-4c1e-8425-bee76e0e6e0c", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 93, "novel_recall": 0.1935, "qid": "8583374e-188b-4367-b71d-012233babc67", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 105, "novel_recall": 0.181, "qid": "2c81ecc5-8b05-424e-b630-f1fe8df21050", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 80, "novel_recall": 0.2, "qid": "8f3dd074-94bd-4c41-99c5-1272a9efce9f", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 36, "novel_recall": 0.2222, "qid": "43fbe771-a65a-45a6-9a72-1a0e480b42dd", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 79, "novel_recall": 0.2785, "qid": "b524eef1-e28e-480a-ba93-a1469d79da23", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 90, "novel_recall": 0.2667, "qid": "a7aa4c28-4c00-4abc-9623-3fb8d9024fa0", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 41, "novel_recall": 0.5366, "qid": "699afd61-5b10-4300-b44b-070e7d978e03", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 66, "novel_recall": 0.3788, "qid": "b6b240a2-1457-4a86-8ec6-3e72eaf13b6b", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "119", "full_hit": true, "n_novel": 14, "novel_recall": 1.0, "qid": "46e6b92b-191d-4eec-bbde-a7e81c914cf8", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 22, "novel_recall": 0.5909, "qid": "4ccd31da-d4d5-4a9b-a6b3-2b31c0cc330f", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 66, "novel_recall": 0.3485, "qid": "59b20f0d-1ffb-4292-aebf-b3f58ecb5d23", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 107, "novel_recall": 0.1589, "qid": "b2911515-3a13-4c1e-8425-bee76e0e6e0c", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 93, "novel_recall": 0.172, "qid": "8583374e-188b-4367-b71d-012233babc67", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 105, "novel_recall": 0.1619, "qid": "2c81ecc5-8b05-424e-b630-f1fe8df21050", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 80, "novel_recall": 0.2, "qid": "8f3dd074-94bd-4c41-99c5-1272a9efce9f", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 36, "novel_recall": 0.1389, "qid": "43fbe771-a65a-45a6-9a72-1a0e480b42dd", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 79, "novel_recall": 0.2278, "qid": "b524eef1-e28e-480a-ba93-a1469d79da23", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 90, "novel_recall": 0.2667, "qid": "a7aa4c28-4c00-4abc-9623-3fb8d9024fa0", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 41, "novel_recall": 0.439, "qid": "699afd61-5b10-4300-b44b-070e7d978e03", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 66, "novel_recall": 0.3182, "qid": "b6b240a2-1457-4a86-8ec6-3e72eaf13b6b", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "119", "full_hit": true, "n_novel": 14, "novel_recall": 1.0, "qid": "46e6b92b-191d-4eec-bbde-a7e81c914cf8", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 22, "novel_recall": 0.5909, "qid": "4ccd31da-d4d5-4a9b-a6b3-2b31c0cc330f", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 66, "novel_recall": 0.3485, "qid": "59b20f0d-1ffb-4292-aebf-b3f58ecb5d23", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 107, "novel_recall": 0.1589, "qid": "b2911515-3a13-4c1e-8425-bee76e0e6e0c", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 93, "novel_recall": 0.172, "qid": "8583374e-188b-4367-b71d-012233babc67", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 105, "novel_recall": 0.1619, "qid": "2c81ecc5-8b05-424e-b630-f1fe8df21050", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 80, "novel_recall": 0.2, "qid": "8f3dd074-94bd-4c41-99c5-1272a9efce9f", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 36, "novel_recall": 0.1389, "qid": "43fbe771-a65a-45a6-9a72-1a0e480b42dd", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 79, "novel_recall": 0.2278, "qid": "b524eef1-e28e-480a-ba93-a1469d79da23", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 90, "novel_recall": 0.2667, "qid": "a7aa4c28-4c00-4abc-9623-3fb8d9024fa0", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 41, "novel_recall": 0.439, "qid": "699afd61-5b10-4300-b44b-070e7d978e03", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 66, "novel_recall": 0.3182, "qid": "b6b240a2-1457-4a86-8ec6-3e72eaf13b6b", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 14, "novel_recall": 0.6429, "qid": "46e6b92b-191d-4eec-bbde-a7e81c914cf8", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 22, "novel_recall": 0.5, "qid": "4ccd31da-d4d5-4a9b-a6b3-2b31c0cc330f", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 66, "novel_recall": 0.2121, "qid": "59b20f0d-1ffb-4292-aebf-b3f58ecb5d23", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 107, "novel_recall": 0.1402, "qid": "b2911515-3a13-4c1e-8425-bee76e0e6e0c", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 93, "novel_recall": 0.1505, "qid": "8583374e-188b-4367-b71d-012233babc67", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 105, "novel_recall": 0.1524, "qid": "2c81ecc5-8b05-424e-b630-f1fe8df21050", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 80, "novel_recall": 0.1375, "qid": "8f3dd074-94bd-4c41-99c5-1272a9efce9f", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 36, "novel_recall": 0.2778, "qid": "43fbe771-a65a-45a6-9a72-1a0e480b42dd", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 79, "novel_recall": 0.1646, "qid": "b524eef1-e28e-480a-ba93-a1469d79da23", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 90, "novel_recall": 0.2444, "qid": "a7aa4c28-4c00-4abc-9623-3fb8d9024fa0", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 41, "novel_recall": 0.3659, "qid": "699afd61-5b10-4300-b44b-070e7d978e03", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "119", "full_hit": false, "n_novel": 66, "novel_recall": 0.2727, "qid": "b6b240a2-1457-4a86-8ec6-3e72eaf13b6b", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 28, "novel_recall": 0.0, "qid": "87b76347-2590-46e9-b5a1-68cd4ed502d7", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 73, "novel_recall": 0.0, "qid": "3f137593-2c0e-4191-95e5-dab2c83557a9", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 94, "novel_recall": 0.0, "qid": "99a35d10-abc4-48ba-af7c-7c66580f2e01", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 62, "novel_recall": 0.0, "qid": "f14d8e93-22af-4d53-9a1c-0e603dca9030", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 121, "novel_recall": 0.0, "qid": "9e61394e-ddf7-468c-ae43-175afecc243c", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 85, "novel_recall": 0.0, "qid": "5e1bb310-3636-4b48-bb41-cb86229f2cd2", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 95, "novel_recall": 0.0, "qid": "d557fc95-1ebc-4a14-b64c-4ed4e4f56e24", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 52, "novel_recall": 0.0, "qid": "c2a3e0da-ab70-4491-94d7-654d7c25bbf6", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 59, "novel_recall": 0.0, "qid": "ab1876a3-7d29-410d-a8c5-e111a8ea21ac", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 76, "novel_recall": 0.0, "qid": "1949e7d3-9c50-4dee-9dcb-f7fce137c3b4", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 34, "novel_recall": 0.0, "qid": "3d338161-778b-41f1-a114-15a20d43b386", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 63, "novel_recall": 0.0, "qid": "6a173546-5d1f-4358-bd96-8b0f50f25ce6", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 28, "novel_recall": 0.5714, "qid": "87b76347-2590-46e9-b5a1-68cd4ed502d7", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 73, "novel_recall": 0.2877, "qid": "3f137593-2c0e-4191-95e5-dab2c83557a9", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 94, "novel_recall": 0.1596, "qid": "99a35d10-abc4-48ba-af7c-7c66580f2e01", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 62, "novel_recall": 0.4839, "qid": "f14d8e93-22af-4d53-9a1c-0e603dca9030", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 121, "novel_recall": 0.2397, "qid": "9e61394e-ddf7-468c-ae43-175afecc243c", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 85, "novel_recall": 0.2353, "qid": "5e1bb310-3636-4b48-bb41-cb86229f2cd2", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 95, "novel_recall": 0.2211, "qid": "d557fc95-1ebc-4a14-b64c-4ed4e4f56e24", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 52, "novel_recall": 0.2115, "qid": "c2a3e0da-ab70-4491-94d7-654d7c25bbf6", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 59, "novel_recall": 0.2203, "qid": "ab1876a3-7d29-410d-a8c5-e111a8ea21ac", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 76, "novel_recall": 0.1974, "qid": "1949e7d3-9c50-4dee-9dcb-f7fce137c3b4", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 34, "novel_recall": 0.6176, "qid": "3d338161-778b-41f1-a114-15a20d43b386", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 63, "novel_recall": 0.254, "qid": "6a173546-5d1f-4358-bd96-8b0f50f25ce6", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 28, "novel_recall": 0.5714, "qid": "87b76347-2590-46e9-b5a1-68cd4ed502d7", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 73, "novel_recall": 0.2877, "qid": "3f137593-2c0e-4191-95e5-dab2c83557a9", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 94, "novel_recall": 0.1596, "qid": "99a35d10-abc4-48ba-af7c-7c66580f2e01", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 62, "novel_recall": 0.4839, "qid": "f14d8e93-22af-4d53-9a1c-0e603dca9030", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 121, "novel_recall": 0.2397, "qid": "9e61394e-ddf7-468c-ae43-175afecc243c", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 85, "novel_recall": 0.2353, "qid": "5e1bb310-3636-4b48-bb41-cb86229f2cd2", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 95, "novel_recall": 0.2211, "qid": "d557fc95-1ebc-4a14-b64c-4ed4e4f56e24", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 52, "novel_recall": 0.2115, "qid": "c2a3e0da-ab70-4491-94d7-654d7c25bbf6", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 59, "novel_recall": 0.2203, "qid": "ab1876a3-7d29-410d-a8c5-e111a8ea21ac", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 76, "novel_recall": 0.1974, "qid": "1949e7d3-9c50-4dee-9dcb-f7fce137c3b4", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 34, "novel_recall": 0.6176, "qid": "3d338161-778b-41f1-a114-15a20d43b386", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 63, "novel_recall": 0.254, "qid": "6a173546-5d1f-4358-bd96-8b0f50f25ce6", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 28, "novel_recall": 0.5357, "qid": "87b76347-2590-46e9-b5a1-68cd4ed502d7", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 73, "novel_recall": 0.2877, "qid": "3f137593-2c0e-4191-95e5-dab2c83557a9", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 94, "novel_recall": 0.1596, "qid": "99a35d10-abc4-48ba-af7c-7c66580f2e01", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 62, "novel_recall": 0.4839, "qid": "f14d8e93-22af-4d53-9a1c-0e603dca9030", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 121, "novel_recall": 0.2231, "qid": "9e61394e-ddf7-468c-ae43-175afecc243c", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 85, "novel_recall": 0.2118, "qid": "5e1bb310-3636-4b48-bb41-cb86229f2cd2", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 95, "novel_recall": 0.1895, "qid": "d557fc95-1ebc-4a14-b64c-4ed4e4f56e24", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 52, "novel_recall": 0.1923, "qid": "c2a3e0da-ab70-4491-94d7-654d7c25bbf6", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 59, "novel_recall": 0.2203, "qid": "ab1876a3-7d29-410d-a8c5-e111a8ea21ac", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 76, "novel_recall": 0.1974, "qid": "1949e7d3-9c50-4dee-9dcb-f7fce137c3b4", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 34, "novel_recall": 0.5, "qid": "3d338161-778b-41f1-a114-15a20d43b386", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 63, "novel_recall": 0.254, "qid": "6a173546-5d1f-4358-bd96-8b0f50f25ce6", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 28, "novel_recall": 0.5357, "qid": "87b76347-2590-46e9-b5a1-68cd4ed502d7", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 73, "novel_recall": 0.2877, "qid": "3f137593-2c0e-4191-95e5-dab2c83557a9", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 94, "novel_recall": 0.1596, "qid": "99a35d10-abc4-48ba-af7c-7c66580f2e01", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 62, "novel_recall": 0.4839, "qid": "f14d8e93-22af-4d53-9a1c-0e603dca9030", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 121, "novel_recall": 0.2231, "qid": "9e61394e-ddf7-468c-ae43-175afecc243c", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 85, "novel_recall": 0.2118, "qid": "5e1bb310-3636-4b48-bb41-cb86229f2cd2", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 95, "novel_recall": 0.1895, "qid": "d557fc95-1ebc-4a14-b64c-4ed4e4f56e24", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 52, "novel_recall": 0.1923, "qid": "c2a3e0da-ab70-4491-94d7-654d7c25bbf6", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 59, "novel_recall": 0.2203, "qid": "ab1876a3-7d29-410d-a8c5-e111a8ea21ac", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 76, "novel_recall": 0.1974, "qid": "1949e7d3-9c50-4dee-9dcb-f7fce137c3b4", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 34, "novel_recall": 0.5, "qid": "3d338161-778b-41f1-a114-15a20d43b386", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 63, "novel_recall": 0.254, "qid": "6a173546-5d1f-4358-bd96-8b0f50f25ce6", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 28, "novel_recall": 0.5357, "qid": "87b76347-2590-46e9-b5a1-68cd4ed502d7", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 73, "novel_recall": 0.274, "qid": "3f137593-2c0e-4191-95e5-dab2c83557a9", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 94, "novel_recall": 0.1596, "qid": "99a35d10-abc4-48ba-af7c-7c66580f2e01", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 62, "novel_recall": 0.4194, "qid": "f14d8e93-22af-4d53-9a1c-0e603dca9030", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 121, "novel_recall": 0.2645, "qid": "9e61394e-ddf7-468c-ae43-175afecc243c", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 85, "novel_recall": 0.2353, "qid": "5e1bb310-3636-4b48-bb41-cb86229f2cd2", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 95, "novel_recall": 0.1789, "qid": "d557fc95-1ebc-4a14-b64c-4ed4e4f56e24", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 52, "novel_recall": 0.1731, "qid": "c2a3e0da-ab70-4491-94d7-654d7c25bbf6", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 59, "novel_recall": 0.1864, "qid": "ab1876a3-7d29-410d-a8c5-e111a8ea21ac", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 76, "novel_recall": 0.1184, "qid": "1949e7d3-9c50-4dee-9dcb-f7fce137c3b4", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 34, "novel_recall": 0.7059, "qid": "3d338161-778b-41f1-a114-15a20d43b386", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "122", "full_hit": false, "n_novel": 63, "novel_recall": 0.3492, "qid": "6a173546-5d1f-4358-bd96-8b0f50f25ce6", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 36, "novel_recall": 0.0, "qid": "f9900af8-e09b-4595-a724-ef7e1df0e290", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 19, "novel_recall": 0.0, "qid": "f9b2b367-924d-4700-a4d5-a66a62be73de", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 113, "novel_recall": 0.0, "qid": "35bc5e55-215a-4396-a5a7-659a51c46256", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "f624d12d-fa05-4002-a2ef-70339660702c", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 82, "novel_recall": 0.0, "qid": "d0c3055f-4065-4424-85a9-c85152b50560", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 77, "novel_recall": 0.0, "qid": "65134317-fc8b-493b-8e4b-0fb29bdd03e9", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 80, "novel_recall": 0.0, "qid": "f0f33dd1-62fe-4a01-8453-b5b2124846c0", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 67, "novel_recall": 0.0, "qid": "dfd9dbbf-aec0-4cce-a273-4113c275444a", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 33, "novel_recall": 0.0, "qid": "ec445f52-6df0-4e2b-9529-937bfaed932b", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 55, "novel_recall": 0.0, "qid": "c5b17b2d-ecb0-4156-8f62-1cf607a4e16c", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 30, "novel_recall": 0.0, "qid": "451f6528-1637-4287-bea0-675958fb2720", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 72, "novel_recall": 0.0, "qid": "17ac1fe3-1451-482d-850f-0089f5c509ce", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 36, "novel_recall": 0.5556, "qid": "f9900af8-e09b-4595-a724-ef7e1df0e290", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 19, "novel_recall": 0.8947, "qid": "f9b2b367-924d-4700-a4d5-a66a62be73de", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 113, "novel_recall": 0.3009, "qid": "35bc5e55-215a-4396-a5a7-659a51c46256", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "f624d12d-fa05-4002-a2ef-70339660702c", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 82, "novel_recall": 0.1341, "qid": "d0c3055f-4065-4424-85a9-c85152b50560", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 77, "novel_recall": 0.1429, "qid": "65134317-fc8b-493b-8e4b-0fb29bdd03e9", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 80, "novel_recall": 0.25, "qid": "f0f33dd1-62fe-4a01-8453-b5b2124846c0", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 67, "novel_recall": 0.3881, "qid": "dfd9dbbf-aec0-4cce-a273-4113c275444a", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 33, "novel_recall": 0.3333, "qid": "ec445f52-6df0-4e2b-9529-937bfaed932b", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 55, "novel_recall": 0.2545, "qid": "c5b17b2d-ecb0-4156-8f62-1cf607a4e16c", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 30, "novel_recall": 0.7, "qid": "451f6528-1637-4287-bea0-675958fb2720", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 72, "novel_recall": 0.3472, "qid": "17ac1fe3-1451-482d-850f-0089f5c509ce", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 36, "novel_recall": 0.5556, "qid": "f9900af8-e09b-4595-a724-ef7e1df0e290", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 19, "novel_recall": 0.8947, "qid": "f9b2b367-924d-4700-a4d5-a66a62be73de", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 113, "novel_recall": 0.3009, "qid": "35bc5e55-215a-4396-a5a7-659a51c46256", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "f624d12d-fa05-4002-a2ef-70339660702c", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 82, "novel_recall": 0.1341, "qid": "d0c3055f-4065-4424-85a9-c85152b50560", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 77, "novel_recall": 0.1429, "qid": "65134317-fc8b-493b-8e4b-0fb29bdd03e9", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 80, "novel_recall": 0.25, "qid": "f0f33dd1-62fe-4a01-8453-b5b2124846c0", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 67, "novel_recall": 0.3881, "qid": "dfd9dbbf-aec0-4cce-a273-4113c275444a", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 33, "novel_recall": 0.3333, "qid": "ec445f52-6df0-4e2b-9529-937bfaed932b", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 55, "novel_recall": 0.2545, "qid": "c5b17b2d-ecb0-4156-8f62-1cf607a4e16c", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 30, "novel_recall": 0.7, "qid": "451f6528-1637-4287-bea0-675958fb2720", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 72, "novel_recall": 0.3472, "qid": "17ac1fe3-1451-482d-850f-0089f5c509ce", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 36, "novel_recall": 0.5556, "qid": "f9900af8-e09b-4595-a724-ef7e1df0e290", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 19, "novel_recall": 0.8947, "qid": "f9b2b367-924d-4700-a4d5-a66a62be73de", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 113, "novel_recall": 0.2212, "qid": "35bc5e55-215a-4396-a5a7-659a51c46256", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "f624d12d-fa05-4002-a2ef-70339660702c", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 82, "novel_recall": 0.122, "qid": "d0c3055f-4065-4424-85a9-c85152b50560", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 77, "novel_recall": 0.1299, "qid": "65134317-fc8b-493b-8e4b-0fb29bdd03e9", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 80, "novel_recall": 0.2125, "qid": "f0f33dd1-62fe-4a01-8453-b5b2124846c0", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 67, "novel_recall": 0.3881, "qid": "dfd9dbbf-aec0-4cce-a273-4113c275444a", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 33, "novel_recall": 0.2727, "qid": "ec445f52-6df0-4e2b-9529-937bfaed932b", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 55, "novel_recall": 0.1455, "qid": "c5b17b2d-ecb0-4156-8f62-1cf607a4e16c", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 30, "novel_recall": 0.5667, "qid": "451f6528-1637-4287-bea0-675958fb2720", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 72, "novel_recall": 0.3472, "qid": "17ac1fe3-1451-482d-850f-0089f5c509ce", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 36, "novel_recall": 0.5556, "qid": "f9900af8-e09b-4595-a724-ef7e1df0e290", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 19, "novel_recall": 0.8947, "qid": "f9b2b367-924d-4700-a4d5-a66a62be73de", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 113, "novel_recall": 0.2212, "qid": "35bc5e55-215a-4396-a5a7-659a51c46256", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "f624d12d-fa05-4002-a2ef-70339660702c", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 82, "novel_recall": 0.122, "qid": "d0c3055f-4065-4424-85a9-c85152b50560", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 77, "novel_recall": 0.1299, "qid": "65134317-fc8b-493b-8e4b-0fb29bdd03e9", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 80, "novel_recall": 0.2125, "qid": "f0f33dd1-62fe-4a01-8453-b5b2124846c0", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 67, "novel_recall": 0.3881, "qid": "dfd9dbbf-aec0-4cce-a273-4113c275444a", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 33, "novel_recall": 0.2727, "qid": "ec445f52-6df0-4e2b-9529-937bfaed932b", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 55, "novel_recall": 0.1455, "qid": "c5b17b2d-ecb0-4156-8f62-1cf607a4e16c", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 30, "novel_recall": 0.5667, "qid": "451f6528-1637-4287-bea0-675958fb2720", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 72, "novel_recall": 0.3472, "qid": "17ac1fe3-1451-482d-850f-0089f5c509ce", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 36, "novel_recall": 0.5556, "qid": "f9900af8-e09b-4595-a724-ef7e1df0e290", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 19, "novel_recall": 0.8947, "qid": "f9b2b367-924d-4700-a4d5-a66a62be73de", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 113, "novel_recall": 0.3186, "qid": "35bc5e55-215a-4396-a5a7-659a51c46256", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "f624d12d-fa05-4002-a2ef-70339660702c", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 82, "novel_recall": 0.1341, "qid": "d0c3055f-4065-4424-85a9-c85152b50560", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 77, "novel_recall": 0.1429, "qid": "65134317-fc8b-493b-8e4b-0fb29bdd03e9", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 80, "novel_recall": 0.2375, "qid": "f0f33dd1-62fe-4a01-8453-b5b2124846c0", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 67, "novel_recall": 0.3731, "qid": "dfd9dbbf-aec0-4cce-a273-4113c275444a", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 33, "novel_recall": 0.3333, "qid": "ec445f52-6df0-4e2b-9529-937bfaed932b", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 55, "novel_recall": 0.2364, "qid": "c5b17b2d-ecb0-4156-8f62-1cf607a4e16c", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 30, "novel_recall": 0.7333, "qid": "451f6528-1637-4287-bea0-675958fb2720", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "125", "full_hit": false, "n_novel": 72, "novel_recall": 0.375, "qid": "17ac1fe3-1451-482d-850f-0089f5c509ce", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 30, "novel_recall": 0.0, "qid": "078634cf-e374-44e6-ac55-b23b7c7e3df2", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 119, "novel_recall": 0.0, "qid": "640556fc-2f3e-44d5-9caa-a3c282c55fe5", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 94, "novel_recall": 0.0, "qid": "439dede1-fc86-4098-b4ea-e5b9f7c82246", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 50, "novel_recall": 0.0, "qid": "a21e22ef-0659-4d09-b95d-afd396bb51ac", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 96, "novel_recall": 0.0, "qid": "3f4583b7-d480-47fc-84b1-13a67ac6a52b", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 92, "novel_recall": 0.0, "qid": "e730f309-db26-42a7-951b-6d550a41bc7d", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 104, "novel_recall": 0.0, "qid": "11ba3c77-3be1-4969-a533-5c6cdbcd87fc", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 58, "novel_recall": 0.0, "qid": "47349701-9fd5-4a57-9382-898a238c7973", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 48, "novel_recall": 0.0, "qid": "15164441-18b1-4b9e-8074-23f6cd3de102", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 51, "novel_recall": 0.0, "qid": "70634a33-db60-47d4-baab-a71c5323bcdb", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 36, "novel_recall": 0.0, "qid": "74f3071a-69dc-49a6-9434-72355d55d5f4", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 56, "novel_recall": 0.0, "qid": "aa85bfa3-519d-44ed-bd80-e213936619e3", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 30, "novel_recall": 0.8, "qid": "078634cf-e374-44e6-ac55-b23b7c7e3df2", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 119, "novel_recall": 0.3697, "qid": "640556fc-2f3e-44d5-9caa-a3c282c55fe5", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 94, "novel_recall": 0.1915, "qid": "439dede1-fc86-4098-b4ea-e5b9f7c82246", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 50, "novel_recall": 0.58, "qid": "a21e22ef-0659-4d09-b95d-afd396bb51ac", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 96, "novel_recall": 0.375, "qid": "3f4583b7-d480-47fc-84b1-13a67ac6a52b", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 92, "novel_recall": 0.2826, "qid": "e730f309-db26-42a7-951b-6d550a41bc7d", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 104, "novel_recall": 0.2404, "qid": "11ba3c77-3be1-4969-a533-5c6cdbcd87fc", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 58, "novel_recall": 0.1552, "qid": "47349701-9fd5-4a57-9382-898a238c7973", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 48, "novel_recall": 0.3958, "qid": "15164441-18b1-4b9e-8074-23f6cd3de102", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 51, "novel_recall": 0.3922, "qid": "70634a33-db60-47d4-baab-a71c5323bcdb", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 36, "novel_recall": 0.7222, "qid": "74f3071a-69dc-49a6-9434-72355d55d5f4", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 56, "novel_recall": 0.5357, "qid": "aa85bfa3-519d-44ed-bd80-e213936619e3", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 30, "novel_recall": 0.8, "qid": "078634cf-e374-44e6-ac55-b23b7c7e3df2", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 119, "novel_recall": 0.3697, "qid": "640556fc-2f3e-44d5-9caa-a3c282c55fe5", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 94, "novel_recall": 0.1915, "qid": "439dede1-fc86-4098-b4ea-e5b9f7c82246", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 50, "novel_recall": 0.58, "qid": "a21e22ef-0659-4d09-b95d-afd396bb51ac", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 96, "novel_recall": 0.375, "qid": "3f4583b7-d480-47fc-84b1-13a67ac6a52b", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 92, "novel_recall": 0.2826, "qid": "e730f309-db26-42a7-951b-6d550a41bc7d", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 104, "novel_recall": 0.2404, "qid": "11ba3c77-3be1-4969-a533-5c6cdbcd87fc", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 58, "novel_recall": 0.1552, "qid": "47349701-9fd5-4a57-9382-898a238c7973", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 48, "novel_recall": 0.3958, "qid": "15164441-18b1-4b9e-8074-23f6cd3de102", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 51, "novel_recall": 0.3922, "qid": "70634a33-db60-47d4-baab-a71c5323bcdb", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 36, "novel_recall": 0.7222, "qid": "74f3071a-69dc-49a6-9434-72355d55d5f4", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 56, "novel_recall": 0.5357, "qid": "aa85bfa3-519d-44ed-bd80-e213936619e3", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 30, "novel_recall": 0.8, "qid": "078634cf-e374-44e6-ac55-b23b7c7e3df2", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 119, "novel_recall": 0.3613, "qid": "640556fc-2f3e-44d5-9caa-a3c282c55fe5", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 94, "novel_recall": 0.1809, "qid": "439dede1-fc86-4098-b4ea-e5b9f7c82246", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 50, "novel_recall": 0.6, "qid": "a21e22ef-0659-4d09-b95d-afd396bb51ac", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 96, "novel_recall": 0.3542, "qid": "3f4583b7-d480-47fc-84b1-13a67ac6a52b", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 92, "novel_recall": 0.2717, "qid": "e730f309-db26-42a7-951b-6d550a41bc7d", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 104, "novel_recall": 0.2308, "qid": "11ba3c77-3be1-4969-a533-5c6cdbcd87fc", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 58, "novel_recall": 0.1552, "qid": "47349701-9fd5-4a57-9382-898a238c7973", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 48, "novel_recall": 0.3958, "qid": "15164441-18b1-4b9e-8074-23f6cd3de102", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 51, "novel_recall": 0.3529, "qid": "70634a33-db60-47d4-baab-a71c5323bcdb", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 36, "novel_recall": 0.6944, "qid": "74f3071a-69dc-49a6-9434-72355d55d5f4", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 56, "novel_recall": 0.5179, "qid": "aa85bfa3-519d-44ed-bd80-e213936619e3", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 30, "novel_recall": 0.8, "qid": "078634cf-e374-44e6-ac55-b23b7c7e3df2", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 119, "novel_recall": 0.3613, "qid": "640556fc-2f3e-44d5-9caa-a3c282c55fe5", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 94, "novel_recall": 0.1809, "qid": "439dede1-fc86-4098-b4ea-e5b9f7c82246", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 50, "novel_recall": 0.6, "qid": "a21e22ef-0659-4d09-b95d-afd396bb51ac", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 96, "novel_recall": 0.3542, "qid": "3f4583b7-d480-47fc-84b1-13a67ac6a52b", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 92, "novel_recall": 0.2717, "qid": "e730f309-db26-42a7-951b-6d550a41bc7d", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 104, "novel_recall": 0.2308, "qid": "11ba3c77-3be1-4969-a533-5c6cdbcd87fc", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 58, "novel_recall": 0.1552, "qid": "47349701-9fd5-4a57-9382-898a238c7973", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 48, "novel_recall": 0.3958, "qid": "15164441-18b1-4b9e-8074-23f6cd3de102", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 51, "novel_recall": 0.3529, "qid": "70634a33-db60-47d4-baab-a71c5323bcdb", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 36, "novel_recall": 0.6944, "qid": "74f3071a-69dc-49a6-9434-72355d55d5f4", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 56, "novel_recall": 0.5179, "qid": "aa85bfa3-519d-44ed-bd80-e213936619e3", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 30, "novel_recall": 0.8, "qid": "078634cf-e374-44e6-ac55-b23b7c7e3df2", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 119, "novel_recall": 0.3866, "qid": "640556fc-2f3e-44d5-9caa-a3c282c55fe5", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 94, "novel_recall": 0.1702, "qid": "439dede1-fc86-4098-b4ea-e5b9f7c82246", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 50, "novel_recall": 0.56, "qid": "a21e22ef-0659-4d09-b95d-afd396bb51ac", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 96, "novel_recall": 0.3542, "qid": "3f4583b7-d480-47fc-84b1-13a67ac6a52b", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 92, "novel_recall": 0.3261, "qid": "e730f309-db26-42a7-951b-6d550a41bc7d", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 104, "novel_recall": 0.2596, "qid": "11ba3c77-3be1-4969-a533-5c6cdbcd87fc", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 58, "novel_recall": 0.2241, "qid": "47349701-9fd5-4a57-9382-898a238c7973", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 48, "novel_recall": 0.4375, "qid": "15164441-18b1-4b9e-8074-23f6cd3de102", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 51, "novel_recall": 0.3725, "qid": "70634a33-db60-47d4-baab-a71c5323bcdb", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 36, "novel_recall": 0.7778, "qid": "74f3071a-69dc-49a6-9434-72355d55d5f4", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "127", "full_hit": false, "n_novel": 56, "novel_recall": 0.5179, "qid": "aa85bfa3-519d-44ed-bd80-e213936619e3", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 30, "novel_recall": 0.0, "qid": "2e523756-ed25-472b-bc22-9b73f5173ebe", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 67, "novel_recall": 0.0, "qid": "2d9f6311-be8e-46cd-a441-0bb6f75d23d0", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 53, "novel_recall": 0.0, "qid": "3aea1901-18e8-4dcb-9984-74c91a0030d0", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 33, "novel_recall": 0.0, "qid": "bcdc1c1b-7a89-4b73-ac04-f9ffbb0bc033", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 77, "novel_recall": 0.0, "qid": "5713e03e-d00e-4dc1-bec3-4ee715f86541", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 69, "novel_recall": 0.0, "qid": "ea9e6e64-1fcf-4a77-936c-eb1edc7f5e63", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 60, "novel_recall": 0.0, "qid": "1b1c06c1-aec4-4edf-9009-d3b96c711016", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 23, "novel_recall": 0.0, "qid": "84217982-e12d-4cbc-9b88-2e50b6f9a527", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 76, "novel_recall": 0.0, "qid": "c82b3be0-13a2-467d-af04-3aba96d595cf", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 62, "novel_recall": 0.0, "qid": "0d0b29ed-bf7e-45b2-97a6-6a9f0c0f4dd1", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 32, "novel_recall": 0.0, "qid": "855552b8-db96-41e9-a675-d0e5ff26d3cb", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 51, "novel_recall": 0.0, "qid": "09897247-e1c6-4f2c-a427-e43f2220cabb", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 30, "novel_recall": 0.2667, "qid": "2e523756-ed25-472b-bc22-9b73f5173ebe", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 67, "novel_recall": 0.0746, "qid": "2d9f6311-be8e-46cd-a441-0bb6f75d23d0", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 53, "novel_recall": 0.1321, "qid": "3aea1901-18e8-4dcb-9984-74c91a0030d0", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 33, "novel_recall": 0.0, "qid": "bcdc1c1b-7a89-4b73-ac04-f9ffbb0bc033", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 77, "novel_recall": 0.1039, "qid": "5713e03e-d00e-4dc1-bec3-4ee715f86541", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 69, "novel_recall": 0.0435, "qid": "ea9e6e64-1fcf-4a77-936c-eb1edc7f5e63", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 60, "novel_recall": 0.0167, "qid": "1b1c06c1-aec4-4edf-9009-d3b96c711016", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 23, "novel_recall": 0.2609, "qid": "84217982-e12d-4cbc-9b88-2e50b6f9a527", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 76, "novel_recall": 0.0526, "qid": "c82b3be0-13a2-467d-af04-3aba96d595cf", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 62, "novel_recall": 0.0645, "qid": "0d0b29ed-bf7e-45b2-97a6-6a9f0c0f4dd1", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 32, "novel_recall": 0.5, "qid": "855552b8-db96-41e9-a675-d0e5ff26d3cb", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 51, "novel_recall": 0.0, "qid": "09897247-e1c6-4f2c-a427-e43f2220cabb", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 30, "novel_recall": 0.2667, "qid": "2e523756-ed25-472b-bc22-9b73f5173ebe", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 67, "novel_recall": 0.0746, "qid": "2d9f6311-be8e-46cd-a441-0bb6f75d23d0", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 53, "novel_recall": 0.1321, "qid": "3aea1901-18e8-4dcb-9984-74c91a0030d0", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 33, "novel_recall": 0.0, "qid": "bcdc1c1b-7a89-4b73-ac04-f9ffbb0bc033", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 77, "novel_recall": 0.1039, "qid": "5713e03e-d00e-4dc1-bec3-4ee715f86541", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 69, "novel_recall": 0.0435, "qid": "ea9e6e64-1fcf-4a77-936c-eb1edc7f5e63", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 60, "novel_recall": 0.0167, "qid": "1b1c06c1-aec4-4edf-9009-d3b96c711016", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 23, "novel_recall": 0.2609, "qid": "84217982-e12d-4cbc-9b88-2e50b6f9a527", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 76, "novel_recall": 0.0526, "qid": "c82b3be0-13a2-467d-af04-3aba96d595cf", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 62, "novel_recall": 0.0645, "qid": "0d0b29ed-bf7e-45b2-97a6-6a9f0c0f4dd1", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 32, "novel_recall": 0.5, "qid": "855552b8-db96-41e9-a675-d0e5ff26d3cb", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 51, "novel_recall": 0.0, "qid": "09897247-e1c6-4f2c-a427-e43f2220cabb", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 30, "novel_recall": 0.2667, "qid": "2e523756-ed25-472b-bc22-9b73f5173ebe", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 67, "novel_recall": 0.0746, "qid": "2d9f6311-be8e-46cd-a441-0bb6f75d23d0", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 53, "novel_recall": 0.1321, "qid": "3aea1901-18e8-4dcb-9984-74c91a0030d0", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 33, "novel_recall": 0.0, "qid": "bcdc1c1b-7a89-4b73-ac04-f9ffbb0bc033", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 77, "novel_recall": 0.1039, "qid": "5713e03e-d00e-4dc1-bec3-4ee715f86541", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 69, "novel_recall": 0.0435, "qid": "ea9e6e64-1fcf-4a77-936c-eb1edc7f5e63", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 60, "novel_recall": 0.0167, "qid": "1b1c06c1-aec4-4edf-9009-d3b96c711016", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 23, "novel_recall": 0.2609, "qid": "84217982-e12d-4cbc-9b88-2e50b6f9a527", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 76, "novel_recall": 0.0526, "qid": "c82b3be0-13a2-467d-af04-3aba96d595cf", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 62, "novel_recall": 0.0645, "qid": "0d0b29ed-bf7e-45b2-97a6-6a9f0c0f4dd1", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 32, "novel_recall": 0.5, "qid": "855552b8-db96-41e9-a675-d0e5ff26d3cb", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 51, "novel_recall": 0.0, "qid": "09897247-e1c6-4f2c-a427-e43f2220cabb", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 30, "novel_recall": 0.2667, "qid": "2e523756-ed25-472b-bc22-9b73f5173ebe", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 67, "novel_recall": 0.0746, "qid": "2d9f6311-be8e-46cd-a441-0bb6f75d23d0", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 53, "novel_recall": 0.1321, "qid": "3aea1901-18e8-4dcb-9984-74c91a0030d0", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 33, "novel_recall": 0.0, "qid": "bcdc1c1b-7a89-4b73-ac04-f9ffbb0bc033", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 77, "novel_recall": 0.1039, "qid": "5713e03e-d00e-4dc1-bec3-4ee715f86541", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 69, "novel_recall": 0.0435, "qid": "ea9e6e64-1fcf-4a77-936c-eb1edc7f5e63", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 60, "novel_recall": 0.0167, "qid": "1b1c06c1-aec4-4edf-9009-d3b96c711016", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 23, "novel_recall": 0.2609, "qid": "84217982-e12d-4cbc-9b88-2e50b6f9a527", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 76, "novel_recall": 0.0526, "qid": "c82b3be0-13a2-467d-af04-3aba96d595cf", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 62, "novel_recall": 0.0645, "qid": "0d0b29ed-bf7e-45b2-97a6-6a9f0c0f4dd1", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 32, "novel_recall": 0.5, "qid": "855552b8-db96-41e9-a675-d0e5ff26d3cb", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 51, "novel_recall": 0.0, "qid": "09897247-e1c6-4f2c-a427-e43f2220cabb", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 30, "novel_recall": 0.2667, "qid": "2e523756-ed25-472b-bc22-9b73f5173ebe", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 67, "novel_recall": 0.0746, "qid": "2d9f6311-be8e-46cd-a441-0bb6f75d23d0", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 53, "novel_recall": 0.1321, "qid": "3aea1901-18e8-4dcb-9984-74c91a0030d0", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 33, "novel_recall": 0.2424, "qid": "bcdc1c1b-7a89-4b73-ac04-f9ffbb0bc033", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 77, "novel_recall": 0.1169, "qid": "5713e03e-d00e-4dc1-bec3-4ee715f86541", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 69, "novel_recall": 0.0435, "qid": "ea9e6e64-1fcf-4a77-936c-eb1edc7f5e63", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 60, "novel_recall": 0.0333, "qid": "1b1c06c1-aec4-4edf-9009-d3b96c711016", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 23, "novel_recall": 0.2609, "qid": "84217982-e12d-4cbc-9b88-2e50b6f9a527", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 76, "novel_recall": 0.0658, "qid": "c82b3be0-13a2-467d-af04-3aba96d595cf", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 62, "novel_recall": 0.0645, "qid": "0d0b29ed-bf7e-45b2-97a6-6a9f0c0f4dd1", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 32, "novel_recall": 0.5, "qid": "855552b8-db96-41e9-a675-d0e5ff26d3cb", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "128", "full_hit": false, "n_novel": 51, "novel_recall": 0.2353, "qid": "09897247-e1c6-4f2c-a427-e43f2220cabb", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 34, "novel_recall": 0.0, "qid": "da2772d0-f0b8-49cd-8d29-bb42aa88aeb5", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 45, "novel_recall": 0.0, "qid": "f8ce9d54-1b1c-462e-9381-da9019b6dfe2", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 67, "novel_recall": 0.0, "qid": "8dae2615-a204-480d-ac55-c81ff824b380", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 62, "novel_recall": 0.0, "qid": "41acab7c-4f5e-4746-8add-409d20e3e628", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 85, "novel_recall": 0.0, "qid": "671de1ba-f22f-4033-9ba6-40f7c1cf9213", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 74, "novel_recall": 0.0, "qid": "f78616a8-7eb8-470a-8c0f-5faa584ee075", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 48, "novel_recall": 0.0, "qid": "9741af6c-be08-4fde-ab6c-5958d820054f", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 40, "novel_recall": 0.0, "qid": "7eae7087-6faf-4dd4-b5a3-8d0a3ca1c700", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 58, "novel_recall": 0.0, "qid": "f1aa1046-361c-4f18-8104-2300e7687454", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 88, "novel_recall": 0.0, "qid": "69737846-c781-4e22-b206-d485fc1b4c3d", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 28, "novel_recall": 0.0, "qid": "f3d2d33b-9c21-46ef-bfaa-1c48b0c1f17c", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 62, "novel_recall": 0.0, "qid": "d7634cbd-6847-4eab-b104-3cdd30345070", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 34, "novel_recall": 0.8235, "qid": "da2772d0-f0b8-49cd-8d29-bb42aa88aeb5", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 45, "novel_recall": 0.6222, "qid": "f8ce9d54-1b1c-462e-9381-da9019b6dfe2", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 67, "novel_recall": 0.209, "qid": "8dae2615-a204-480d-ac55-c81ff824b380", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 62, "novel_recall": 0.1452, "qid": "41acab7c-4f5e-4746-8add-409d20e3e628", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 85, "novel_recall": 0.3529, "qid": "671de1ba-f22f-4033-9ba6-40f7c1cf9213", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 74, "novel_recall": 0.3108, "qid": "f78616a8-7eb8-470a-8c0f-5faa584ee075", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 48, "novel_recall": 0.375, "qid": "9741af6c-be08-4fde-ab6c-5958d820054f", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 40, "novel_recall": 0.3, "qid": "7eae7087-6faf-4dd4-b5a3-8d0a3ca1c700", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 58, "novel_recall": 0.2586, "qid": "f1aa1046-361c-4f18-8104-2300e7687454", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 88, "novel_recall": 0.125, "qid": "69737846-c781-4e22-b206-d485fc1b4c3d", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 28, "novel_recall": 0.6786, "qid": "f3d2d33b-9c21-46ef-bfaa-1c48b0c1f17c", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 62, "novel_recall": 0.3065, "qid": "d7634cbd-6847-4eab-b104-3cdd30345070", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 34, "novel_recall": 0.8235, "qid": "da2772d0-f0b8-49cd-8d29-bb42aa88aeb5", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 45, "novel_recall": 0.6222, "qid": "f8ce9d54-1b1c-462e-9381-da9019b6dfe2", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 67, "novel_recall": 0.209, "qid": "8dae2615-a204-480d-ac55-c81ff824b380", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 62, "novel_recall": 0.1452, "qid": "41acab7c-4f5e-4746-8add-409d20e3e628", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 85, "novel_recall": 0.3529, "qid": "671de1ba-f22f-4033-9ba6-40f7c1cf9213", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 74, "novel_recall": 0.3108, "qid": "f78616a8-7eb8-470a-8c0f-5faa584ee075", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 48, "novel_recall": 0.375, "qid": "9741af6c-be08-4fde-ab6c-5958d820054f", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 40, "novel_recall": 0.3, "qid": "7eae7087-6faf-4dd4-b5a3-8d0a3ca1c700", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 58, "novel_recall": 0.2586, "qid": "f1aa1046-361c-4f18-8104-2300e7687454", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 88, "novel_recall": 0.125, "qid": "69737846-c781-4e22-b206-d485fc1b4c3d", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 28, "novel_recall": 0.6786, "qid": "f3d2d33b-9c21-46ef-bfaa-1c48b0c1f17c", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 62, "novel_recall": 0.3065, "qid": "d7634cbd-6847-4eab-b104-3cdd30345070", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 34, "novel_recall": 0.8235, "qid": "da2772d0-f0b8-49cd-8d29-bb42aa88aeb5", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 45, "novel_recall": 0.6222, "qid": "f8ce9d54-1b1c-462e-9381-da9019b6dfe2", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 67, "novel_recall": 0.194, "qid": "8dae2615-a204-480d-ac55-c81ff824b380", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 62, "novel_recall": 0.1452, "qid": "41acab7c-4f5e-4746-8add-409d20e3e628", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 85, "novel_recall": 0.3529, "qid": "671de1ba-f22f-4033-9ba6-40f7c1cf9213", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 74, "novel_recall": 0.3108, "qid": "f78616a8-7eb8-470a-8c0f-5faa584ee075", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 48, "novel_recall": 0.375, "qid": "9741af6c-be08-4fde-ab6c-5958d820054f", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 40, "novel_recall": 0.3, "qid": "7eae7087-6faf-4dd4-b5a3-8d0a3ca1c700", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 58, "novel_recall": 0.2586, "qid": "f1aa1046-361c-4f18-8104-2300e7687454", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 88, "novel_recall": 0.125, "qid": "69737846-c781-4e22-b206-d485fc1b4c3d", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 28, "novel_recall": 0.6786, "qid": "f3d2d33b-9c21-46ef-bfaa-1c48b0c1f17c", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 62, "novel_recall": 0.3065, "qid": "d7634cbd-6847-4eab-b104-3cdd30345070", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 34, "novel_recall": 0.8235, "qid": "da2772d0-f0b8-49cd-8d29-bb42aa88aeb5", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 45, "novel_recall": 0.6222, "qid": "f8ce9d54-1b1c-462e-9381-da9019b6dfe2", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 67, "novel_recall": 0.194, "qid": "8dae2615-a204-480d-ac55-c81ff824b380", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 62, "novel_recall": 0.1452, "qid": "41acab7c-4f5e-4746-8add-409d20e3e628", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 85, "novel_recall": 0.3529, "qid": "671de1ba-f22f-4033-9ba6-40f7c1cf9213", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 74, "novel_recall": 0.3108, "qid": "f78616a8-7eb8-470a-8c0f-5faa584ee075", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 48, "novel_recall": 0.375, "qid": "9741af6c-be08-4fde-ab6c-5958d820054f", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 40, "novel_recall": 0.3, "qid": "7eae7087-6faf-4dd4-b5a3-8d0a3ca1c700", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 58, "novel_recall": 0.2586, "qid": "f1aa1046-361c-4f18-8104-2300e7687454", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 88, "novel_recall": 0.125, "qid": "69737846-c781-4e22-b206-d485fc1b4c3d", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 28, "novel_recall": 0.6786, "qid": "f3d2d33b-9c21-46ef-bfaa-1c48b0c1f17c", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 62, "novel_recall": 0.3065, "qid": "d7634cbd-6847-4eab-b104-3cdd30345070", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 34, "novel_recall": 0.8235, "qid": "da2772d0-f0b8-49cd-8d29-bb42aa88aeb5", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 45, "novel_recall": 0.6222, "qid": "f8ce9d54-1b1c-462e-9381-da9019b6dfe2", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 67, "novel_recall": 0.209, "qid": "8dae2615-a204-480d-ac55-c81ff824b380", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 62, "novel_recall": 0.1452, "qid": "41acab7c-4f5e-4746-8add-409d20e3e628", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 85, "novel_recall": 0.3529, "qid": "671de1ba-f22f-4033-9ba6-40f7c1cf9213", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 74, "novel_recall": 0.3108, "qid": "f78616a8-7eb8-470a-8c0f-5faa584ee075", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 48, "novel_recall": 0.375, "qid": "9741af6c-be08-4fde-ab6c-5958d820054f", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 40, "novel_recall": 0.3, "qid": "7eae7087-6faf-4dd4-b5a3-8d0a3ca1c700", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 58, "novel_recall": 0.2586, "qid": "f1aa1046-361c-4f18-8104-2300e7687454", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 88, "novel_recall": 0.125, "qid": "69737846-c781-4e22-b206-d485fc1b4c3d", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 28, "novel_recall": 0.6786, "qid": "f3d2d33b-9c21-46ef-bfaa-1c48b0c1f17c", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "133", "full_hit": false, "n_novel": 62, "novel_recall": 0.3065, "qid": "d7634cbd-6847-4eab-b104-3cdd30345070", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "d31098e0-1a6c-40ba-9ddd-878d9981bde2", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 67, "novel_recall": 0.0, "qid": "65901435-a24a-40e0-bb1c-1356e5e29a15", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 52, "novel_recall": 0.0, "qid": "88508d32-61a9-42e7-bbfb-70eb753b4f78", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 49, "novel_recall": 0.0, "qid": "7b5a6a6f-770c-409e-b252-57c0b3b35a4e", "type": "A"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 93, "novel_recall": 0.0, "qid": "dd9645ea-300a-4e52-b8b7-6b53852bafb1", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 104, "novel_recall": 0.0, "qid": "fc2ab808-e91a-405c-9b5b-ae2c375be12d", "type": "B"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 18, "novel_recall": 0.0, "qid": "4f54586c-7a32-4c8c-a058-a62b2152e088", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 31, "novel_recall": 0.0, "qid": "690b0640-3ce7-451c-80e3-e15ce98a6253", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 71, "novel_recall": 0.0, "qid": "2077da74-50c7-4775-a66c-79e16d6b6b7e", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 71, "novel_recall": 0.0, "qid": "cb4f7877-6ec6-4bac-858d-d7d1f4cdbcad", "type": "C"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 46, "novel_recall": 0.0, "qid": "1881db1f-a112-4357-bae7-aa882fba4bad", "type": "D"}, {"arm": "M0_none", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 91, "novel_recall": 0.0, "qid": "e966a582-abc0-42d3-8bb1-68f6ea1af463", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 12, "novel_recall": 0.75, "qid": "d31098e0-1a6c-40ba-9ddd-878d9981bde2", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 67, "novel_recall": 0.194, "qid": "65901435-a24a-40e0-bb1c-1356e5e29a15", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 52, "novel_recall": 0.1346, "qid": "88508d32-61a9-42e7-bbfb-70eb753b4f78", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 49, "novel_recall": 0.4286, "qid": "7b5a6a6f-770c-409e-b252-57c0b3b35a4e", "type": "A"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 93, "novel_recall": 0.1828, "qid": "dd9645ea-300a-4e52-b8b7-6b53852bafb1", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 104, "novel_recall": 0.1635, "qid": "fc2ab808-e91a-405c-9b5b-ae2c375be12d", "type": "B"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 18, "novel_recall": 0.2222, "qid": "4f54586c-7a32-4c8c-a058-a62b2152e088", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 31, "novel_recall": 0.2903, "qid": "690b0640-3ce7-451c-80e3-e15ce98a6253", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 71, "novel_recall": 0.169, "qid": "2077da74-50c7-4775-a66c-79e16d6b6b7e", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 71, "novel_recall": 0.1972, "qid": "cb4f7877-6ec6-4bac-858d-d7d1f4cdbcad", "type": "C"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 46, "novel_recall": 0.413, "qid": "1881db1f-a112-4357-bae7-aa882fba4bad", "type": "D"}, {"arm": "M1_naive", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 91, "novel_recall": 0.2857, "qid": "e966a582-abc0-42d3-8bb1-68f6ea1af463", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 12, "novel_recall": 0.75, "qid": "d31098e0-1a6c-40ba-9ddd-878d9981bde2", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 67, "novel_recall": 0.194, "qid": "65901435-a24a-40e0-bb1c-1356e5e29a15", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 52, "novel_recall": 0.1346, "qid": "88508d32-61a9-42e7-bbfb-70eb753b4f78", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 49, "novel_recall": 0.4286, "qid": "7b5a6a6f-770c-409e-b252-57c0b3b35a4e", "type": "A"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 93, "novel_recall": 0.1828, "qid": "dd9645ea-300a-4e52-b8b7-6b53852bafb1", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 104, "novel_recall": 0.1635, "qid": "fc2ab808-e91a-405c-9b5b-ae2c375be12d", "type": "B"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 18, "novel_recall": 0.2222, "qid": "4f54586c-7a32-4c8c-a058-a62b2152e088", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 31, "novel_recall": 0.2903, "qid": "690b0640-3ce7-451c-80e3-e15ce98a6253", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 71, "novel_recall": 0.169, "qid": "2077da74-50c7-4775-a66c-79e16d6b6b7e", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 71, "novel_recall": 0.1972, "qid": "cb4f7877-6ec6-4bac-858d-d7d1f4cdbcad", "type": "C"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 46, "novel_recall": 0.6087, "qid": "1881db1f-a112-4357-bae7-aa882fba4bad", "type": "D"}, {"arm": "M2_evidence", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 91, "novel_recall": 0.2857, "qid": "e966a582-abc0-42d3-8bb1-68f6ea1af463", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 12, "novel_recall": 0.75, "qid": "d31098e0-1a6c-40ba-9ddd-878d9981bde2", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 67, "novel_recall": 0.1642, "qid": "65901435-a24a-40e0-bb1c-1356e5e29a15", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 52, "novel_recall": 0.1346, "qid": "88508d32-61a9-42e7-bbfb-70eb753b4f78", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 49, "novel_recall": 0.4286, "qid": "7b5a6a6f-770c-409e-b252-57c0b3b35a4e", "type": "A"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 93, "novel_recall": 0.172, "qid": "dd9645ea-300a-4e52-b8b7-6b53852bafb1", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 104, "novel_recall": 0.125, "qid": "fc2ab808-e91a-405c-9b5b-ae2c375be12d", "type": "B"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 18, "novel_recall": 0.2222, "qid": "4f54586c-7a32-4c8c-a058-a62b2152e088", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 31, "novel_recall": 0.1935, "qid": "690b0640-3ce7-451c-80e3-e15ce98a6253", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 71, "novel_recall": 0.1549, "qid": "2077da74-50c7-4775-a66c-79e16d6b6b7e", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 71, "novel_recall": 0.169, "qid": "cb4f7877-6ec6-4bac-858d-d7d1f4cdbcad", "type": "C"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 46, "novel_recall": 0.587, "qid": "1881db1f-a112-4357-bae7-aa882fba4bad", "type": "D"}, {"arm": "M3_causal", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 91, "novel_recall": 0.2637, "qid": "e966a582-abc0-42d3-8bb1-68f6ea1af463", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 12, "novel_recall": 0.75, "qid": "d31098e0-1a6c-40ba-9ddd-878d9981bde2", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 67, "novel_recall": 0.1642, "qid": "65901435-a24a-40e0-bb1c-1356e5e29a15", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 52, "novel_recall": 0.1346, "qid": "88508d32-61a9-42e7-bbfb-70eb753b4f78", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 49, "novel_recall": 0.4286, "qid": "7b5a6a6f-770c-409e-b252-57c0b3b35a4e", "type": "A"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 93, "novel_recall": 0.172, "qid": "dd9645ea-300a-4e52-b8b7-6b53852bafb1", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 104, "novel_recall": 0.125, "qid": "fc2ab808-e91a-405c-9b5b-ae2c375be12d", "type": "B"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 18, "novel_recall": 0.2222, "qid": "4f54586c-7a32-4c8c-a058-a62b2152e088", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 31, "novel_recall": 0.1935, "qid": "690b0640-3ce7-451c-80e3-e15ce98a6253", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 71, "novel_recall": 0.1549, "qid": "2077da74-50c7-4775-a66c-79e16d6b6b7e", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 71, "novel_recall": 0.169, "qid": "cb4f7877-6ec6-4bac-858d-d7d1f4cdbcad", "type": "C"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 46, "novel_recall": 0.587, "qid": "1881db1f-a112-4357-bae7-aa882fba4bad", "type": "D"}, {"arm": "M4_validated", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 91, "novel_recall": 0.2637, "qid": "e966a582-abc0-42d3-8bb1-68f6ea1af463", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "136", "full_hit": true, "n_novel": 12, "novel_recall": 1.0, "qid": "d31098e0-1a6c-40ba-9ddd-878d9981bde2", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 67, "novel_recall": 0.1791, "qid": "65901435-a24a-40e0-bb1c-1356e5e29a15", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 52, "novel_recall": 0.1154, "qid": "88508d32-61a9-42e7-bbfb-70eb753b4f78", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 49, "novel_recall": 0.4286, "qid": "7b5a6a6f-770c-409e-b252-57c0b3b35a4e", "type": "A"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 93, "novel_recall": 0.2151, "qid": "dd9645ea-300a-4e52-b8b7-6b53852bafb1", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 104, "novel_recall": 0.1635, "qid": "fc2ab808-e91a-405c-9b5b-ae2c375be12d", "type": "B"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 18, "novel_recall": 0.4444, "qid": "4f54586c-7a32-4c8c-a058-a62b2152e088", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 31, "novel_recall": 0.4516, "qid": "690b0640-3ce7-451c-80e3-e15ce98a6253", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 71, "novel_recall": 0.169, "qid": "2077da74-50c7-4775-a66c-79e16d6b6b7e", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 71, "novel_recall": 0.2113, "qid": "cb4f7877-6ec6-4bac-858d-d7d1f4cdbcad", "type": "C"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 46, "novel_recall": 0.4783, "qid": "1881db1f-a112-4357-bae7-aa882fba4bad", "type": "D"}, {"arm": "R_recent", "domain": "TEXT2SQL", "episode": "136", "full_hit": false, "n_novel": 91, "novel_recall": 0.3077, "qid": "e966a582-abc0-42d3-8bb1-68f6ea1af463", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "9b45a224-f043-4434-a5bd-e50e67d8fd5c", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "af826533-56ca-4601-ace1-a755db235fe7", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "b5196c95-648f-4139-9cb2-eeeea966ad7d", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "474ffb9a-85d6-48e8-a4c5-d64f46254a94", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "5a0a6862-bc7e-4760-ad1e-ae152eba2a74", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "243c25c0-15ba-4cf0-aaca-d7fed558b921", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "ef306413-875b-4f62-b561-79f4ba50d35b", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "05674af5-6120-472c-b7c5-f6643a22a02b", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "8b4070ed-1bfb-47dd-bb79-922b1e40c675", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "769200f6-752e-4499-aecd-d72243b31f9f", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 31, "novel_recall": 0.0, "qid": "d6622720-fa53-488e-8be6-4bf0b3d341c0", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 53, "novel_recall": 0.0, "qid": "2bdf5cec-98f3-4479-82c9-cd3da74f9727", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "9b45a224-f043-4434-a5bd-e50e67d8fd5c", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "af826533-56ca-4601-ace1-a755db235fe7", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "b5196c95-648f-4139-9cb2-eeeea966ad7d", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "474ffb9a-85d6-48e8-a4c5-d64f46254a94", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "5a0a6862-bc7e-4760-ad1e-ae152eba2a74", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "243c25c0-15ba-4cf0-aaca-d7fed558b921", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "ef306413-875b-4f62-b561-79f4ba50d35b", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "05674af5-6120-472c-b7c5-f6643a22a02b", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "8b4070ed-1bfb-47dd-bb79-922b1e40c675", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "769200f6-752e-4499-aecd-d72243b31f9f", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 31, "novel_recall": 0.8065, "qid": "d6622720-fa53-488e-8be6-4bf0b3d341c0", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 53, "novel_recall": 0.6038, "qid": "2bdf5cec-98f3-4479-82c9-cd3da74f9727", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "9b45a224-f043-4434-a5bd-e50e67d8fd5c", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "af826533-56ca-4601-ace1-a755db235fe7", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "b5196c95-648f-4139-9cb2-eeeea966ad7d", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "474ffb9a-85d6-48e8-a4c5-d64f46254a94", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "5a0a6862-bc7e-4760-ad1e-ae152eba2a74", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "243c25c0-15ba-4cf0-aaca-d7fed558b921", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "ef306413-875b-4f62-b561-79f4ba50d35b", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "05674af5-6120-472c-b7c5-f6643a22a02b", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "8b4070ed-1bfb-47dd-bb79-922b1e40c675", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "769200f6-752e-4499-aecd-d72243b31f9f", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 31, "novel_recall": 0.7419, "qid": "d6622720-fa53-488e-8be6-4bf0b3d341c0", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 53, "novel_recall": 0.6038, "qid": "2bdf5cec-98f3-4479-82c9-cd3da74f9727", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "9b45a224-f043-4434-a5bd-e50e67d8fd5c", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "af826533-56ca-4601-ace1-a755db235fe7", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "b5196c95-648f-4139-9cb2-eeeea966ad7d", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "474ffb9a-85d6-48e8-a4c5-d64f46254a94", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "5a0a6862-bc7e-4760-ad1e-ae152eba2a74", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "243c25c0-15ba-4cf0-aaca-d7fed558b921", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "ef306413-875b-4f62-b561-79f4ba50d35b", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "05674af5-6120-472c-b7c5-f6643a22a02b", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "8b4070ed-1bfb-47dd-bb79-922b1e40c675", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "769200f6-752e-4499-aecd-d72243b31f9f", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 31, "novel_recall": 0.7742, "qid": "d6622720-fa53-488e-8be6-4bf0b3d341c0", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 53, "novel_recall": 0.5472, "qid": "2bdf5cec-98f3-4479-82c9-cd3da74f9727", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "9b45a224-f043-4434-a5bd-e50e67d8fd5c", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "af826533-56ca-4601-ace1-a755db235fe7", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "b5196c95-648f-4139-9cb2-eeeea966ad7d", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "474ffb9a-85d6-48e8-a4c5-d64f46254a94", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "5a0a6862-bc7e-4760-ad1e-ae152eba2a74", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "243c25c0-15ba-4cf0-aaca-d7fed558b921", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "ef306413-875b-4f62-b561-79f4ba50d35b", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "05674af5-6120-472c-b7c5-f6643a22a02b", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "8b4070ed-1bfb-47dd-bb79-922b1e40c675", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "769200f6-752e-4499-aecd-d72243b31f9f", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 31, "novel_recall": 0.7742, "qid": "d6622720-fa53-488e-8be6-4bf0b3d341c0", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 53, "novel_recall": 0.5472, "qid": "2bdf5cec-98f3-4479-82c9-cd3da74f9727", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "9b45a224-f043-4434-a5bd-e50e67d8fd5c", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "af826533-56ca-4601-ace1-a755db235fe7", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "b5196c95-648f-4139-9cb2-eeeea966ad7d", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "149", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "474ffb9a-85d6-48e8-a4c5-d64f46254a94", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "5a0a6862-bc7e-4760-ad1e-ae152eba2a74", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 7, "novel_recall": 0.1429, "qid": "243c25c0-15ba-4cf0-aaca-d7fed558b921", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "ef306413-875b-4f62-b561-79f4ba50d35b", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "05674af5-6120-472c-b7c5-f6643a22a02b", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "8b4070ed-1bfb-47dd-bb79-922b1e40c675", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "149", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "769200f6-752e-4499-aecd-d72243b31f9f", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 31, "novel_recall": 0.3548, "qid": "d6622720-fa53-488e-8be6-4bf0b3d341c0", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "149", "full_hit": false, "n_novel": 53, "novel_recall": 0.3585, "qid": "2bdf5cec-98f3-4479-82c9-cd3da74f9727", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "3d7adf77-1499-4c95-881e-561078a22d5b", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "021381fb-6069-4b51-aec5-fbc82418cbbe", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "961b1182-db95-40a7-97a1-3d3a1d689f03", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 20, "novel_recall": 0.0, "qid": "c124bd9e-9f65-4ccf-b49a-f48e56e18ef0", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "424fe085-75d7-4f76-b1ee-c2078c04f22c", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "6d9e9cd9-23ac-4869-aa6e-8493a78ab5a7", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "d0abf1d4-0a46-4af9-b6ff-f2d46f999e2a", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "8c8bf59a-4679-4b9e-b3db-85b897edee4e", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "1e91425c-687c-4c2a-a986-d1584fc2d31e", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "112bde3f-2ed2-4886-8cfb-4ac7572cb1ea", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 22, "novel_recall": 0.0, "qid": "1bc1dcaf-8db4-4039-a1c7-a9829730f15a", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 37, "novel_recall": 0.0, "qid": "a261e69d-a3e6-4d1d-9772-8a671b27344d", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "3d7adf77-1499-4c95-881e-561078a22d5b", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "021381fb-6069-4b51-aec5-fbc82418cbbe", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "961b1182-db95-40a7-97a1-3d3a1d689f03", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 20, "novel_recall": 0.35, "qid": "c124bd9e-9f65-4ccf-b49a-f48e56e18ef0", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.1429, "qid": "424fe085-75d7-4f76-b1ee-c2078c04f22c", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "6d9e9cd9-23ac-4869-aa6e-8493a78ab5a7", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.7143, "qid": "d0abf1d4-0a46-4af9-b6ff-f2d46f999e2a", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 10, "novel_recall": 0.9, "qid": "8c8bf59a-4679-4b9e-b3db-85b897edee4e", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "1e91425c-687c-4c2a-a986-d1584fc2d31e", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "112bde3f-2ed2-4886-8cfb-4ac7572cb1ea", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 22, "novel_recall": 0.8636, "qid": "1bc1dcaf-8db4-4039-a1c7-a9829730f15a", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 37, "novel_recall": 0.6486, "qid": "a261e69d-a3e6-4d1d-9772-8a671b27344d", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "3d7adf77-1499-4c95-881e-561078a22d5b", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "021381fb-6069-4b51-aec5-fbc82418cbbe", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "961b1182-db95-40a7-97a1-3d3a1d689f03", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 20, "novel_recall": 0.35, "qid": "c124bd9e-9f65-4ccf-b49a-f48e56e18ef0", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.1429, "qid": "424fe085-75d7-4f76-b1ee-c2078c04f22c", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "6d9e9cd9-23ac-4869-aa6e-8493a78ab5a7", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.7143, "qid": "d0abf1d4-0a46-4af9-b6ff-f2d46f999e2a", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 10, "novel_recall": 0.9, "qid": "8c8bf59a-4679-4b9e-b3db-85b897edee4e", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "1e91425c-687c-4c2a-a986-d1584fc2d31e", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "112bde3f-2ed2-4886-8cfb-4ac7572cb1ea", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 22, "novel_recall": 0.8636, "qid": "1bc1dcaf-8db4-4039-a1c7-a9829730f15a", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 37, "novel_recall": 0.6486, "qid": "a261e69d-a3e6-4d1d-9772-8a671b27344d", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "3d7adf77-1499-4c95-881e-561078a22d5b", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "021381fb-6069-4b51-aec5-fbc82418cbbe", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "961b1182-db95-40a7-97a1-3d3a1d689f03", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 20, "novel_recall": 0.3, "qid": "c124bd9e-9f65-4ccf-b49a-f48e56e18ef0", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.1429, "qid": "424fe085-75d7-4f76-b1ee-c2078c04f22c", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "6d9e9cd9-23ac-4869-aa6e-8493a78ab5a7", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.7143, "qid": "d0abf1d4-0a46-4af9-b6ff-f2d46f999e2a", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 10, "novel_recall": 0.8, "qid": "8c8bf59a-4679-4b9e-b3db-85b897edee4e", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "1e91425c-687c-4c2a-a986-d1584fc2d31e", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "112bde3f-2ed2-4886-8cfb-4ac7572cb1ea", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 22, "novel_recall": 0.8636, "qid": "1bc1dcaf-8db4-4039-a1c7-a9829730f15a", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 37, "novel_recall": 0.6216, "qid": "a261e69d-a3e6-4d1d-9772-8a671b27344d", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "3d7adf77-1499-4c95-881e-561078a22d5b", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "021381fb-6069-4b51-aec5-fbc82418cbbe", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "961b1182-db95-40a7-97a1-3d3a1d689f03", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 20, "novel_recall": 0.3, "qid": "c124bd9e-9f65-4ccf-b49a-f48e56e18ef0", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.1429, "qid": "424fe085-75d7-4f76-b1ee-c2078c04f22c", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "6d9e9cd9-23ac-4869-aa6e-8493a78ab5a7", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.7143, "qid": "d0abf1d4-0a46-4af9-b6ff-f2d46f999e2a", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 10, "novel_recall": 0.8, "qid": "8c8bf59a-4679-4b9e-b3db-85b897edee4e", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "1e91425c-687c-4c2a-a986-d1584fc2d31e", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "112bde3f-2ed2-4886-8cfb-4ac7572cb1ea", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 22, "novel_recall": 0.8636, "qid": "1bc1dcaf-8db4-4039-a1c7-a9829730f15a", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 37, "novel_recall": 0.6216, "qid": "a261e69d-a3e6-4d1d-9772-8a671b27344d", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.7143, "qid": "3d7adf77-1499-4c95-881e-561078a22d5b", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "021381fb-6069-4b51-aec5-fbc82418cbbe", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "961b1182-db95-40a7-97a1-3d3a1d689f03", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 20, "novel_recall": 0.25, "qid": "c124bd9e-9f65-4ccf-b49a-f48e56e18ef0", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "424fe085-75d7-4f76-b1ee-c2078c04f22c", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "6d9e9cd9-23ac-4869-aa6e-8493a78ab5a7", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 7, "novel_recall": 0.7143, "qid": "d0abf1d4-0a46-4af9-b6ff-f2d46f999e2a", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 10, "novel_recall": 0.2, "qid": "8c8bf59a-4679-4b9e-b3db-85b897edee4e", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "153", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "1e91425c-687c-4c2a-a986-d1584fc2d31e", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "112bde3f-2ed2-4886-8cfb-4ac7572cb1ea", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 22, "novel_recall": 0.6364, "qid": "1bc1dcaf-8db4-4039-a1c7-a9829730f15a", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "153", "full_hit": false, "n_novel": 37, "novel_recall": 0.5135, "qid": "a261e69d-a3e6-4d1d-9772-8a671b27344d", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "1810e294-af7f-485e-8be1-1856c7b34079", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "b2d5f6bb-f433-4f2f-9ba4-9f68a8c4f120", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "e676656d-27d2-4d22-9c45-d948ad410702", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "05a4993d-60d5-4dfa-aa1a-f3213c41f875", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "d20c2590-ea7a-468d-ad83-ceb7404cb5aa", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "fb92d6e8-fdcc-4850-b71c-0c588c87e906", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "684cbc45-faf6-4f75-bd14-d7ead55a19c6", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "9f520f66-3d34-46b5-84f1-540451aada25", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "e068c23d-466e-4e6c-96ec-fddd7fe012bf", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "f21a2c86-9388-4190-8f48-eb3ecc9f117c", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 34, "novel_recall": 0.0, "qid": "ea7e4604-f65c-4009-9729-82feaf7acc66", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 43, "novel_recall": 0.0, "qid": "d9b076ee-7d47-449d-8cb7-828af43cfa14", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "1810e294-af7f-485e-8be1-1856c7b34079", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "b2d5f6bb-f433-4f2f-9ba4-9f68a8c4f120", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 6, "novel_recall": 0.8333, "qid": "e676656d-27d2-4d22-9c45-d948ad410702", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "05a4993d-60d5-4dfa-aa1a-f3213c41f875", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "d20c2590-ea7a-468d-ad83-ceb7404cb5aa", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "fb92d6e8-fdcc-4850-b71c-0c588c87e906", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "684cbc45-faf6-4f75-bd14-d7ead55a19c6", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "9f520f66-3d34-46b5-84f1-540451aada25", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "e068c23d-466e-4e6c-96ec-fddd7fe012bf", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "f21a2c86-9388-4190-8f48-eb3ecc9f117c", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 34, "novel_recall": 0.7059, "qid": "ea7e4604-f65c-4009-9729-82feaf7acc66", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 43, "novel_recall": 0.5349, "qid": "d9b076ee-7d47-449d-8cb7-828af43cfa14", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "1810e294-af7f-485e-8be1-1856c7b34079", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "b2d5f6bb-f433-4f2f-9ba4-9f68a8c4f120", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 6, "novel_recall": 0.8333, "qid": "e676656d-27d2-4d22-9c45-d948ad410702", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "05a4993d-60d5-4dfa-aa1a-f3213c41f875", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "d20c2590-ea7a-468d-ad83-ceb7404cb5aa", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "fb92d6e8-fdcc-4850-b71c-0c588c87e906", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "684cbc45-faf6-4f75-bd14-d7ead55a19c6", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "9f520f66-3d34-46b5-84f1-540451aada25", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "e068c23d-466e-4e6c-96ec-fddd7fe012bf", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "f21a2c86-9388-4190-8f48-eb3ecc9f117c", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 34, "novel_recall": 0.7059, "qid": "ea7e4604-f65c-4009-9729-82feaf7acc66", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 43, "novel_recall": 0.5349, "qid": "d9b076ee-7d47-449d-8cb7-828af43cfa14", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "1810e294-af7f-485e-8be1-1856c7b34079", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "b2d5f6bb-f433-4f2f-9ba4-9f68a8c4f120", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 6, "novel_recall": 0.8333, "qid": "e676656d-27d2-4d22-9c45-d948ad410702", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "05a4993d-60d5-4dfa-aa1a-f3213c41f875", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "d20c2590-ea7a-468d-ad83-ceb7404cb5aa", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "fb92d6e8-fdcc-4850-b71c-0c588c87e906", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "684cbc45-faf6-4f75-bd14-d7ead55a19c6", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "9f520f66-3d34-46b5-84f1-540451aada25", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "e068c23d-466e-4e6c-96ec-fddd7fe012bf", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "f21a2c86-9388-4190-8f48-eb3ecc9f117c", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 34, "novel_recall": 0.7059, "qid": "ea7e4604-f65c-4009-9729-82feaf7acc66", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 43, "novel_recall": 0.5349, "qid": "d9b076ee-7d47-449d-8cb7-828af43cfa14", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "1810e294-af7f-485e-8be1-1856c7b34079", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "b2d5f6bb-f433-4f2f-9ba4-9f68a8c4f120", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 6, "novel_recall": 0.8333, "qid": "e676656d-27d2-4d22-9c45-d948ad410702", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "05a4993d-60d5-4dfa-aa1a-f3213c41f875", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "d20c2590-ea7a-468d-ad83-ceb7404cb5aa", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "fb92d6e8-fdcc-4850-b71c-0c588c87e906", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "684cbc45-faf6-4f75-bd14-d7ead55a19c6", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "9f520f66-3d34-46b5-84f1-540451aada25", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "e068c23d-466e-4e6c-96ec-fddd7fe012bf", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "f21a2c86-9388-4190-8f48-eb3ecc9f117c", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 34, "novel_recall": 0.7059, "qid": "ea7e4604-f65c-4009-9729-82feaf7acc66", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 43, "novel_recall": 0.5349, "qid": "d9b076ee-7d47-449d-8cb7-828af43cfa14", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "1810e294-af7f-485e-8be1-1856c7b34079", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "b2d5f6bb-f433-4f2f-9ba4-9f68a8c4f120", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 6, "novel_recall": 0.8333, "qid": "e676656d-27d2-4d22-9c45-d948ad410702", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "05a4993d-60d5-4dfa-aa1a-f3213c41f875", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "d20c2590-ea7a-468d-ad83-ceb7404cb5aa", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "fb92d6e8-fdcc-4850-b71c-0c588c87e906", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "684cbc45-faf6-4f75-bd14-d7ead55a19c6", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "9f520f66-3d34-46b5-84f1-540451aada25", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "e068c23d-466e-4e6c-96ec-fddd7fe012bf", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "f21a2c86-9388-4190-8f48-eb3ecc9f117c", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 34, "novel_recall": 0.5294, "qid": "ea7e4604-f65c-4009-9729-82feaf7acc66", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "155", "full_hit": false, "n_novel": 43, "novel_recall": 0.3953, "qid": "d9b076ee-7d47-449d-8cb7-828af43cfa14", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "7843c7a8-e99f-40f2-ae92-151931f6aebc", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "ac15fbaa-0ab7-4ade-b332-2738e97f03c9", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "d947e1f3-f4c7-4268-829e-f29a458ef94a", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "820838ee-f27b-4ed8-b4c1-5b5c27e64c17", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "f1e6fa94-8c97-4b84-be91-41aae6f02dbe", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "0045dd35-6fba-46c1-a240-40689e07aaa3", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "18c8556e-53f5-4ad0-a9ab-957b9725045c", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "06cfc806-1b16-4c49-b8d3-30f07ff9d5d0", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "7e426f34-55cf-4b57-b547-713ce8070470", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "d30709fb-3680-4d96-be72-b46ac290054c", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 27, "novel_recall": 0.0, "qid": "3e7319b3-eea5-46de-8bdd-281b6f5354fd", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 23, "novel_recall": 0.0, "qid": "f50951b0-df99-460d-adf4-f4dbdc4227b4", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "7843c7a8-e99f-40f2-ae92-151931f6aebc", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "ac15fbaa-0ab7-4ade-b332-2738e97f03c9", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "d947e1f3-f4c7-4268-829e-f29a458ef94a", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "163", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "820838ee-f27b-4ed8-b4c1-5b5c27e64c17", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "f1e6fa94-8c97-4b84-be91-41aae6f02dbe", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "163", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "0045dd35-6fba-46c1-a240-40689e07aaa3", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "18c8556e-53f5-4ad0-a9ab-957b9725045c", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "06cfc806-1b16-4c49-b8d3-30f07ff9d5d0", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "7e426f34-55cf-4b57-b547-713ce8070470", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "d30709fb-3680-4d96-be72-b46ac290054c", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 27, "novel_recall": 0.5185, "qid": "3e7319b3-eea5-46de-8bdd-281b6f5354fd", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 23, "novel_recall": 0.8261, "qid": "f50951b0-df99-460d-adf4-f4dbdc4227b4", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "7843c7a8-e99f-40f2-ae92-151931f6aebc", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "ac15fbaa-0ab7-4ade-b332-2738e97f03c9", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "d947e1f3-f4c7-4268-829e-f29a458ef94a", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "163", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "820838ee-f27b-4ed8-b4c1-5b5c27e64c17", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "f1e6fa94-8c97-4b84-be91-41aae6f02dbe", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "163", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "0045dd35-6fba-46c1-a240-40689e07aaa3", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "18c8556e-53f5-4ad0-a9ab-957b9725045c", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "06cfc806-1b16-4c49-b8d3-30f07ff9d5d0", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "7e426f34-55cf-4b57-b547-713ce8070470", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "d30709fb-3680-4d96-be72-b46ac290054c", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 27, "novel_recall": 0.5185, "qid": "3e7319b3-eea5-46de-8bdd-281b6f5354fd", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 23, "novel_recall": 0.8261, "qid": "f50951b0-df99-460d-adf4-f4dbdc4227b4", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "7843c7a8-e99f-40f2-ae92-151931f6aebc", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "ac15fbaa-0ab7-4ade-b332-2738e97f03c9", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "d947e1f3-f4c7-4268-829e-f29a458ef94a", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "163", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "820838ee-f27b-4ed8-b4c1-5b5c27e64c17", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "f1e6fa94-8c97-4b84-be91-41aae6f02dbe", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "163", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "0045dd35-6fba-46c1-a240-40689e07aaa3", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "18c8556e-53f5-4ad0-a9ab-957b9725045c", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "06cfc806-1b16-4c49-b8d3-30f07ff9d5d0", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "7e426f34-55cf-4b57-b547-713ce8070470", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "d30709fb-3680-4d96-be72-b46ac290054c", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 27, "novel_recall": 0.5185, "qid": "3e7319b3-eea5-46de-8bdd-281b6f5354fd", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 23, "novel_recall": 0.8261, "qid": "f50951b0-df99-460d-adf4-f4dbdc4227b4", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "7843c7a8-e99f-40f2-ae92-151931f6aebc", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "ac15fbaa-0ab7-4ade-b332-2738e97f03c9", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "d947e1f3-f4c7-4268-829e-f29a458ef94a", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "163", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "820838ee-f27b-4ed8-b4c1-5b5c27e64c17", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "f1e6fa94-8c97-4b84-be91-41aae6f02dbe", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "163", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "0045dd35-6fba-46c1-a240-40689e07aaa3", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "18c8556e-53f5-4ad0-a9ab-957b9725045c", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "06cfc806-1b16-4c49-b8d3-30f07ff9d5d0", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "7e426f34-55cf-4b57-b547-713ce8070470", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "d30709fb-3680-4d96-be72-b46ac290054c", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 27, "novel_recall": 0.5185, "qid": "3e7319b3-eea5-46de-8bdd-281b6f5354fd", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 23, "novel_recall": 0.8261, "qid": "f50951b0-df99-460d-adf4-f4dbdc4227b4", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "7843c7a8-e99f-40f2-ae92-151931f6aebc", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "ac15fbaa-0ab7-4ade-b332-2738e97f03c9", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "d947e1f3-f4c7-4268-829e-f29a458ef94a", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "820838ee-f27b-4ed8-b4c1-5b5c27e64c17", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "f1e6fa94-8c97-4b84-be91-41aae6f02dbe", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "163", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "0045dd35-6fba-46c1-a240-40689e07aaa3", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "18c8556e-53f5-4ad0-a9ab-957b9725045c", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "06cfc806-1b16-4c49-b8d3-30f07ff9d5d0", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "7e426f34-55cf-4b57-b547-713ce8070470", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "d30709fb-3680-4d96-be72-b46ac290054c", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 27, "novel_recall": 0.2963, "qid": "3e7319b3-eea5-46de-8bdd-281b6f5354fd", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "163", "full_hit": false, "n_novel": 23, "novel_recall": 0.8261, "qid": "f50951b0-df99-460d-adf4-f4dbdc4227b4", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "7e47fb6d-4267-49a2-ad12-6113bf6c7d4e", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "6a2edcb3-0c5d-46d5-a2ca-4c3f393edb9b", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "6f1d3f00-e094-42b2-8d00-6d4f20cec77b", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "6fd67407-8ee5-4acc-a747-a21c9517c14c", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "9c8a8523-7c2f-4887-983a-323a4b88138c", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "555b2acd-2259-4059-93a3-cea017aa641d", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "c5abccaa-0f42-4925-91a4-66c6ea4c7809", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "af4723eb-6cc3-4771-b090-daca02c2f57b", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "0c117b39-f90d-484b-8dec-5a66b5c81498", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 31, "novel_recall": 0.0, "qid": "e82dfc9b-94cd-4c5c-9618-482414f3d2a4", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 34, "novel_recall": 0.0, "qid": "f0d458f9-4005-487c-b498-2bf192506b63", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 8, "novel_recall": 0.875, "qid": "7e47fb6d-4267-49a2-ad12-6113bf6c7d4e", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "6a2edcb3-0c5d-46d5-a2ca-4c3f393edb9b", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 8, "novel_recall": 0.875, "qid": "6f1d3f00-e094-42b2-8d00-6d4f20cec77b", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "6fd67407-8ee5-4acc-a747-a21c9517c14c", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "9c8a8523-7c2f-4887-983a-323a4b88138c", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "555b2acd-2259-4059-93a3-cea017aa641d", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "c5abccaa-0f42-4925-91a4-66c6ea4c7809", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "166", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "af4723eb-6cc3-4771-b090-daca02c2f57b", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "166", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "0c117b39-f90d-484b-8dec-5a66b5c81498", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 31, "novel_recall": 0.9677, "qid": "e82dfc9b-94cd-4c5c-9618-482414f3d2a4", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 34, "novel_recall": 0.7647, "qid": "f0d458f9-4005-487c-b498-2bf192506b63", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 8, "novel_recall": 0.875, "qid": "7e47fb6d-4267-49a2-ad12-6113bf6c7d4e", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "6a2edcb3-0c5d-46d5-a2ca-4c3f393edb9b", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 8, "novel_recall": 0.875, "qid": "6f1d3f00-e094-42b2-8d00-6d4f20cec77b", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "6fd67407-8ee5-4acc-a747-a21c9517c14c", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "9c8a8523-7c2f-4887-983a-323a4b88138c", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "555b2acd-2259-4059-93a3-cea017aa641d", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "c5abccaa-0f42-4925-91a4-66c6ea4c7809", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "166", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "af4723eb-6cc3-4771-b090-daca02c2f57b", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "166", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "0c117b39-f90d-484b-8dec-5a66b5c81498", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 31, "novel_recall": 0.9677, "qid": "e82dfc9b-94cd-4c5c-9618-482414f3d2a4", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 34, "novel_recall": 0.7647, "qid": "f0d458f9-4005-487c-b498-2bf192506b63", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 8, "novel_recall": 0.875, "qid": "7e47fb6d-4267-49a2-ad12-6113bf6c7d4e", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "6a2edcb3-0c5d-46d5-a2ca-4c3f393edb9b", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 8, "novel_recall": 0.875, "qid": "6f1d3f00-e094-42b2-8d00-6d4f20cec77b", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "6fd67407-8ee5-4acc-a747-a21c9517c14c", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "9c8a8523-7c2f-4887-983a-323a4b88138c", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "555b2acd-2259-4059-93a3-cea017aa641d", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "c5abccaa-0f42-4925-91a4-66c6ea4c7809", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "166", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "af4723eb-6cc3-4771-b090-daca02c2f57b", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "166", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "0c117b39-f90d-484b-8dec-5a66b5c81498", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 31, "novel_recall": 0.9677, "qid": "e82dfc9b-94cd-4c5c-9618-482414f3d2a4", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 34, "novel_recall": 0.7647, "qid": "f0d458f9-4005-487c-b498-2bf192506b63", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 8, "novel_recall": 0.875, "qid": "7e47fb6d-4267-49a2-ad12-6113bf6c7d4e", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "6a2edcb3-0c5d-46d5-a2ca-4c3f393edb9b", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 8, "novel_recall": 0.875, "qid": "6f1d3f00-e094-42b2-8d00-6d4f20cec77b", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "6fd67407-8ee5-4acc-a747-a21c9517c14c", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "9c8a8523-7c2f-4887-983a-323a4b88138c", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "555b2acd-2259-4059-93a3-cea017aa641d", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "c5abccaa-0f42-4925-91a4-66c6ea4c7809", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "166", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "af4723eb-6cc3-4771-b090-daca02c2f57b", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "166", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "0c117b39-f90d-484b-8dec-5a66b5c81498", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 31, "novel_recall": 0.9677, "qid": "e82dfc9b-94cd-4c5c-9618-482414f3d2a4", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 34, "novel_recall": 0.7647, "qid": "f0d458f9-4005-487c-b498-2bf192506b63", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "7e47fb6d-4267-49a2-ad12-6113bf6c7d4e", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "6a2edcb3-0c5d-46d5-a2ca-4c3f393edb9b", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "6f1d3f00-e094-42b2-8d00-6d4f20cec77b", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "6fd67407-8ee5-4acc-a747-a21c9517c14c", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "9c8a8523-7c2f-4887-983a-323a4b88138c", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "555b2acd-2259-4059-93a3-cea017aa641d", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "c5abccaa-0f42-4925-91a4-66c6ea4c7809", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "166", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "af4723eb-6cc3-4771-b090-daca02c2f57b", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "166", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "0c117b39-f90d-484b-8dec-5a66b5c81498", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 31, "novel_recall": 0.7419, "qid": "e82dfc9b-94cd-4c5c-9618-482414f3d2a4", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "166", "full_hit": false, "n_novel": 34, "novel_recall": 0.6765, "qid": "f0d458f9-4005-487c-b498-2bf192506b63", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "a02e57c7-4652-47e1-a328-a9bb00317ac2", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "530114fb-6597-4c26-af06-e61e3f468ac1", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "98bf10e4-ee24-46b7-8845-0e2875ef1778", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "e3e26ec0-3910-4c02-a0ec-10d24ba718d4", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "a5db3226-faf6-42a5-95ef-4c8ecfc99d10", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "676cf5e8-5f49-4f38-b41b-e45b078efa55", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "c2ae9610-bf64-463e-8d14-91a078ae6014", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "6694e1de-db31-4251-a432-6a391d7293c5", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "c571a5b7-2473-4258-8dbd-025011584645", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "bf6f526a-0e90-4764-b572-ed98ef8f8fb1", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "fca55b30-8c1e-4428-bf1f-498ceb051ec3", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 59, "novel_recall": 0.0, "qid": "84080577-6420-46ba-81bc-ee1e3b456f37", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 9, "novel_recall": 0.5556, "qid": "a02e57c7-4652-47e1-a328-a9bb00317ac2", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "530114fb-6597-4c26-af06-e61e3f468ac1", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "98bf10e4-ee24-46b7-8845-0e2875ef1778", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "e3e26ec0-3910-4c02-a0ec-10d24ba718d4", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "a5db3226-faf6-42a5-95ef-4c8ecfc99d10", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "676cf5e8-5f49-4f38-b41b-e45b078efa55", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 9, "novel_recall": 0.2222, "qid": "c2ae9610-bf64-463e-8d14-91a078ae6014", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "6694e1de-db31-4251-a432-6a391d7293c5", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "c571a5b7-2473-4258-8dbd-025011584645", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "bf6f526a-0e90-4764-b572-ed98ef8f8fb1", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 24, "novel_recall": 0.5833, "qid": "fca55b30-8c1e-4428-bf1f-498ceb051ec3", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 59, "novel_recall": 0.4746, "qid": "84080577-6420-46ba-81bc-ee1e3b456f37", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 9, "novel_recall": 0.5556, "qid": "a02e57c7-4652-47e1-a328-a9bb00317ac2", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "530114fb-6597-4c26-af06-e61e3f468ac1", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "98bf10e4-ee24-46b7-8845-0e2875ef1778", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "e3e26ec0-3910-4c02-a0ec-10d24ba718d4", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "a5db3226-faf6-42a5-95ef-4c8ecfc99d10", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "676cf5e8-5f49-4f38-b41b-e45b078efa55", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 9, "novel_recall": 0.2222, "qid": "c2ae9610-bf64-463e-8d14-91a078ae6014", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "6694e1de-db31-4251-a432-6a391d7293c5", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "c571a5b7-2473-4258-8dbd-025011584645", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "bf6f526a-0e90-4764-b572-ed98ef8f8fb1", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 24, "novel_recall": 0.5833, "qid": "fca55b30-8c1e-4428-bf1f-498ceb051ec3", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 59, "novel_recall": 0.4746, "qid": "84080577-6420-46ba-81bc-ee1e3b456f37", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 9, "novel_recall": 0.5556, "qid": "a02e57c7-4652-47e1-a328-a9bb00317ac2", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "530114fb-6597-4c26-af06-e61e3f468ac1", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "98bf10e4-ee24-46b7-8845-0e2875ef1778", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "e3e26ec0-3910-4c02-a0ec-10d24ba718d4", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "a5db3226-faf6-42a5-95ef-4c8ecfc99d10", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "676cf5e8-5f49-4f38-b41b-e45b078efa55", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 9, "novel_recall": 0.2222, "qid": "c2ae9610-bf64-463e-8d14-91a078ae6014", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "6694e1de-db31-4251-a432-6a391d7293c5", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "c571a5b7-2473-4258-8dbd-025011584645", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "bf6f526a-0e90-4764-b572-ed98ef8f8fb1", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 24, "novel_recall": 0.5, "qid": "fca55b30-8c1e-4428-bf1f-498ceb051ec3", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 59, "novel_recall": 0.4576, "qid": "84080577-6420-46ba-81bc-ee1e3b456f37", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 9, "novel_recall": 0.5556, "qid": "a02e57c7-4652-47e1-a328-a9bb00317ac2", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "530114fb-6597-4c26-af06-e61e3f468ac1", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "98bf10e4-ee24-46b7-8845-0e2875ef1778", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "e3e26ec0-3910-4c02-a0ec-10d24ba718d4", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "a5db3226-faf6-42a5-95ef-4c8ecfc99d10", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "676cf5e8-5f49-4f38-b41b-e45b078efa55", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 9, "novel_recall": 0.2222, "qid": "c2ae9610-bf64-463e-8d14-91a078ae6014", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "6694e1de-db31-4251-a432-6a391d7293c5", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "c571a5b7-2473-4258-8dbd-025011584645", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "bf6f526a-0e90-4764-b572-ed98ef8f8fb1", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 24, "novel_recall": 0.5, "qid": "fca55b30-8c1e-4428-bf1f-498ceb051ec3", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 59, "novel_recall": 0.4576, "qid": "84080577-6420-46ba-81bc-ee1e3b456f37", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 9, "novel_recall": 0.1111, "qid": "a02e57c7-4652-47e1-a328-a9bb00317ac2", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "530114fb-6597-4c26-af06-e61e3f468ac1", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "98bf10e4-ee24-46b7-8845-0e2875ef1778", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "e3e26ec0-3910-4c02-a0ec-10d24ba718d4", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 8, "novel_recall": 0.125, "qid": "a5db3226-faf6-42a5-95ef-4c8ecfc99d10", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "676cf5e8-5f49-4f38-b41b-e45b078efa55", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 9, "novel_recall": 0.1111, "qid": "c2ae9610-bf64-463e-8d14-91a078ae6014", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "6694e1de-db31-4251-a432-6a391d7293c5", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "171", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "c571a5b7-2473-4258-8dbd-025011584645", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "bf6f526a-0e90-4764-b572-ed98ef8f8fb1", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 24, "novel_recall": 0.1667, "qid": "fca55b30-8c1e-4428-bf1f-498ceb051ec3", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "171", "full_hit": false, "n_novel": 59, "novel_recall": 0.1525, "qid": "84080577-6420-46ba-81bc-ee1e3b456f37", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "6a5dc563-8ba5-47a6-b813-20151878c4b0", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "c8efd99c-bf2f-4d37-baa5-5114332ca71c", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "b7231bd5-22b4-4357-babb-63d98067a15a", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "7647a14f-2ba8-4e07-9c4a-770b0664c5ae", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "a6a065ff-c5e4-47ca-9f36-4de65ad57940", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "ce1ff2c3-8d65-4a31-af26-96ceeaf9272c", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "092aafad-a3ef-4884-b228-f2d7cd01ddde", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "13f1c38b-bfa4-42ce-948d-64af36fd20cd", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "9d604156-85ea-4c43-b669-262dec064739", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "add8b58c-d106-40c2-b474-93dbf5d0b758", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 43, "novel_recall": 0.0, "qid": "a4482165-9e7a-44a3-b22f-586bf44b9901", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 42, "novel_recall": 0.0, "qid": "9c2f7a02-bc68-4ed6-9dee-e87cc508ea34", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "6a5dc563-8ba5-47a6-b813-20151878c4b0", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "c8efd99c-bf2f-4d37-baa5-5114332ca71c", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "b7231bd5-22b4-4357-babb-63d98067a15a", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "7647a14f-2ba8-4e07-9c4a-770b0664c5ae", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 7, "novel_recall": 0.1429, "qid": "a6a065ff-c5e4-47ca-9f36-4de65ad57940", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "ce1ff2c3-8d65-4a31-af26-96ceeaf9272c", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "092aafad-a3ef-4884-b228-f2d7cd01ddde", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "174", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "13f1c38b-bfa4-42ce-948d-64af36fd20cd", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "9d604156-85ea-4c43-b669-262dec064739", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "174", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "add8b58c-d106-40c2-b474-93dbf5d0b758", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 43, "novel_recall": 0.5814, "qid": "a4482165-9e7a-44a3-b22f-586bf44b9901", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 42, "novel_recall": 0.6905, "qid": "9c2f7a02-bc68-4ed6-9dee-e87cc508ea34", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "6a5dc563-8ba5-47a6-b813-20151878c4b0", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "c8efd99c-bf2f-4d37-baa5-5114332ca71c", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "b7231bd5-22b4-4357-babb-63d98067a15a", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "7647a14f-2ba8-4e07-9c4a-770b0664c5ae", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 7, "novel_recall": 0.1429, "qid": "a6a065ff-c5e4-47ca-9f36-4de65ad57940", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "174", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "ce1ff2c3-8d65-4a31-af26-96ceeaf9272c", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "092aafad-a3ef-4884-b228-f2d7cd01ddde", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "13f1c38b-bfa4-42ce-948d-64af36fd20cd", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "9d604156-85ea-4c43-b669-262dec064739", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "174", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "add8b58c-d106-40c2-b474-93dbf5d0b758", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 43, "novel_recall": 0.5814, "qid": "a4482165-9e7a-44a3-b22f-586bf44b9901", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 42, "novel_recall": 0.6429, "qid": "9c2f7a02-bc68-4ed6-9dee-e87cc508ea34", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "6a5dc563-8ba5-47a6-b813-20151878c4b0", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "c8efd99c-bf2f-4d37-baa5-5114332ca71c", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "b7231bd5-22b4-4357-babb-63d98067a15a", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "7647a14f-2ba8-4e07-9c4a-770b0664c5ae", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 7, "novel_recall": 0.1429, "qid": "a6a065ff-c5e4-47ca-9f36-4de65ad57940", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "174", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "ce1ff2c3-8d65-4a31-af26-96ceeaf9272c", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "092aafad-a3ef-4884-b228-f2d7cd01ddde", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "13f1c38b-bfa4-42ce-948d-64af36fd20cd", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "9d604156-85ea-4c43-b669-262dec064739", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "174", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "add8b58c-d106-40c2-b474-93dbf5d0b758", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 43, "novel_recall": 0.5814, "qid": "a4482165-9e7a-44a3-b22f-586bf44b9901", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 42, "novel_recall": 0.6429, "qid": "9c2f7a02-bc68-4ed6-9dee-e87cc508ea34", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "6a5dc563-8ba5-47a6-b813-20151878c4b0", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "c8efd99c-bf2f-4d37-baa5-5114332ca71c", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "b7231bd5-22b4-4357-babb-63d98067a15a", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "7647a14f-2ba8-4e07-9c4a-770b0664c5ae", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 7, "novel_recall": 0.1429, "qid": "a6a065ff-c5e4-47ca-9f36-4de65ad57940", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "174", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "ce1ff2c3-8d65-4a31-af26-96ceeaf9272c", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "092aafad-a3ef-4884-b228-f2d7cd01ddde", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "13f1c38b-bfa4-42ce-948d-64af36fd20cd", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "9d604156-85ea-4c43-b669-262dec064739", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "174", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "add8b58c-d106-40c2-b474-93dbf5d0b758", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 43, "novel_recall": 0.5814, "qid": "a4482165-9e7a-44a3-b22f-586bf44b9901", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 42, "novel_recall": 0.6429, "qid": "9c2f7a02-bc68-4ed6-9dee-e87cc508ea34", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "6a5dc563-8ba5-47a6-b813-20151878c4b0", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "c8efd99c-bf2f-4d37-baa5-5114332ca71c", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "b7231bd5-22b4-4357-babb-63d98067a15a", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "7647a14f-2ba8-4e07-9c4a-770b0664c5ae", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 7, "novel_recall": 0.1429, "qid": "a6a065ff-c5e4-47ca-9f36-4de65ad57940", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "ce1ff2c3-8d65-4a31-af26-96ceeaf9272c", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "092aafad-a3ef-4884-b228-f2d7cd01ddde", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "13f1c38b-bfa4-42ce-948d-64af36fd20cd", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "9d604156-85ea-4c43-b669-262dec064739", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "add8b58c-d106-40c2-b474-93dbf5d0b758", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 43, "novel_recall": 0.3488, "qid": "a4482165-9e7a-44a3-b22f-586bf44b9901", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "174", "full_hit": false, "n_novel": 42, "novel_recall": 0.6429, "qid": "9c2f7a02-bc68-4ed6-9dee-e87cc508ea34", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "493202d3-4ff9-4cd1-b70d-eb0b93133a50", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "17b52317-9e3a-4e11-9b6b-256a7b5b6eb8", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "bd5d9977-8454-4229-ae72-2e90cf73f3a6", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "12d1b904-353d-4fce-aafe-2f22805e7c67", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "4bde4534-f396-45c9-9a20-33d015cd448b", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "43882b27-faa1-4840-aaed-acee71a00c8a", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "5208872c-376f-4e47-9812-eac818b484e5", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "0ba3d8f5-f4c7-4acc-9f3a-556278e808bc", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "34f75353-7fd1-4285-ab91-64a8278ddbc3", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "570c451a-a908-4bcc-9acd-5080ee4bc9d6", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 35, "novel_recall": 0.0, "qid": "e2234099-a770-427f-a15a-8a981a316296", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 43, "novel_recall": 0.0, "qid": "942a32bd-b395-4643-a22c-661360f9f8a4", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "493202d3-4ff9-4cd1-b70d-eb0b93133a50", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "17b52317-9e3a-4e11-9b6b-256a7b5b6eb8", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "bd5d9977-8454-4229-ae72-2e90cf73f3a6", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "12d1b904-353d-4fce-aafe-2f22805e7c67", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 11, "novel_recall": 0.7273, "qid": "4bde4534-f396-45c9-9a20-33d015cd448b", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 9, "novel_recall": 0.8889, "qid": "43882b27-faa1-4840-aaed-acee71a00c8a", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "5208872c-376f-4e47-9812-eac818b484e5", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "0ba3d8f5-f4c7-4acc-9f3a-556278e808bc", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "34f75353-7fd1-4285-ab91-64a8278ddbc3", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "570c451a-a908-4bcc-9acd-5080ee4bc9d6", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 35, "novel_recall": 0.6857, "qid": "e2234099-a770-427f-a15a-8a981a316296", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 43, "novel_recall": 0.7209, "qid": "942a32bd-b395-4643-a22c-661360f9f8a4", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "493202d3-4ff9-4cd1-b70d-eb0b93133a50", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "17b52317-9e3a-4e11-9b6b-256a7b5b6eb8", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "bd5d9977-8454-4229-ae72-2e90cf73f3a6", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "12d1b904-353d-4fce-aafe-2f22805e7c67", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 11, "novel_recall": 0.7273, "qid": "4bde4534-f396-45c9-9a20-33d015cd448b", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "43882b27-faa1-4840-aaed-acee71a00c8a", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "5208872c-376f-4e47-9812-eac818b484e5", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "0ba3d8f5-f4c7-4acc-9f3a-556278e808bc", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "34f75353-7fd1-4285-ab91-64a8278ddbc3", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "570c451a-a908-4bcc-9acd-5080ee4bc9d6", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 35, "novel_recall": 0.6857, "qid": "e2234099-a770-427f-a15a-8a981a316296", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 43, "novel_recall": 0.7209, "qid": "942a32bd-b395-4643-a22c-661360f9f8a4", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "493202d3-4ff9-4cd1-b70d-eb0b93133a50", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "17b52317-9e3a-4e11-9b6b-256a7b5b6eb8", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "bd5d9977-8454-4229-ae72-2e90cf73f3a6", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "12d1b904-353d-4fce-aafe-2f22805e7c67", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 11, "novel_recall": 0.7273, "qid": "4bde4534-f396-45c9-9a20-33d015cd448b", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "43882b27-faa1-4840-aaed-acee71a00c8a", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "5208872c-376f-4e47-9812-eac818b484e5", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "0ba3d8f5-f4c7-4acc-9f3a-556278e808bc", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "34f75353-7fd1-4285-ab91-64a8278ddbc3", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "570c451a-a908-4bcc-9acd-5080ee4bc9d6", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 35, "novel_recall": 0.6857, "qid": "e2234099-a770-427f-a15a-8a981a316296", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 43, "novel_recall": 0.7209, "qid": "942a32bd-b395-4643-a22c-661360f9f8a4", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "493202d3-4ff9-4cd1-b70d-eb0b93133a50", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "17b52317-9e3a-4e11-9b6b-256a7b5b6eb8", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "bd5d9977-8454-4229-ae72-2e90cf73f3a6", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "12d1b904-353d-4fce-aafe-2f22805e7c67", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 11, "novel_recall": 0.7273, "qid": "4bde4534-f396-45c9-9a20-33d015cd448b", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "43882b27-faa1-4840-aaed-acee71a00c8a", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "5208872c-376f-4e47-9812-eac818b484e5", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "0ba3d8f5-f4c7-4acc-9f3a-556278e808bc", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "34f75353-7fd1-4285-ab91-64a8278ddbc3", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "570c451a-a908-4bcc-9acd-5080ee4bc9d6", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 35, "novel_recall": 0.6857, "qid": "e2234099-a770-427f-a15a-8a981a316296", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 43, "novel_recall": 0.7209, "qid": "942a32bd-b395-4643-a22c-661360f9f8a4", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "493202d3-4ff9-4cd1-b70d-eb0b93133a50", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "17b52317-9e3a-4e11-9b6b-256a7b5b6eb8", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "bd5d9977-8454-4229-ae72-2e90cf73f3a6", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "12d1b904-353d-4fce-aafe-2f22805e7c67", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 11, "novel_recall": 0.8182, "qid": "4bde4534-f396-45c9-9a20-33d015cd448b", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "43882b27-faa1-4840-aaed-acee71a00c8a", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "5208872c-376f-4e47-9812-eac818b484e5", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 9, "novel_recall": 0.5556, "qid": "0ba3d8f5-f4c7-4acc-9f3a-556278e808bc", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "175", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "34f75353-7fd1-4285-ab91-64a8278ddbc3", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "570c451a-a908-4bcc-9acd-5080ee4bc9d6", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 35, "novel_recall": 0.6286, "qid": "e2234099-a770-427f-a15a-8a981a316296", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "175", "full_hit": false, "n_novel": 43, "novel_recall": 0.7209, "qid": "942a32bd-b395-4643-a22c-661360f9f8a4", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "79a2ab3d-7640-41c4-9900-1f29dfd8fd81", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "4ad63014-cbd3-4363-8dab-76477df86302", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "2f94ea9c-1574-4671-87da-f87d9b654f3d", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "0c499f10-0b55-41eb-b766-b1f9ddf06fb2", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 19, "novel_recall": 0.0, "qid": "4af176e2-fea2-45e5-af7c-520214507f33", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "8309c673-76bb-49af-b224-68942359e24d", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "75829a79-fe93-4f22-80c6-6d8195cf02aa", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "4a14d100-a7df-4b0c-914f-98da38d7453b", "type": "C"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "02070dcf-4566-4d6f-822c-28e5d3e7d90f", "type": "B"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "3f9865e3-d391-4c10-840c-9c51bf669aaf", "type": "A"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 38, "novel_recall": 0.0, "qid": "bdc479ce-9f75-4a07-90a0-662e4f4e66de", "type": "D"}, {"arm": "M0_none", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 46, "novel_recall": 0.0, "qid": "32813da2-4cd4-4258-95c8-abd60edece01", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.125, "qid": "79a2ab3d-7640-41c4-9900-1f29dfd8fd81", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "4ad63014-cbd3-4363-8dab-76477df86302", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "2f94ea9c-1574-4671-87da-f87d9b654f3d", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.7143, "qid": "0c499f10-0b55-41eb-b766-b1f9ddf06fb2", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 19, "novel_recall": 0.7368, "qid": "4af176e2-fea2-45e5-af7c-520214507f33", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "8309c673-76bb-49af-b224-68942359e24d", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "75829a79-fe93-4f22-80c6-6d8195cf02aa", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "4a14d100-a7df-4b0c-914f-98da38d7453b", "type": "C"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "02070dcf-4566-4d6f-822c-28e5d3e7d90f", "type": "B"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "3f9865e3-d391-4c10-840c-9c51bf669aaf", "type": "A"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 38, "novel_recall": 0.7632, "qid": "bdc479ce-9f75-4a07-90a0-662e4f4e66de", "type": "D"}, {"arm": "M1_naive", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 46, "novel_recall": 0.4783, "qid": "32813da2-4cd4-4258-95c8-abd60edece01", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.125, "qid": "79a2ab3d-7640-41c4-9900-1f29dfd8fd81", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "4ad63014-cbd3-4363-8dab-76477df86302", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "2f94ea9c-1574-4671-87da-f87d9b654f3d", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.7143, "qid": "0c499f10-0b55-41eb-b766-b1f9ddf06fb2", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 19, "novel_recall": 0.7368, "qid": "4af176e2-fea2-45e5-af7c-520214507f33", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "8309c673-76bb-49af-b224-68942359e24d", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "75829a79-fe93-4f22-80c6-6d8195cf02aa", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "4a14d100-a7df-4b0c-914f-98da38d7453b", "type": "C"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "02070dcf-4566-4d6f-822c-28e5d3e7d90f", "type": "B"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "3f9865e3-d391-4c10-840c-9c51bf669aaf", "type": "A"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 38, "novel_recall": 0.7632, "qid": "bdc479ce-9f75-4a07-90a0-662e4f4e66de", "type": "D"}, {"arm": "M2_evidence", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 46, "novel_recall": 0.4783, "qid": "32813da2-4cd4-4258-95c8-abd60edece01", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.125, "qid": "79a2ab3d-7640-41c4-9900-1f29dfd8fd81", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "4ad63014-cbd3-4363-8dab-76477df86302", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "2f94ea9c-1574-4671-87da-f87d9b654f3d", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.7143, "qid": "0c499f10-0b55-41eb-b766-b1f9ddf06fb2", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 19, "novel_recall": 0.7368, "qid": "4af176e2-fea2-45e5-af7c-520214507f33", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "8309c673-76bb-49af-b224-68942359e24d", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "75829a79-fe93-4f22-80c6-6d8195cf02aa", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "4a14d100-a7df-4b0c-914f-98da38d7453b", "type": "C"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "02070dcf-4566-4d6f-822c-28e5d3e7d90f", "type": "B"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "3f9865e3-d391-4c10-840c-9c51bf669aaf", "type": "A"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 38, "novel_recall": 0.7632, "qid": "bdc479ce-9f75-4a07-90a0-662e4f4e66de", "type": "D"}, {"arm": "M3_causal", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 46, "novel_recall": 0.4783, "qid": "32813da2-4cd4-4258-95c8-abd60edece01", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.125, "qid": "79a2ab3d-7640-41c4-9900-1f29dfd8fd81", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "4ad63014-cbd3-4363-8dab-76477df86302", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "2f94ea9c-1574-4671-87da-f87d9b654f3d", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.7143, "qid": "0c499f10-0b55-41eb-b766-b1f9ddf06fb2", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 19, "novel_recall": 0.7368, "qid": "4af176e2-fea2-45e5-af7c-520214507f33", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "8309c673-76bb-49af-b224-68942359e24d", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "75829a79-fe93-4f22-80c6-6d8195cf02aa", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "4a14d100-a7df-4b0c-914f-98da38d7453b", "type": "C"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "02070dcf-4566-4d6f-822c-28e5d3e7d90f", "type": "B"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "3f9865e3-d391-4c10-840c-9c51bf669aaf", "type": "A"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 38, "novel_recall": 0.7632, "qid": "bdc479ce-9f75-4a07-90a0-662e4f4e66de", "type": "D"}, {"arm": "M4_validated", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 46, "novel_recall": 0.4783, "qid": "32813da2-4cd4-4258-95c8-abd60edece01", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "79a2ab3d-7640-41c4-9900-1f29dfd8fd81", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "4ad63014-cbd3-4363-8dab-76477df86302", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "2f94ea9c-1574-4671-87da-f87d9b654f3d", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "0c499f10-0b55-41eb-b766-b1f9ddf06fb2", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 19, "novel_recall": 0.4737, "qid": "4af176e2-fea2-45e5-af7c-520214507f33", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 7, "novel_recall": 0.4286, "qid": "8309c673-76bb-49af-b224-68942359e24d", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "75829a79-fe93-4f22-80c6-6d8195cf02aa", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "4a14d100-a7df-4b0c-914f-98da38d7453b", "type": "C"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "02070dcf-4566-4d6f-822c-28e5d3e7d90f", "type": "B"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "3f9865e3-d391-4c10-840c-9c51bf669aaf", "type": "A"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 38, "novel_recall": 0.4737, "qid": "bdc479ce-9f75-4a07-90a0-662e4f4e66de", "type": "D"}, {"arm": "R_recent", "domain": "SOFTWARE", "episode": "176", "full_hit": false, "n_novel": 46, "novel_recall": 0.3913, "qid": "32813da2-4cd4-4258-95c8-abd60edece01", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "41611ff3-6b23-4086-bdc5-940f659b5e10", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "359be10e-f56f-4c0b-a867-53f2baf359cf", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "a96f87ef-7b58-475c-bfc1-97ca86b05b84", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "7ccbe1de-265d-49b7-8bac-65be98e52d3f", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 14, "novel_recall": 0.0, "qid": "6617bf63-81f5-4362-840f-cd33743e07f4", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "6eae2f68-e1ee-4b2d-bdf2-18ef03a55e04", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "93c986de-df73-41d7-ae52-1af2fd94c0c2", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "3d0e1c9d-1ef1-4791-a1d6-e5353c89ad14", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "ec9680dc-dc65-4847-8e50-c2ac678e4563", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "06cb9623-c40d-4f39-a7e9-7c0b26b4408c", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "305aa4d5-25eb-45d0-b6ac-84258b9bb61b", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "1dbb2ecb-583b-49ff-b74d-7ca27fba4e7e", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "41611ff3-6b23-4086-bdc5-940f659b5e10", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 15, "novel_recall": 0.2, "qid": "359be10e-f56f-4c0b-a867-53f2baf359cf", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "a96f87ef-7b58-475c-bfc1-97ca86b05b84", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "177", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "7ccbe1de-265d-49b7-8bac-65be98e52d3f", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 14, "novel_recall": 0.3571, "qid": "6617bf63-81f5-4362-840f-cd33743e07f4", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 10, "novel_recall": 0.7, "qid": "6eae2f68-e1ee-4b2d-bdf2-18ef03a55e04", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 12, "novel_recall": 0.0833, "qid": "93c986de-df73-41d7-ae52-1af2fd94c0c2", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "3d0e1c9d-1ef1-4791-a1d6-e5353c89ad14", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 8, "novel_recall": 0.125, "qid": "ec9680dc-dc65-4847-8e50-c2ac678e4563", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 13, "novel_recall": 0.6923, "qid": "06cb9623-c40d-4f39-a7e9-7c0b26b4408c", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "305aa4d5-25eb-45d0-b6ac-84258b9bb61b", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "1dbb2ecb-583b-49ff-b74d-7ca27fba4e7e", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.5, "qid": "41611ff3-6b23-4086-bdc5-940f659b5e10", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 15, "novel_recall": 0.2, "qid": "359be10e-f56f-4c0b-a867-53f2baf359cf", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "a96f87ef-7b58-475c-bfc1-97ca86b05b84", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "177", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "7ccbe1de-265d-49b7-8bac-65be98e52d3f", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 14, "novel_recall": 0.3571, "qid": "6617bf63-81f5-4362-840f-cd33743e07f4", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 10, "novel_recall": 0.7, "qid": "6eae2f68-e1ee-4b2d-bdf2-18ef03a55e04", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 12, "novel_recall": 0.0833, "qid": "93c986de-df73-41d7-ae52-1af2fd94c0c2", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "3d0e1c9d-1ef1-4791-a1d6-e5353c89ad14", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 8, "novel_recall": 0.125, "qid": "ec9680dc-dc65-4847-8e50-c2ac678e4563", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 13, "novel_recall": 0.6923, "qid": "06cb9623-c40d-4f39-a7e9-7c0b26b4408c", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "305aa4d5-25eb-45d0-b6ac-84258b9bb61b", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 4, "novel_recall": 0.75, "qid": "1dbb2ecb-583b-49ff-b74d-7ca27fba4e7e", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "41611ff3-6b23-4086-bdc5-940f659b5e10", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 15, "novel_recall": 0.0667, "qid": "359be10e-f56f-4c0b-a867-53f2baf359cf", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "a96f87ef-7b58-475c-bfc1-97ca86b05b84", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "7ccbe1de-265d-49b7-8bac-65be98e52d3f", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 14, "novel_recall": 0.1429, "qid": "6617bf63-81f5-4362-840f-cd33743e07f4", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 10, "novel_recall": 0.2, "qid": "6eae2f68-e1ee-4b2d-bdf2-18ef03a55e04", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 12, "novel_recall": 0.0833, "qid": "93c986de-df73-41d7-ae52-1af2fd94c0c2", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "3d0e1c9d-1ef1-4791-a1d6-e5353c89ad14", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "ec9680dc-dc65-4847-8e50-c2ac678e4563", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 13, "novel_recall": 0.4615, "qid": "06cb9623-c40d-4f39-a7e9-7c0b26b4408c", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "305aa4d5-25eb-45d0-b6ac-84258b9bb61b", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "1dbb2ecb-583b-49ff-b74d-7ca27fba4e7e", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "41611ff3-6b23-4086-bdc5-940f659b5e10", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 15, "novel_recall": 0.0667, "qid": "359be10e-f56f-4c0b-a867-53f2baf359cf", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "a96f87ef-7b58-475c-bfc1-97ca86b05b84", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "7ccbe1de-265d-49b7-8bac-65be98e52d3f", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 14, "novel_recall": 0.1429, "qid": "6617bf63-81f5-4362-840f-cd33743e07f4", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 10, "novel_recall": 0.2, "qid": "6eae2f68-e1ee-4b2d-bdf2-18ef03a55e04", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 12, "novel_recall": 0.0833, "qid": "93c986de-df73-41d7-ae52-1af2fd94c0c2", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.1667, "qid": "3d0e1c9d-1ef1-4791-a1d6-e5353c89ad14", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "ec9680dc-dc65-4847-8e50-c2ac678e4563", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 13, "novel_recall": 0.4615, "qid": "06cb9623-c40d-4f39-a7e9-7c0b26b4408c", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "305aa4d5-25eb-45d0-b6ac-84258b9bb61b", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "1dbb2ecb-583b-49ff-b74d-7ca27fba4e7e", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "41611ff3-6b23-4086-bdc5-940f659b5e10", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 15, "novel_recall": 0.2667, "qid": "359be10e-f56f-4c0b-a867-53f2baf359cf", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "a96f87ef-7b58-475c-bfc1-97ca86b05b84", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "177", "full_hit": true, "n_novel": 6, "novel_recall": 1.0, "qid": "7ccbe1de-265d-49b7-8bac-65be98e52d3f", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 14, "novel_recall": 0.3571, "qid": "6617bf63-81f5-4362-840f-cd33743e07f4", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "6eae2f68-e1ee-4b2d-bdf2-18ef03a55e04", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 12, "novel_recall": 0.1667, "qid": "93c986de-df73-41d7-ae52-1af2fd94c0c2", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "3d0e1c9d-1ef1-4791-a1d6-e5353c89ad14", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "ec9680dc-dc65-4847-8e50-c2ac678e4563", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 13, "novel_recall": 0.6923, "qid": "06cb9623-c40d-4f39-a7e9-7c0b26b4408c", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "177", "full_hit": false, "n_novel": 13, "novel_recall": 0.2308, "qid": "305aa4d5-25eb-45d0-b6ac-84258b9bb61b", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "177", "full_hit": true, "n_novel": 4, "novel_recall": 1.0, "qid": "1dbb2ecb-583b-49ff-b74d-7ca27fba4e7e", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 30, "novel_recall": 0.0, "qid": "7a4800f6-9680-4a6d-ab9c-59d41f0f6bc0", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 18, "novel_recall": 0.0, "qid": "f1e61956-3736-43fa-ad69-b6cc79ec1f38", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 25, "novel_recall": 0.0, "qid": "36ef0891-761a-43d8-a488-76afbdd70881", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "5ddbf86a-b6d3-4cdf-9ba9-3a39564889dc", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 27, "novel_recall": 0.0, "qid": "d292bd83-3c69-4dd3-bf5a-90030fb8a608", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "eaec3128-5a9f-4368-a6e2-4d99e96ccc8e", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 20, "novel_recall": 0.0, "qid": "61318761-708a-4c59-a261-43f781ae5847", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "1d001a38-5c73-4a20-8dca-416d1047879c", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "a2053e04-251c-4e9a-bfad-897b3690afd1", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "9ddd6216-2272-4cc1-aab9-a2f21f664675", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 19, "novel_recall": 0.0, "qid": "3067ead9-e82d-4a99-8578-6708c52448e7", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 25, "novel_recall": 0.0, "qid": "728d5ee3-36af-456c-8ee2-26ef43c0e0ce", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 30, "novel_recall": 0.3667, "qid": "7a4800f6-9680-4a6d-ab9c-59d41f0f6bc0", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 18, "novel_recall": 0.3333, "qid": "f1e61956-3736-43fa-ad69-b6cc79ec1f38", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 25, "novel_recall": 0.68, "qid": "36ef0891-761a-43d8-a488-76afbdd70881", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 24, "novel_recall": 0.375, "qid": "5ddbf86a-b6d3-4cdf-9ba9-3a39564889dc", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 27, "novel_recall": 0.3704, "qid": "d292bd83-3c69-4dd3-bf5a-90030fb8a608", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 12, "novel_recall": 0.25, "qid": "eaec3128-5a9f-4368-a6e2-4d99e96ccc8e", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 20, "novel_recall": 0.4, "qid": "61318761-708a-4c59-a261-43f781ae5847", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 15, "novel_recall": 0.5333, "qid": "1d001a38-5c73-4a20-8dca-416d1047879c", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "a2053e04-251c-4e9a-bfad-897b3690afd1", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 15, "novel_recall": 0.3333, "qid": "9ddd6216-2272-4cc1-aab9-a2f21f664675", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 19, "novel_recall": 0.4211, "qid": "3067ead9-e82d-4a99-8578-6708c52448e7", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 25, "novel_recall": 0.24, "qid": "728d5ee3-36af-456c-8ee2-26ef43c0e0ce", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 30, "novel_recall": 0.3667, "qid": "7a4800f6-9680-4a6d-ab9c-59d41f0f6bc0", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 18, "novel_recall": 0.3333, "qid": "f1e61956-3736-43fa-ad69-b6cc79ec1f38", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 25, "novel_recall": 0.68, "qid": "36ef0891-761a-43d8-a488-76afbdd70881", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 24, "novel_recall": 0.375, "qid": "5ddbf86a-b6d3-4cdf-9ba9-3a39564889dc", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 27, "novel_recall": 0.3704, "qid": "d292bd83-3c69-4dd3-bf5a-90030fb8a608", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 12, "novel_recall": 0.25, "qid": "eaec3128-5a9f-4368-a6e2-4d99e96ccc8e", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 20, "novel_recall": 0.4, "qid": "61318761-708a-4c59-a261-43f781ae5847", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 15, "novel_recall": 0.5333, "qid": "1d001a38-5c73-4a20-8dca-416d1047879c", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "a2053e04-251c-4e9a-bfad-897b3690afd1", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 15, "novel_recall": 0.3333, "qid": "9ddd6216-2272-4cc1-aab9-a2f21f664675", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 19, "novel_recall": 0.4211, "qid": "3067ead9-e82d-4a99-8578-6708c52448e7", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 25, "novel_recall": 0.24, "qid": "728d5ee3-36af-456c-8ee2-26ef43c0e0ce", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 30, "novel_recall": 0.3667, "qid": "7a4800f6-9680-4a6d-ab9c-59d41f0f6bc0", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 18, "novel_recall": 0.3333, "qid": "f1e61956-3736-43fa-ad69-b6cc79ec1f38", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 25, "novel_recall": 0.4, "qid": "36ef0891-761a-43d8-a488-76afbdd70881", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 24, "novel_recall": 0.375, "qid": "5ddbf86a-b6d3-4cdf-9ba9-3a39564889dc", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 27, "novel_recall": 0.3704, "qid": "d292bd83-3c69-4dd3-bf5a-90030fb8a608", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 12, "novel_recall": 0.25, "qid": "eaec3128-5a9f-4368-a6e2-4d99e96ccc8e", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 20, "novel_recall": 0.4, "qid": "61318761-708a-4c59-a261-43f781ae5847", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 15, "novel_recall": 0.4667, "qid": "1d001a38-5c73-4a20-8dca-416d1047879c", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "a2053e04-251c-4e9a-bfad-897b3690afd1", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 15, "novel_recall": 0.3333, "qid": "9ddd6216-2272-4cc1-aab9-a2f21f664675", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 19, "novel_recall": 0.4211, "qid": "3067ead9-e82d-4a99-8578-6708c52448e7", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 25, "novel_recall": 0.24, "qid": "728d5ee3-36af-456c-8ee2-26ef43c0e0ce", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 30, "novel_recall": 0.3667, "qid": "7a4800f6-9680-4a6d-ab9c-59d41f0f6bc0", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 18, "novel_recall": 0.3333, "qid": "f1e61956-3736-43fa-ad69-b6cc79ec1f38", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 25, "novel_recall": 0.4, "qid": "36ef0891-761a-43d8-a488-76afbdd70881", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 24, "novel_recall": 0.375, "qid": "5ddbf86a-b6d3-4cdf-9ba9-3a39564889dc", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 27, "novel_recall": 0.3704, "qid": "d292bd83-3c69-4dd3-bf5a-90030fb8a608", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 12, "novel_recall": 0.25, "qid": "eaec3128-5a9f-4368-a6e2-4d99e96ccc8e", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 20, "novel_recall": 0.4, "qid": "61318761-708a-4c59-a261-43f781ae5847", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 15, "novel_recall": 0.4667, "qid": "1d001a38-5c73-4a20-8dca-416d1047879c", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "a2053e04-251c-4e9a-bfad-897b3690afd1", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 15, "novel_recall": 0.3333, "qid": "9ddd6216-2272-4cc1-aab9-a2f21f664675", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 19, "novel_recall": 0.4211, "qid": "3067ead9-e82d-4a99-8578-6708c52448e7", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 25, "novel_recall": 0.24, "qid": "728d5ee3-36af-456c-8ee2-26ef43c0e0ce", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 30, "novel_recall": 0.3333, "qid": "7a4800f6-9680-4a6d-ab9c-59d41f0f6bc0", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 18, "novel_recall": 0.1667, "qid": "f1e61956-3736-43fa-ad69-b6cc79ec1f38", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 25, "novel_recall": 0.16, "qid": "36ef0891-761a-43d8-a488-76afbdd70881", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 24, "novel_recall": 0.3333, "qid": "5ddbf86a-b6d3-4cdf-9ba9-3a39564889dc", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 27, "novel_recall": 0.2963, "qid": "d292bd83-3c69-4dd3-bf5a-90030fb8a608", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 12, "novel_recall": 0.25, "qid": "eaec3128-5a9f-4368-a6e2-4d99e96ccc8e", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 20, "novel_recall": 0.15, "qid": "61318761-708a-4c59-a261-43f781ae5847", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 15, "novel_recall": 0.6, "qid": "1d001a38-5c73-4a20-8dca-416d1047879c", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "a2053e04-251c-4e9a-bfad-897b3690afd1", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 15, "novel_recall": 0.2667, "qid": "9ddd6216-2272-4cc1-aab9-a2f21f664675", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 19, "novel_recall": 0.3158, "qid": "3067ead9-e82d-4a99-8578-6708c52448e7", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "178", "full_hit": false, "n_novel": 25, "novel_recall": 0.08, "qid": "728d5ee3-36af-456c-8ee2-26ef43c0e0ce", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 34, "novel_recall": 0.0, "qid": "5798d2dd-cd62-4cad-b442-9b24bc0f56b1", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 31, "novel_recall": 0.0, "qid": "5a0bec64-4ef6-41b0-a845-a4514df165de", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.0, "qid": "d3e85f8d-27dc-4d61-acca-975e469a99ad", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.0, "qid": "3a4966b2-e5de-4472-8624-98216c4fa382", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 35, "novel_recall": 0.0, "qid": "cda578b8-791e-420d-ad07-6b8842596662", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 28, "novel_recall": 0.0, "qid": "8d6f2ecf-6097-4f46-8431-fe1a47be8df6", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.0, "qid": "af240ed4-ed3f-40c3-ac33-67ca411917e9", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.0, "qid": "259ffe85-a35b-40e4-8908-2687c034f1c4", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.0, "qid": "8967a313-ff24-43f8-9f50-3d8babfd8693", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 25, "novel_recall": 0.0, "qid": "d21bcf31-d1e7-4095-9a90-9a575566c924", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.0, "qid": "eefc8e60-f0ee-400e-b408-9ad53fdb4933", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "c37691cf-2c1b-4978-a599-94c250ad69e0", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 34, "novel_recall": 0.2353, "qid": "5798d2dd-cd62-4cad-b442-9b24bc0f56b1", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 31, "novel_recall": 0.3871, "qid": "5a0bec64-4ef6-41b0-a845-a4514df165de", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.5263, "qid": "d3e85f8d-27dc-4d61-acca-975e469a99ad", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.3913, "qid": "3a4966b2-e5de-4472-8624-98216c4fa382", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 35, "novel_recall": 0.2286, "qid": "cda578b8-791e-420d-ad07-6b8842596662", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 28, "novel_recall": 0.3214, "qid": "8d6f2ecf-6097-4f46-8431-fe1a47be8df6", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.2174, "qid": "af240ed4-ed3f-40c3-ac33-67ca411917e9", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.0526, "qid": "259ffe85-a35b-40e4-8908-2687c034f1c4", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.5263, "qid": "8967a313-ff24-43f8-9f50-3d8babfd8693", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 25, "novel_recall": 0.28, "qid": "d21bcf31-d1e7-4095-9a90-9a575566c924", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.2609, "qid": "eefc8e60-f0ee-400e-b408-9ad53fdb4933", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 24, "novel_recall": 0.375, "qid": "c37691cf-2c1b-4978-a599-94c250ad69e0", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 34, "novel_recall": 0.2353, "qid": "5798d2dd-cd62-4cad-b442-9b24bc0f56b1", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 31, "novel_recall": 0.3871, "qid": "5a0bec64-4ef6-41b0-a845-a4514df165de", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.5263, "qid": "d3e85f8d-27dc-4d61-acca-975e469a99ad", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.3913, "qid": "3a4966b2-e5de-4472-8624-98216c4fa382", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 35, "novel_recall": 0.2286, "qid": "cda578b8-791e-420d-ad07-6b8842596662", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 28, "novel_recall": 0.3214, "qid": "8d6f2ecf-6097-4f46-8431-fe1a47be8df6", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.2174, "qid": "af240ed4-ed3f-40c3-ac33-67ca411917e9", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.5789, "qid": "259ffe85-a35b-40e4-8908-2687c034f1c4", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.5263, "qid": "8967a313-ff24-43f8-9f50-3d8babfd8693", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 25, "novel_recall": 0.28, "qid": "d21bcf31-d1e7-4095-9a90-9a575566c924", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.2609, "qid": "eefc8e60-f0ee-400e-b408-9ad53fdb4933", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 24, "novel_recall": 0.375, "qid": "c37691cf-2c1b-4978-a599-94c250ad69e0", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 34, "novel_recall": 0.2353, "qid": "5798d2dd-cd62-4cad-b442-9b24bc0f56b1", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 31, "novel_recall": 0.2581, "qid": "5a0bec64-4ef6-41b0-a845-a4514df165de", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.5263, "qid": "d3e85f8d-27dc-4d61-acca-975e469a99ad", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.3913, "qid": "3a4966b2-e5de-4472-8624-98216c4fa382", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 35, "novel_recall": 0.2286, "qid": "cda578b8-791e-420d-ad07-6b8842596662", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 28, "novel_recall": 0.3214, "qid": "8d6f2ecf-6097-4f46-8431-fe1a47be8df6", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.2174, "qid": "af240ed4-ed3f-40c3-ac33-67ca411917e9", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.5789, "qid": "259ffe85-a35b-40e4-8908-2687c034f1c4", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.4211, "qid": "8967a313-ff24-43f8-9f50-3d8babfd8693", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 25, "novel_recall": 0.28, "qid": "d21bcf31-d1e7-4095-9a90-9a575566c924", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.1739, "qid": "eefc8e60-f0ee-400e-b408-9ad53fdb4933", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 24, "novel_recall": 0.2917, "qid": "c37691cf-2c1b-4978-a599-94c250ad69e0", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 34, "novel_recall": 0.2353, "qid": "5798d2dd-cd62-4cad-b442-9b24bc0f56b1", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 31, "novel_recall": 0.2581, "qid": "5a0bec64-4ef6-41b0-a845-a4514df165de", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.5263, "qid": "d3e85f8d-27dc-4d61-acca-975e469a99ad", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.3913, "qid": "3a4966b2-e5de-4472-8624-98216c4fa382", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 35, "novel_recall": 0.2286, "qid": "cda578b8-791e-420d-ad07-6b8842596662", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 28, "novel_recall": 0.3214, "qid": "8d6f2ecf-6097-4f46-8431-fe1a47be8df6", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.2174, "qid": "af240ed4-ed3f-40c3-ac33-67ca411917e9", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.5789, "qid": "259ffe85-a35b-40e4-8908-2687c034f1c4", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.4211, "qid": "8967a313-ff24-43f8-9f50-3d8babfd8693", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 25, "novel_recall": 0.28, "qid": "d21bcf31-d1e7-4095-9a90-9a575566c924", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.1739, "qid": "eefc8e60-f0ee-400e-b408-9ad53fdb4933", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 24, "novel_recall": 0.2917, "qid": "c37691cf-2c1b-4978-a599-94c250ad69e0", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 34, "novel_recall": 0.1471, "qid": "5798d2dd-cd62-4cad-b442-9b24bc0f56b1", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 31, "novel_recall": 0.1935, "qid": "5a0bec64-4ef6-41b0-a845-a4514df165de", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.2632, "qid": "d3e85f8d-27dc-4d61-acca-975e469a99ad", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.3478, "qid": "3a4966b2-e5de-4472-8624-98216c4fa382", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 35, "novel_recall": 0.1143, "qid": "cda578b8-791e-420d-ad07-6b8842596662", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 28, "novel_recall": 0.25, "qid": "8d6f2ecf-6097-4f46-8431-fe1a47be8df6", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.1739, "qid": "af240ed4-ed3f-40c3-ac33-67ca411917e9", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.2105, "qid": "259ffe85-a35b-40e4-8908-2687c034f1c4", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 19, "novel_recall": 0.1579, "qid": "8967a313-ff24-43f8-9f50-3d8babfd8693", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 25, "novel_recall": 0.16, "qid": "d21bcf31-d1e7-4095-9a90-9a575566c924", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 23, "novel_recall": 0.1304, "qid": "eefc8e60-f0ee-400e-b408-9ad53fdb4933", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "180", "full_hit": false, "n_novel": 24, "novel_recall": 0.2083, "qid": "c37691cf-2c1b-4978-a599-94c250ad69e0", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "85771bf8-c84d-421a-8dee-bb902c01faa3", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "10482397-4a46-4dff-af4a-bb95595b999c", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "210ca28a-059c-456c-98da-a23f2beed02f", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "91ffdd81-0d59-4c6e-9b41-60eaed93c2cf", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 20, "novel_recall": 0.0, "qid": "13ddd10b-06b7-48f5-a0ab-b5b11efd7e11", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "4d3f2a31-f145-458f-b0ef-71019f650625", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 14, "novel_recall": 0.0, "qid": "270ae720-3393-44e4-b38c-015fe1f4ec6b", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "f9018f29-0631-4054-ac0b-cbcfc58ef79e", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "f5fc7704-ddc3-4e55-afe5-c3d95d03bdee", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "e0430e5e-9e7b-4d64-9d26-7a6f53542402", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "29aa94df-befb-4e1b-90d4-a65c2ffa6713", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "6fb9bbbc-0d80-4cc2-9112-19800b593037", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "85771bf8-c84d-421a-8dee-bb902c01faa3", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 9, "novel_recall": 0.4444, "qid": "10482397-4a46-4dff-af4a-bb95595b999c", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "210ca28a-059c-456c-98da-a23f2beed02f", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 15, "novel_recall": 0.1333, "qid": "91ffdd81-0d59-4c6e-9b41-60eaed93c2cf", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 20, "novel_recall": 0.0, "qid": "13ddd10b-06b7-48f5-a0ab-b5b11efd7e11", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 13, "novel_recall": 0.3846, "qid": "4d3f2a31-f145-458f-b0ef-71019f650625", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 14, "novel_recall": 0.1429, "qid": "270ae720-3393-44e4-b38c-015fe1f4ec6b", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "f9018f29-0631-4054-ac0b-cbcfc58ef79e", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "f5fc7704-ddc3-4e55-afe5-c3d95d03bdee", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "e0430e5e-9e7b-4d64-9d26-7a6f53542402", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 16, "novel_recall": 0.1875, "qid": "29aa94df-befb-4e1b-90d4-a65c2ffa6713", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 17, "novel_recall": 0.2941, "qid": "6fb9bbbc-0d80-4cc2-9112-19800b593037", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "85771bf8-c84d-421a-8dee-bb902c01faa3", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 9, "novel_recall": 0.4444, "qid": "10482397-4a46-4dff-af4a-bb95595b999c", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "210ca28a-059c-456c-98da-a23f2beed02f", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 15, "novel_recall": 0.1333, "qid": "91ffdd81-0d59-4c6e-9b41-60eaed93c2cf", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 20, "novel_recall": 0.0, "qid": "13ddd10b-06b7-48f5-a0ab-b5b11efd7e11", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 13, "novel_recall": 0.3846, "qid": "4d3f2a31-f145-458f-b0ef-71019f650625", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 14, "novel_recall": 0.1429, "qid": "270ae720-3393-44e4-b38c-015fe1f4ec6b", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "f9018f29-0631-4054-ac0b-cbcfc58ef79e", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "f5fc7704-ddc3-4e55-afe5-c3d95d03bdee", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "e0430e5e-9e7b-4d64-9d26-7a6f53542402", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 16, "novel_recall": 0.1875, "qid": "29aa94df-befb-4e1b-90d4-a65c2ffa6713", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 17, "novel_recall": 0.2941, "qid": "6fb9bbbc-0d80-4cc2-9112-19800b593037", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "85771bf8-c84d-421a-8dee-bb902c01faa3", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 9, "novel_recall": 0.4444, "qid": "10482397-4a46-4dff-af4a-bb95595b999c", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "210ca28a-059c-456c-98da-a23f2beed02f", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 15, "novel_recall": 0.1333, "qid": "91ffdd81-0d59-4c6e-9b41-60eaed93c2cf", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 20, "novel_recall": 0.0, "qid": "13ddd10b-06b7-48f5-a0ab-b5b11efd7e11", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 13, "novel_recall": 0.3846, "qid": "4d3f2a31-f145-458f-b0ef-71019f650625", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 14, "novel_recall": 0.1429, "qid": "270ae720-3393-44e4-b38c-015fe1f4ec6b", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "f9018f29-0631-4054-ac0b-cbcfc58ef79e", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "f5fc7704-ddc3-4e55-afe5-c3d95d03bdee", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "e0430e5e-9e7b-4d64-9d26-7a6f53542402", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 16, "novel_recall": 0.1875, "qid": "29aa94df-befb-4e1b-90d4-a65c2ffa6713", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 17, "novel_recall": 0.2941, "qid": "6fb9bbbc-0d80-4cc2-9112-19800b593037", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "85771bf8-c84d-421a-8dee-bb902c01faa3", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 9, "novel_recall": 0.4444, "qid": "10482397-4a46-4dff-af4a-bb95595b999c", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "210ca28a-059c-456c-98da-a23f2beed02f", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 15, "novel_recall": 0.1333, "qid": "91ffdd81-0d59-4c6e-9b41-60eaed93c2cf", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 20, "novel_recall": 0.0, "qid": "13ddd10b-06b7-48f5-a0ab-b5b11efd7e11", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 13, "novel_recall": 0.3846, "qid": "4d3f2a31-f145-458f-b0ef-71019f650625", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 14, "novel_recall": 0.1429, "qid": "270ae720-3393-44e4-b38c-015fe1f4ec6b", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "f9018f29-0631-4054-ac0b-cbcfc58ef79e", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "f5fc7704-ddc3-4e55-afe5-c3d95d03bdee", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "e0430e5e-9e7b-4d64-9d26-7a6f53542402", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 16, "novel_recall": 0.1875, "qid": "29aa94df-befb-4e1b-90d4-a65c2ffa6713", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 17, "novel_recall": 0.2941, "qid": "6fb9bbbc-0d80-4cc2-9112-19800b593037", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "85771bf8-c84d-421a-8dee-bb902c01faa3", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "10482397-4a46-4dff-af4a-bb95595b999c", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "210ca28a-059c-456c-98da-a23f2beed02f", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "91ffdd81-0d59-4c6e-9b41-60eaed93c2cf", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 20, "novel_recall": 0.0, "qid": "13ddd10b-06b7-48f5-a0ab-b5b11efd7e11", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "4d3f2a31-f145-458f-b0ef-71019f650625", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 14, "novel_recall": 0.0, "qid": "270ae720-3393-44e4-b38c-015fe1f4ec6b", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "f9018f29-0631-4054-ac0b-cbcfc58ef79e", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "f5fc7704-ddc3-4e55-afe5-c3d95d03bdee", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "e0430e5e-9e7b-4d64-9d26-7a6f53542402", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "29aa94df-befb-4e1b-90d4-a65c2ffa6713", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "184", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "6fb9bbbc-0d80-4cc2-9112-19800b593037", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "cf4ede8a-ed72-4507-a501-20c5fc8b6c02", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "d793fb13-ccef-4636-be64-8f4429ddad5a", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "9993d94b-5ad3-44ab-911b-9c3a979ff855", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "791e8660-bc32-43aa-b686-397f312bf393", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "7d8c3a8c-2764-40ff-a69b-479043a166fd", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "6bd8487e-241f-4289-bb6b-1702d8181349", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "9f940622-1514-43f5-a952-48ab488c5a64", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "051aa367-5276-4def-94a0-21125250bc37", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "528f36f8-6a59-47cd-9888-f03da60aee38", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "7a9bcbd4-35be-4fe1-a919-a9a83534c13a", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "f3a0ee36-eae4-4613-9b44-9b91b5ec7c07", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "3152ca63-4918-452d-93d4-865614bb6095", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "cf4ede8a-ed72-4507-a501-20c5fc8b6c02", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "187", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "d793fb13-ccef-4636-be64-8f4429ddad5a", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "187", "full_hit": true, "n_novel": 3, "novel_recall": 1.0, "qid": "9993d94b-5ad3-44ab-911b-9c3a979ff855", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "791e8660-bc32-43aa-b686-397f312bf393", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 13, "novel_recall": 0.3077, "qid": "7d8c3a8c-2764-40ff-a69b-479043a166fd", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "6bd8487e-241f-4289-bb6b-1702d8181349", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 11, "novel_recall": 0.0909, "qid": "9f940622-1514-43f5-a952-48ab488c5a64", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "051aa367-5276-4def-94a0-21125250bc37", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "528f36f8-6a59-47cd-9888-f03da60aee38", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "187", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "7a9bcbd4-35be-4fe1-a919-a9a83534c13a", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "f3a0ee36-eae4-4613-9b44-9b91b5ec7c07", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "3152ca63-4918-452d-93d4-865614bb6095", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "cf4ede8a-ed72-4507-a501-20c5fc8b6c02", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "187", "full_hit": true, "n_novel": 5, "novel_recall": 1.0, "qid": "d793fb13-ccef-4636-be64-8f4429ddad5a", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "187", "full_hit": true, "n_novel": 3, "novel_recall": 1.0, "qid": "9993d94b-5ad3-44ab-911b-9c3a979ff855", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 10, "novel_recall": 0.6, "qid": "791e8660-bc32-43aa-b686-397f312bf393", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 13, "novel_recall": 0.3077, "qid": "7d8c3a8c-2764-40ff-a69b-479043a166fd", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "6bd8487e-241f-4289-bb6b-1702d8181349", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 11, "novel_recall": 0.0909, "qid": "9f940622-1514-43f5-a952-48ab488c5a64", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "051aa367-5276-4def-94a0-21125250bc37", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "528f36f8-6a59-47cd-9888-f03da60aee38", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "187", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "7a9bcbd4-35be-4fe1-a919-a9a83534c13a", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "f3a0ee36-eae4-4613-9b44-9b91b5ec7c07", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "3152ca63-4918-452d-93d4-865614bb6095", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "cf4ede8a-ed72-4507-a501-20c5fc8b6c02", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "d793fb13-ccef-4636-be64-8f4429ddad5a", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "187", "full_hit": true, "n_novel": 3, "novel_recall": 1.0, "qid": "9993d94b-5ad3-44ab-911b-9c3a979ff855", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 10, "novel_recall": 0.2, "qid": "791e8660-bc32-43aa-b686-397f312bf393", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 13, "novel_recall": 0.1538, "qid": "7d8c3a8c-2764-40ff-a69b-479043a166fd", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "6bd8487e-241f-4289-bb6b-1702d8181349", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 11, "novel_recall": 0.0909, "qid": "9f940622-1514-43f5-a952-48ab488c5a64", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "051aa367-5276-4def-94a0-21125250bc37", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "528f36f8-6a59-47cd-9888-f03da60aee38", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "187", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "7a9bcbd4-35be-4fe1-a919-a9a83534c13a", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "f3a0ee36-eae4-4613-9b44-9b91b5ec7c07", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "3152ca63-4918-452d-93d4-865614bb6095", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 5, "novel_recall": 0.6, "qid": "cf4ede8a-ed72-4507-a501-20c5fc8b6c02", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 5, "novel_recall": 0.8, "qid": "d793fb13-ccef-4636-be64-8f4429ddad5a", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "187", "full_hit": true, "n_novel": 3, "novel_recall": 1.0, "qid": "9993d94b-5ad3-44ab-911b-9c3a979ff855", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 10, "novel_recall": 0.2, "qid": "791e8660-bc32-43aa-b686-397f312bf393", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 13, "novel_recall": 0.1538, "qid": "7d8c3a8c-2764-40ff-a69b-479043a166fd", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "6bd8487e-241f-4289-bb6b-1702d8181349", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 11, "novel_recall": 0.0909, "qid": "9f940622-1514-43f5-a952-48ab488c5a64", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "051aa367-5276-4def-94a0-21125250bc37", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "528f36f8-6a59-47cd-9888-f03da60aee38", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "187", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "7a9bcbd4-35be-4fe1-a919-a9a83534c13a", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "f3a0ee36-eae4-4613-9b44-9b91b5ec7c07", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "3152ca63-4918-452d-93d4-865614bb6095", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "cf4ede8a-ed72-4507-a501-20c5fc8b6c02", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "d793fb13-ccef-4636-be64-8f4429ddad5a", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "187", "full_hit": true, "n_novel": 3, "novel_recall": 1.0, "qid": "9993d94b-5ad3-44ab-911b-9c3a979ff855", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "791e8660-bc32-43aa-b686-397f312bf393", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 13, "novel_recall": 0.3846, "qid": "7d8c3a8c-2764-40ff-a69b-479043a166fd", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 4, "novel_recall": 0.5, "qid": "6bd8487e-241f-4289-bb6b-1702d8181349", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 11, "novel_recall": 0.0909, "qid": "9f940622-1514-43f5-a952-48ab488c5a64", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "051aa367-5276-4def-94a0-21125250bc37", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "528f36f8-6a59-47cd-9888-f03da60aee38", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "187", "full_hit": true, "n_novel": 2, "novel_recall": 1.0, "qid": "7a9bcbd4-35be-4fe1-a919-a9a83534c13a", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "f3a0ee36-eae4-4613-9b44-9b91b5ec7c07", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "187", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "3152ca63-4918-452d-93d4-865614bb6095", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "4cb81964-c8de-4013-a019-50ef654e6526", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "872a01bc-4e56-4c80-9fd5-d81702a22865", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "394a9c2c-a49f-4f92-abae-db2e63c9f90b", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "4b606836-b7bb-4580-ae18-c84f578239ec", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "a91baddd-5285-423a-b377-afcb914d1439", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "9403bfc4-5fdd-4a57-af11-07ba4201d10d", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "1c97f650-b514-4299-a3de-c53093d84c2e", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "fbf2e32f-e3ca-4bd2-a45c-1ee500d4ca1e", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "9bcfbe48-eecb-4d07-b65b-01cddbf8c1f1", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e8ec0652-ce2c-48e4-8690-e63f3755977c", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "dd0c5a16-2812-41c1-b57d-32cfab75a6a5", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "9a6783e2-60fc-4183-9906-ebe0e7a97aad", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "194", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "4cb81964-c8de-4013-a019-50ef654e6526", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "872a01bc-4e56-4c80-9fd5-d81702a22865", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "394a9c2c-a49f-4f92-abae-db2e63c9f90b", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "4b606836-b7bb-4580-ae18-c84f578239ec", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 6, "novel_recall": 0.8333, "qid": "a91baddd-5285-423a-b377-afcb914d1439", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "9403bfc4-5fdd-4a57-af11-07ba4201d10d", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "1c97f650-b514-4299-a3de-c53093d84c2e", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "fbf2e32f-e3ca-4bd2-a45c-1ee500d4ca1e", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "9bcfbe48-eecb-4d07-b65b-01cddbf8c1f1", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e8ec0652-ce2c-48e4-8690-e63f3755977c", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "dd0c5a16-2812-41c1-b57d-32cfab75a6a5", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "9a6783e2-60fc-4183-9906-ebe0e7a97aad", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "194", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "4cb81964-c8de-4013-a019-50ef654e6526", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "872a01bc-4e56-4c80-9fd5-d81702a22865", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "394a9c2c-a49f-4f92-abae-db2e63c9f90b", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "4b606836-b7bb-4580-ae18-c84f578239ec", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 6, "novel_recall": 0.8333, "qid": "a91baddd-5285-423a-b377-afcb914d1439", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "9403bfc4-5fdd-4a57-af11-07ba4201d10d", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "1c97f650-b514-4299-a3de-c53093d84c2e", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "fbf2e32f-e3ca-4bd2-a45c-1ee500d4ca1e", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "9bcfbe48-eecb-4d07-b65b-01cddbf8c1f1", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e8ec0652-ce2c-48e4-8690-e63f3755977c", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "dd0c5a16-2812-41c1-b57d-32cfab75a6a5", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "9a6783e2-60fc-4183-9906-ebe0e7a97aad", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "194", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "4cb81964-c8de-4013-a019-50ef654e6526", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "872a01bc-4e56-4c80-9fd5-d81702a22865", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "394a9c2c-a49f-4f92-abae-db2e63c9f90b", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "4b606836-b7bb-4580-ae18-c84f578239ec", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 6, "novel_recall": 0.8333, "qid": "a91baddd-5285-423a-b377-afcb914d1439", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "9403bfc4-5fdd-4a57-af11-07ba4201d10d", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "1c97f650-b514-4299-a3de-c53093d84c2e", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "fbf2e32f-e3ca-4bd2-a45c-1ee500d4ca1e", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "9bcfbe48-eecb-4d07-b65b-01cddbf8c1f1", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e8ec0652-ce2c-48e4-8690-e63f3755977c", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "dd0c5a16-2812-41c1-b57d-32cfab75a6a5", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "9a6783e2-60fc-4183-9906-ebe0e7a97aad", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "194", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "4cb81964-c8de-4013-a019-50ef654e6526", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "872a01bc-4e56-4c80-9fd5-d81702a22865", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "394a9c2c-a49f-4f92-abae-db2e63c9f90b", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "4b606836-b7bb-4580-ae18-c84f578239ec", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 6, "novel_recall": 0.8333, "qid": "a91baddd-5285-423a-b377-afcb914d1439", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "9403bfc4-5fdd-4a57-af11-07ba4201d10d", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "1c97f650-b514-4299-a3de-c53093d84c2e", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "fbf2e32f-e3ca-4bd2-a45c-1ee500d4ca1e", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "9bcfbe48-eecb-4d07-b65b-01cddbf8c1f1", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e8ec0652-ce2c-48e4-8690-e63f3755977c", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 11, "novel_recall": 0.6364, "qid": "dd0c5a16-2812-41c1-b57d-32cfab75a6a5", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "9a6783e2-60fc-4183-9906-ebe0e7a97aad", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "194", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "4cb81964-c8de-4013-a019-50ef654e6526", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "872a01bc-4e56-4c80-9fd5-d81702a22865", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "394a9c2c-a49f-4f92-abae-db2e63c9f90b", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 3, "novel_recall": 0.6667, "qid": "4b606836-b7bb-4580-ae18-c84f578239ec", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "a91baddd-5285-423a-b377-afcb914d1439", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 6, "novel_recall": 0.6667, "qid": "9403bfc4-5fdd-4a57-af11-07ba4201d10d", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "1c97f650-b514-4299-a3de-c53093d84c2e", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "fbf2e32f-e3ca-4bd2-a45c-1ee500d4ca1e", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "9bcfbe48-eecb-4d07-b65b-01cddbf8c1f1", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "e8ec0652-ce2c-48e4-8690-e63f3755977c", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 11, "novel_recall": 0.4545, "qid": "dd0c5a16-2812-41c1-b57d-32cfab75a6a5", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "194", "full_hit": false, "n_novel": 7, "novel_recall": 0.1429, "qid": "9a6783e2-60fc-4183-9906-ebe0e7a97aad", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 20, "novel_recall": 0.0, "qid": "4cdf5ee7-e4fa-4294-822f-87005ab32ba4", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "3c33b35f-202e-4e2c-b2bd-15fde177b847", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "7e733549-78b0-4522-9005-86ebee0a5f9e", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 28, "novel_recall": 0.0, "qid": "125880f4-76a6-47bd-bfe4-8480b267499b", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "3d398b83-6d00-4114-848e-15148273ef71", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "90605869-0a08-4ac1-bd57-67d264393e9a", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "9c6b769f-653c-4ec4-a155-0a544237bc3f", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "09332439-361e-47aa-ae77-97ae00dbd6a7", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 23, "novel_recall": 0.0, "qid": "c2c41867-775e-4697-8634-f828f4dee13b", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "5fd0a363-67b2-4fc3-8870-01ab5fb4eb74", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "f49e5eec-13a9-4878-9e9d-087cb3c4ba44", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 11, "novel_recall": 0.0, "qid": "3b828783-a329-4a20-a1f8-b512f7f72b1f", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 20, "novel_recall": 0.3, "qid": "4cdf5ee7-e4fa-4294-822f-87005ab32ba4", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "3c33b35f-202e-4e2c-b2bd-15fde177b847", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "7e733549-78b0-4522-9005-86ebee0a5f9e", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 28, "novel_recall": 0.1786, "qid": "125880f4-76a6-47bd-bfe4-8480b267499b", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "3d398b83-6d00-4114-848e-15148273ef71", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "90605869-0a08-4ac1-bd57-67d264393e9a", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 13, "novel_recall": 0.0769, "qid": "9c6b769f-653c-4ec4-a155-0a544237bc3f", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "09332439-361e-47aa-ae77-97ae00dbd6a7", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 23, "novel_recall": 0.4783, "qid": "c2c41867-775e-4697-8634-f828f4dee13b", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "5fd0a363-67b2-4fc3-8870-01ab5fb4eb74", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "f49e5eec-13a9-4878-9e9d-087cb3c4ba44", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "3b828783-a329-4a20-a1f8-b512f7f72b1f", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 20, "novel_recall": 0.3, "qid": "4cdf5ee7-e4fa-4294-822f-87005ab32ba4", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "3c33b35f-202e-4e2c-b2bd-15fde177b847", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "7e733549-78b0-4522-9005-86ebee0a5f9e", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 28, "novel_recall": 0.1786, "qid": "125880f4-76a6-47bd-bfe4-8480b267499b", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "3d398b83-6d00-4114-848e-15148273ef71", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "90605869-0a08-4ac1-bd57-67d264393e9a", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 13, "novel_recall": 0.0769, "qid": "9c6b769f-653c-4ec4-a155-0a544237bc3f", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "09332439-361e-47aa-ae77-97ae00dbd6a7", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 23, "novel_recall": 0.4783, "qid": "c2c41867-775e-4697-8634-f828f4dee13b", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "5fd0a363-67b2-4fc3-8870-01ab5fb4eb74", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 8, "novel_recall": 0.5, "qid": "f49e5eec-13a9-4878-9e9d-087cb3c4ba44", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "3b828783-a329-4a20-a1f8-b512f7f72b1f", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 20, "novel_recall": 0.3, "qid": "4cdf5ee7-e4fa-4294-822f-87005ab32ba4", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "3c33b35f-202e-4e2c-b2bd-15fde177b847", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "7e733549-78b0-4522-9005-86ebee0a5f9e", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 28, "novel_recall": 0.1786, "qid": "125880f4-76a6-47bd-bfe4-8480b267499b", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "3d398b83-6d00-4114-848e-15148273ef71", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "90605869-0a08-4ac1-bd57-67d264393e9a", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 13, "novel_recall": 0.0769, "qid": "9c6b769f-653c-4ec4-a155-0a544237bc3f", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "09332439-361e-47aa-ae77-97ae00dbd6a7", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 23, "novel_recall": 0.3913, "qid": "c2c41867-775e-4697-8634-f828f4dee13b", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "5fd0a363-67b2-4fc3-8870-01ab5fb4eb74", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "f49e5eec-13a9-4878-9e9d-087cb3c4ba44", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "3b828783-a329-4a20-a1f8-b512f7f72b1f", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 20, "novel_recall": 0.3, "qid": "4cdf5ee7-e4fa-4294-822f-87005ab32ba4", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "3c33b35f-202e-4e2c-b2bd-15fde177b847", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "7e733549-78b0-4522-9005-86ebee0a5f9e", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 28, "novel_recall": 0.1786, "qid": "125880f4-76a6-47bd-bfe4-8480b267499b", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "3d398b83-6d00-4114-848e-15148273ef71", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "90605869-0a08-4ac1-bd57-67d264393e9a", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 13, "novel_recall": 0.0769, "qid": "9c6b769f-653c-4ec4-a155-0a544237bc3f", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "09332439-361e-47aa-ae77-97ae00dbd6a7", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 23, "novel_recall": 0.3913, "qid": "c2c41867-775e-4697-8634-f828f4dee13b", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 6, "novel_recall": 0.3333, "qid": "5fd0a363-67b2-4fc3-8870-01ab5fb4eb74", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 8, "novel_recall": 0.375, "qid": "f49e5eec-13a9-4878-9e9d-087cb3c4ba44", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "3b828783-a329-4a20-a1f8-b512f7f72b1f", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 20, "novel_recall": 0.3, "qid": "4cdf5ee7-e4fa-4294-822f-87005ab32ba4", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "3c33b35f-202e-4e2c-b2bd-15fde177b847", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "7e733549-78b0-4522-9005-86ebee0a5f9e", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 28, "novel_recall": 0.1429, "qid": "125880f4-76a6-47bd-bfe4-8480b267499b", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 4, "novel_recall": 0.25, "qid": "3d398b83-6d00-4114-848e-15148273ef71", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 4, "novel_recall": 0.0, "qid": "90605869-0a08-4ac1-bd57-67d264393e9a", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 13, "novel_recall": 0.1538, "qid": "9c6b769f-653c-4ec4-a155-0a544237bc3f", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "09332439-361e-47aa-ae77-97ae00dbd6a7", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 23, "novel_recall": 0.3478, "qid": "c2c41867-775e-4697-8634-f828f4dee13b", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 6, "novel_recall": 0.0, "qid": "5fd0a363-67b2-4fc3-8870-01ab5fb4eb74", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 8, "novel_recall": 0.625, "qid": "f49e5eec-13a9-4878-9e9d-087cb3c4ba44", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "202", "full_hit": false, "n_novel": 11, "novel_recall": 0.2727, "qid": "3b828783-a329-4a20-a1f8-b512f7f72b1f", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 20, "novel_recall": 0.0, "qid": "993270e0-c1fd-444a-b44b-4bc1713ea36c", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 22, "novel_recall": 0.0, "qid": "5f1a3db1-0254-48b0-b31b-87f67eda23ee", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "fd2fd01f-6374-4c74-b00d-dc353d872215", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 14, "novel_recall": 0.0, "qid": "02e69507-acf9-4d25-a7fe-0c35c0e8a1f7", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "313accbd-ddcd-4b72-93a5-2fd6b827d690", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 25, "novel_recall": 0.0, "qid": "417f8c3e-e82a-4b0c-a0f4-00577a26fe29", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 18, "novel_recall": 0.0, "qid": "efbad052-9e6d-43cb-b056-39ff9a8cb8bf", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 3, "novel_recall": 0.0, "qid": "b3ef2778-5b99-428a-88a7-bbdbb66244e1", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "8c6b93b8-e42a-40e8-86b5-295c2dee387a", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 12, "novel_recall": 0.0, "qid": "ef8c8c27-fea6-4af1-863d-d6b637914a9c", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "f2e07949-4004-4fe5-a86e-4e3a47daeaf3", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "1735f547-10d8-4b78-8f77-e5f8a124cfd5", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 20, "novel_recall": 0.1, "qid": "993270e0-c1fd-444a-b44b-4bc1713ea36c", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 22, "novel_recall": 0.2273, "qid": "5f1a3db1-0254-48b0-b31b-87f67eda23ee", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 9, "novel_recall": 0.1111, "qid": "fd2fd01f-6374-4c74-b00d-dc353d872215", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 14, "novel_recall": 0.2857, "qid": "02e69507-acf9-4d25-a7fe-0c35c0e8a1f7", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "313accbd-ddcd-4b72-93a5-2fd6b827d690", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 25, "novel_recall": 0.16, "qid": "417f8c3e-e82a-4b0c-a0f4-00577a26fe29", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 18, "novel_recall": 0.1667, "qid": "efbad052-9e6d-43cb-b056-39ff9a8cb8bf", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "b3ef2778-5b99-428a-88a7-bbdbb66244e1", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "8c6b93b8-e42a-40e8-86b5-295c2dee387a", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 12, "novel_recall": 0.0833, "qid": "ef8c8c27-fea6-4af1-863d-d6b637914a9c", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "f2e07949-4004-4fe5-a86e-4e3a47daeaf3", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "1735f547-10d8-4b78-8f77-e5f8a124cfd5", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 20, "novel_recall": 0.1, "qid": "993270e0-c1fd-444a-b44b-4bc1713ea36c", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 22, "novel_recall": 0.2273, "qid": "5f1a3db1-0254-48b0-b31b-87f67eda23ee", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "fd2fd01f-6374-4c74-b00d-dc353d872215", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 14, "novel_recall": 0.2857, "qid": "02e69507-acf9-4d25-a7fe-0c35c0e8a1f7", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "313accbd-ddcd-4b72-93a5-2fd6b827d690", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 25, "novel_recall": 0.2, "qid": "417f8c3e-e82a-4b0c-a0f4-00577a26fe29", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 18, "novel_recall": 0.1667, "qid": "efbad052-9e6d-43cb-b056-39ff9a8cb8bf", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "b3ef2778-5b99-428a-88a7-bbdbb66244e1", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "8c6b93b8-e42a-40e8-86b5-295c2dee387a", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 12, "novel_recall": 0.0833, "qid": "ef8c8c27-fea6-4af1-863d-d6b637914a9c", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "f2e07949-4004-4fe5-a86e-4e3a47daeaf3", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "1735f547-10d8-4b78-8f77-e5f8a124cfd5", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 20, "novel_recall": 0.1, "qid": "993270e0-c1fd-444a-b44b-4bc1713ea36c", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 22, "novel_recall": 0.2273, "qid": "5f1a3db1-0254-48b0-b31b-87f67eda23ee", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 9, "novel_recall": 0.1111, "qid": "fd2fd01f-6374-4c74-b00d-dc353d872215", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 14, "novel_recall": 0.2857, "qid": "02e69507-acf9-4d25-a7fe-0c35c0e8a1f7", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "313accbd-ddcd-4b72-93a5-2fd6b827d690", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 25, "novel_recall": 0.2, "qid": "417f8c3e-e82a-4b0c-a0f4-00577a26fe29", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 18, "novel_recall": 0.1667, "qid": "efbad052-9e6d-43cb-b056-39ff9a8cb8bf", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "b3ef2778-5b99-428a-88a7-bbdbb66244e1", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "8c6b93b8-e42a-40e8-86b5-295c2dee387a", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 12, "novel_recall": 0.0833, "qid": "ef8c8c27-fea6-4af1-863d-d6b637914a9c", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "f2e07949-4004-4fe5-a86e-4e3a47daeaf3", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "1735f547-10d8-4b78-8f77-e5f8a124cfd5", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 20, "novel_recall": 0.1, "qid": "993270e0-c1fd-444a-b44b-4bc1713ea36c", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 22, "novel_recall": 0.2273, "qid": "5f1a3db1-0254-48b0-b31b-87f67eda23ee", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 9, "novel_recall": 0.1111, "qid": "fd2fd01f-6374-4c74-b00d-dc353d872215", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 14, "novel_recall": 0.2857, "qid": "02e69507-acf9-4d25-a7fe-0c35c0e8a1f7", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "313accbd-ddcd-4b72-93a5-2fd6b827d690", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 25, "novel_recall": 0.2, "qid": "417f8c3e-e82a-4b0c-a0f4-00577a26fe29", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 18, "novel_recall": 0.1667, "qid": "efbad052-9e6d-43cb-b056-39ff9a8cb8bf", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "b3ef2778-5b99-428a-88a7-bbdbb66244e1", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "8c6b93b8-e42a-40e8-86b5-295c2dee387a", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 12, "novel_recall": 0.0833, "qid": "ef8c8c27-fea6-4af1-863d-d6b637914a9c", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "f2e07949-4004-4fe5-a86e-4e3a47daeaf3", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 2, "novel_recall": 0.5, "qid": "1735f547-10d8-4b78-8f77-e5f8a124cfd5", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 20, "novel_recall": 0.05, "qid": "993270e0-c1fd-444a-b44b-4bc1713ea36c", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 22, "novel_recall": 0.0909, "qid": "5f1a3db1-0254-48b0-b31b-87f67eda23ee", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "fd2fd01f-6374-4c74-b00d-dc353d872215", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 14, "novel_recall": 0.2143, "qid": "02e69507-acf9-4d25-a7fe-0c35c0e8a1f7", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "313accbd-ddcd-4b72-93a5-2fd6b827d690", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 25, "novel_recall": 0.24, "qid": "417f8c3e-e82a-4b0c-a0f4-00577a26fe29", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 18, "novel_recall": 0.1111, "qid": "efbad052-9e6d-43cb-b056-39ff9a8cb8bf", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 3, "novel_recall": 0.3333, "qid": "b3ef2778-5b99-428a-88a7-bbdbb66244e1", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "8c6b93b8-e42a-40e8-86b5-295c2dee387a", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 12, "novel_recall": 0.1667, "qid": "ef8c8c27-fea6-4af1-863d-d6b637914a9c", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "f2e07949-4004-4fe5-a86e-4e3a47daeaf3", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "203", "full_hit": false, "n_novel": 2, "novel_recall": 0.0, "qid": "1735f547-10d8-4b78-8f77-e5f8a124cfd5", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "71767248-ea90-4bea-b83d-071d8acdf09c", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "d237b1b8-ae9b-4739-b803-41e0a8476700", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "2929b5fd-0435-49f1-9460-9ec5c35192ff", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 18, "novel_recall": 0.0, "qid": "1c30a352-12f4-4f46-863a-b76f610fb46f", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 21, "novel_recall": 0.0, "qid": "e04bcf79-32c6-44be-8c55-2fd63756d251", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 16, "novel_recall": 0.0, "qid": "8c3969db-1fcc-4bfd-a209-f307f25037d9", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "a33477d6-3212-4770-903f-d3614514d30b", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "733a51ff-9e61-40f4-a90f-67d45781aad1", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "c80efe12-c371-4b56-b6cf-5ab41bf4e1a6", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 8, "novel_recall": 0.0, "qid": "a6e8ea29-d1a2-48b4-aa82-514d28e8fcd2", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 19, "novel_recall": 0.0, "qid": "f3bc9326-7404-42b7-967b-bd45dc2c0a2a", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 21, "novel_recall": 0.0, "qid": "a6f1e054-3a69-4bca-a437-1776222860f7", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 13, "novel_recall": 0.1538, "qid": "71767248-ea90-4bea-b83d-071d8acdf09c", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 16, "novel_recall": 0.25, "qid": "d237b1b8-ae9b-4739-b803-41e0a8476700", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 16, "novel_recall": 0.4375, "qid": "2929b5fd-0435-49f1-9460-9ec5c35192ff", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 18, "novel_recall": 0.1667, "qid": "1c30a352-12f4-4f46-863a-b76f610fb46f", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 21, "novel_recall": 0.2857, "qid": "e04bcf79-32c6-44be-8c55-2fd63756d251", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "204", "full_hit": true, "n_novel": 16, "novel_recall": 1.0, "qid": "8c3969db-1fcc-4bfd-a209-f307f25037d9", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 9, "novel_recall": 0.2222, "qid": "a33477d6-3212-4770-903f-d3614514d30b", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 15, "novel_recall": 0.1333, "qid": "733a51ff-9e61-40f4-a90f-67d45781aad1", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "c80efe12-c371-4b56-b6cf-5ab41bf4e1a6", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "a6e8ea29-d1a2-48b4-aa82-514d28e8fcd2", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 19, "novel_recall": 0.1579, "qid": "f3bc9326-7404-42b7-967b-bd45dc2c0a2a", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 21, "novel_recall": 0.381, "qid": "a6f1e054-3a69-4bca-a437-1776222860f7", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 13, "novel_recall": 0.1538, "qid": "71767248-ea90-4bea-b83d-071d8acdf09c", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 16, "novel_recall": 0.25, "qid": "d237b1b8-ae9b-4739-b803-41e0a8476700", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 16, "novel_recall": 0.4375, "qid": "2929b5fd-0435-49f1-9460-9ec5c35192ff", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 18, "novel_recall": 0.1667, "qid": "1c30a352-12f4-4f46-863a-b76f610fb46f", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 21, "novel_recall": 0.2857, "qid": "e04bcf79-32c6-44be-8c55-2fd63756d251", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "204", "full_hit": true, "n_novel": 16, "novel_recall": 1.0, "qid": "8c3969db-1fcc-4bfd-a209-f307f25037d9", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 9, "novel_recall": 0.2222, "qid": "a33477d6-3212-4770-903f-d3614514d30b", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 15, "novel_recall": 0.1333, "qid": "733a51ff-9e61-40f4-a90f-67d45781aad1", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "c80efe12-c371-4b56-b6cf-5ab41bf4e1a6", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "a6e8ea29-d1a2-48b4-aa82-514d28e8fcd2", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 19, "novel_recall": 0.1579, "qid": "f3bc9326-7404-42b7-967b-bd45dc2c0a2a", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 21, "novel_recall": 0.381, "qid": "a6f1e054-3a69-4bca-a437-1776222860f7", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 13, "novel_recall": 0.1538, "qid": "71767248-ea90-4bea-b83d-071d8acdf09c", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 16, "novel_recall": 0.25, "qid": "d237b1b8-ae9b-4739-b803-41e0a8476700", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 16, "novel_recall": 0.4375, "qid": "2929b5fd-0435-49f1-9460-9ec5c35192ff", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 18, "novel_recall": 0.1667, "qid": "1c30a352-12f4-4f46-863a-b76f610fb46f", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 21, "novel_recall": 0.2857, "qid": "e04bcf79-32c6-44be-8c55-2fd63756d251", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "204", "full_hit": true, "n_novel": 16, "novel_recall": 1.0, "qid": "8c3969db-1fcc-4bfd-a209-f307f25037d9", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 9, "novel_recall": 0.2222, "qid": "a33477d6-3212-4770-903f-d3614514d30b", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 15, "novel_recall": 0.1333, "qid": "733a51ff-9e61-40f4-a90f-67d45781aad1", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "c80efe12-c371-4b56-b6cf-5ab41bf4e1a6", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "a6e8ea29-d1a2-48b4-aa82-514d28e8fcd2", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 19, "novel_recall": 0.1579, "qid": "f3bc9326-7404-42b7-967b-bd45dc2c0a2a", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 21, "novel_recall": 0.3333, "qid": "a6f1e054-3a69-4bca-a437-1776222860f7", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 13, "novel_recall": 0.1538, "qid": "71767248-ea90-4bea-b83d-071d8acdf09c", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 16, "novel_recall": 0.25, "qid": "d237b1b8-ae9b-4739-b803-41e0a8476700", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 16, "novel_recall": 0.4375, "qid": "2929b5fd-0435-49f1-9460-9ec5c35192ff", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 18, "novel_recall": 0.1667, "qid": "1c30a352-12f4-4f46-863a-b76f610fb46f", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 21, "novel_recall": 0.2857, "qid": "e04bcf79-32c6-44be-8c55-2fd63756d251", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "204", "full_hit": true, "n_novel": 16, "novel_recall": 1.0, "qid": "8c3969db-1fcc-4bfd-a209-f307f25037d9", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 9, "novel_recall": 0.2222, "qid": "a33477d6-3212-4770-903f-d3614514d30b", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 15, "novel_recall": 0.1333, "qid": "733a51ff-9e61-40f4-a90f-67d45781aad1", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "c80efe12-c371-4b56-b6cf-5ab41bf4e1a6", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "a6e8ea29-d1a2-48b4-aa82-514d28e8fcd2", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 19, "novel_recall": 0.1579, "qid": "f3bc9326-7404-42b7-967b-bd45dc2c0a2a", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 21, "novel_recall": 0.3333, "qid": "a6f1e054-3a69-4bca-a437-1776222860f7", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 13, "novel_recall": 0.1538, "qid": "71767248-ea90-4bea-b83d-071d8acdf09c", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 16, "novel_recall": 0.25, "qid": "d237b1b8-ae9b-4739-b803-41e0a8476700", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 16, "novel_recall": 0.375, "qid": "2929b5fd-0435-49f1-9460-9ec5c35192ff", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 18, "novel_recall": 0.2222, "qid": "1c30a352-12f4-4f46-863a-b76f610fb46f", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 21, "novel_recall": 0.3333, "qid": "e04bcf79-32c6-44be-8c55-2fd63756d251", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "204", "full_hit": true, "n_novel": 16, "novel_recall": 1.0, "qid": "8c3969db-1fcc-4bfd-a209-f307f25037d9", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 9, "novel_recall": 0.2222, "qid": "a33477d6-3212-4770-903f-d3614514d30b", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 15, "novel_recall": 0.1333, "qid": "733a51ff-9e61-40f4-a90f-67d45781aad1", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "c80efe12-c371-4b56-b6cf-5ab41bf4e1a6", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 8, "novel_recall": 0.25, "qid": "a6e8ea29-d1a2-48b4-aa82-514d28e8fcd2", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 19, "novel_recall": 0.1579, "qid": "f3bc9326-7404-42b7-967b-bd45dc2c0a2a", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "204", "full_hit": false, "n_novel": 21, "novel_recall": 0.3333, "qid": "a6f1e054-3a69-4bca-a437-1776222860f7", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "fcba5fe3-25ab-4abe-9eb2-55e2fe10d439", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 13, "novel_recall": 0.0, "qid": "aa769196-b02b-4b6b-9aa9-523f0e083142", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 9, "novel_recall": 0.0, "qid": "c7711845-a2e4-41b7-ae16-54a7158ba8bd", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 18, "novel_recall": 0.0, "qid": "6dae0b4b-464f-4b17-93e8-02f847a2a3e6", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 7, "novel_recall": 0.0, "qid": "0a452495-543e-41ff-a625-8cdee936c16e", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 15, "novel_recall": 0.0, "qid": "d478c2a3-23f7-4ef1-877d-3cca04b29b1a", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "d78106b0-fe08-4e77-94dd-05841b1b7df7", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 10, "novel_recall": 0.0, "qid": "3705a072-c8c2-4029-afec-0610a5a8e119", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 1, "novel_recall": 0.0, "qid": "f38c4403-9c57-49b7-853b-7b1e9132980d", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "2aad213f-5c3b-40be-a3f8-4497e8c56bee", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 17, "novel_recall": 0.0, "qid": "c5bd328f-67e4-4b6a-9bef-33bd997824a9", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 14, "novel_recall": 0.0, "qid": "65901be4-4cad-47d2-96da-54fb2df15e0c", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 9, "novel_recall": 0.8889, "qid": "fcba5fe3-25ab-4abe-9eb2-55e2fe10d439", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 13, "novel_recall": 0.6923, "qid": "aa769196-b02b-4b6b-9aa9-523f0e083142", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 9, "novel_recall": 0.6667, "qid": "c7711845-a2e4-41b7-ae16-54a7158ba8bd", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 18, "novel_recall": 0.2222, "qid": "6dae0b4b-464f-4b17-93e8-02f847a2a3e6", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 7, "novel_recall": 0.7143, "qid": "0a452495-543e-41ff-a625-8cdee936c16e", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 15, "novel_recall": 0.4667, "qid": "d478c2a3-23f7-4ef1-877d-3cca04b29b1a", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "d78106b0-fe08-4e77-94dd-05841b1b7df7", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 10, "novel_recall": 0.3, "qid": "3705a072-c8c2-4029-afec-0610a5a8e119", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "205", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "f38c4403-9c57-49b7-853b-7b1e9132980d", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "2aad213f-5c3b-40be-a3f8-4497e8c56bee", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 17, "novel_recall": 0.5294, "qid": "c5bd328f-67e4-4b6a-9bef-33bd997824a9", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 14, "novel_recall": 0.7143, "qid": "65901be4-4cad-47d2-96da-54fb2df15e0c", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 9, "novel_recall": 0.8889, "qid": "fcba5fe3-25ab-4abe-9eb2-55e2fe10d439", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 13, "novel_recall": 0.6923, "qid": "aa769196-b02b-4b6b-9aa9-523f0e083142", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "c7711845-a2e4-41b7-ae16-54a7158ba8bd", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 18, "novel_recall": 0.2222, "qid": "6dae0b4b-464f-4b17-93e8-02f847a2a3e6", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "0a452495-543e-41ff-a625-8cdee936c16e", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 15, "novel_recall": 0.4667, "qid": "d478c2a3-23f7-4ef1-877d-3cca04b29b1a", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "d78106b0-fe08-4e77-94dd-05841b1b7df7", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "3705a072-c8c2-4029-afec-0610a5a8e119", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "205", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "f38c4403-9c57-49b7-853b-7b1e9132980d", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "2aad213f-5c3b-40be-a3f8-4497e8c56bee", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 17, "novel_recall": 0.5294, "qid": "c5bd328f-67e4-4b6a-9bef-33bd997824a9", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 14, "novel_recall": 0.7143, "qid": "65901be4-4cad-47d2-96da-54fb2df15e0c", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 9, "novel_recall": 0.8889, "qid": "fcba5fe3-25ab-4abe-9eb2-55e2fe10d439", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 13, "novel_recall": 0.6923, "qid": "aa769196-b02b-4b6b-9aa9-523f0e083142", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "c7711845-a2e4-41b7-ae16-54a7158ba8bd", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 18, "novel_recall": 0.2222, "qid": "6dae0b4b-464f-4b17-93e8-02f847a2a3e6", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "0a452495-543e-41ff-a625-8cdee936c16e", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 15, "novel_recall": 0.4, "qid": "d478c2a3-23f7-4ef1-877d-3cca04b29b1a", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "d78106b0-fe08-4e77-94dd-05841b1b7df7", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "3705a072-c8c2-4029-afec-0610a5a8e119", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "205", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "f38c4403-9c57-49b7-853b-7b1e9132980d", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "2aad213f-5c3b-40be-a3f8-4497e8c56bee", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 17, "novel_recall": 0.5294, "qid": "c5bd328f-67e4-4b6a-9bef-33bd997824a9", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 14, "novel_recall": 0.7143, "qid": "65901be4-4cad-47d2-96da-54fb2df15e0c", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 9, "novel_recall": 0.8889, "qid": "fcba5fe3-25ab-4abe-9eb2-55e2fe10d439", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 13, "novel_recall": 0.6923, "qid": "aa769196-b02b-4b6b-9aa9-523f0e083142", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "c7711845-a2e4-41b7-ae16-54a7158ba8bd", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 18, "novel_recall": 0.2222, "qid": "6dae0b4b-464f-4b17-93e8-02f847a2a3e6", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 7, "novel_recall": 0.5714, "qid": "0a452495-543e-41ff-a625-8cdee936c16e", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 15, "novel_recall": 0.4, "qid": "d478c2a3-23f7-4ef1-877d-3cca04b29b1a", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "d78106b0-fe08-4e77-94dd-05841b1b7df7", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 10, "novel_recall": 0.4, "qid": "3705a072-c8c2-4029-afec-0610a5a8e119", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "205", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "f38c4403-9c57-49b7-853b-7b1e9132980d", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 5, "novel_recall": 0.4, "qid": "2aad213f-5c3b-40be-a3f8-4497e8c56bee", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 17, "novel_recall": 0.5294, "qid": "c5bd328f-67e4-4b6a-9bef-33bd997824a9", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 14, "novel_recall": 0.7143, "qid": "65901be4-4cad-47d2-96da-54fb2df15e0c", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 9, "novel_recall": 0.7778, "qid": "fcba5fe3-25ab-4abe-9eb2-55e2fe10d439", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 13, "novel_recall": 0.0769, "qid": "aa769196-b02b-4b6b-9aa9-523f0e083142", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 9, "novel_recall": 0.5556, "qid": "c7711845-a2e4-41b7-ae16-54a7158ba8bd", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 18, "novel_recall": 0.0556, "qid": "6dae0b4b-464f-4b17-93e8-02f847a2a3e6", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 7, "novel_recall": 0.2857, "qid": "0a452495-543e-41ff-a625-8cdee936c16e", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 15, "novel_recall": 0.2667, "qid": "d478c2a3-23f7-4ef1-877d-3cca04b29b1a", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 5, "novel_recall": 0.0, "qid": "d78106b0-fe08-4e77-94dd-05841b1b7df7", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 10, "novel_recall": 0.1, "qid": "3705a072-c8c2-4029-afec-0610a5a8e119", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "205", "full_hit": true, "n_novel": 1, "novel_recall": 1.0, "qid": "f38c4403-9c57-49b7-853b-7b1e9132980d", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 5, "novel_recall": 0.2, "qid": "2aad213f-5c3b-40be-a3f8-4497e8c56bee", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 17, "novel_recall": 0.4706, "qid": "c5bd328f-67e4-4b6a-9bef-33bd997824a9", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "205", "full_hit": false, "n_novel": 14, "novel_recall": 0.7143, "qid": "65901be4-4cad-47d2-96da-54fb2df15e0c", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 34, "novel_recall": 0.0, "qid": "cc347473-5c28-4888-b36f-56f2744547a7", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "a82020f7-609b-459c-83d5-1a368878a9e2", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "b7b9e4d3-0c7c-4e50-9dca-4fe394c9ee21", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 34, "novel_recall": 0.0, "qid": "be10bd78-2a49-406a-8176-e5de94ba3f09", "type": "A"}, {"arm": "M0_none", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 26, "novel_recall": 0.0, "qid": "dbb0e6e1-dae2-4135-9d46-791283baac62", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 22, "novel_recall": 0.0, "qid": "84156030-18b3-43f5-b2b2-202b59399612", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 29, "novel_recall": 0.0, "qid": "d28b9e48-1689-41df-b5d1-740a5fe0afbb", "type": "B"}, {"arm": "M0_none", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 26, "novel_recall": 0.0, "qid": "cf179869-67db-4c97-a7eb-7cec2179f28c", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 23, "novel_recall": 0.0, "qid": "51d0bc76-2ae6-44aa-a645-9e9a7376b9e5", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.0, "qid": "73f2b7b8-11d1-43a3-8c00-09f392c6cdd8", "type": "C"}, {"arm": "M0_none", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 19, "novel_recall": 0.0, "qid": "d4ec4324-b34c-4145-a975-2449082ef6cd", "type": "D"}, {"arm": "M0_none", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 31, "novel_recall": 0.0, "qid": "d694db63-aff9-4f12-a901-44578248a9c7", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 34, "novel_recall": 0.2059, "qid": "cc347473-5c28-4888-b36f-56f2744547a7", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.4167, "qid": "a82020f7-609b-459c-83d5-1a368878a9e2", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.4583, "qid": "b7b9e4d3-0c7c-4e50-9dca-4fe394c9ee21", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 34, "novel_recall": 0.4118, "qid": "be10bd78-2a49-406a-8176-e5de94ba3f09", "type": "A"}, {"arm": "M1_naive", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 26, "novel_recall": 0.1923, "qid": "dbb0e6e1-dae2-4135-9d46-791283baac62", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 22, "novel_recall": 0.2273, "qid": "84156030-18b3-43f5-b2b2-202b59399612", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 29, "novel_recall": 0.4483, "qid": "d28b9e48-1689-41df-b5d1-740a5fe0afbb", "type": "B"}, {"arm": "M1_naive", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 26, "novel_recall": 0.2692, "qid": "cf179869-67db-4c97-a7eb-7cec2179f28c", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 23, "novel_recall": 0.1304, "qid": "51d0bc76-2ae6-44aa-a645-9e9a7376b9e5", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.3333, "qid": "73f2b7b8-11d1-43a3-8c00-09f392c6cdd8", "type": "C"}, {"arm": "M1_naive", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 19, "novel_recall": 0.4211, "qid": "d4ec4324-b34c-4145-a975-2449082ef6cd", "type": "D"}, {"arm": "M1_naive", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 31, "novel_recall": 0.3548, "qid": "d694db63-aff9-4f12-a901-44578248a9c7", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 34, "novel_recall": 0.2059, "qid": "cc347473-5c28-4888-b36f-56f2744547a7", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.4167, "qid": "a82020f7-609b-459c-83d5-1a368878a9e2", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.4583, "qid": "b7b9e4d3-0c7c-4e50-9dca-4fe394c9ee21", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 34, "novel_recall": 0.4118, "qid": "be10bd78-2a49-406a-8176-e5de94ba3f09", "type": "A"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 26, "novel_recall": 0.1538, "qid": "dbb0e6e1-dae2-4135-9d46-791283baac62", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 22, "novel_recall": 0.2273, "qid": "84156030-18b3-43f5-b2b2-202b59399612", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 29, "novel_recall": 0.4483, "qid": "d28b9e48-1689-41df-b5d1-740a5fe0afbb", "type": "B"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 26, "novel_recall": 0.2692, "qid": "cf179869-67db-4c97-a7eb-7cec2179f28c", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 23, "novel_recall": 0.1304, "qid": "51d0bc76-2ae6-44aa-a645-9e9a7376b9e5", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.3333, "qid": "73f2b7b8-11d1-43a3-8c00-09f392c6cdd8", "type": "C"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 19, "novel_recall": 0.4211, "qid": "d4ec4324-b34c-4145-a975-2449082ef6cd", "type": "D"}, {"arm": "M2_evidence", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 31, "novel_recall": 0.3548, "qid": "d694db63-aff9-4f12-a901-44578248a9c7", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 34, "novel_recall": 0.2059, "qid": "cc347473-5c28-4888-b36f-56f2744547a7", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.4167, "qid": "a82020f7-609b-459c-83d5-1a368878a9e2", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.4167, "qid": "b7b9e4d3-0c7c-4e50-9dca-4fe394c9ee21", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 34, "novel_recall": 0.3529, "qid": "be10bd78-2a49-406a-8176-e5de94ba3f09", "type": "A"}, {"arm": "M3_causal", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 26, "novel_recall": 0.1538, "qid": "dbb0e6e1-dae2-4135-9d46-791283baac62", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 22, "novel_recall": 0.2273, "qid": "84156030-18b3-43f5-b2b2-202b59399612", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 29, "novel_recall": 0.4483, "qid": "d28b9e48-1689-41df-b5d1-740a5fe0afbb", "type": "B"}, {"arm": "M3_causal", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 26, "novel_recall": 0.2692, "qid": "cf179869-67db-4c97-a7eb-7cec2179f28c", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 23, "novel_recall": 0.1304, "qid": "51d0bc76-2ae6-44aa-a645-9e9a7376b9e5", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.3333, "qid": "73f2b7b8-11d1-43a3-8c00-09f392c6cdd8", "type": "C"}, {"arm": "M3_causal", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 19, "novel_recall": 0.3684, "qid": "d4ec4324-b34c-4145-a975-2449082ef6cd", "type": "D"}, {"arm": "M3_causal", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 31, "novel_recall": 0.3226, "qid": "d694db63-aff9-4f12-a901-44578248a9c7", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 34, "novel_recall": 0.2059, "qid": "cc347473-5c28-4888-b36f-56f2744547a7", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.4167, "qid": "a82020f7-609b-459c-83d5-1a368878a9e2", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.4167, "qid": "b7b9e4d3-0c7c-4e50-9dca-4fe394c9ee21", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 34, "novel_recall": 0.3529, "qid": "be10bd78-2a49-406a-8176-e5de94ba3f09", "type": "A"}, {"arm": "M4_validated", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 26, "novel_recall": 0.1538, "qid": "dbb0e6e1-dae2-4135-9d46-791283baac62", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 22, "novel_recall": 0.2273, "qid": "84156030-18b3-43f5-b2b2-202b59399612", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 29, "novel_recall": 0.4483, "qid": "d28b9e48-1689-41df-b5d1-740a5fe0afbb", "type": "B"}, {"arm": "M4_validated", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 26, "novel_recall": 0.2692, "qid": "cf179869-67db-4c97-a7eb-7cec2179f28c", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 23, "novel_recall": 0.1304, "qid": "51d0bc76-2ae6-44aa-a645-9e9a7376b9e5", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.3333, "qid": "73f2b7b8-11d1-43a3-8c00-09f392c6cdd8", "type": "C"}, {"arm": "M4_validated", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 19, "novel_recall": 0.3684, "qid": "d4ec4324-b34c-4145-a975-2449082ef6cd", "type": "D"}, {"arm": "M4_validated", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 31, "novel_recall": 0.3226, "qid": "d694db63-aff9-4f12-a901-44578248a9c7", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 34, "novel_recall": 0.2353, "qid": "cc347473-5c28-4888-b36f-56f2744547a7", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.4167, "qid": "a82020f7-609b-459c-83d5-1a368878a9e2", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.4167, "qid": "b7b9e4d3-0c7c-4e50-9dca-4fe394c9ee21", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 34, "novel_recall": 0.3529, "qid": "be10bd78-2a49-406a-8176-e5de94ba3f09", "type": "A"}, {"arm": "R_recent", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 26, "novel_recall": 0.1538, "qid": "dbb0e6e1-dae2-4135-9d46-791283baac62", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 22, "novel_recall": 0.2273, "qid": "84156030-18b3-43f5-b2b2-202b59399612", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 29, "novel_recall": 0.4483, "qid": "d28b9e48-1689-41df-b5d1-740a5fe0afbb", "type": "B"}, {"arm": "R_recent", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 26, "novel_recall": 0.3077, "qid": "cf179869-67db-4c97-a7eb-7cec2179f28c", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 23, "novel_recall": 0.1304, "qid": "51d0bc76-2ae6-44aa-a645-9e9a7376b9e5", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 24, "novel_recall": 0.3333, "qid": "73f2b7b8-11d1-43a3-8c00-09f392c6cdd8", "type": "C"}, {"arm": "R_recent", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 19, "novel_recall": 0.3684, "qid": "d4ec4324-b34c-4145-a975-2449082ef6cd", "type": "D"}, {"arm": "R_recent", "domain": "WEB", "episode": "206", "full_hit": false, "n_novel": 31, "novel_recall": 0.3226, "qid": "d694db63-aff9-4f12-a901-44578248a9c7", "type": "D"}], "schema": "cain.e115.trackA.retrieval_diag.v1", "split": "dev", "split_hash": "0027643f7de9e0d34520c45f7bb7169d9eb6e3871e856667161a9da60e6a8506", "summary": {"M0_none": {"full_hit": 0.0, "novel_recall": 0.0, "questions": 632}, "M1_naive": {"full_hit": 0.068, "novel_recall": 0.3934, "questions": 632}, "M2_evidence": {"full_hit": 0.068, "novel_recall": 0.4092, "questions": 632}, "M3_causal": {"full_hit": 0.0633, "novel_recall": 0.3908, "questions": 632}, "M4_validated": {"full_hit": 0.0633, "novel_recall": 0.3908, "questions": 632}, "R_recent": {"full_hit": 0.0649, "novel_recall": 0.351, "questions": 632}}, "wall_s": 79.1}