[{"acc_matrix": [[0.9625, 0.0, 0.0, 0.0, 0.0], [0.62, 0.9475, 0.0, 0.0, 0.0], [0.49, 0.6825, 0.93, 0.0, 0.0], [0.545, 0.5375, 0.76, 0.9325, 0.0], [0.53, 0.4625, 0.635, 0.545, 0.885]], "bwt": -0.39999999999999997, "cpu_s": 0.395, "dataset_hashes": ["1763e3ad199c0541cee1c7afb1550d3c5dd2e367e96c9c647bc573d9b84b6bd6", "39ef399dfa355a4c7587367528e84339faca2ecc156ff5ce578e290fc0de6b14", "839ae9f4d382126953c8cde38f7ac8c8aeb6355a1e41849dd578f26c5239c51a", "cf59cd936c49d683739dcf4626d59137777f2a359032091296bc41aeba765a17", "647d40cd81a4c2ea323cece20ac0277164c9c9fc8d8292f6bc39c7ddbfb89817"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=1, T=5); synthetic, no external data", "final_mean_acc": 0.6115, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.485, "mean_forgetting": 0.39999999999999997, "memory_bytes": 0, "model_version": "208caa649293342afe5eebe20bac147710e0aed31be1dd4d77c83f4c40656e4a", "seed": 1, "train_sample_steps": 90000, "zero_shot_mean": 0.513125}, {"acc_matrix": [[0.89, 0.0, 0.0, 0.0, 0.0], [0.51, 0.955, 0.0, 0.0, 0.0], [0.655, 0.76, 0.9075, 0.0, 0.0], [0.545, 0.775, 0.8275, 0.8225, 0.0], [0.5325, 0.74, 0.6075, 0.6275, 0.905]], "bwt": -0.266875, "cpu_s": 0.413, "dataset_hashes": ["b5ae6ef12b3fe70545431c625aacc0fac2b665714e1e696d72c4a3172987e1b6", "0475c525b42c118dc488986ba633c3bc8b2b30c18c3134c6705f8c6180c6a236", "4cdae5d59ca14ae7fa026a157abf14ab3642a0e2327b11079424b35032b37764", "87dc6d565da745b304e6a56dd017b913d71d29c12267bf8582cb9e772972ea63", "eb387c2dc797c4526a8845690496547806507d18518b7d4fb1689a3ea0ced8a6"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=2, T=5); synthetic, no external data", "final_mean_acc": 0.6824999999999999, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.35750000000000004, "mean_forgetting": 0.266875, "memory_bytes": 0, "model_version": "925ba0fbe1d849cf61ff3f198d6c89ac2eb0bb5bf8067ff28b10f8970505d77e", "seed": 2, "train_sample_steps": 90000, "zero_shot_mean": 0.55}, {"acc_matrix": [[0.945, 0.0, 0.0, 0.0, 0.0], [0.55, 0.925, 0.0, 0.0, 0.0], [0.6375, 0.7875, 0.8725, 0.0, 0.0], [0.585, 0.6375, 0.815, 0.8875, 0.0], [0.645, 0.6425, 0.6375, 0.7175, 0.8625]], "bwt": -0.246875, "cpu_s": 0.423, "dataset_hashes": ["1ae5498d43a7de131455502d7e201f830fbc9fc83f0b46ce4dbbf439b9df6788", "1b7bfd93ac4e3158fa398db6270320b3fccd18363b8f21fa67cf622dda1e308f", "0f421997ab00bb16a6e9f285b03a1fc6ad3a7c6c644525a4042b35402ba903c0", "07c33c5d80819888a86e637d49aa6b11db456f2d5ab3c3ec8d93ca59c349bebf", "ae35ac6c156af000742d2891478ff5442b764b12dd8f3004f9960b001d8db196"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=3, T=5); synthetic, no external data", "final_mean_acc": 0.701, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.29999999999999993, "mean_forgetting": 0.246875, "memory_bytes": 0, "model_version": "925736271733adb010eff4fb8153c90ecb0ae411a3ad1e5bba17c149d28fa0f0", "seed": 3, "train_sample_steps": 90000, "zero_shot_mean": 0.595}, {"acc_matrix": [[0.935, 0.0, 0.0, 0.0, 0.0], [0.71, 0.905, 0.0, 0.0, 0.0], [0.645, 0.805, 0.93, 0.0, 0.0], [0.5925, 0.5, 0.535, 0.945, 0.0], [0.675, 0.605, 0.5225, 0.825, 0.8775]], "bwt": -0.27187500000000003, "cpu_s": 0.422, "dataset_hashes": ["545b09cd57fc021bb240131299dd0efe7d2dd0bd79950af2cb6f531a64cd816c", "2f0f3b07df589bf72ce937754e7d426d09100a0eb47356ed406bcf3def9e76cc", "1b3a416238ff5b191d0d2f6fddb29bdffb2083d8d8b44be99684e2bfe5081304", "f5fa6523dd0fbdf2ff8df999a423c0952b2b55ddfae9e5eec2cd1bce7eac147d", "a88ee39310a11f89883386e9761789974618f8c015ae22e402ada8182a345c33"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=4, T=5); synthetic, no external data", "final_mean_acc": 0.701, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.4075000000000001, "mean_forgetting": 0.27187500000000003, "memory_bytes": 0, "model_version": "d79c127b3349c5b50efbf13d2c831ed2b071ed03eca4a65ab2cc6d2aec1ae08d", "seed": 4, "train_sample_steps": 90000, "zero_shot_mean": 0.6068749999999999}, {"acc_matrix": [[0.9225, 0.0, 0.0, 0.0, 0.0], [0.72, 0.9025, 0.0, 0.0, 0.0], [0.7475, 0.685, 0.9575, 0.0, 0.0], [0.6225, 0.465, 0.78, 0.9075, 0.0], [0.7075, 0.4675, 0.7075, 0.7625, 0.8975]], "bwt": -0.26125, "cpu_s": 0.384, "dataset_hashes": ["bbf0176ddb0c3209ff8728a2b97e3a27e6daab2daeb97b50d757cc13980fd9a6", "d4d201450e93c6d1f4ab9034c21f76454957ae11f87b9cfd6d092a63da8a50fd", "f8c3987387b7df37f7bb68a171117cdff99c5d7b8fbf68593aa6489e5e001656", "15449aef827fd44c0107f6b9339801e41fbc95fc9b9e66f61dbaf7ddfe9d2150", "c2e21ac54b6a218e1691f9ce411cf231023a2a083698a149f6130731ac57f3bb"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=5, T=5); synthetic, no external data", "final_mean_acc": 0.7085, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.43499999999999994, "mean_forgetting": 0.26125, "memory_bytes": 0, "model_version": "491cf5635936d2b2c06617de0e28eb3d1106dac428fa05ef52db0109427f70fd", "seed": 5, "train_sample_steps": 90000, "zero_shot_mean": 0.5543750000000001}, {"acc_matrix": [[0.9625, 0.0, 0.0, 0.0, 0.0], [0.7875, 0.9325, 0.0, 0.0, 0.0], [0.76, 0.81, 0.9175, 0.0, 0.0], [0.7275, 0.795, 0.835, 0.905, 0.0], [0.68, 0.82, 0.785, 0.83, 0.8525]], "bwt": -0.150625, "cpu_s": 0.544, "dataset_hashes": ["1763e3ad199c0541cee1c7afb1550d3c5dd2e367e96c9c647bc573d9b84b6bd6", "39ef399dfa355a4c7587367528e84339faca2ecc156ff5ce578e290fc0de6b14", "839ae9f4d382126953c8cde38f7ac8c8aeb6355a1e41849dd578f26c5239c51a", "cf59cd936c49d683739dcf4626d59137777f2a359032091296bc41aeba765a17", "647d40cd81a4c2ea323cece20ac0277164c9c9fc8d8292f6bc39c7ddbfb89817"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=1, T=5); synthetic, no external data", "final_mean_acc": 0.7935000000000001, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.2825, "mean_forgetting": 0.150625, "memory_bytes": 50400, "model_version": "8b5247a5da16af2353a51fbeb03ff47124afb36b23130ff1604f9534af661a09", "seed": 1, "train_sample_steps": 126000, "zero_shot_mean": 0.520625}, {"acc_matrix": [[0.89, 0.0, 0.0, 0.0, 0.0], [0.8625, 0.95, 0.0, 0.0, 0.0], [0.8275, 0.885, 0.945, 0.0, 0.0], [0.8175, 0.8625, 0.8675, 0.8175, 0.0], [0.8175, 0.87, 0.815, 0.77, 0.8875]], "bwt": -0.08249999999999999, "cpu_s": 0.544, "dataset_hashes": ["b5ae6ef12b3fe70545431c625aacc0fac2b665714e1e696d72c4a3172987e1b6", "0475c525b42c118dc488986ba633c3bc8b2b30c18c3134c6705f8c6180c6a236", "4cdae5d59ca14ae7fa026a157abf14ab3642a0e2327b11079424b35032b37764", "87dc6d565da745b304e6a56dd017b913d71d29c12267bf8582cb9e772972ea63", "eb387c2dc797c4526a8845690496547806507d18518b7d4fb1689a3ea0ced8a6"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=2, T=5); synthetic, no external data", "final_mean_acc": 0.8320000000000001, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.13, "mean_forgetting": 0.08249999999999999, "memory_bytes": 50400, "model_version": "5e4b71f14189082649eaac19795aebdf149ac0ba016d8f13672f882b8d26b545", "seed": 2, "train_sample_steps": 126000, "zero_shot_mean": 0.559375}, {"acc_matrix": [[0.945, 0.0, 0.0, 0.0, 0.0], [0.8625, 0.9125, 0.0, 0.0, 0.0], [0.87, 0.84, 0.8875, 0.0, 0.0], [0.845, 0.84, 0.8575, 0.87, 0.0], [0.8075, 0.78, 0.8275, 0.8175, 0.855]], "bwt": -0.09562499999999996, "cpu_s": 0.544, "dataset_hashes": ["1ae5498d43a7de131455502d7e201f830fbc9fc83f0b46ce4dbbf439b9df6788", "1b7bfd93ac4e3158fa398db6270320b3fccd18363b8f21fa67cf622dda1e308f", "0f421997ab00bb16a6e9f285b03a1fc6ad3a7c6c644525a4042b35402ba903c0", "07c33c5d80819888a86e637d49aa6b11db456f2d5ab3c3ec8d93ca59c349bebf", "ae35ac6c156af000742d2891478ff5442b764b12dd8f3004f9960b001d8db196"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=3, T=5); synthetic, no external data", "final_mean_acc": 0.8175000000000001, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.13749999999999996, "mean_forgetting": 0.09562499999999996, "memory_bytes": 50400, "model_version": "a3bb6a8b1929dc2db98beb0e12d93f445223a82e608a83c238a40421e940313e", "seed": 3, "train_sample_steps": 126000, "zero_shot_mean": 0.559375}, {"acc_matrix": [[0.935, 0.0, 0.0, 0.0, 0.0], [0.84, 0.8875, 0.0, 0.0, 0.0], [0.7925, 0.8575, 0.8725, 0.0, 0.0], [0.7625, 0.8175, 0.8775, 0.91, 0.0], [0.8075, 0.7975, 0.87, 0.8425, 0.845]], "bwt": -0.07187500000000002, "cpu_s": 0.562, "dataset_hashes": ["545b09cd57fc021bb240131299dd0efe7d2dd0bd79950af2cb6f531a64cd816c", "2f0f3b07df589bf72ce937754e7d426d09100a0eb47356ed406bcf3def9e76cc", "1b3a416238ff5b191d0d2f6fddb29bdffb2083d8d8b44be99684e2bfe5081304", "f5fa6523dd0fbdf2ff8df999a423c0952b2b55ddfae9e5eec2cd1bce7eac147d", "a88ee39310a11f89883386e9761789974618f8c015ae22e402ada8182a345c33"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=4, T=5); synthetic, no external data", "final_mean_acc": 0.8324999999999999, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.12750000000000006, "mean_forgetting": 0.073125, "memory_bytes": 50400, "model_version": "7edcf13eb44956d056b88382895a5f583b6706eb11e70b363d36f60a6601512f", "seed": 4, "train_sample_steps": 126000, "zero_shot_mean": 0.594375}, {"acc_matrix": [[0.9225, 0.0, 0.0, 0.0, 0.0], [0.85, 0.9, 0.0, 0.0, 0.0], [0.8475, 0.85, 0.935, 0.0, 0.0], [0.8575, 0.8625, 0.89, 0.92, 0.0], [0.85, 0.8225, 0.8025, 0.8275, 0.8875]], "bwt": -0.09375000000000003, "cpu_s": 0.494, "dataset_hashes": ["bbf0176ddb0c3209ff8728a2b97e3a27e6daab2daeb97b50d757cc13980fd9a6", "d4d201450e93c6d1f4ab9034c21f76454957ae11f87b9cfd6d092a63da8a50fd", "f8c3987387b7df37f7bb68a171117cdff99c5d7b8fbf68593aa6489e5e001656", "15449aef827fd44c0107f6b9339801e41fbc95fc9b9e66f61dbaf7ddfe9d2150", "c2e21ac54b6a218e1691f9ce411cf231023a2a083698a149f6130731ac57f3bb"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=5, T=5); synthetic, no external data", "final_mean_acc": 0.8379999999999999, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.13250000000000006, "mean_forgetting": 0.09375000000000003, "memory_bytes": 50400, "model_version": "b851e45f635b78755dad83a910291da36cd0f47d54028b3ca40f8eaa0fe5c31d", "seed": 5, "train_sample_steps": 126000, "zero_shot_mean": 0.566875}, {"acc_matrix": [[0.8025, 0.0, 0.0, 0.0, 0.0], [0.8025, 0.7325, 0.0, 0.0, 0.0], [0.7975, 0.7325, 0.8, 0.0, 0.0], [0.7975, 0.7325, 0.8, 0.8175, 0.0], [0.8, 0.73, 0.8025, 0.8175, 0.8375]], "bwt": -0.0006250000000000144, "cpu_s": 1.433, "dataset_hashes": ["1763e3ad199c0541cee1c7afb1550d3c5dd2e367e96c9c647bc573d9b84b6bd6", "39ef399dfa355a4c7587367528e84339faca2ecc156ff5ce578e290fc0de6b14", "839ae9f4d382126953c8cde38f7ac8c8aeb6355a1e41849dd578f26c5239c51a", "cf59cd936c49d683739dcf4626d59137777f2a359032091296bc41aeba765a17", "647d40cd81a4c2ea323cece20ac0277164c9c9fc8d8292f6bc39c7ddbfb89817"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=1, T=5); synthetic, no external data", "final_mean_acc": 0.7975, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0025000000000000577, "mean_forgetting": 0.0006250000000000144, "memory_bytes": 252000, "seed": 1, "train_sample_steps": 0, "zero_shot_mean": 0.49312500000000004}, {"acc_matrix": [[0.785, 0.0, 0.0, 0.0, 0.0], [0.785, 0.8225, 0.0, 0.0, 0.0], [0.785, 0.8225, 0.7725, 0.0, 0.0], [0.785, 0.82, 0.7725, 0.7825, 0.0], [0.785, 0.82, 0.7725, 0.7825, 0.815]], "bwt": -0.0006250000000000144, "cpu_s": 1.57, "dataset_hashes": ["b5ae6ef12b3fe70545431c625aacc0fac2b665714e1e696d72c4a3172987e1b6", "0475c525b42c118dc488986ba633c3bc8b2b30c18c3134c6705f8c6180c6a236", "4cdae5d59ca14ae7fa026a157abf14ab3642a0e2327b11079424b35032b37764", "87dc6d565da745b304e6a56dd017b913d71d29c12267bf8582cb9e772972ea63", "eb387c2dc797c4526a8845690496547806507d18518b7d4fb1689a3ea0ced8a6"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=2, T=5); synthetic, no external data", "final_mean_acc": 0.795, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0025000000000000577, "mean_forgetting": 0.0006250000000000144, "memory_bytes": 252000, "seed": 2, "train_sample_steps": 0, "zero_shot_mean": 0.5375000000000001}, {"acc_matrix": [[0.8175, 0.0, 0.0, 0.0, 0.0], [0.8175, 0.8175, 0.0, 0.0, 0.0], [0.8175, 0.8175, 0.7625, 0.0, 0.0], [0.8175, 0.8175, 0.7625, 0.78, 0.0], [0.8175, 0.8175, 0.7625, 0.7825, 0.81]], "bwt": 0.0006249999999999867, "cpu_s": 1.605, "dataset_hashes": ["1ae5498d43a7de131455502d7e201f830fbc9fc83f0b46ce4dbbf439b9df6788", "1b7bfd93ac4e3158fa398db6270320b3fccd18363b8f21fa67cf622dda1e308f", "0f421997ab00bb16a6e9f285b03a1fc6ad3a7c6c644525a4042b35402ba903c0", "07c33c5d80819888a86e637d49aa6b11db456f2d5ab3c3ec8d93ca59c349bebf", "ae35ac6c156af000742d2891478ff5442b764b12dd8f3004f9960b001d8db196"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=3, T=5); synthetic, no external data", "final_mean_acc": 0.7979999999999999, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0, "mean_forgetting": -0.0006249999999999867, "memory_bytes": 252000, "seed": 3, "train_sample_steps": 0, "zero_shot_mean": 0.565}, {"acc_matrix": [[0.84, 0.0, 0.0, 0.0, 0.0], [0.845, 0.805, 0.0, 0.0, 0.0], [0.845, 0.805, 0.7925, 0.0, 0.0], [0.845, 0.805, 0.7925, 0.7725, 0.0], [0.845, 0.805, 0.7925, 0.7725, 0.815]], "bwt": 0.0012500000000000011, "cpu_s": 1.563, "dataset_hashes": ["545b09cd57fc021bb240131299dd0efe7d2dd0bd79950af2cb6f531a64cd816c", "2f0f3b07df589bf72ce937754e7d426d09100a0eb47356ed406bcf3def9e76cc", "1b3a416238ff5b191d0d2f6fddb29bdffb2083d8d8b44be99684e2bfe5081304", "f5fa6523dd0fbdf2ff8df999a423c0952b2b55ddfae9e5eec2cd1bce7eac147d", "a88ee39310a11f89883386e9761789974618f8c015ae22e402ada8182a345c33"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=4, T=5); synthetic, no external data", "final_mean_acc": 0.8059999999999998, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 252000, "seed": 4, "train_sample_steps": 0, "zero_shot_mean": 0.5575}, {"acc_matrix": [[0.7875, 0.0, 0.0, 0.0, 0.0], [0.7875, 0.83, 0.0, 0.0, 0.0], [0.7875, 0.83, 0.805, 0.0, 0.0], [0.7875, 0.83, 0.805, 0.7825, 0.0], [0.7875, 0.83, 0.805, 0.7825, 0.8075]], "bwt": 0.0, "cpu_s": 1.506, "dataset_hashes": ["bbf0176ddb0c3209ff8728a2b97e3a27e6daab2daeb97b50d757cc13980fd9a6", "d4d201450e93c6d1f4ab9034c21f76454957ae11f87b9cfd6d092a63da8a50fd", "f8c3987387b7df37f7bb68a171117cdff99c5d7b8fbf68593aa6489e5e001656", "15449aef827fd44c0107f6b9339801e41fbc95fc9b9e66f61dbaf7ddfe9d2150", "c2e21ac54b6a218e1691f9ce411cf231023a2a083698a149f6130731ac57f3bb"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=5, T=5); synthetic, no external data", "final_mean_acc": 0.8025, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 252000, "seed": 5, "train_sample_steps": 0, "zero_shot_mean": 0.5}, {"acc_matrix": [[0.9625, 0.0, 0.0, 0.0, 0.0], [0.9625, 0.9275, 0.0, 0.0, 0.0], [0.9625, 0.9275, 0.9475, 0.0, 0.0], [0.9625, 0.9275, 0.9475, 0.955, 0.0], [0.9625, 0.9275, 0.9475, 0.955, 0.9325]], "bwt": 0.0, "cpu_s": 0.384, "dataset_hashes": ["1763e3ad199c0541cee1c7afb1550d3c5dd2e367e96c9c647bc573d9b84b6bd6", "39ef399dfa355a4c7587367528e84339faca2ecc156ff5ce578e290fc0de6b14", "839ae9f4d382126953c8cde38f7ac8c8aeb6355a1e41849dd578f26c5239c51a", "cf59cd936c49d683739dcf4626d59137777f2a359032091296bc41aeba765a17", "647d40cd81a4c2ea323cece20ac0277164c9c9fc8d8292f6bc39c7ddbfb89817"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=1, T=5); synthetic, no external data", "final_mean_acc": 0.9450000000000001, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7892105263157895, "seed": 1, "train_sample_steps": 90000, "zero_shot_mean": 0.499375}, {"acc_matrix": [[0.89, 0.0, 0.0, 0.0, 0.0], [0.89, 0.945, 0.0, 0.0, 0.0], [0.89, 0.945, 0.9075, 0.0, 0.0], [0.89, 0.945, 0.9075, 0.8625, 0.0], [0.89, 0.945, 0.9075, 0.8625, 0.9225]], "bwt": 0.0, "cpu_s": 0.395, "dataset_hashes": ["b5ae6ef12b3fe70545431c625aacc0fac2b665714e1e696d72c4a3172987e1b6", "0475c525b42c118dc488986ba633c3bc8b2b30c18c3134c6705f8c6180c6a236", "4cdae5d59ca14ae7fa026a157abf14ab3642a0e2327b11079424b35032b37764", "87dc6d565da745b304e6a56dd017b913d71d29c12267bf8582cb9e772972ea63", "eb387c2dc797c4526a8845690496547806507d18518b7d4fb1689a3ea0ced8a6"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=2, T=5); synthetic, no external data", "final_mean_acc": 0.9055, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7889473684210526, "seed": 2, "train_sample_steps": 90000, "zero_shot_mean": 0.548125}, {"acc_matrix": [[0.945, 0.0, 0.0, 0.0, 0.0], [0.945, 0.915, 0.0, 0.0, 0.0], [0.945, 0.915, 0.865, 0.0, 0.0], [0.945, 0.915, 0.865, 0.8725, 0.0], [0.945, 0.915, 0.865, 0.8725, 0.85]], "bwt": 0.0, "cpu_s": 0.398, "dataset_hashes": ["1ae5498d43a7de131455502d7e201f830fbc9fc83f0b46ce4dbbf439b9df6788", "1b7bfd93ac4e3158fa398db6270320b3fccd18363b8f21fa67cf622dda1e308f", "0f421997ab00bb16a6e9f285b03a1fc6ad3a7c6c644525a4042b35402ba903c0", "07c33c5d80819888a86e637d49aa6b11db456f2d5ab3c3ec8d93ca59c349bebf", "ae35ac6c156af000742d2891478ff5442b764b12dd8f3004f9960b001d8db196"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=3, T=5); synthetic, no external data", "final_mean_acc": 0.8895, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7894736842105263, "seed": 3, "train_sample_steps": 90000, "zero_shot_mean": 0.57125}, {"acc_matrix": [[0.935, 0.0, 0.0, 0.0, 0.0], [0.935, 0.945, 0.0, 0.0, 0.0], [0.935, 0.945, 0.94, 0.0, 0.0], [0.935, 0.945, 0.94, 0.955, 0.0], [0.935, 0.945, 0.94, 0.955, 0.8725]], "bwt": 0.0, "cpu_s": 0.435, "dataset_hashes": ["545b09cd57fc021bb240131299dd0efe7d2dd0bd79950af2cb6f531a64cd816c", "2f0f3b07df589bf72ce937754e7d426d09100a0eb47356ed406bcf3def9e76cc", "1b3a416238ff5b191d0d2f6fddb29bdffb2083d8d8b44be99684e2bfe5081304", "f5fa6523dd0fbdf2ff8df999a423c0952b2b55ddfae9e5eec2cd1bce7eac147d", "a88ee39310a11f89883386e9761789974618f8c015ae22e402ada8182a345c33"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=4, T=5); synthetic, no external data", "final_mean_acc": 0.9295, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7894736842105263, "seed": 4, "train_sample_steps": 90000, "zero_shot_mean": 0.58}, {"acc_matrix": [[0.9225, 0.0, 0.0, 0.0, 0.0], [0.9225, 0.9025, 0.0, 0.0, 0.0], [0.9225, 0.9025, 0.915, 0.0, 0.0], [0.9225, 0.9025, 0.915, 0.8975, 0.0], [0.9225, 0.9025, 0.915, 0.8975, 0.9075]], "bwt": 0.0, "cpu_s": 0.404, "dataset_hashes": ["bbf0176ddb0c3209ff8728a2b97e3a27e6daab2daeb97b50d757cc13980fd9a6", "d4d201450e93c6d1f4ab9034c21f76454957ae11f87b9cfd6d092a63da8a50fd", "f8c3987387b7df37f7bb68a171117cdff99c5d7b8fbf68593aa6489e5e001656", "15449aef827fd44c0107f6b9339801e41fbc95fc9b9e66f61dbaf7ddfe9d2150", "c2e21ac54b6a218e1691f9ce411cf231023a2a083698a149f6130731ac57f3bb"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=5, T=5); synthetic, no external data", "final_mean_acc": 0.909, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7894736842105263, "seed": 5, "train_sample_steps": 90000, "zero_shot_mean": 0.536875}, {"acc_matrix": [[0.89, 0.0, 0.0, 0.0, 0.0], [0.8, 0.95, 0.0, 0.0, 0.0], [0.64, 0.7425, 0.88, 0.0, 0.0], [0.685, 0.695, 0.765, 0.93, 0.0], [0.7575, 0.6825, 0.5475, 0.8075, 0.9375]], "bwt": -0.21375000000000002, "cpu_s": 0.39, "dataset_hashes": ["7692e94c41331a04d068e5504f68c35a8dfffd3099e7d68df41a2c6adc042fb2", "7fe893a0822d5d03682474c63a4b42208f1344b054cb1413c7f6ad7fef8e495c", "f73b75264e6c6dfc14e473ab2c37af855a9be0c5453073ebeb1f7adf5bd1f8d1", "81f3dbb7c61c0c5aecc2771d8212462a01584d9e9b226f3fae2d8b5a1c952435", "ed53ba3ae8a520e1aa9bf9a2932c742ec83d98d078a72ee136556213610421d1"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=101, T=5); synthetic, no external data", "final_mean_acc": 0.7464999999999999, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.3325, "mean_forgetting": 0.21375000000000002, "memory_bytes": 0, "model_version": "12498c6d41e5586dc8028251500a78087e105afddcc4a75e8cf9bfac1c90dd3d", "seed": 101, "train_sample_steps": 90000, "zero_shot_mean": 0.5249999999999999}, {"acc_matrix": [[0.95, 0.0, 0.0, 0.0, 0.0], [0.6175, 0.8925, 0.0, 0.0, 0.0], [0.545, 0.7575, 0.8625, 0.0, 0.0], [0.615, 0.7125, 0.7625, 0.915, 0.0], [0.505, 0.6625, 0.68, 0.8, 0.9125]], "bwt": -0.24312499999999998, "cpu_s": 0.415, "dataset_hashes": ["70ec99a7d167496916fae64f7b30048f7c73ccf788980d47ac1299d63669cd11", "e850d1772baac847ee2940440577a7589941192dc698435641d00a34f7d5a62b", "5e1d9e27dea13d9e7fd1e1d70a2c83bf36e6990021d6df94ccc1868945ba67f3", "c63af5f19bb8a1eec9c2e49e1bda5551728678a31d6dc2cadd99df032df2e1ec", "0d7ac133d50e5d6492856b402803a138431cafaf415b9901ec4da47d50371e39"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=102, T=5); synthetic, no external data", "final_mean_acc": 0.712, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.44499999999999995, "mean_forgetting": 0.24312499999999998, "memory_bytes": 0, "model_version": "9a63a2819097aa7d9ebe3dcc9b912ab683767d29403a694e022eb835bcda33ba", "seed": 102, "train_sample_steps": 90000, "zero_shot_mean": 0.634375}, {"acc_matrix": [[0.9375, 0.0, 0.0, 0.0, 0.0], [0.5275, 0.9125, 0.0, 0.0, 0.0], [0.5525, 0.6125, 0.8725, 0.0, 0.0], [0.57, 0.735, 0.7925, 0.925, 0.0], [0.6625, 0.5325, 0.6425, 0.62, 0.945]], "bwt": -0.29750000000000004, "cpu_s": 0.375, "dataset_hashes": ["f2783f09c146e59d34aff8909af6b8597d0e2a8f44eb00baf4c725c0c67936b0", "310f7ca6c82c564e84b23f0a0f8118b581a5c7d8266f9c7e9d34af5b83946084", "92259ccf5a5f4df7efba2a46c4dc80e63226a812035d9ed3ed8c02f64abc15ed", "e20fbd692eeb55c6e7eed9771a72bce2afa4520c04382a3775a1e92cb6f30da1", "fec9f4e8850c6220f70257bca7da49f1026aff4f3bb7881917f6d240c6fbfd9b"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=103, T=5); synthetic, no external data", "final_mean_acc": 0.6805, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.38, "mean_forgetting": 0.29750000000000004, "memory_bytes": 0, "model_version": "8a5333d1fff535409cb3c7971e3a162cb155ce30840d78a3673e2009ea1c8dd0", "seed": 103, "train_sample_steps": 90000, "zero_shot_mean": 0.621875}, {"acc_matrix": [[0.95, 0.0, 0.0, 0.0, 0.0], [0.6125, 0.9225, 0.0, 0.0, 0.0], [0.52, 0.77, 0.945, 0.0, 0.0], [0.575, 0.6475, 0.6925, 0.9225, 0.0], [0.5, 0.525, 0.5775, 0.7625, 0.9225]], "bwt": -0.34375, "cpu_s": 0.383, "dataset_hashes": ["50a2692f150818326dece0d3e2d0f928bd8889f120c3440198179ffcc0fe4415", "f5c3ced4b920a8e66cb09dfe82ff6703ebe31252e1a11e76d4c79622b197aef9", "f42cb106b97255c1fb872fd3d971f2896ea5c51f422c5a717615e1db56972c3a", "e472b7ef4bed3fb3ef8c801b6098a6164023240a87df127e08e547469f9ed072", "0493e8453f02084503ce14cee6838aaa4cd4f5c48c3c73386e6dfa978188f44f"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=104, T=5); synthetic, no external data", "final_mean_acc": 0.6575, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.44999999999999996, "mean_forgetting": 0.34375, "memory_bytes": 0, "model_version": "be290ff2261c512cd0fae694b5a6d34948fd84bad1a6b18819c829759f7bd429", "seed": 104, "train_sample_steps": 90000, "zero_shot_mean": 0.63}, {"acc_matrix": [[0.92, 0.0, 0.0, 0.0, 0.0], [0.58, 0.895, 0.0, 0.0, 0.0], [0.6475, 0.6425, 0.915, 0.0, 0.0], [0.5375, 0.52, 0.85, 0.9325, 0.0], [0.5725, 0.51, 0.7175, 0.5975, 0.8525]], "bwt": -0.31625000000000003, "cpu_s": 0.425, "dataset_hashes": ["de63a04f1e7cd650d86f03df806e6414f31da0ca55039a4acb9daeb0666e4894", "c52ddc8080f6e873750d9ef3c611923099ea990d7a1a93347d31f17c3d3b5cd1", "22cd8a0cee3b3403e92c43a804099559d119ce50f223d4184e952e778a76da60", "d743fcfcee554108664e16454e06f66b9226a4894fdbbe2e4f997785187db04a", "32f688ca2f4f053c421322ac3d77177545b8a3a3e441e7972bea08a18ecd134e"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=105, T=5); synthetic, no external data", "final_mean_acc": 0.65, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.385, "mean_forgetting": 0.31625000000000003, "memory_bytes": 0, "model_version": "15391617d69f65c52a795bb34a1eae5176e9020265bdce04d91d3faf2e538a66", "seed": 105, "train_sample_steps": 90000, "zero_shot_mean": 0.571875}, {"acc_matrix": [[0.955, 0.0, 0.0, 0.0, 0.0], [0.7025, 0.9625, 0.0, 0.0, 0.0], [0.6175, 0.7425, 0.845, 0.0, 0.0], [0.59, 0.7775, 0.5125, 0.8975, 0.0], [0.5775, 0.7375, 0.745, 0.6325, 0.9375]], "bwt": -0.24187499999999998, "cpu_s": 0.381, "dataset_hashes": ["c9d790f978251db8c5588837af54deb32f1b0090f78d62c62b47cd3edc12f23a", "f60c20c74ea8f42dad87dcc18f8ba7d4db90c908cf0581caecb4046346e862a9", "8c470026d9048aef1cf15e639d873aca77accceb00e4611911b17bcdf5b428ac", "8bf5fff2a1bd277b6e4ad3f7f1517836de6c7c6cea12c048cf498bf43b0d3445", "845ff8f52d01160500c9168301a1df005c3354ca6b33395a5b739645fffe9dd0"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=106, T=5); synthetic, no external data", "final_mean_acc": 0.726, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.37749999999999995, "mean_forgetting": 0.24187499999999998, "memory_bytes": 0, "model_version": "e81eb3d60982f39c274144b4d8560fb238df529b58104a3a73f17263a718153a", "seed": 106, "train_sample_steps": 90000, "zero_shot_mean": 0.554375}, {"acc_matrix": [[0.9425, 0.0, 0.0, 0.0, 0.0], [0.5625, 0.92, 0.0, 0.0, 0.0], [0.5225, 0.775, 0.9225, 0.0, 0.0], [0.6425, 0.57, 0.6275, 0.8475, 0.0], [0.51, 0.6025, 0.5725, 0.6575, 0.8925]], "bwt": -0.3225, "cpu_s": 0.375, "dataset_hashes": ["23816b26028461935e27bb81124b583a607c2cfa3c9022e010237bac7f99b222", "f8948ec48d4e972808611ed71527f7aa3bc41cac9f67b4c5a2ddcf42195f4f69", "6f8d78bc83414fa92f5a84df3b6e36b7521d032fc153b7435e578f7441461842", "5f915d98b5319257c3a7b8d0dc12042381c86da9b102d03d9ce1ef4bf67da301", "d526f8127b75bbf433b3ecca71144431c7cbdd5aa3d92dd4fc07f7586af8b667"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=107, T=5); synthetic, no external data", "final_mean_acc": 0.647, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.4325, "mean_forgetting": 0.3225, "memory_bytes": 0, "model_version": "aa18f022e3f8f38044d35fcbfc1dcd90d1398813c3288496c70c2b54e4bdfbbd", "seed": 107, "train_sample_steps": 90000, "zero_shot_mean": 0.5431250000000001}, {"acc_matrix": [[0.865, 0.0, 0.0, 0.0, 0.0], [0.62, 0.945, 0.0, 0.0, 0.0], [0.7325, 0.7925, 0.835, 0.0, 0.0], [0.685, 0.8, 0.6675, 0.9325, 0.0], [0.6725, 0.735, 0.6025, 0.7925, 0.91]], "bwt": -0.19374999999999998, "cpu_s": 0.401, "dataset_hashes": ["e3bf75538b2746e82cd186afc09269ceeee42d82e045c4a483af6dbd89e2e648", "5a59dad6960d17b399334c3f9143dc08862e5191168779f60b509cd5c3556268", "69b1063cca61e3ee1758fe905ea62308283437641156ab7ada4501b6bcfa5cdb", "ad05443578a5ae10a2649521cff86408bc1794efd54695965de9ca236ce7fe45", "a6baa45a5dc1141d86f108b61944ae1fbeef1def1405fb74a0ac1fc287fe3dfc"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=108, T=5); synthetic, no external data", "final_mean_acc": 0.7424999999999999, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.23249999999999993, "mean_forgetting": 0.19374999999999998, "memory_bytes": 0, "model_version": "1c27701e9ddef09b1060fe031a1d844a4fc1c731ace4d6eef10af8b618402343", "seed": 108, "train_sample_steps": 90000, "zero_shot_mean": 0.615}, {"acc_matrix": [[0.9375, 0.0, 0.0, 0.0, 0.0], [0.5525, 0.9025, 0.0, 0.0, 0.0], [0.595, 0.5725, 0.935, 0.0, 0.0], [0.5575, 0.72, 0.6675, 0.9025, 0.0], [0.49, 0.5175, 0.705, 0.6725, 0.9325]], "bwt": -0.323125, "cpu_s": 0.414, "dataset_hashes": ["ec29c4cd31353448288bc5c99cb63362de8fc3cff512741b8d94ddbb73ab6bdc", "149695feda0cb91bde33bc70747c4abf57fd2b8ceaa4e94a726ba1667302acd6", "faed3664f5b90eb67e45edea1e4623119a9297f04112fc28d50b3e547e5408e2", "7a9b8904fbc4e7f6bf254f2ad40558d673a44fba2457c2713488f73fc5fd14e1", "3cb6ef11f54899a08550ecbd1e6242ad6fd946a840678bcbe7426b70282de28d"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=109, T=5); synthetic, no external data", "final_mean_acc": 0.6635, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.4475, "mean_forgetting": 0.323125, "memory_bytes": 0, "model_version": "643112eaffcd119c33a353fe35c168f56e207195e3984d9dfdf6c90a3c937cb3", "seed": 109, "train_sample_steps": 90000, "zero_shot_mean": 0.5700000000000001}, {"acc_matrix": [[0.92, 0.0, 0.0, 0.0, 0.0], [0.705, 0.9175, 0.0, 0.0, 0.0], [0.485, 0.67, 0.8875, 0.0, 0.0], [0.485, 0.48, 0.73, 0.9375, 0.0], [0.5125, 0.48, 0.5725, 0.625, 0.8925]], "bwt": -0.36812500000000004, "cpu_s": 0.397, "dataset_hashes": ["fe918b74357265067f71b30380773ed71d4355ca531a8aa78e9b4df0ab331bc4", "6fd1cd0bac3efd3a0f2389c2e2a7852a225597e2b9b9460ed202553690e0b317", "19b369566bcf3f33d07e69e688454005a09514020670cf3ce8ff03ed848d537d", "f9809614391b8b4862aaf6fe9c0947831d17f2440def008ca60a33ba48af5698", "1d7823fc8cd89c5c11cc30886ae3d57be9947ec6ab0726e0dc29d6834dab2c79"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=110, T=5); synthetic, no external data", "final_mean_acc": 0.6165, "inference_distance_ops": 0, "learner": "naive", "max_forgetting": 0.4375, "mean_forgetting": 0.36812500000000004, "memory_bytes": 0, "model_version": "68b73e5661d800a4f50fc7adf301666afee1e166680db8a6fd863f944657aec1", "seed": 110, "train_sample_steps": 90000, "zero_shot_mean": 0.56125}, {"acc_matrix": [[0.89, 0.0, 0.0, 0.0, 0.0], [0.8825, 0.9425, 0.0, 0.0, 0.0], [0.8625, 0.8425, 0.8225, 0.0, 0.0], [0.8575, 0.8125, 0.8175, 0.93, 0.0], [0.86, 0.7975, 0.7675, 0.9025, 0.9]], "bwt": -0.06437500000000004, "cpu_s": 0.518, "dataset_hashes": ["7692e94c41331a04d068e5504f68c35a8dfffd3099e7d68df41a2c6adc042fb2", "7fe893a0822d5d03682474c63a4b42208f1344b054cb1413c7f6ad7fef8e495c", "f73b75264e6c6dfc14e473ab2c37af855a9be0c5453073ebeb1f7adf5bd1f8d1", "81f3dbb7c61c0c5aecc2771d8212462a01584d9e9b226f3fae2d8b5a1c952435", "ed53ba3ae8a520e1aa9bf9a2932c742ec83d98d078a72ee136556213610421d1"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=101, T=5); synthetic, no external data", "final_mean_acc": 0.8455, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.14500000000000002, "mean_forgetting": 0.06437500000000004, "memory_bytes": 50400, "model_version": "36bcbc1cd4cd439461574fc8d368f99bdd956149b1b0372936a15f3507a8b678", "seed": 101, "train_sample_steps": 126000, "zero_shot_mean": 0.5443749999999999}, {"acc_matrix": [[0.95, 0.0, 0.0, 0.0, 0.0], [0.895, 0.9125, 0.0, 0.0, 0.0], [0.8775, 0.86, 0.855, 0.0, 0.0], [0.825, 0.855, 0.8525, 0.895, 0.0], [0.8175, 0.81, 0.79, 0.8175, 0.8925]], "bwt": -0.09437499999999996, "cpu_s": 0.517, "dataset_hashes": ["70ec99a7d167496916fae64f7b30048f7c73ccf788980d47ac1299d63669cd11", "e850d1772baac847ee2940440577a7589941192dc698435641d00a34f7d5a62b", "5e1d9e27dea13d9e7fd1e1d70a2c83bf36e6990021d6df94ccc1868945ba67f3", "c63af5f19bb8a1eec9c2e49e1bda5551728678a31d6dc2cadd99df032df2e1ec", "0d7ac133d50e5d6492856b402803a138431cafaf415b9901ec4da47d50371e39"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=102, T=5); synthetic, no external data", "final_mean_acc": 0.8254999999999999, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.13249999999999995, "mean_forgetting": 0.09437499999999996, "memory_bytes": 50400, "model_version": "572a06ad7dcae329df2e52f4a5d0248de44d91df7d6056faf28911676c07737d", "seed": 102, "train_sample_steps": 126000, "zero_shot_mean": 0.604375}, {"acc_matrix": [[0.9375, 0.0, 0.0, 0.0, 0.0], [0.8625, 0.895, 0.0, 0.0, 0.0], [0.8475, 0.8075, 0.8825, 0.0, 0.0], [0.8675, 0.795, 0.8775, 0.905, 0.0], [0.865, 0.74, 0.8225, 0.82, 0.9]], "bwt": -0.09312500000000001, "cpu_s": 0.562, "dataset_hashes": ["f2783f09c146e59d34aff8909af6b8597d0e2a8f44eb00baf4c725c0c67936b0", "310f7ca6c82c564e84b23f0a0f8118b581a5c7d8266f9c7e9d34af5b83946084", "92259ccf5a5f4df7efba2a46c4dc80e63226a812035d9ed3ed8c02f64abc15ed", "e20fbd692eeb55c6e7eed9771a72bce2afa4520c04382a3775a1e92cb6f30da1", "fec9f4e8850c6220f70257bca7da49f1026aff4f3bb7881917f6d240c6fbfd9b"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=103, T=5); synthetic, no external data", "final_mean_acc": 0.8295, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.15500000000000003, "mean_forgetting": 0.09312500000000001, "memory_bytes": 50400, "model_version": "71900a6be5f166fb4a34e36053c848d7810917561c827aa5fd2a3f79e068a3c1", "seed": 103, "train_sample_steps": 126000, "zero_shot_mean": 0.59875}, {"acc_matrix": [[0.95, 0.0, 0.0, 0.0, 0.0], [0.9225, 0.9425, 0.0, 0.0, 0.0], [0.865, 0.8625, 0.945, 0.0, 0.0], [0.8575, 0.8625, 0.89, 0.9, 0.0], [0.8175, 0.8125, 0.85, 0.79, 0.9]], "bwt": -0.11687499999999998, "cpu_s": 0.576, "dataset_hashes": ["50a2692f150818326dece0d3e2d0f928bd8889f120c3440198179ffcc0fe4415", "f5c3ced4b920a8e66cb09dfe82ff6703ebe31252e1a11e76d4c79622b197aef9", "f42cb106b97255c1fb872fd3d971f2896ea5c51f422c5a717615e1db56972c3a", "e472b7ef4bed3fb3ef8c801b6098a6164023240a87df127e08e547469f9ed072", "0493e8453f02084503ce14cee6838aaa4cd4f5c48c3c73386e6dfa978188f44f"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=104, T=5); synthetic, no external data", "final_mean_acc": 0.834, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.13249999999999995, "mean_forgetting": 0.11687499999999998, "memory_bytes": 50400, "model_version": "4071e2952251bdb144537f21791288ea4a631731067cdc40cbbebe7495f9a48f", "seed": 104, "train_sample_steps": 126000, "zero_shot_mean": 0.6325}, {"acc_matrix": [[0.92, 0.0, 0.0, 0.0, 0.0], [0.85, 0.865, 0.0, 0.0, 0.0], [0.8775, 0.8025, 0.9425, 0.0, 0.0], [0.8125, 0.81, 0.8825, 0.8825, 0.0], [0.8075, 0.7625, 0.875, 0.82, 0.785]], "bwt": -0.08625000000000002, "cpu_s": 0.56, "dataset_hashes": ["de63a04f1e7cd650d86f03df806e6414f31da0ca55039a4acb9daeb0666e4894", "c52ddc8080f6e873750d9ef3c611923099ea990d7a1a93347d31f17c3d3b5cd1", "22cd8a0cee3b3403e92c43a804099559d119ce50f223d4184e952e778a76da60", "d743fcfcee554108664e16454e06f66b9226a4894fdbbe2e4f997785187db04a", "32f688ca2f4f053c421322ac3d77177545b8a3a3e441e7972bea08a18ecd134e"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=105, T=5); synthetic, no external data", "final_mean_acc": 0.8099999999999999, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.11250000000000004, "mean_forgetting": 0.08625000000000002, "memory_bytes": 50400, "model_version": "f27832e67b2afb6f4bd32c7fa50f3826177be90d945ffca41c39f8ffa9b6d120", "seed": 105, "train_sample_steps": 126000, "zero_shot_mean": 0.5700000000000001}, {"acc_matrix": [[0.955, 0.0, 0.0, 0.0, 0.0], [0.8675, 0.945, 0.0, 0.0, 0.0], [0.8275, 0.8175, 0.88, 0.0, 0.0], [0.8, 0.8525, 0.8725, 0.8925, 0.0], [0.7925, 0.8175, 0.845, 0.8175, 0.8975]], "bwt": -0.09999999999999998, "cpu_s": 0.536, "dataset_hashes": ["c9d790f978251db8c5588837af54deb32f1b0090f78d62c62b47cd3edc12f23a", "f60c20c74ea8f42dad87dcc18f8ba7d4db90c908cf0581caecb4046346e862a9", "8c470026d9048aef1cf15e639d873aca77accceb00e4611911b17bcdf5b428ac", "8bf5fff2a1bd277b6e4ad3f7f1517836de6c7c6cea12c048cf498bf43b0d3445", "845ff8f52d01160500c9168301a1df005c3354ca6b33395a5b739645fffe9dd0"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=106, T=5); synthetic, no external data", "final_mean_acc": 0.834, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.16249999999999998, "mean_forgetting": 0.09999999999999998, "memory_bytes": 50400, "model_version": "6eac75fe199a48356bb113912a90ccf1b67fded8d740d6290749cf59dd7cfb6e", "seed": 106, "train_sample_steps": 126000, "zero_shot_mean": 0.556875}, {"acc_matrix": [[0.9425, 0.0, 0.0, 0.0, 0.0], [0.8825, 0.95, 0.0, 0.0, 0.0], [0.8825, 0.9075, 0.88, 0.0, 0.0], [0.7825, 0.825, 0.81, 0.845, 0.0], [0.77, 0.8125, 0.83, 0.7825, 0.8425]], "bwt": -0.105625, "cpu_s": 0.531, "dataset_hashes": ["23816b26028461935e27bb81124b583a607c2cfa3c9022e010237bac7f99b222", "f8948ec48d4e972808611ed71527f7aa3bc41cac9f67b4c5a2ddcf42195f4f69", "6f8d78bc83414fa92f5a84df3b6e36b7521d032fc153b7435e578f7441461842", "5f915d98b5319257c3a7b8d0dc12042381c86da9b102d03d9ce1ef4bf67da301", "d526f8127b75bbf433b3ecca71144431c7cbdd5aa3d92dd4fc07f7586af8b667"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=107, T=5); synthetic, no external data", "final_mean_acc": 0.8075000000000001, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.1725, "mean_forgetting": 0.105625, "memory_bytes": 50400, "model_version": "d98f6862485d25fc339336a971897faa55e24deaa75862a99027e487504ec4d3", "seed": 107, "train_sample_steps": 126000, "zero_shot_mean": 0.545625}, {"acc_matrix": [[0.865, 0.0, 0.0, 0.0, 0.0], [0.82, 0.955, 0.0, 0.0, 0.0], [0.805, 0.8275, 0.835, 0.0, 0.0], [0.81, 0.825, 0.785, 0.9225, 0.0], [0.815, 0.8475, 0.735, 0.87, 0.895]], "bwt": -0.07749999999999999, "cpu_s": 0.531, "dataset_hashes": ["e3bf75538b2746e82cd186afc09269ceeee42d82e045c4a483af6dbd89e2e648", "5a59dad6960d17b399334c3f9143dc08862e5191168779f60b509cd5c3556268", "69b1063cca61e3ee1758fe905ea62308283437641156ab7ada4501b6bcfa5cdb", "ad05443578a5ae10a2649521cff86408bc1794efd54695965de9ca236ce7fe45", "a6baa45a5dc1141d86f108b61944ae1fbeef1def1405fb74a0ac1fc287fe3dfc"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=108, T=5); synthetic, no external data", "final_mean_acc": 0.8324999999999999, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.10749999999999993, "mean_forgetting": 0.07749999999999999, "memory_bytes": 50400, "model_version": "dfae47ed76afd08734a3305294fc3e3dc670d9c115d5f417a515e99b39de5348", "seed": 108, "train_sample_steps": 126000, "zero_shot_mean": 0.6174999999999999}, {"acc_matrix": [[0.9375, 0.0, 0.0, 0.0, 0.0], [0.8875, 0.9, 0.0, 0.0, 0.0], [0.835, 0.8575, 0.94, 0.0, 0.0], [0.8225, 0.82, 0.78, 0.895, 0.0], [0.835, 0.77, 0.735, 0.7925, 0.8575]], "bwt": -0.135, "cpu_s": 0.545, "dataset_hashes": ["ec29c4cd31353448288bc5c99cb63362de8fc3cff512741b8d94ddbb73ab6bdc", "149695feda0cb91bde33bc70747c4abf57fd2b8ceaa4e94a726ba1667302acd6", "faed3664f5b90eb67e45edea1e4623119a9297f04112fc28d50b3e547e5408e2", "7a9b8904fbc4e7f6bf254f2ad40558d673a44fba2457c2713488f73fc5fd14e1", "3cb6ef11f54899a08550ecbd1e6242ad6fd946a840678bcbe7426b70282de28d"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=109, T=5); synthetic, no external data", "final_mean_acc": 0.7979999999999999, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.20499999999999996, "mean_forgetting": 0.135, "memory_bytes": 50400, "model_version": "f0a519a4228a5444a1cdb7ca29e803cde0e353d73ec7b17be8c19acd78c91a9a", "seed": 109, "train_sample_steps": 126000, "zero_shot_mean": 0.5487500000000001}, {"acc_matrix": [[0.92, 0.0, 0.0, 0.0, 0.0], [0.8325, 0.915, 0.0, 0.0, 0.0], [0.77, 0.8975, 0.8625, 0.0, 0.0], [0.69, 0.86, 0.8225, 0.9025, 0.0], [0.68, 0.845, 0.825, 0.855, 0.8625]], "bwt": -0.09875000000000003, "cpu_s": 0.591, "dataset_hashes": ["fe918b74357265067f71b30380773ed71d4355ca531a8aa78e9b4df0ab331bc4", "6fd1cd0bac3efd3a0f2389c2e2a7852a225597e2b9b9460ed202553690e0b317", "19b369566bcf3f33d07e69e688454005a09514020670cf3ce8ff03ed848d537d", "f9809614391b8b4862aaf6fe9c0947831d17f2440def008ca60a33ba48af5698", "1d7823fc8cd89c5c11cc30886ae3d57be9947ec6ab0726e0dc29d6834dab2c79"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=110, T=5); synthetic, no external data", "final_mean_acc": 0.8135, "inference_distance_ops": 0, "learner": "replay", "max_forgetting": 0.24, "mean_forgetting": 0.09875000000000003, "memory_bytes": 50400, "model_version": "ff30a8216039584ffe42a1494ffb74230dfd3e7370ee3a182247e814630658da", "seed": 110, "train_sample_steps": 126000, "zero_shot_mean": 0.583125}, {"acc_matrix": [[0.79, 0.0, 0.0, 0.0, 0.0], [0.79, 0.7825, 0.0, 0.0, 0.0], [0.79, 0.7825, 0.8225, 0.0, 0.0], [0.79, 0.7825, 0.8225, 0.805, 0.0], [0.79, 0.7825, 0.8225, 0.805, 0.805]], "bwt": 0.0, "cpu_s": 1.555, "dataset_hashes": ["7692e94c41331a04d068e5504f68c35a8dfffd3099e7d68df41a2c6adc042fb2", "7fe893a0822d5d03682474c63a4b42208f1344b054cb1413c7f6ad7fef8e495c", "f73b75264e6c6dfc14e473ab2c37af855a9be0c5453073ebeb1f7adf5bd1f8d1", "81f3dbb7c61c0c5aecc2771d8212462a01584d9e9b226f3fae2d8b5a1c952435", "ed53ba3ae8a520e1aa9bf9a2932c742ec83d98d078a72ee136556213610421d1"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=101, T=5); synthetic, no external data", "final_mean_acc": 0.8009999999999999, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 252000, "seed": 101, "train_sample_steps": 0, "zero_shot_mean": 0.5175}, {"acc_matrix": [[0.78, 0.0, 0.0, 0.0, 0.0], [0.78, 0.7825, 0.0, 0.0, 0.0], [0.78, 0.7825, 0.775, 0.0, 0.0], [0.78, 0.7825, 0.775, 0.77, 0.0], [0.78, 0.7825, 0.775, 0.77, 0.8325]], "bwt": 0.0, "cpu_s": 1.541, "dataset_hashes": ["70ec99a7d167496916fae64f7b30048f7c73ccf788980d47ac1299d63669cd11", "e850d1772baac847ee2940440577a7589941192dc698435641d00a34f7d5a62b", "5e1d9e27dea13d9e7fd1e1d70a2c83bf36e6990021d6df94ccc1868945ba67f3", "c63af5f19bb8a1eec9c2e49e1bda5551728678a31d6dc2cadd99df032df2e1ec", "0d7ac133d50e5d6492856b402803a138431cafaf415b9901ec4da47d50371e39"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=102, T=5); synthetic, no external data", "final_mean_acc": 0.788, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 252000, "seed": 102, "train_sample_steps": 0, "zero_shot_mean": 0.5625}, {"acc_matrix": [[0.7925, 0.0, 0.0, 0.0, 0.0], [0.7925, 0.8125, 0.0, 0.0, 0.0], [0.7925, 0.8125, 0.8475, 0.0, 0.0], [0.7925, 0.8125, 0.8475, 0.83, 0.0], [0.7925, 0.8125, 0.8475, 0.83, 0.825]], "bwt": 0.0, "cpu_s": 1.529, "dataset_hashes": ["f2783f09c146e59d34aff8909af6b8597d0e2a8f44eb00baf4c725c0c67936b0", "310f7ca6c82c564e84b23f0a0f8118b581a5c7d8266f9c7e9d34af5b83946084", "92259ccf5a5f4df7efba2a46c4dc80e63226a812035d9ed3ed8c02f64abc15ed", "e20fbd692eeb55c6e7eed9771a72bce2afa4520c04382a3775a1e92cb6f30da1", "fec9f4e8850c6220f70257bca7da49f1026aff4f3bb7881917f6d240c6fbfd9b"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=103, T=5); synthetic, no external data", "final_mean_acc": 0.8215, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 252000, "seed": 103, "train_sample_steps": 0, "zero_shot_mean": 0.5275}, {"acc_matrix": [[0.78, 0.0, 0.0, 0.0, 0.0], [0.78, 0.7675, 0.0, 0.0, 0.0], [0.78, 0.7675, 0.805, 0.0, 0.0], [0.78, 0.7675, 0.805, 0.8, 0.0], [0.78, 0.7675, 0.805, 0.8, 0.805]], "bwt": 0.0, "cpu_s": 1.512, "dataset_hashes": ["50a2692f150818326dece0d3e2d0f928bd8889f120c3440198179ffcc0fe4415", "f5c3ced4b920a8e66cb09dfe82ff6703ebe31252e1a11e76d4c79622b197aef9", "f42cb106b97255c1fb872fd3d971f2896ea5c51f422c5a717615e1db56972c3a", "e472b7ef4bed3fb3ef8c801b6098a6164023240a87df127e08e547469f9ed072", "0493e8453f02084503ce14cee6838aaa4cd4f5c48c3c73386e6dfa978188f44f"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=104, T=5); synthetic, no external data", "final_mean_acc": 0.7915, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 252000, "seed": 104, "train_sample_steps": 0, "zero_shot_mean": 0.566875}, {"acc_matrix": [[0.8, 0.0, 0.0, 0.0, 0.0], [0.8, 0.7425, 0.0, 0.0, 0.0], [0.8, 0.745, 0.815, 0.0, 0.0], [0.8, 0.745, 0.815, 0.79, 0.0], [0.8, 0.745, 0.815, 0.79, 0.8175]], "bwt": 0.0006249999999999867, "cpu_s": 1.526, "dataset_hashes": ["de63a04f1e7cd650d86f03df806e6414f31da0ca55039a4acb9daeb0666e4894", "c52ddc8080f6e873750d9ef3c611923099ea990d7a1a93347d31f17c3d3b5cd1", "22cd8a0cee3b3403e92c43a804099559d119ce50f223d4184e952e778a76da60", "d743fcfcee554108664e16454e06f66b9226a4894fdbbe2e4f997785187db04a", "32f688ca2f4f053c421322ac3d77177545b8a3a3e441e7972bea08a18ecd134e"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=105, T=5); synthetic, no external data", "final_mean_acc": 0.7935, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 252000, "seed": 105, "train_sample_steps": 0, "zero_shot_mean": 0.485625}, {"acc_matrix": [[0.795, 0.0, 0.0, 0.0, 0.0], [0.795, 0.795, 0.0, 0.0, 0.0], [0.795, 0.795, 0.7825, 0.0, 0.0], [0.7975, 0.795, 0.7825, 0.7775, 0.0], [0.7975, 0.795, 0.7825, 0.7775, 0.79]], "bwt": 0.0006249999999999867, "cpu_s": 1.563, "dataset_hashes": ["c9d790f978251db8c5588837af54deb32f1b0090f78d62c62b47cd3edc12f23a", "f60c20c74ea8f42dad87dcc18f8ba7d4db90c908cf0581caecb4046346e862a9", "8c470026d9048aef1cf15e639d873aca77accceb00e4611911b17bcdf5b428ac", "8bf5fff2a1bd277b6e4ad3f7f1517836de6c7c6cea12c048cf498bf43b0d3445", "845ff8f52d01160500c9168301a1df005c3354ca6b33395a5b739645fffe9dd0"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=106, T=5); synthetic, no external data", "final_mean_acc": 0.7885, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 252000, "seed": 106, "train_sample_steps": 0, "zero_shot_mean": 0.563125}, {"acc_matrix": [[0.8225, 0.0, 0.0, 0.0, 0.0], [0.8225, 0.82, 0.0, 0.0, 0.0], [0.8225, 0.82, 0.7525, 0.0, 0.0], [0.8225, 0.82, 0.7525, 0.7875, 0.0], [0.8225, 0.82, 0.7525, 0.7875, 0.795]], "bwt": 0.0, "cpu_s": 1.597, "dataset_hashes": ["23816b26028461935e27bb81124b583a607c2cfa3c9022e010237bac7f99b222", "f8948ec48d4e972808611ed71527f7aa3bc41cac9f67b4c5a2ddcf42195f4f69", "6f8d78bc83414fa92f5a84df3b6e36b7521d032fc153b7435e578f7441461842", "5f915d98b5319257c3a7b8d0dc12042381c86da9b102d03d9ce1ef4bf67da301", "d526f8127b75bbf433b3ecca71144431c7cbdd5aa3d92dd4fc07f7586af8b667"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=107, T=5); synthetic, no external data", "final_mean_acc": 0.7955, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 252000, "seed": 107, "train_sample_steps": 0, "zero_shot_mean": 0.529375}, {"acc_matrix": [[0.7925, 0.0, 0.0, 0.0, 0.0], [0.7925, 0.77, 0.0, 0.0, 0.0], [0.7925, 0.77, 0.785, 0.0, 0.0], [0.7925, 0.77, 0.785, 0.7875, 0.0], [0.7925, 0.77, 0.785, 0.7875, 0.7825]], "bwt": 0.0, "cpu_s": 1.458, "dataset_hashes": ["e3bf75538b2746e82cd186afc09269ceeee42d82e045c4a483af6dbd89e2e648", "5a59dad6960d17b399334c3f9143dc08862e5191168779f60b509cd5c3556268", "69b1063cca61e3ee1758fe905ea62308283437641156ab7ada4501b6bcfa5cdb", "ad05443578a5ae10a2649521cff86408bc1794efd54695965de9ca236ce7fe45", "a6baa45a5dc1141d86f108b61944ae1fbeef1def1405fb74a0ac1fc287fe3dfc"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=108, T=5); synthetic, no external data", "final_mean_acc": 0.7835000000000001, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 252000, "seed": 108, "train_sample_steps": 0, "zero_shot_mean": 0.5406249999999999}, {"acc_matrix": [[0.81, 0.0, 0.0, 0.0, 0.0], [0.81, 0.79, 0.0, 0.0, 0.0], [0.81, 0.79, 0.795, 0.0, 0.0], [0.81, 0.79, 0.795, 0.7775, 0.0], [0.8125, 0.79, 0.795, 0.7825, 0.8625]], "bwt": 0.0018749999999999878, "cpu_s": 1.536, "dataset_hashes": ["ec29c4cd31353448288bc5c99cb63362de8fc3cff512741b8d94ddbb73ab6bdc", "149695feda0cb91bde33bc70747c4abf57fd2b8ceaa4e94a726ba1667302acd6", "faed3664f5b90eb67e45edea1e4623119a9297f04112fc28d50b3e547e5408e2", "7a9b8904fbc4e7f6bf254f2ad40558d673a44fba2457c2713488f73fc5fd14e1", "3cb6ef11f54899a08550ecbd1e6242ad6fd946a840678bcbe7426b70282de28d"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=109, T=5); synthetic, no external data", "final_mean_acc": 0.8084999999999999, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0, "mean_forgetting": -0.0018749999999999878, "memory_bytes": 252000, "seed": 109, "train_sample_steps": 0, "zero_shot_mean": 0.5349999999999999}, {"acc_matrix": [[0.7875, 0.0, 0.0, 0.0, 0.0], [0.7875, 0.81, 0.0, 0.0, 0.0], [0.785, 0.81, 0.805, 0.0, 0.0], [0.785, 0.81, 0.805, 0.815, 0.0], [0.7825, 0.81, 0.805, 0.815, 0.8275]], "bwt": -0.0012500000000000011, "cpu_s": 1.527, "dataset_hashes": ["fe918b74357265067f71b30380773ed71d4355ca531a8aa78e9b4df0ab331bc4", "6fd1cd0bac3efd3a0f2389c2e2a7852a225597e2b9b9460ed202553690e0b317", "19b369566bcf3f33d07e69e688454005a09514020670cf3ce8ff03ed848d537d", "f9809614391b8b4862aaf6fe9c0947831d17f2440def008ca60a33ba48af5698", "1d7823fc8cd89c5c11cc30886ae3d57be9947ec6ab0726e0dc29d6834dab2c79"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=110, T=5); synthetic, no external data", "final_mean_acc": 0.808, "inference_distance_ops": 7800000, "learner": "knn_memory", "max_forgetting": 0.0050000000000000044, "mean_forgetting": 0.0012500000000000011, "memory_bytes": 252000, "seed": 110, "train_sample_steps": 0, "zero_shot_mean": 0.5387500000000001}, {"acc_matrix": [[0.89, 0.0, 0.0, 0.0, 0.0], [0.89, 0.9425, 0.0, 0.0, 0.0], [0.89, 0.9425, 0.8875, 0.0, 0.0], [0.89, 0.9425, 0.8875, 0.9525, 0.0], [0.89, 0.9425, 0.8875, 0.9525, 0.955]], "bwt": 0.0, "cpu_s": 0.408, "dataset_hashes": ["7692e94c41331a04d068e5504f68c35a8dfffd3099e7d68df41a2c6adc042fb2", "7fe893a0822d5d03682474c63a4b42208f1344b054cb1413c7f6ad7fef8e495c", "f73b75264e6c6dfc14e473ab2c37af855a9be0c5453073ebeb1f7adf5bd1f8d1", "81f3dbb7c61c0c5aecc2771d8212462a01584d9e9b226f3fae2d8b5a1c952435", "ed53ba3ae8a520e1aa9bf9a2932c742ec83d98d078a72ee136556213610421d1"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=101, T=5); synthetic, no external data", "final_mean_acc": 0.9254999999999999, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7894736842105263, "seed": 101, "train_sample_steps": 90000, "zero_shot_mean": 0.5293749999999999}, {"acc_matrix": [[0.95, 0.0, 0.0, 0.0, 0.0], [0.95, 0.9075, 0.0, 0.0, 0.0], [0.95, 0.9075, 0.88, 0.0, 0.0], [0.95, 0.9075, 0.88, 0.9175, 0.0], [0.95, 0.9075, 0.88, 0.9175, 0.9225]], "bwt": 0.0, "cpu_s": 0.401, "dataset_hashes": ["70ec99a7d167496916fae64f7b30048f7c73ccf788980d47ac1299d63669cd11", "e850d1772baac847ee2940440577a7589941192dc698435641d00a34f7d5a62b", "5e1d9e27dea13d9e7fd1e1d70a2c83bf36e6990021d6df94ccc1868945ba67f3", "c63af5f19bb8a1eec9c2e49e1bda5551728678a31d6dc2cadd99df032df2e1ec", "0d7ac133d50e5d6492856b402803a138431cafaf415b9901ec4da47d50371e39"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=102, T=5); synthetic, no external data", "final_mean_acc": 0.9155, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7894736842105263, "seed": 102, "train_sample_steps": 90000, "zero_shot_mean": 0.6143750000000001}, {"acc_matrix": [[0.9375, 0.0, 0.0, 0.0, 0.0], [0.9375, 0.885, 0.0, 0.0, 0.0], [0.9375, 0.885, 0.8825, 0.0, 0.0], [0.9375, 0.885, 0.8825, 0.95, 0.0], [0.9375, 0.885, 0.8825, 0.95, 0.9425]], "bwt": 0.0, "cpu_s": 0.379, "dataset_hashes": ["f2783f09c146e59d34aff8909af6b8597d0e2a8f44eb00baf4c725c0c67936b0", "310f7ca6c82c564e84b23f0a0f8118b581a5c7d8266f9c7e9d34af5b83946084", "92259ccf5a5f4df7efba2a46c4dc80e63226a812035d9ed3ed8c02f64abc15ed", "e20fbd692eeb55c6e7eed9771a72bce2afa4520c04382a3775a1e92cb6f30da1", "fec9f4e8850c6220f70257bca7da49f1026aff4f3bb7881917f6d240c6fbfd9b"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=103, T=5); synthetic, no external data", "final_mean_acc": 0.9195, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7894736842105263, "seed": 103, "train_sample_steps": 90000, "zero_shot_mean": 0.561875}, {"acc_matrix": [[0.95, 0.0, 0.0, 0.0, 0.0], [0.95, 0.9475, 0.0, 0.0, 0.0], [0.95, 0.9475, 0.9475, 0.0, 0.0], [0.95, 0.9475, 0.9475, 0.9325, 0.0], [0.95, 0.9475, 0.9475, 0.9325, 0.9275]], "bwt": 0.0, "cpu_s": 0.394, "dataset_hashes": ["50a2692f150818326dece0d3e2d0f928bd8889f120c3440198179ffcc0fe4415", "f5c3ced4b920a8e66cb09dfe82ff6703ebe31252e1a11e76d4c79622b197aef9", "f42cb106b97255c1fb872fd3d971f2896ea5c51f422c5a717615e1db56972c3a", "e472b7ef4bed3fb3ef8c801b6098a6164023240a87df127e08e547469f9ed072", "0493e8453f02084503ce14cee6838aaa4cd4f5c48c3c73386e6dfa978188f44f"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=104, T=5); synthetic, no external data", "final_mean_acc": 0.9410000000000001, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7894736842105263, "seed": 104, "train_sample_steps": 90000, "zero_shot_mean": 0.64375}, {"acc_matrix": [[0.92, 0.0, 0.0, 0.0, 0.0], [0.92, 0.86, 0.0, 0.0, 0.0], [0.92, 0.86, 0.9175, 0.0, 0.0], [0.92, 0.86, 0.9175, 0.9225, 0.0], [0.92, 0.86, 0.9175, 0.9225, 0.8975]], "bwt": 0.0, "cpu_s": 0.467, "dataset_hashes": ["de63a04f1e7cd650d86f03df806e6414f31da0ca55039a4acb9daeb0666e4894", "c52ddc8080f6e873750d9ef3c611923099ea990d7a1a93347d31f17c3d3b5cd1", "22cd8a0cee3b3403e92c43a804099559d119ce50f223d4184e952e778a76da60", "d743fcfcee554108664e16454e06f66b9226a4894fdbbe2e4f997785187db04a", "32f688ca2f4f053c421322ac3d77177545b8a3a3e441e7972bea08a18ecd134e"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=105, T=5); synthetic, no external data", "final_mean_acc": 0.9035, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7893421052631578, "seed": 105, "train_sample_steps": 90000, "zero_shot_mean": 0.516875}, {"acc_matrix": [[0.955, 0.0, 0.0, 0.0, 0.0], [0.955, 0.96, 0.0, 0.0, 0.0], [0.955, 0.96, 0.845, 0.0, 0.0], [0.955, 0.96, 0.845, 0.915, 0.0], [0.955, 0.96, 0.845, 0.915, 0.9525]], "bwt": 0.0, "cpu_s": 0.487, "dataset_hashes": ["c9d790f978251db8c5588837af54deb32f1b0090f78d62c62b47cd3edc12f23a", "f60c20c74ea8f42dad87dcc18f8ba7d4db90c908cf0581caecb4046346e862a9", "8c470026d9048aef1cf15e639d873aca77accceb00e4611911b17bcdf5b428ac", "8bf5fff2a1bd277b6e4ad3f7f1517836de6c7c6cea12c048cf498bf43b0d3445", "845ff8f52d01160500c9168301a1df005c3354ca6b33395a5b739645fffe9dd0"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=106, T=5); synthetic, no external data", "final_mean_acc": 0.9254999999999999, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7894736842105263, "seed": 106, "train_sample_steps": 90000, "zero_shot_mean": 0.584375}, {"acc_matrix": [[0.9425, 0.0, 0.0, 0.0, 0.0], [0.9425, 0.905, 0.0, 0.0, 0.0], [0.9425, 0.905, 0.92, 0.0, 0.0], [0.9425, 0.905, 0.92, 0.85, 0.0], [0.9425, 0.905, 0.92, 0.85, 0.905]], "bwt": 0.0, "cpu_s": 0.439, "dataset_hashes": ["23816b26028461935e27bb81124b583a607c2cfa3c9022e010237bac7f99b222", "f8948ec48d4e972808611ed71527f7aa3bc41cac9f67b4c5a2ddcf42195f4f69", "6f8d78bc83414fa92f5a84df3b6e36b7521d032fc153b7435e578f7441461842", "5f915d98b5319257c3a7b8d0dc12042381c86da9b102d03d9ce1ef4bf67da301", "d526f8127b75bbf433b3ecca71144431c7cbdd5aa3d92dd4fc07f7586af8b667"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=107, T=5); synthetic, no external data", "final_mean_acc": 0.9045, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7894736842105263, "seed": 107, "train_sample_steps": 90000, "zero_shot_mean": 0.536875}, {"acc_matrix": [[0.865, 0.0, 0.0, 0.0, 0.0], [0.865, 0.93, 0.0, 0.0, 0.0], [0.865, 0.93, 0.8625, 0.0, 0.0], [0.865, 0.93, 0.8625, 0.9425, 0.0], [0.865, 0.93, 0.8625, 0.9425, 0.92]], "bwt": 0.0, "cpu_s": 0.399, "dataset_hashes": ["e3bf75538b2746e82cd186afc09269ceeee42d82e045c4a483af6dbd89e2e648", "5a59dad6960d17b399334c3f9143dc08862e5191168779f60b509cd5c3556268", "69b1063cca61e3ee1758fe905ea62308283437641156ab7ada4501b6bcfa5cdb", "ad05443578a5ae10a2649521cff86408bc1794efd54695965de9ca236ce7fe45", "a6baa45a5dc1141d86f108b61944ae1fbeef1def1405fb74a0ac1fc287fe3dfc"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=108, T=5); synthetic, no external data", "final_mean_acc": 0.9039999999999999, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7894736842105263, "seed": 108, "train_sample_steps": 90000, "zero_shot_mean": 0.643125}, {"acc_matrix": [[0.9375, 0.0, 0.0, 0.0, 0.0], [0.9375, 0.9, 0.0, 0.0, 0.0], [0.9375, 0.9, 0.94, 0.0, 0.0], [0.9375, 0.9, 0.94, 0.9375, 0.0], [0.9375, 0.9, 0.94, 0.9375, 0.97]], "bwt": 0.0, "cpu_s": 0.421, "dataset_hashes": ["ec29c4cd31353448288bc5c99cb63362de8fc3cff512741b8d94ddbb73ab6bdc", "149695feda0cb91bde33bc70747c4abf57fd2b8ceaa4e94a726ba1667302acd6", "faed3664f5b90eb67e45edea1e4623119a9297f04112fc28d50b3e547e5408e2", "7a9b8904fbc4e7f6bf254f2ad40558d673a44fba2457c2713488f73fc5fd14e1", "3cb6ef11f54899a08550ecbd1e6242ad6fd946a840678bcbe7426b70282de28d"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=109, T=5); synthetic, no external data", "final_mean_acc": 0.9369999999999999, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7892105263157895, "seed": 109, "train_sample_steps": 90000, "zero_shot_mean": 0.5443749999999999}, {"acc_matrix": [[0.92, 0.0, 0.0, 0.0, 0.0], [0.92, 0.9175, 0.0, 0.0, 0.0], [0.92, 0.9175, 0.9025, 0.0, 0.0], [0.92, 0.9175, 0.9025, 0.9325, 0.0], [0.92, 0.9175, 0.9025, 0.9325, 0.8975]], "bwt": 0.0, "cpu_s": 0.42, "dataset_hashes": ["fe918b74357265067f71b30380773ed71d4355ca531a8aa78e9b4df0ab331bc4", "6fd1cd0bac3efd3a0f2389c2e2a7852a225597e2b9b9460ed202553690e0b317", "19b369566bcf3f33d07e69e688454005a09514020670cf3ce8ff03ed848d537d", "f9809614391b8b4862aaf6fe9c0947831d17f2440def008ca60a33ba48af5698", "1d7823fc8cd89c5c11cc30886ae3d57be9947ec6ab0726e0dc29d6834dab2c79"], "dataset_provenance": "generated by cain.e114.continual.make_stream(seed=110, T=5); synthetic, no external data", "final_mean_acc": 0.914, "inference_distance_ops": 0, "learner": "experts", "max_forgetting": 0.0, "mean_forgetting": 0.0, "memory_bytes": 26880, "router_accuracy": 0.7890789473684211, "seed": 110, "train_sample_steps": 90000, "zero_shot_mean": 0.5506249999999999}]