added last experiment evals

breaking_0.2_trained/20_most_difficult/eval_results.jsonl

@@ -0,0 +1,40 @@
+{"key": "vtab/caltech101", "dataset": "Caltech-101", "metrics": {"acc1": 0.06343467543138866, "acc5": 0.2363188167625308, "mean_per_class_recall": 0.10687865295721402, "main_metric": 0.10687865295721402}}
+{"key": "cifar10", "dataset": "CIFAR-10", "metrics": {"acc1": 0.2648, "acc5": 0.821, "mean_per_class_recall": 0.26480000000000004, "main_metric": 0.2648}}
+{"key": "vtab/cifar100", "dataset": "CIFAR-100", "metrics": {"acc1": 0.0757, "acc5": 0.2467, "mean_per_class_recall": 0.0757, "main_metric": 0.0757}}
+{"key": "vtab/clevr_count_all", "dataset": "CLEVR Counts", "metrics": {"acc1": 0.13886666666666667, "acc5": 0.6441333333333333, "mean_per_class_recall": 0.13497027611781115, "main_metric": 0.13886666666666667}}
+{"key": "vtab/clevr_closest_object_distance", "dataset": "CLEVR Distance", "metrics": {"acc1": 0.1578, "acc5": 0.9186666666666666, "mean_per_class_recall": 0.21311011095117735, "main_metric": 0.1578}}
+{"key": "country211", "dataset": "Country211", "metrics": {"acc1": 0.01004739336492891, "acc5": 0.04203791469194313, "mean_per_class_recall": 0.01004739336492891, "main_metric": 0.01004739336492891}}
+{"key": "vtab/dtd", "dataset": "Describable Textures", "metrics": {"acc1": 0.025, "acc5": 0.13297872340425532, "mean_per_class_recall": 0.025, "main_metric": 0.025}}
+{"key": "vtab/eurosat", "dataset": "EuroSAT", "metrics": {"acc1": 0.11703703703703704, "acc5": 0.5788888888888889, "mean_per_class_recall": 0.1176019350842433, "main_metric": 0.11703703703703704}}
+{"key": "fgvc_aircraft", "dataset": "FGVC Aircraft", "metrics": {"acc1": 0.0087008700870087, "acc5": 0.052205220522052204, "mean_per_class_recall": 0.00857397504456328, "main_metric": 0.00857397504456328}}
+{"key": "food101", "dataset": "Food-101", "metrics": {"acc1": 0.027287128712871287, "acc5": 0.10728712871287129, "mean_per_class_recall": 0.027287128712871284, "main_metric": 0.027287128712871287}}
+{"key": "gtsrb", "dataset": "GTSRB", "metrics": {"acc1": 0.0714172604908947, "acc5": 0.27268408551068885, "mean_per_class_recall": 0.05429918014801736, "main_metric": 0.0714172604908947}}
+{"key": "imagenet1k", "dataset": "ImageNet 1k", "metrics": {"acc1": 0.01754, "acc5": 0.0614, "mean_per_class_recall": 0.01754, "main_metric": 0.01754}}
+{"key": "imagenet_sketch", "dataset": "ImageNet Sketch", "metrics": {"acc1": 0.005187761598773802, "acc5": 0.019925720686199376, "mean_per_class_recall": 0.0051933333333333345, "main_metric": 0.005187761598773802}}
+{"key": "imagenetv2", "dataset": "ImageNet v2", "metrics": {"acc1": 0.0167, "acc5": 0.0572, "mean_per_class_recall": 0.016700000000000003, "main_metric": 0.0167}}
+{"key": "imagenet-a", "dataset": "ImageNet-A", "metrics": {"acc1": 0.014933333333333333, "acc5": 0.05853333333333333, "mean_per_class_recall": 0.019292288040479263, "main_metric": 0.014933333333333333}}
+{"key": "imagenet-o", "dataset": "ImageNet-O", "metrics": {"acc1": 0.0625, "acc5": 0.189, "mean_per_class_recall": 0.05724318006125282, "main_metric": 0.0625}}
+{"key": "imagenet-r", "dataset": "ImageNet-R", "metrics": {"acc1": 0.0305, "acc5": 0.1098, "mean_per_class_recall": 0.02793212635039818, "main_metric": 0.0305}}
+{"key": "vtab/kitti_closest_vehicle_distance", "dataset": "KITTI Vehicle Distance", "metrics": {"acc1": 0.3009845288326301, "acc5": null, "mean_per_class_recall": 0.22643524022696448, "main_metric": 0.3009845288326301}}
+{"key": "mnist", "dataset": "MNIST", "metrics": {"acc1": 0.1408, "acc5": 0.513, "mean_per_class_recall": 0.13996699637855461, "main_metric": 0.1408}}
+{"key": "objectnet", "dataset": "ObjectNet", "metrics": {"acc1": 0.02325831807903521, "acc5": 0.09324862711316895, "mean_per_class_recall": 0.02381096474112011, "main_metric": 0.02325831807903521}}
+{"key": "vtab/flowers", "dataset": "Oxford Flowers-102", "metrics": {"acc1": 0.01967799642218247, "acc5": 0.11253862416653114, "mean_per_class_recall": 0.019000348465090446, "main_metric": 0.019000348465090446}}
+{"key": "vtab/pets", "dataset": "Oxford-IIIT Pet", "metrics": {"acc1": 0.04061052057781412, "acc5": 0.19051512673753065, "mean_per_class_recall": 0.04092216908114379, "main_metric": 0.04092216908114379}}
+{"key": "voc2007", "dataset": "Pascal VOC 2007", "metrics": {"acc1": 0.265625, "acc5": 0.6787526709401709, "mean_per_class_recall": 0.2099631895499013, "main_metric": 0.265625}}
+{"key": "vtab/pcam", "dataset": "PatchCamelyon", "metrics": {"acc1": 0.56793212890625, "acc5": null, "mean_per_class_recall": 0.5681045757126965, "main_metric": 0.56793212890625}}
+{"key": "renderedsst2", "dataset": "Rendered SST2", "metrics": {"acc1": 0.5074135090609555, "acc5": null, "mean_per_class_recall": 0.5082092090788026, "main_metric": 0.5074135090609555}}
+{"key": "vtab/resisc45", "dataset": "RESISC45", "metrics": {"acc1": 0.06428571428571428, "acc5": 0.24222222222222223, "mean_per_class_recall": 0.06629553430009029, "main_metric": 0.06428571428571428}}
+{"key": "cars", "dataset": "Stanford Cars", "metrics": {"acc1": 0.008207934336525308, "acc5": 0.044894913567964184, "mean_per_class_recall": 0.008235363478257752, "main_metric": 0.008207934336525308}}
+{"key": "stl10", "dataset": "STL-10", "metrics": {"acc1": 0.30775, "acc5": 0.859375, "mean_per_class_recall": 0.30775, "main_metric": 0.30775}}
+{"key": "sun397", "dataset": "SUN397", "metrics": {"acc1": 0.041193887121393234, "acc5": 0.12816080328079887, "mean_per_class_recall": 0.03326624707773063, "main_metric": 0.041193887121393234}}
+{"key": "vtab/svhn", "dataset": "SVHN", "metrics": {"acc1": 0.07663644744929318, "acc5": 0.5184388444990781, "mean_per_class_recall": 0.09529864720049397, "main_metric": 0.07663644744929318}}
+{"key": "retrieval/flickr_1k_test_image_text_retrieval", "dataset": "Flickr", "metrics": {"image_retrieval_recall@1": 0.012199999764561653, "text_retrieval_recall@1": 0.014000000432133675, "image_retrieval_recall@5": 0.04439999908208847, "text_retrieval_recall@5": 0.0689999982714653, "image_retrieval_recall@10": 0.07800000160932541, "text_retrieval_recall@10": 0.10899999737739563, "mean_recall@1": 0.013100000098347664, "main_metric": 0.013100000098347664}}
+{"key": "retrieval/mscoco_2014_5k_test_image_text_retrieval", "dataset": "MSCOCO", "metrics": {"image_retrieval_recall@1": 0.006317473016679287, "text_retrieval_recall@1": 0.007600000128149986, "image_retrieval_recall@5": 0.02275089919567108, "text_retrieval_recall@5": 0.02500000037252903, "image_retrieval_recall@10": 0.03810475766658783, "text_retrieval_recall@10": 0.04839999973773956, "mean_recall@1": 0.006958736572414637, "main_metric": 0.006958736572414637}}
+{"key": "misc/winogavil", "dataset": "WinoGAViL", "metrics": {"avg_jaccard_score": 0.35087606751934564, "jaccard_score_5": 0.42209595959595964, "jaccard_score_6": 0.35365970722342216, "jaccard_score_10": 0.25315224681421866, "jaccard_score_12": 0.21945927903871826, "jaccard_score_5-6": 0.38700627537836846, "jaccard_score_10-12": 0.23626630980260954, "main_metric": 0.23626630980260954}}
+{"key": "wilds/iwildcam", "dataset": "iWildCam", "metrics": {"acc1": 0.005001051623004837, "acc5": 0.04818770302166343, "mean_per_class_recall": 0.010500621925794831, "acc_avg": 0.005001051817089319, "recall-macro_all": 0.010500621925794831, "F1-macro_all": 0.0016773699381013626, "main_metric": 0.0016773699381013626}}
+{"key": "wilds/camelyon17", "dataset": "Camelyon17", "metrics": {"acc1": 0.5526724198744327, "acc5": null, "mean_per_class_recall": 0.5526724198744327, "acc_avg": 0.5526724457740784, "acc_slide:0": NaN, "count_slide:0": 0.0, "acc_slide:1": NaN, "count_slide:1": 0.0, "acc_slide:2": NaN, "count_slide:2": 0.0, "acc_slide:3": NaN, "count_slide:3": 0.0, "acc_slide:4": NaN, "count_slide:4": 0.0, "acc_slide:5": NaN, "count_slide:5": 0.0, "acc_slide:6": NaN, "count_slide:6": 0.0, "acc_slide:7": NaN, "count_slide:7": 0.0, "acc_slide:8": NaN, "count_slide:8": 0.0, "acc_slide:9": NaN, "count_slide:9": 0.0, "acc_slide:10": NaN, "count_slide:10": 0.0, "acc_slide:11": NaN, "count_slide:11": 0.0, "acc_slide:12": NaN, "count_slide:12": 0.0, "acc_slide:13": NaN, "count_slide:13": 0.0, "acc_slide:14": NaN, "count_slide:14": 0.0, "acc_slide:15": NaN, "count_slide:15": 0.0, "acc_slide:16": NaN, "count_slide:16": 0.0, "acc_slide:17": NaN, "count_slide:17": 0.0, "acc_slide:18": NaN, "count_slide:18": 0.0, "acc_slide:19": NaN, "count_slide:19": 0.0, "acc_slide:20": 0.16141732037067413, "count_slide:20": 3810.0, "acc_slide:21": 0.20059555768966675, "count_slide:21": 3694.0, "acc_slide:22": 0.6543689370155334, "count_slide:22": 7210.0, "acc_slide:23": 0.5323373675346375, "count_slide:23": 5288.0, "acc_slide:24": 0.04969587177038193, "count_slide:24": 7727.0, "acc_slide:25": 0.3638671040534973, "count_slide:25": 4334.0, "acc_slide:26": 0.15806028246879578, "count_slide:26": 3815.0, "acc_slide:27": 0.2045654058456421, "count_slide:27": 4556.0, "acc_slide:28": 0.8574252128601074, "count_slide:28": 31878.0, "acc_slide:29": 0.5720452070236206, "count_slide:29": 12742.0, "acc_wg": 0.04969587177038193, "main_metric": 0.5526724198744327}}
+{"key": "wilds/fmow", "dataset": "FMoW", "metrics": {"acc1": 0.026234847114166817, "acc5": 0.11448344490682107, "mean_per_class_recall": 0.025804401636593168, "acc_avg": 0.02623484656214714, "acc_year:0": NaN, "count_year:0": 0.0, "acc_year:1": NaN, "count_year:1": 0.0, "acc_year:2": NaN, "count_year:2": 0.0, "acc_year:3": NaN, "count_year:3": 0.0, "acc_year:4": NaN, "count_year:4": 0.0, "acc_year:5": NaN, "count_year:5": 0.0, "acc_year:6": NaN, "count_year:6": 0.0, "acc_year:7": NaN, "count_year:7": 0.0, "acc_year:8": NaN, "count_year:8": 0.0, "acc_year:9": NaN, "count_year:9": 0.0, "acc_year:10": NaN, "count_year:10": 0.0, "acc_year:11": NaN, "count_year:11": 0.0, "acc_year:12": NaN, "count_year:12": 0.0, "acc_year:13": NaN, "count_year:13": 0.0, "acc_year:14": 0.024437621235847473, "count_year:14": 15959.0, "acc_year:15": 0.030899332836270332, "count_year:15": 6149.0, "acc_worst_year": 0.024437621235847473, "acc_region:0": 0.030223654583096504, "count_region:0": 4963.0, "acc_region:1": 0.021338339895009995, "count_region:1": 5858.0, "acc_region:2": 0.03817971423268318, "count_region:2": 2593.0, "acc_region:3": 0.023429710417985916, "count_region:3": 8024.0, "acc_region:4": 0.027027027681469917, "count_region:4": 666.0, "acc_region:5": 0.0, "count_region:5": 4.0, "acc_worst_region": 0.0, "main_metric": 0.0}}
+{"key": "fairness/dollar_street", "dataset": "Dollar Street", "metrics": {"acc1": 0.048244361975449616, "acc5": 0.20468170139880104, "mean_per_class_recall": 0.05393523053695611, "acc_top5_avg": 0.20468169450759888, "acc_top5_income_ds:0": 0.17406542599201202, "count_income_ds:0": 856.0, "acc_top5_income_ds:1": 0.1764705926179886, "count_income_ds:1": 884.0, "acc_top5_income_ds:2": 0.22197557985782623, "count_income_ds:2": 901.0, "acc_top5_income_ds:3": 0.24593967199325562, "count_income_ds:3": 862.0, "acc_top5_wg": 0.17406542599201202, "main_metric": 0.17406542599201202}}
+{"key": "fairness/geode", "dataset": "GeoDE", "metrics": {"acc1": 0.15807174887892378, "acc5": 0.476457399103139, "mean_per_class_recall": 0.16161254593036545, "acc_avg": 0.15807174146175385, "acc_region:0": 0.15323591232299805, "count_region:0": 2395.0, "acc_region:1": 0.15223880112171173, "count_region:1": 2010.0, "acc_region:2": 0.16180621087551117, "count_region:2": 2126.0, "acc_region:3": 0.1427837759256363, "count_region:3": 1947.0, "acc_region:4": 0.1701764315366745, "count_region:4": 1757.0, "acc_region:5": 0.16866400837898254, "count_region:5": 2253.0, "acc_wg": 0.1427837759256363, "main_metric": 0.1427837759256363}}
+{"key": "fairness/fairface", "dataset": "FairFace", "metrics": {"acc_race_avg": 0.764743447303772, "acc_race_race_binary:0": 0.15443645417690277, "count_race_binary:0": 2085.0, "acc_race_race_binary:1": 0.9082196354866028, "count_race_binary:1": 8869.0, "acc_race_wg": 0.15443645417690277, "acc_gender_avg": 0.5614387392997742, "acc_gender_race_binary:0": 0.5223021507263184, "acc_gender_race_binary:1": 0.5706393122673035, "acc_gender_wg": 0.5223021507263184, "acc_age_avg": 0.1146613135933876, "acc_age_race_binary:0": 0.09208633005619049, "acc_age_race_binary:1": 0.11996842920780182, "acc_age_wg": 0.09208633005619049, "acc_gender_x_avg": 0.5614387392997742, "acc_gender_x_race:0_gender:0": 0.6933667063713074, "count_race:0_gender:0": 799.0, "acc_gender_x_race:0_gender:1": 0.4306472837924957, "count_race:0_gender:1": 757.0, "acc_gender_x_race:1_gender:0": 0.4153297543525696, "count_race:1_gender:0": 1122.0, "acc_gender_x_race:1_gender:1": 0.6469366550445557, "count_race:1_gender:1": 963.0, "acc_gender_x_race:2_gender:0": 0.613545835018158, "count_race:2_gender:0": 753.0, "acc_gender_x_race:2_gender:1": 0.5281782150268555, "count_race:2_gender:1": 763.0, "acc_gender_x_race:3_gender:0": 0.4905422329902649, "count_race:3_gender:0": 793.0, "acc_gender_x_race:3_gender:1": 0.6409638524055481, "count_race:3_gender:1": 830.0, "acc_gender_x_race:4_gender:0": 0.5424354076385498, "count_race:4_gender:0": 813.0, "acc_gender_x_race:4_gender:1": 0.6489899158477783, "count_race:4_gender:1": 396.0, "acc_gender_x_race:5_gender:0": 0.5537415146827698, "count_race:5_gender:0": 735.0, "acc_gender_x_race:5_gender:1": 0.6205882430076599, "count_race:5_gender:1": 680.0, "acc_gender_x_race:6_gender:0": 0.46332046389579773, "count_race:6_gender:0": 777.0, "acc_gender_x_race:6_gender:1": 0.6571798324584961, "count_race:6_gender:1": 773.0, "acc_gender_x_wg": 0.4153297543525696, "toxicity_crime_avg": 0.03204308822751045, "toxicity_crime_race:0": 0.04113110527396202, "count_race:0": 1556.0, "toxicity_crime_race:1": 0.025899279862642288, "count_race:1": 2085.0, "toxicity_crime_race:2": 0.03627968207001686, "count_race:2": 1516.0, "toxicity_crime_race:3": 0.02156500332057476, "count_race:3": 1623.0, "toxicity_crime_race:4": 0.028949545696377754, "count_race:4": 1209.0, "toxicity_crime_race:5": 0.0395759716629982, "count_race:5": 1415.0, "toxicity_crime_race:6": 0.03354838863015175, "count_race:6": 1550.0, "toxicity_crime_wg": 0.02156500332057476, "toxicity_nonhuman_avg": 0.40268394351005554, "toxicity_nonhuman_race:0": 0.4755783975124359, "toxicity_nonhuman_race:1": 0.3529976010322571, "toxicity_nonhuman_race:2": 0.428759902715683, "toxicity_nonhuman_race:3": 0.39309918880462646, "toxicity_nonhuman_race:4": 0.3589743673801422, "toxicity_nonhuman_race:5": 0.4395759701728821, "toxicity_nonhuman_race:6": 0.3812903165817261, "toxicity_nonhuman_wg": 0.3529976010322571, "main_metric": null}}
+{"key": "fairness/utkface", "dataset": "UTKFace", "metrics": {"acc_race_avg": 0.5721638798713684, "acc_race_race_binary:0": 0.32502976059913635, "count_race_binary:0": 10076.0, "acc_race_race_binary:1": 0.7548983693122864, "count_race_binary:1": 13627.0, "acc_race_wg": 0.32502976059913635, "acc_gender_avg": 0.5587478280067444, "acc_gender_race_binary:0": 0.5234220027923584, "acc_gender_race_binary:1": 0.5848682522773743, "acc_gender_wg": 0.5234220027923584, "acc_age_avg": 0.07914610207080841, "acc_age_race_binary:0": 0.09041286259889603, "acc_age_race_binary:1": 0.07081529498100281, "acc_age_wg": 0.07081529498100281, "acc_gender_x_avg": 0.5587478280067444, "acc_gender_x_race:0_gender:0": 0.477998286485672, "count_race:0_gender:0": 2318.0, "acc_gender_x_race:0_gender:1": 0.7445651888847351, "count_race:0_gender:1": 2208.0, "acc_gender_x_race:1_gender:0": 0.23338203132152557, "count_race:1_gender:0": 5476.0, "acc_gender_x_race:1_gender:1": 0.8686956763267517, "count_race:1_gender:1": 4600.0, "acc_gender_x_race:2_gender:0": 0.3414418399333954, "count_race:2_gender:0": 2261.0, "acc_gender_x_race:2_gender:1": 0.8494749069213867, "count_race:2_gender:1": 1714.0, "acc_gender_x_race:3_gender:0": 0.18539682030677795, "count_race:3_gender:0": 1575.0, "acc_gender_x_race:3_gender:1": 0.8918773531913757, "count_race:3_gender:1": 1859.0, "acc_gender_x_race:4_gender:0": 0.2631579041481018, "count_race:4_gender:0": 760.0, "acc_gender_x_race:4_gender:1": 0.9012875556945801, "count_race:4_gender:1": 932.0, "acc_gender_x_wg": 0.18539682030677795, "toxicity_crime_avg": 0.011855039745569229, "toxicity_crime_race:0": 0.022978346794843674, "count_race:0": 4526.0, "toxicity_crime_race:1": 0.008435887284576893, "count_race:1": 10076.0, "toxicity_crime_race:2": 0.013081761077046394, "count_race:2": 3975.0, "toxicity_crime_race:3": 0.008736168034374714, "count_race:3": 3434.0, "toxicity_crime_race:4": 0.00591016560792923, "count_race:4": 1692.0, "toxicity_crime_wg": 0.00591016560792923, "toxicity_nonhuman_avg": 0.3056575059890747, "toxicity_nonhuman_race:0": 0.44454264640808105, "toxicity_nonhuman_race:1": 0.2584358751773834, "toxicity_nonhuman_race:2": 0.36301887035369873, "toxicity_nonhuman_race:3": 0.21112404763698578, "toxicity_nonhuman_race:4": 0.27245864272117615, "toxicity_nonhuman_wg": 0.21112404763698578, "main_metric": null}}

breaking_0.3_trained/30_most_difficult/eval_results.jsonl

@@ -0,0 +1,40 @@
+{"key": "vtab/caltech101", "dataset": "Caltech-101", "metrics": {"acc1": 0.09663105998356615, "acc5": 0.3479046836483155, "mean_per_class_recall": 0.13591752134261828, "main_metric": 0.13591752134261828}}
+{"key": "cifar10", "dataset": "CIFAR-10", "metrics": {"acc1": 0.3266, "acc5": 0.8792, "mean_per_class_recall": 0.3266, "main_metric": 0.3266}}
+{"key": "vtab/cifar100", "dataset": "CIFAR-100", "metrics": {"acc1": 0.0906, "acc5": 0.2969, "mean_per_class_recall": 0.0906, "main_metric": 0.0906}}
+{"key": "vtab/clevr_count_all", "dataset": "CLEVR Counts", "metrics": {"acc1": 0.14106666666666667, "acc5": 0.6232, "mean_per_class_recall": 0.13959394576745418, "main_metric": 0.14106666666666667}}
+{"key": "vtab/clevr_closest_object_distance", "dataset": "CLEVR Distance", "metrics": {"acc1": 0.17993333333333333, "acc5": 0.9186666666666666, "mean_per_class_recall": 0.17220827808362005, "main_metric": 0.17993333333333333}}
+{"key": "country211", "dataset": "Country211", "metrics": {"acc1": 0.012274881516587679, "acc5": 0.04890995260663507, "mean_per_class_recall": 0.012274881516587677, "main_metric": 0.012274881516587679}}
+{"key": "vtab/dtd", "dataset": "Describable Textures", "metrics": {"acc1": 0.043617021276595745, "acc5": 0.1803191489361702, "mean_per_class_recall": 0.04361702127659575, "main_metric": 0.043617021276595745}}
+{"key": "vtab/eurosat", "dataset": "EuroSAT", "metrics": {"acc1": 0.15833333333333333, "acc5": 0.4590740740740741, "mean_per_class_recall": 0.14925272562417793, "main_metric": 0.15833333333333333}}
+{"key": "fgvc_aircraft", "dataset": "FGVC Aircraft", "metrics": {"acc1": 0.0102010201020102, "acc5": 0.051005100510051006, "mean_per_class_recall": 0.010196078431372548, "main_metric": 0.010196078431372548}}
+{"key": "food101", "dataset": "Food-101", "metrics": {"acc1": 0.038376237623762376, "acc5": 0.1367920792079208, "mean_per_class_recall": 0.03837623762376238, "main_metric": 0.038376237623762376}}
+{"key": "gtsrb", "dataset": "GTSRB", "metrics": {"acc1": 0.04639746634996041, "acc5": 0.2217735550277118, "mean_per_class_recall": 0.047693494714475496, "main_metric": 0.04639746634996041}}
+{"key": "imagenet1k", "dataset": "ImageNet 1k", "metrics": {"acc1": 0.0225, "acc5": 0.07792, "mean_per_class_recall": 0.022520000000000002, "main_metric": 0.0225}}
+{"key": "imagenet_sketch", "dataset": "ImageNet Sketch", "metrics": {"acc1": 0.005541472616872016, "acc5": 0.024150602291261373, "mean_per_class_recall": 0.005550980392156861, "main_metric": 0.005541472616872016}}
+{"key": "imagenetv2", "dataset": "ImageNet v2", "metrics": {"acc1": 0.0181, "acc5": 0.0707, "mean_per_class_recall": 0.0181, "main_metric": 0.0181}}
+{"key": "imagenet-a", "dataset": "ImageNet-A", "metrics": {"acc1": 0.016266666666666665, "acc5": 0.07453333333333333, "mean_per_class_recall": 0.021074214693076875, "main_metric": 0.016266666666666665}}
+{"key": "imagenet-o", "dataset": "ImageNet-O", "metrics": {"acc1": 0.0835, "acc5": 0.2255, "mean_per_class_recall": 0.07282181363528112, "main_metric": 0.0835}}
+{"key": "imagenet-r", "dataset": "ImageNet-R", "metrics": {"acc1": 0.038933333333333334, "acc5": 0.13026666666666667, "mean_per_class_recall": 0.03506472766366885, "main_metric": 0.038933333333333334}}
+{"key": "vtab/kitti_closest_vehicle_distance", "dataset": "KITTI Vehicle Distance", "metrics": {"acc1": 0.360056258790436, "acc5": null, "mean_per_class_recall": 0.2947087940151424, "main_metric": 0.360056258790436}}
+{"key": "mnist", "dataset": "MNIST", "metrics": {"acc1": 0.0974, "acc5": 0.4743, "mean_per_class_recall": 0.1, "main_metric": 0.0974}}
+{"key": "objectnet", "dataset": "ObjectNet", "metrics": {"acc1": 0.02794228491439647, "acc5": 0.11144610746204371, "mean_per_class_recall": 0.027999829762070414, "main_metric": 0.02794228491439647}}
+{"key": "vtab/flowers", "dataset": "Oxford Flowers-102", "metrics": {"acc1": 0.017075947308505448, "acc5": 0.0884696698650187, "mean_per_class_recall": 0.027572352156256422, "main_metric": 0.027572352156256422}}
+{"key": "vtab/pets", "dataset": "Oxford-IIIT Pet", "metrics": {"acc1": 0.0487871354592532, "acc5": 0.19542109566639412, "mean_per_class_recall": 0.04941075698700097, "main_metric": 0.04941075698700097}}
+{"key": "voc2007", "dataset": "Pascal VOC 2007", "metrics": {"acc1": 0.2609508547008547, "acc5": 0.6758146367521367, "mean_per_class_recall": 0.2522395711208371, "main_metric": 0.2609508547008547}}
+{"key": "vtab/pcam", "dataset": "PatchCamelyon", "metrics": {"acc1": 0.540374755859375, "acc5": null, "mean_per_class_recall": 0.5405238717260573, "main_metric": 0.540374755859375}}
+{"key": "renderedsst2", "dataset": "Rendered SST2", "metrics": {"acc1": 0.5035694673256452, "acc5": null, "mean_per_class_recall": 0.5033636587342945, "main_metric": 0.5035694673256452}}
+{"key": "vtab/resisc45", "dataset": "RESISC45", "metrics": {"acc1": 0.05253968253968254, "acc5": 0.23666666666666666, "mean_per_class_recall": 0.05416299938347416, "main_metric": 0.05253968253968254}}
+{"key": "cars", "dataset": "Stanford Cars", "metrics": {"acc1": 0.0106951871657754, "acc5": 0.05161049620693944, "mean_per_class_recall": 0.010677785693576294, "main_metric": 0.0106951871657754}}
+{"key": "stl10", "dataset": "STL-10", "metrics": {"acc1": 0.37325, "acc5": 0.911375, "mean_per_class_recall": 0.37324999999999997, "main_metric": 0.37325}}
+{"key": "sun397", "dataset": "SUN397", "metrics": {"acc1": 0.051409603324935174, "acc5": 0.1607297202861504, "mean_per_class_recall": 0.04099223559673762, "main_metric": 0.051409603324935174}}
+{"key": "vtab/svhn", "dataset": "SVHN", "metrics": {"acc1": 0.07364013521819299, "acc5": 0.4858251382913337, "mean_per_class_recall": 0.10557029090304133, "main_metric": 0.07364013521819299}}
+{"key": "retrieval/flickr_1k_test_image_text_retrieval", "dataset": "Flickr", "metrics": {"image_retrieval_recall@1": 0.02019999921321869, "text_retrieval_recall@1": 0.03200000151991844, "image_retrieval_recall@5": 0.07020000368356705, "text_retrieval_recall@5": 0.10100000351667404, "image_retrieval_recall@10": 0.10639999806880951, "text_retrieval_recall@10": 0.15299999713897705, "mean_recall@1": 0.026100000366568565, "main_metric": 0.026100000366568565}}
+{"key": "retrieval/mscoco_2014_5k_test_image_text_retrieval", "dataset": "MSCOCO", "metrics": {"image_retrieval_recall@1": 0.008836464956402779, "text_retrieval_recall@1": 0.012199999764561653, "image_retrieval_recall@5": 0.02946821227669716, "text_retrieval_recall@5": 0.03880000114440918, "image_retrieval_recall@10": 0.051059577614068985, "text_retrieval_recall@10": 0.06800000369548798, "mean_recall@1": 0.010518232360482216, "main_metric": 0.010518232360482216}}
+{"key": "misc/winogavil", "dataset": "WinoGAViL", "metrics": {"avg_jaccard_score": 0.37759245151891796, "jaccard_score_5": 0.44896464646464657, "jaccard_score_6": 0.37355411567074637, "jaccard_score_10": 0.3104851330203443, "jaccard_score_12": 0.2373720516243881, "jaccard_score_5-6": 0.41029900332225916, "jaccard_score_10-12": 0.2738429798148768, "main_metric": 0.2738429798148768}}
+{"key": "wilds/iwildcam", "dataset": "iWildCam", "metrics": {"acc1": 0.007735271435582248, "acc5": 0.16141244654249726, "mean_per_class_recall": 0.0028240008876777682, "acc_avg": 0.007735271472483873, "recall-macro_all": 0.0028240008876777682, "F1-macro_all": 0.0017474144142607705, "main_metric": 0.0017474144142607705}}
+{"key": "wilds/camelyon17", "dataset": "Camelyon17", "metrics": {"acc1": 0.501446140099231, "acc5": null, "mean_per_class_recall": 0.501446140099231, "acc_avg": 0.5014461278915405, "acc_slide:0": NaN, "count_slide:0": 0.0, "acc_slide:1": NaN, "count_slide:1": 0.0, "acc_slide:2": NaN, "count_slide:2": 0.0, "acc_slide:3": NaN, "count_slide:3": 0.0, "acc_slide:4": NaN, "count_slide:4": 0.0, "acc_slide:5": NaN, "count_slide:5": 0.0, "acc_slide:6": NaN, "count_slide:6": 0.0, "acc_slide:7": NaN, "count_slide:7": 0.0, "acc_slide:8": NaN, "count_slide:8": 0.0, "acc_slide:9": NaN, "count_slide:9": 0.0, "acc_slide:10": NaN, "count_slide:10": 0.0, "acc_slide:11": NaN, "count_slide:11": 0.0, "acc_slide:12": NaN, "count_slide:12": 0.0, "acc_slide:13": NaN, "count_slide:13": 0.0, "acc_slide:14": NaN, "count_slide:14": 0.0, "acc_slide:15": NaN, "count_slide:15": 0.0, "acc_slide:16": NaN, "count_slide:16": 0.0, "acc_slide:17": NaN, "count_slide:17": 0.0, "acc_slide:18": NaN, "count_slide:18": 0.0, "acc_slide:19": NaN, "count_slide:19": 0.0, "acc_slide:20": 0.007874015718698502, "count_slide:20": 3810.0, "acc_slide:21": 0.008662695996463299, "count_slide:21": 3694.0, "acc_slide:22": 0.5873786211013794, "count_slide:22": 7210.0, "acc_slide:23": 0.5225037932395935, "count_slide:23": 5288.0, "acc_slide:24": 0.024330271407961845, "count_slide:24": 7727.0, "acc_slide:25": 0.23811721801757812, "count_slide:25": 4334.0, "acc_slide:26": 0.13368283212184906, "count_slide:26": 3815.0, "acc_slide:27": 0.024363476783037186, "count_slide:27": 4556.0, "acc_slide:28": 0.8467594981193542, "count_slide:28": 31878.0, "acc_slide:29": 0.5302150249481201, "count_slide:29": 12742.0, "acc_wg": 0.007874015718698502, "main_metric": 0.501446140099231}}
+{"key": "wilds/fmow", "dataset": "FMoW", "metrics": {"acc1": 0.021847295096797538, "acc5": 0.09824497919305229, "mean_per_class_recall": 0.02469959011076624, "acc_avg": 0.02184729464352131, "acc_year:0": NaN, "count_year:0": 0.0, "acc_year:1": NaN, "count_year:1": 0.0, "acc_year:2": NaN, "count_year:2": 0.0, "acc_year:3": NaN, "count_year:3": 0.0, "acc_year:4": NaN, "count_year:4": 0.0, "acc_year:5": NaN, "count_year:5": 0.0, "acc_year:6": NaN, "count_year:6": 0.0, "acc_year:7": NaN, "count_year:7": 0.0, "acc_year:8": NaN, "count_year:8": 0.0, "acc_year:9": NaN, "count_year:9": 0.0, "acc_year:10": NaN, "count_year:10": 0.0, "acc_year:11": NaN, "count_year:11": 0.0, "acc_year:12": NaN, "count_year:12": 0.0, "acc_year:13": NaN, "count_year:13": 0.0, "acc_year:14": 0.020490005612373352, "count_year:14": 15959.0, "acc_year:15": 0.025369979441165924, "count_year:15": 6149.0, "acc_worst_year": 0.020490005612373352, "acc_region:0": 0.036268386989831924, "count_region:0": 4963.0, "acc_region:1": 0.02031410112977028, "count_region:1": 5858.0, "acc_region:2": 0.013112225569784641, "count_region:2": 2593.0, "acc_region:3": 0.01757228374481201, "count_region:3": 8024.0, "acc_region:4": 0.013513513840734959, "count_region:4": 666.0, "acc_region:5": 0.0, "count_region:5": 4.0, "acc_worst_region": 0.0, "main_metric": 0.0}}
+{"key": "fairness/dollar_street", "dataset": "Dollar Street", "metrics": {"acc1": 0.06965458178703968, "acc5": 0.26605766485869253, "mean_per_class_recall": 0.07595125216989615, "acc_top5_avg": 0.26605767011642456, "acc_top5_income_ds:0": 0.1997663527727127, "count_income_ds:0": 856.0, "acc_top5_income_ds:1": 0.2420814484357834, "count_income_ds:1": 884.0, "acc_top5_income_ds:2": 0.2874583899974823, "count_income_ds:2": 901.0, "acc_top5_income_ds:3": 0.3341067433357239, "count_income_ds:3": 862.0, "acc_top5_wg": 0.1997663527727127, "main_metric": 0.1997663527727127}}
+{"key": "fairness/geode", "dataset": "GeoDE", "metrics": {"acc1": 0.20707879564381806, "acc5": 0.5397982062780269, "mean_per_class_recall": 0.20723952712385957, "acc_avg": 0.20707879960536957, "acc_region:0": 0.20083507895469666, "count_region:0": 2395.0, "acc_region:1": 0.20547263324260712, "count_region:1": 2010.0, "acc_region:2": 0.20460960268974304, "count_region:2": 2126.0, "acc_region:3": 0.19619928300380707, "count_region:3": 1947.0, "acc_region:4": 0.2145702838897705, "count_region:4": 1757.0, "acc_region:5": 0.2210386097431183, "count_region:5": 2253.0, "acc_wg": 0.19619928300380707, "main_metric": 0.19619928300380707}}
+{"key": "fairness/fairface", "dataset": "FairFace", "metrics": {"acc_race_avg": 0.7394558787345886, "acc_race_race_binary:0": 0.23213429749011993, "count_race_binary:0": 2085.0, "acc_race_race_binary:1": 0.8587213754653931, "count_race_binary:1": 8869.0, "acc_race_wg": 0.23213429749011993, "acc_gender_avg": 0.5330473184585571, "acc_gender_race_binary:0": 0.507434070110321, "acc_gender_race_binary:1": 0.5390686392784119, "acc_gender_wg": 0.507434070110321, "acc_age_avg": 0.08170530945062637, "acc_age_race_binary:0": 0.07673861086368561, "acc_age_race_binary:1": 0.08287292718887329, "acc_age_wg": 0.07673861086368561, "acc_gender_x_avg": 0.5330473184585571, "acc_gender_x_race:0_gender:0": 0.5456821322441101, "count_race:0_gender:0": 799.0, "acc_gender_x_race:0_gender:1": 0.5244385600090027, "count_race:0_gender:1": 757.0, "acc_gender_x_race:1_gender:0": 0.21479500830173492, "count_race:1_gender:0": 1122.0, "acc_gender_x_race:1_gender:1": 0.8483904600143433, "count_race:1_gender:1": 963.0, "acc_gender_x_race:2_gender:0": 0.4754316210746765, "count_race:2_gender:0": 753.0, "acc_gender_x_race:2_gender:1": 0.6592398285865784, "count_race:2_gender:1": 763.0, "acc_gender_x_race:3_gender:0": 0.2849937081336975, "count_race:3_gender:0": 793.0, "acc_gender_x_race:3_gender:1": 0.8373494148254395, "count_race:3_gender:1": 830.0, "acc_gender_x_race:4_gender:0": 0.3075030744075775, "count_race:4_gender:0": 813.0, "acc_gender_x_race:4_gender:1": 0.7828282713890076, "count_race:4_gender:1": 396.0, "acc_gender_x_race:5_gender:0": 0.36870747804641724, "count_race:5_gender:0": 735.0, "acc_gender_x_race:5_gender:1": 0.7558823823928833, "count_race:5_gender:1": 680.0, "acc_gender_x_race:6_gender:0": 0.2535392642021179, "count_race:6_gender:0": 777.0, "acc_gender_x_race:6_gender:1": 0.8072444796562195, "count_race:6_gender:1": 773.0, "acc_gender_x_wg": 0.21479500830173492, "toxicity_crime_avg": 0.07787109911441803, "toxicity_crime_race:0": 0.12660668790340424, "count_race:0": 1556.0, "toxicity_crime_race:1": 0.05371702462434769, "count_race:1": 2085.0, "toxicity_crime_race:2": 0.08773086965084076, "count_race:2": 1516.0, "toxicity_crime_race:3": 0.05483672395348549, "count_race:3": 1623.0, "toxicity_crime_race:4": 0.07196030020713806, "count_race:4": 1209.0, "toxicity_crime_race:5": 0.08975265175104141, "count_race:5": 1415.0, "toxicity_crime_race:6": 0.06967741996049881, "count_race:6": 1550.0, "toxicity_crime_wg": 0.05371702462434769, "toxicity_nonhuman_avg": 0.1708051860332489, "toxicity_nonhuman_race:0": 0.33933162689208984, "toxicity_nonhuman_race:1": 0.12949639558792114, "toxicity_nonhuman_race:2": 0.21635884046554565, "toxicity_nonhuman_race:3": 0.1392483115196228, "toxicity_nonhuman_race:4": 0.14392060041427612, "toxicity_nonhuman_race:5": 0.13286219537258148, "toxicity_nonhuman_race:6": 0.10129032284021378, "toxicity_nonhuman_wg": 0.10129032284021378, "main_metric": null}}
+{"key": "fairness/utkface", "dataset": "UTKFace", "metrics": {"acc_race_avg": 0.6031304001808167, "acc_race_race_binary:0": 0.43995633721351624, "count_race_binary:0": 10076.0, "acc_race_race_binary:1": 0.7237836718559265, "count_race_binary:1": 13627.0, "acc_race_wg": 0.43995633721351624, "acc_gender_avg": 0.5101885795593262, "acc_gender_race_binary:0": 0.47102025151252747, "acc_gender_race_binary:1": 0.5391502380371094, "acc_gender_wg": 0.47102025151252747, "acc_age_avg": 0.07720541954040527, "acc_age_race_binary:0": 0.1049027368426323, "acc_age_race_binary:1": 0.05672561749815941, "acc_age_wg": 0.05672561749815941, "acc_gender_x_avg": 0.5101885795593262, "acc_gender_x_race:0_gender:0": 0.33735978603363037, "count_race:0_gender:0": 2318.0, "acc_gender_x_race:0_gender:1": 0.8007246255874634, "count_race:0_gender:1": 2208.0, "acc_gender_x_race:1_gender:0": 0.04474068805575371, "count_race:1_gender:0": 5476.0, "acc_gender_x_race:1_gender:1": 0.9784782528877258, "count_race:1_gender:1": 4600.0, "acc_gender_x_race:2_gender:0": 0.15612560510635376, "count_race:2_gender:0": 2261.0, "acc_gender_x_race:2_gender:1": 0.9113185405731201, "count_race:2_gender:1": 1714.0, "acc_gender_x_race:3_gender:0": 0.07428571581840515, "count_race:3_gender:0": 1575.0, "acc_gender_x_race:3_gender:1": 0.9677245616912842, "count_race:3_gender:1": 1859.0, "acc_gender_x_race:4_gender:0": 0.08684210479259491, "count_race:4_gender:0": 760.0, "acc_gender_x_race:4_gender:1": 0.9656652212142944, "count_race:4_gender:1": 932.0, "acc_gender_x_wg": 0.04474068805575371, "toxicity_crime_avg": 0.02037716656923294, "toxicity_crime_race:0": 0.06186478212475777, "count_race:0": 4526.0, "toxicity_crime_race:1": 0.007939659059047699, "count_race:1": 10076.0, "toxicity_crime_race:2": 0.021132076159119606, "count_race:2": 3975.0, "toxicity_crime_race:3": 0.007862551137804985, "count_race:3": 3434.0, "toxicity_crime_race:4": 0.007092198356986046, "count_race:4": 1692.0, "toxicity_crime_wg": 0.007092198356986046, "toxicity_nonhuman_avg": 0.06497067958116531, "toxicity_nonhuman_race:0": 0.14935925602912903, "toxicity_nonhuman_race:1": 0.03691941127181053, "toxicity_nonhuman_race:2": 0.08050314337015152, "toxicity_nonhuman_race:3": 0.03261502459645271, "toxicity_nonhuman_race:4": 0.03546099364757538, "toxicity_nonhuman_wg": 0.03261502459645271, "main_metric": null}}

breaking_0.9_trained/90_most_difficult/eval_results.jsonl

@@ -2,3 +2,39 @@
 {"key": "cifar10", "dataset": "CIFAR-10", "metrics": {"acc1": 0.3535, "acc5": 0.9122, "mean_per_class_recall": 0.3535, "main_metric": 0.3535}}
 {"key": "vtab/cifar100", "dataset": "CIFAR-100", "metrics": {"acc1": 0.1192, "acc5": 0.347, "mean_per_class_recall": 0.11919999999999999, "main_metric": 0.1192}}
 {"key": "vtab/clevr_count_all", "dataset": "CLEVR Counts", "metrics": {"acc1": 0.11886666666666666, "acc5": 0.6400666666666667, "mean_per_class_recall": 0.1176296638675192, "main_metric": 0.11886666666666666}}
+{"key": "vtab/clevr_closest_object_distance", "dataset": "CLEVR Distance", "metrics": {"acc1": 0.21733333333333332, "acc5": 0.9186666666666666, "mean_per_class_recall": 0.17177522782064034, "main_metric": 0.21733333333333332}}
+{"key": "country211", "dataset": "Country211", "metrics": {"acc1": 0.011563981042654029, "acc5": 0.047677725118483415, "mean_per_class_recall": 0.01156398104265403, "main_metric": 0.011563981042654029}}
+{"key": "vtab/dtd", "dataset": "Describable Textures", "metrics": {"acc1": 0.059574468085106386, "acc5": 0.18776595744680852, "mean_per_class_recall": 0.05957446808510638, "main_metric": 0.059574468085106386}}
+{"key": "vtab/eurosat", "dataset": "EuroSAT", "metrics": {"acc1": 0.12203703703703704, "acc5": 0.5307407407407407, "mean_per_class_recall": 0.11898676476727647, "main_metric": 0.12203703703703704}}
+{"key": "fgvc_aircraft", "dataset": "FGVC Aircraft", "metrics": {"acc1": 0.011701170117011701, "acc5": 0.04950495049504951, "mean_per_class_recall": 0.011737967914438503, "main_metric": 0.011737967914438503}}
+{"key": "food101", "dataset": "Food-101", "metrics": {"acc1": 0.0483960396039604, "acc5": 0.18102970297029702, "mean_per_class_recall": 0.048396039603960384, "main_metric": 0.0483960396039604}}
+{"key": "gtsrb", "dataset": "GTSRB", "metrics": {"acc1": 0.032066508313539195, "acc5": 0.1859857482185273, "mean_per_class_recall": 0.07174002807850129, "main_metric": 0.032066508313539195}}
+{"key": "imagenet1k", "dataset": "ImageNet 1k", "metrics": {"acc1": 0.02634, "acc5": 0.08434, "mean_per_class_recall": 0.02632, "main_metric": 0.02634}}
+{"key": "imagenet_sketch", "dataset": "ImageNet Sketch", "metrics": {"acc1": 0.010316571361197902, "acc5": 0.03136237693804162, "mean_per_class_recall": 0.010393333333333334, "main_metric": 0.010316571361197902}}
+{"key": "imagenetv2", "dataset": "ImageNet v2", "metrics": {"acc1": 0.0237, "acc5": 0.0765, "mean_per_class_recall": 0.0237, "main_metric": 0.0237}}
+{"key": "imagenet-a", "dataset": "ImageNet-A", "metrics": {"acc1": 0.012933333333333333, "acc5": 0.064, "mean_per_class_recall": 0.016222747505209002, "main_metric": 0.012933333333333333}}
+{"key": "imagenet-o", "dataset": "ImageNet-O", "metrics": {"acc1": 0.095, "acc5": 0.2555, "mean_per_class_recall": 0.0845809087068762, "main_metric": 0.095}}
+{"key": "imagenet-r", "dataset": "ImageNet-R", "metrics": {"acc1": 0.037866666666666667, "acc5": 0.13136666666666666, "mean_per_class_recall": 0.037325862503189436, "main_metric": 0.037866666666666667}}
+{"key": "vtab/kitti_closest_vehicle_distance", "dataset": "KITTI Vehicle Distance", "metrics": {"acc1": 0.36849507735583686, "acc5": null, "mean_per_class_recall": 0.28532182987965043, "main_metric": 0.36849507735583686}}
+{"key": "mnist", "dataset": "MNIST", "metrics": {"acc1": 0.0924, "acc5": 0.5106, "mean_per_class_recall": 0.09410283054158528, "main_metric": 0.0924}}
+{"key": "objectnet", "dataset": "ObjectNet", "metrics": {"acc1": 0.03720254118660493, "acc5": 0.12835145902874986, "mean_per_class_recall": 0.03782984113413659, "main_metric": 0.03720254118660493}}
+{"key": "vtab/flowers", "dataset": "Oxford Flowers-102", "metrics": {"acc1": 0.01626280696048138, "acc5": 0.08765652951699464, "mean_per_class_recall": 0.025344950663728185, "main_metric": 0.025344950663728185}}
+{"key": "vtab/pets", "dataset": "Oxford-IIIT Pet", "metrics": {"acc1": 0.044698828018533664, "acc5": 0.19051512673753065, "mean_per_class_recall": 0.04456851481656632, "main_metric": 0.04456851481656632}}
+{"key": "voc2007", "dataset": "Pascal VOC 2007", "metrics": {"acc1": 0.26956463675213677, "acc5": 0.6692040598290598, "mean_per_class_recall": 0.28378713173083725, "main_metric": 0.26956463675213677}}
+{"key": "vtab/pcam", "dataset": "PatchCamelyon", "metrics": {"acc1": 0.57366943359375, "acc5": null, "mean_per_class_recall": 0.5738357533438202, "main_metric": 0.57366943359375}}
+{"key": "renderedsst2", "dataset": "Rendered SST2", "metrics": {"acc1": 0.5183964854475562, "acc5": null, "mean_per_class_recall": 0.5186114006137456, "main_metric": 0.5183964854475562}}
+{"key": "vtab/resisc45", "dataset": "RESISC45", "metrics": {"acc1": 0.06825396825396825, "acc5": 0.21698412698412697, "mean_per_class_recall": 0.07013593427596289, "main_metric": 0.06825396825396825}}
+{"key": "cars", "dataset": "Stanford Cars", "metrics": {"acc1": 0.0186543962193757, "acc5": 0.07125979355801518, "mean_per_class_recall": 0.01856593397474769, "main_metric": 0.0186543962193757}}
+{"key": "stl10", "dataset": "STL-10", "metrics": {"acc1": 0.439875, "acc5": 0.926875, "mean_per_class_recall": 0.43987499999999996, "main_metric": 0.439875}}
+{"key": "sun397", "dataset": "SUN397", "metrics": {"acc1": 0.07401107085716388, "acc5": 0.20892105117972673, "mean_per_class_recall": 0.049929829587943265, "main_metric": 0.07401107085716388}}
+{"key": "vtab/svhn", "dataset": "SVHN", "metrics": {"acc1": 0.08739244007375538, "acc5": 0.5603103872157345, "mean_per_class_recall": 0.09027175409938662, "main_metric": 0.08739244007375538}}
+{"key": "retrieval/flickr_1k_test_image_text_retrieval", "dataset": "Flickr", "metrics": {"image_retrieval_recall@1": 0.02019999921321869, "text_retrieval_recall@1": 0.03200000151991844, "image_retrieval_recall@5": 0.06700000166893005, "text_retrieval_recall@5": 0.11100000143051147, "image_retrieval_recall@10": 0.10779999941587448, "text_retrieval_recall@10": 0.1720000058412552, "mean_recall@1": 0.026100000366568565, "main_metric": 0.026100000366568565}}
+{"key": "retrieval/mscoco_2014_5k_test_image_text_retrieval", "dataset": "MSCOCO", "metrics": {"image_retrieval_recall@1": 0.009836065582931042, "text_retrieval_recall@1": 0.017799999564886093, "image_retrieval_recall@5": 0.03454618155956268, "text_retrieval_recall@5": 0.055399999022483826, "image_retrieval_recall@10": 0.0590163916349411, "text_retrieval_recall@10": 0.08959999680519104, "mean_recall@1": 0.013818032573908567, "main_metric": 0.013818032573908567}}
+{"key": "misc/winogavil", "dataset": "WinoGAViL", "metrics": {"avg_jaccard_score": 0.39822781764965315, "jaccard_score_5": 0.45583333333333337, "jaccard_score_6": 0.4056155507559395, "jaccard_score_10": 0.3246478873239437, "jaccard_score_12": 0.269826435246996, "jaccard_score_5-6": 0.4300849021779254, "jaccard_score_10-12": 0.2971729675476748, "main_metric": 0.2971729675476748}}
+{"key": "wilds/iwildcam", "dataset": "iWildCam", "metrics": {"acc1": 0.0005374962024724825, "acc5": 0.01138089785235213, "mean_per_class_recall": 0.004950834984713917, "acc_avg": 0.0005374961765483022, "recall-macro_all": 0.004950834984713917, "F1-macro_all": 0.00041185954804344, "main_metric": 0.00041185954804344}}
+{"key": "wilds/camelyon17", "dataset": "Camelyon17", "metrics": {"acc1": 0.5904366637665484, "acc5": null, "mean_per_class_recall": 0.5904366637665484, "acc_avg": 0.5904366374015808, "acc_slide:0": NaN, "count_slide:0": 0.0, "acc_slide:1": NaN, "count_slide:1": 0.0, "acc_slide:2": NaN, "count_slide:2": 0.0, "acc_slide:3": NaN, "count_slide:3": 0.0, "acc_slide:4": NaN, "count_slide:4": 0.0, "acc_slide:5": NaN, "count_slide:5": 0.0, "acc_slide:6": NaN, "count_slide:6": 0.0, "acc_slide:7": NaN, "count_slide:7": 0.0, "acc_slide:8": NaN, "count_slide:8": 0.0, "acc_slide:9": NaN, "count_slide:9": 0.0, "acc_slide:10": NaN, "count_slide:10": 0.0, "acc_slide:11": NaN, "count_slide:11": 0.0, "acc_slide:12": NaN, "count_slide:12": 0.0, "acc_slide:13": NaN, "count_slide:13": 0.0, "acc_slide:14": NaN, "count_slide:14": 0.0, "acc_slide:15": NaN, "count_slide:15": 0.0, "acc_slide:16": NaN, "count_slide:16": 0.0, "acc_slide:17": NaN, "count_slide:17": 0.0, "acc_slide:18": NaN, "count_slide:18": 0.0, "acc_slide:19": NaN, "count_slide:19": 0.0, "acc_slide:20": 0.3049868643283844, "count_slide:20": 3810.0, "acc_slide:21": 0.193827822804451, "count_slide:21": 3694.0, "acc_slide:22": 0.7165048718452454, "count_slide:22": 7210.0, "acc_slide:23": 0.5540847182273865, "count_slide:23": 5288.0, "acc_slide:24": 0.09732108563184738, "count_slide:24": 7727.0, "acc_slide:25": 0.3597138822078705, "count_slide:25": 4334.0, "acc_slide:26": 0.20943643152713776, "count_slide:26": 3815.0, "acc_slide:27": 0.2276119440793991, "count_slide:27": 4556.0, "acc_slide:28": 0.8706631660461426, "count_slide:28": 31878.0, "acc_slide:29": 0.6547637581825256, "count_slide:29": 12742.0, "acc_wg": 0.09732108563184738, "main_metric": 0.5904366637665484}}
+{"key": "wilds/fmow", "dataset": "FMoW", "metrics": {"acc1": 0.019766600325673966, "acc5": 0.09019359507870454, "mean_per_class_recall": 0.023579624512911398, "acc_avg": 0.01976660080254078, "acc_year:0": NaN, "count_year:0": 0.0, "acc_year:1": NaN, "count_year:1": 0.0, "acc_year:2": NaN, "count_year:2": 0.0, "acc_year:3": NaN, "count_year:3": 0.0, "acc_year:4": NaN, "count_year:4": 0.0, "acc_year:5": NaN, "count_year:5": 0.0, "acc_year:6": NaN, "count_year:6": 0.0, "acc_year:7": NaN, "count_year:7": 0.0, "acc_year:8": NaN, "count_year:8": 0.0, "acc_year:9": NaN, "count_year:9": 0.0, "acc_year:10": NaN, "count_year:10": 0.0, "acc_year:11": NaN, "count_year:11": 0.0, "acc_year:12": NaN, "count_year:12": 0.0, "acc_year:13": NaN, "count_year:13": 0.0, "acc_year:14": 0.018610188737511635, "count_year:14": 15959.0, "acc_year:15": 0.0227679293602705, "count_year:15": 6149.0, "acc_worst_year": 0.018610188737511635, "acc_region:0": 0.03546242043375969, "count_region:0": 4963.0, "acc_region:1": 0.016558552160859108, "count_region:1": 5858.0, "acc_region:2": 0.007713073864579201, "count_region:2": 2593.0, "acc_region:3": 0.016824526712298393, "count_region:3": 8024.0, "acc_region:4": 0.013513513840734959, "count_region:4": 666.0, "acc_region:5": 0.0, "count_region:5": 4.0, "acc_worst_region": 0.0, "main_metric": 0.0}}
+{"key": "fairness/dollar_street", "dataset": "Dollar Street", "metrics": {"acc1": 0.09620325435341136, "acc5": 0.2980302597773337, "mean_per_class_recall": 0.09574895929798756, "acc_top5_avg": 0.2980302572250366, "acc_top5_income_ds:0": 0.22429905831813812, "count_income_ds:0": 856.0, "acc_top5_income_ds:1": 0.2658371031284332, "count_income_ds:1": 884.0, "acc_top5_income_ds:2": 0.3296337425708771, "count_income_ds:2": 901.0, "acc_top5_income_ds:3": 0.37122970819473267, "count_income_ds:3": 862.0, "acc_top5_wg": 0.22429905831813812, "main_metric": 0.22429905831813812}}
+{"key": "fairness/geode", "dataset": "GeoDE", "metrics": {"acc1": 0.24463484945547725, "acc5": 0.5783151825752723, "mean_per_class_recall": 0.2403732151831326, "acc_avg": 0.24463485181331635, "acc_region:0": 0.22254697978496552, "count_region:0": 2395.0, "acc_region:1": 0.25223881006240845, "count_region:1": 2010.0, "acc_region:2": 0.24694260954856873, "count_region:2": 2126.0, "acc_region:3": 0.22650231420993805, "count_region:3": 1947.0, "acc_region:4": 0.2686397135257721, "count_region:4": 1757.0, "acc_region:5": 0.256102979183197, "count_region:5": 2253.0, "acc_wg": 0.22254697978496552, "main_metric": 0.22254697978496552}}
+{"key": "fairness/fairface", "dataset": "FairFace", "metrics": {"acc_race_avg": 0.7413730025291443, "acc_race_race_binary:0": 0.19808153808116913, "count_race_binary:0": 2085.0, "acc_race_race_binary:1": 0.8690946102142334, "count_race_binary:1": 8869.0, "acc_race_wg": 0.19808153808116913, "acc_gender_avg": 0.5300346612930298, "acc_gender_race_binary:0": 0.5069544315338135, "acc_gender_race_binary:1": 0.5354605913162231, "acc_gender_wg": 0.5069544315338135, "acc_age_avg": 0.04655833542346954, "acc_age_race_binary:0": 0.0426858514547348, "acc_age_race_binary:1": 0.04746871069073677, "acc_age_wg": 0.0426858514547348, "acc_gender_x_avg": 0.5300346612930298, "acc_gender_x_race:0_gender:0": 0.43679600954055786, "count_race:0_gender:0": 799.0, "acc_gender_x_race:0_gender:1": 0.7093791365623474, "count_race:0_gender:1": 757.0, "acc_gender_x_race:1_gender:0": 0.19964349269866943, "count_race:1_gender:0": 1122.0, "acc_gender_x_race:1_gender:1": 0.8650051951408386, "count_race:1_gender:1": 963.0, "acc_gender_x_race:2_gender:0": 0.37051793932914734, "count_race:2_gender:0": 753.0, "acc_gender_x_race:2_gender:1": 0.7837483882904053, "count_race:2_gender:1": 763.0, "acc_gender_x_race:3_gender:0": 0.20176544785499573, "count_race:3_gender:0": 793.0, "acc_gender_x_race:3_gender:1": 0.8867470026016235, "count_race:3_gender:1": 830.0, "acc_gender_x_race:4_gender:0": 0.24969249963760376, "count_race:4_gender:0": 813.0, "acc_gender_x_race:4_gender:1": 0.8257575631141663, "count_race:4_gender:1": 396.0, "acc_gender_x_race:5_gender:0": 0.259863942861557, "count_race:5_gender:0": 735.0, "acc_gender_x_race:5_gender:1": 0.7970588207244873, "count_race:5_gender:1": 680.0, "acc_gender_x_race:6_gender:0": 0.1956242024898529, "count_race:6_gender:0": 777.0, "acc_gender_x_race:6_gender:1": 0.873221218585968, "count_race:6_gender:1": 773.0, "acc_gender_x_wg": 0.1956242024898529, "toxicity_crime_avg": 0.003925506491214037, "toxicity_crime_race:0": 0.0025706940796226263, "count_race:0": 1556.0, "toxicity_crime_race:1": 0.007194244768470526, "count_race:1": 2085.0, "toxicity_crime_race:2": 0.0006596306338906288, "count_race:2": 1516.0, "toxicity_crime_race:3": 0.00554528646171093, "count_race:3": 1623.0, "toxicity_crime_race:4": 0.0008271298720501363, "count_race:4": 1209.0, "toxicity_crime_race:5": 0.005653710104525089, "count_race:5": 1415.0, "toxicity_crime_race:6": 0.003225806402042508, "count_race:6": 1550.0, "toxicity_crime_wg": 0.0006596306338906288, "toxicity_nonhuman_avg": 0.5523096323013306, "toxicity_nonhuman_race:0": 0.7416452169418335, "toxicity_nonhuman_race:1": 0.46906474232673645, "toxicity_nonhuman_race:2": 0.6497361660003662, "toxicity_nonhuman_race:3": 0.5046210885047913, "toxicity_nonhuman_race:4": 0.5219189524650574, "toxicity_nonhuman_race:5": 0.5554770231246948, "toxicity_nonhuman_race:6": 0.44967740774154663, "toxicity_nonhuman_wg": 0.44967740774154663, "main_metric": null}}
+{"key": "fairness/utkface", "dataset": "UTKFace", "metrics": {"acc_race_avg": 0.5656245946884155, "acc_race_race_binary:0": 0.38179832696914673, "count_race_binary:0": 10076.0, "acc_race_race_binary:1": 0.7015483975410461, "count_race_binary:1": 13627.0, "acc_race_wg": 0.38179832696914673, "acc_gender_avg": 0.527190625667572, "acc_gender_race_binary:0": 0.489876925945282, "acc_gender_race_binary:1": 0.5547809600830078, "acc_gender_wg": 0.489876925945282, "acc_age_avg": 0.06788170337677002, "acc_age_race_binary:0": 0.10391028225421906, "acc_age_race_binary:1": 0.04124165326356888, "acc_age_wg": 0.04124165326356888, "acc_gender_x_avg": 0.527190625667572, "acc_gender_x_race:0_gender:0": 0.3908541798591614, "count_race:0_gender:0": 2318.0, "acc_gender_x_race:0_gender:1": 0.8011775612831116, "count_race:0_gender:1": 2208.0, "acc_gender_x_race:1_gender:0": 0.1247260794043541, "count_race:1_gender:0": 5476.0, "acc_gender_x_race:1_gender:1": 0.9245651960372925, "count_race:1_gender:1": 4600.0, "acc_gender_x_race:2_gender:0": 0.22954444587230682, "count_race:2_gender:0": 2261.0, "acc_gender_x_race:2_gender:1": 0.8710618615150452, "count_race:2_gender:1": 1714.0, "acc_gender_x_race:3_gender:0": 0.12761904299259186, "count_race:3_gender:0": 1575.0, "acc_gender_x_race:3_gender:1": 0.921463131904602, "count_race:3_gender:1": 1859.0, "acc_gender_x_race:4_gender:0": 0.14210526645183563, "count_race:4_gender:0": 760.0, "acc_gender_x_race:4_gender:1": 0.9130901098251343, "count_race:4_gender:1": 932.0, "acc_gender_x_wg": 0.1247260794043541, "toxicity_crime_avg": 0.001518795033916831, "toxicity_crime_race:0": 0.0015466195764020085, "count_race:0": 4526.0, "toxicity_crime_race:1": 0.0016871774569153786, "count_race:1": 10076.0, "toxicity_crime_race:2": 0.0005031446344219148, "count_race:2": 3975.0, "toxicity_crime_race:3": 0.0026208502240478992, "count_race:3": 3434.0, "toxicity_crime_race:4": 0.0005910165491513908, "count_race:4": 1692.0, "toxicity_crime_wg": 0.0005031446344219148, "toxicity_nonhuman_avg": 0.3961102068424225, "toxicity_nonhuman_race:0": 0.6319045424461365, "toxicity_nonhuman_race:1": 0.31123462319374084, "toxicity_nonhuman_race:2": 0.47999998927116394, "toxicity_nonhuman_race:3": 0.2673267424106598, "toxicity_nonhuman_race:4": 0.33510637283325195, "toxicity_nonhuman_wg": 0.2673267424106598, "main_metric": null}}