{ "vidore/arxivqa_test_subsampled": { "ndcg_at_1": 0.85, "ndcg_at_3": 0.89228, "ndcg_at_5": 0.89969, "ndcg_at_10": 0.90675, "ndcg_at_20": 0.91237, "ndcg_at_50": 0.91522, "ndcg_at_100": 0.91554, "map_at_1": 0.85, "map_at_3": 0.882, "map_at_5": 0.8861, "map_at_10": 0.88898, "map_at_20": 0.89055, "map_at_50": 0.89104, "map_at_100": 0.89107, "recall_at_1": 0.85, "recall_at_3": 0.922, "recall_at_5": 0.94, "recall_at_10": 0.962, "recall_at_20": 0.984, "recall_at_50": 0.998, "recall_at_100": 1.0, "precision_at_1": 0.85, "precision_at_3": 0.30733, "precision_at_5": 0.188, "precision_at_10": 0.0962, "precision_at_20": 0.0492, "precision_at_50": 0.01996, "precision_at_100": 0.01, "mrr_at_1": 0.846, "mrr_at_3": 0.8793333333333334, "mrr_at_5": 0.8830333333333333, "mrr_at_10": 0.8864992063492062, "mrr_at_20": 0.8880353542656172, "mrr_at_50": 0.8885337581333198, "mrr_at_100": 0.8885597321592938, "naucs_at_1_max": 0.807211194272698, "naucs_at_1_std": -0.05267165636186127, "naucs_at_1_diff1": 0.9369345916042959, "naucs_at_3_max": 0.8579209461562436, "naucs_at_3_std": 0.05307764131293816, "naucs_at_3_diff1": 0.9121956474897643, "naucs_at_5_max": 0.8668689698101463, "naucs_at_5_std": 0.1076719576719556, "naucs_at_5_diff1": 0.9274042950513546, "naucs_at_10_max": 0.9051550444739312, "naucs_at_10_std": 0.11334709322325026, "naucs_at_10_diff1": 0.9363605091159263, "naucs_at_20_max": 0.9115896358543447, "naucs_at_20_std": 0.0822829131652637, "naucs_at_20_diff1": 0.9325980392156931, "naucs_at_50_max": 0.86928104575168, "naucs_at_50_std": -1.151727357609834, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/docvqa_test_subsampled": { "ndcg_at_1": 0.5255, "ndcg_at_3": 0.58323, "ndcg_at_5": 0.60928, "ndcg_at_10": 0.6312, "ndcg_at_20": 0.64271, "ndcg_at_50": 0.65172, "ndcg_at_100": 0.66009, "map_at_1": 0.5255, "map_at_3": 0.56874, "map_at_5": 0.58348, "map_at_10": 0.59235, "map_at_20": 0.59566, "map_at_50": 0.597, "map_at_100": 0.59776, "recall_at_1": 0.5255, "recall_at_3": 0.62528, "recall_at_5": 0.68736, "recall_at_10": 0.7561, "recall_at_20": 0.80044, "recall_at_50": 0.84701, "recall_at_100": 0.898, "precision_at_1": 0.5255, "precision_at_3": 0.20843, "precision_at_5": 0.13747, "precision_at_10": 0.07561, "precision_at_20": 0.04002, "precision_at_50": 0.01694, "precision_at_100": 0.00898, "mrr_at_1": 0.5210643015521065, "mrr_at_3": 0.567257945306726, "mrr_at_5": 0.5807834441980785, "mrr_at_10": 0.5902518213493823, "mrr_at_20": 0.5930426782324527, "mrr_at_50": 0.5946364866259196, "mrr_at_100": 0.5951585532787789, "naucs_at_1_max": 0.17116973406228145, "naucs_at_1_std": 0.19544078728073017, "naucs_at_1_diff1": 0.8677435977596096, "naucs_at_3_max": 0.058557313285818244, "naucs_at_3_std": 0.2717799040343582, "naucs_at_3_diff1": 0.8164636793441545, "naucs_at_5_max": -0.05874522172882286, "naucs_at_5_std": 0.3665163783010554, "naucs_at_5_diff1": 0.7947909776107586, "naucs_at_10_max": -0.25417632083805924, "naucs_at_10_std": 0.4815135951184752, "naucs_at_10_diff1": 0.7793392745175759, "naucs_at_20_max": -0.4248479263934413, "naucs_at_20_std": 0.5025457196714144, "naucs_at_20_diff1": 0.8088576009445793, "naucs_at_50_max": -0.6253326294965954, "naucs_at_50_std": 0.7313108018974633, "naucs_at_50_diff1": 0.8583751909406857, "naucs_at_100_max": -0.8449611456968541, "naucs_at_100_std": 0.9545749692417804, "naucs_at_100_diff1": 0.8604715807203863 }, "vidore/infovqa_test_subsampled": { "ndcg_at_1": 0.87247, "ndcg_at_3": 0.91404, "ndcg_at_5": 0.91918, "ndcg_at_10": 0.92524, "ndcg_at_20": 0.92769, "ndcg_at_50": 0.92926, "ndcg_at_100": 0.92926, "map_at_1": 0.87247, "map_at_3": 0.90452, "map_at_5": 0.90746, "map_at_10": 0.91005, "map_at_20": 0.91067, "map_at_50": 0.9109, "map_at_100": 0.9109, "recall_at_1": 0.87247, "recall_at_3": 0.9413, "recall_at_5": 0.95344, "recall_at_10": 0.97166, "recall_at_20": 0.98178, "recall_at_50": 0.98988, "recall_at_100": 0.98988, "precision_at_1": 0.87247, "precision_at_3": 0.31377, "precision_at_5": 0.19069, "precision_at_10": 0.09717, "precision_at_20": 0.04909, "precision_at_50": 0.0198, "precision_at_100": 0.0099, "mrr_at_1": 0.8704453441295547, "mrr_at_3": 0.9035087719298244, "mrr_at_5": 0.9063427800269903, "mrr_at_10": 0.9085799434483645, "mrr_at_20": 0.9091654351207062, "mrr_at_50": 0.9094025116628965, "mrr_at_100": 0.9094025116628965, "naucs_at_1_max": 0.5862949877081309, "naucs_at_1_std": -0.09978842769982305, "naucs_at_1_diff1": 0.9422575413582027, "naucs_at_3_max": 0.6682958971071765, "naucs_at_3_std": -0.17493815995842807, "naucs_at_3_diff1": 0.9255153259160952, "naucs_at_5_max": 0.7631677959063347, "naucs_at_5_std": 0.10157587361064939, "naucs_at_5_diff1": 0.9117628654622407, "naucs_at_10_max": 0.8838078085686626, "naucs_at_10_std": 0.5343512930451619, "naucs_at_10_diff1": 0.9148610774134698, "naucs_at_20_max": 0.9546293120196974, "naucs_at_20_std": 0.7223350051556908, "naucs_at_20_diff1": 0.9564661819784059, "naucs_at_50_max": 0.9183327616354471, "naucs_at_50_std": 0.6841596004695476, "naucs_at_50_diff1": 0.9477594183740937, "naucs_at_100_max": 0.9183327616354471, "naucs_at_100_std": 0.6841596004695476, "naucs_at_100_diff1": 0.9477594183740937 }, "vidore/tabfquad_test_subsampled": { "ndcg_at_1": 0.9, "ndcg_at_3": 0.94188, "ndcg_at_5": 0.94342, "ndcg_at_10": 0.95164, "ndcg_at_20": 0.95164, "ndcg_at_50": 0.95237, "ndcg_at_100": 0.95237, "map_at_1": 0.9, "map_at_3": 0.93274, "map_at_5": 0.93363, "map_at_10": 0.93711, "map_at_20": 0.93711, "map_at_50": 0.93723, "map_at_100": 0.93723, "recall_at_1": 0.9, "recall_at_3": 0.96786, "recall_at_5": 0.97143, "recall_at_10": 0.99643, "recall_at_20": 0.99643, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.9, "precision_at_3": 0.32262, "precision_at_5": 0.19429, "precision_at_10": 0.09964, "precision_at_20": 0.04982, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.8964285714285715, "mrr_at_3": 0.930952380952381, "mrr_at_5": 0.9318452380952381, "mrr_at_10": 0.9354053287981859, "mrr_at_20": 0.9354053287981859, "mrr_at_50": 0.9355284815075456, "mrr_at_100": 0.9355284815075456, "naucs_at_1_max": 0.6023242630385479, "naucs_at_1_std": 0.1999466453247955, "naucs_at_1_diff1": 0.9321395224756563, "naucs_at_3_max": 0.9709513435003594, "naucs_at_3_std": 0.7533457827575493, "naucs_at_3_diff1": 0.9400871459694935, "naucs_at_5_max": 0.9673202614379075, "naucs_at_5_std": 0.7572362278244656, "naucs_at_5_diff1": 0.932598039215689, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 0.72222222222224, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 0.72222222222224, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/tatdqa_test": { "ndcg_at_1": 0.67922, "ndcg_at_3": 0.77126, "ndcg_at_5": 0.79547, "ndcg_at_10": 0.81213, "ndcg_at_20": 0.81714, "ndcg_at_50": 0.8201, "ndcg_at_100": 0.82157, "map_at_1": 0.67922, "map_at_3": 0.74868, "map_at_5": 0.76223, "map_at_10": 0.7692, "map_at_20": 0.77062, "map_at_50": 0.77108, "map_at_100": 0.7712, "recall_at_1": 0.67922, "recall_at_3": 0.83657, "recall_at_5": 0.8949, "recall_at_10": 0.94593, "recall_at_20": 0.96537, "recall_at_50": 0.98056, "recall_at_100": 0.98967, "precision_at_1": 0.67922, "precision_at_3": 0.27886, "precision_at_5": 0.17898, "precision_at_10": 0.09459, "precision_at_20": 0.04827, "precision_at_50": 0.01961, "precision_at_100": 0.0099, "mrr_at_1": 0.6773997569866342, "mrr_at_3": 0.7477723774807619, "mrr_at_5": 0.7615633859862302, "mrr_at_10": 0.7684981870431451, "mrr_at_20": 0.7700138577184539, "mrr_at_50": 0.7704845824005803, "mrr_at_100": 0.7706159963424961, "naucs_at_1_max": 0.1774117896564107, "naucs_at_1_std": -0.24832936266113984, "naucs_at_1_diff1": 0.7996783800774746, "naucs_at_3_max": 0.26913363145255553, "naucs_at_3_std": -0.17474231347643812, "naucs_at_3_diff1": 0.7326803741116257, "naucs_at_5_max": 0.36576896450364044, "naucs_at_5_std": -0.032454808755900705, "naucs_at_5_diff1": 0.7033807158189981, "naucs_at_10_max": 0.3646700445883288, "naucs_at_10_std": 0.19880405123210632, "naucs_at_10_diff1": 0.7042603308547618, "naucs_at_20_max": 0.4634752837344628, "naucs_at_20_std": 0.4658794215520421, "naucs_at_20_diff1": 0.6598124404673089, "naucs_at_50_max": 0.6289968950467605, "naucs_at_50_std": 0.6642506493403703, "naucs_at_50_diff1": 0.7500502336238475, "naucs_at_100_max": 0.5973270147647666, "naucs_at_100_std": 0.6380160564965871, "naucs_at_100_diff1": 0.7184268296201503 }, "vidore/shiftproject_test": { "ndcg_at_1": 0.8, "ndcg_at_3": 0.87809, "ndcg_at_5": 0.89875, "ndcg_at_10": 0.90231, "ndcg_at_20": 0.90231, "ndcg_at_50": 0.90231, "ndcg_at_100": 0.90392, "map_at_1": 0.8, "map_at_3": 0.86, "map_at_5": 0.8715, "map_at_10": 0.87317, "map_at_20": 0.87317, "map_at_50": 0.87317, "map_at_100": 0.8733, "recall_at_1": 0.8, "recall_at_3": 0.93, "recall_at_5": 0.98, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 0.99, "recall_at_100": 1.0, "precision_at_1": 0.8, "precision_at_3": 0.31, "precision_at_5": 0.196, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.0198, "precision_at_100": 0.01, "mrr_at_1": 0.79, "mrr_at_3": 0.865, "mrr_at_5": 0.872, "mrr_at_10": 0.8736666666666667, "mrr_at_20": 0.8736666666666667, "mrr_at_50": 0.8736666666666667, "mrr_at_100": 0.8738036529680366, "naucs_at_1_max": 0.1717282717282722, "naucs_at_1_std": -0.36158841158841215, "naucs_at_1_diff1": 0.8469030969030963, "naucs_at_3_max": 0.01674002934507224, "naucs_at_3_std": -0.664332399626516, "naucs_at_3_diff1": 0.8195945044684548, "naucs_at_5_max": 0.35807656395892185, "naucs_at_5_std": -0.10270774976657283, "naucs_at_5_diff1": 0.7117180205415541, "naucs_at_10_max": 0.35807656395891135, "naucs_at_10_std": 0.35807656395891135, "naucs_at_10_diff1": 0.5541549953314738, "naucs_at_20_max": 0.35807656395891135, "naucs_at_20_std": 0.35807656395891135, "naucs_at_20_diff1": 0.5541549953314738, "naucs_at_50_max": 0.35807656395892007, "naucs_at_50_std": 0.35807656395892007, "naucs_at_50_diff1": 0.554154995331464, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 0.99, "ndcg_at_3": 0.99631, "ndcg_at_5": 0.99631, "ndcg_at_10": 0.99631, "ndcg_at_20": 0.99631, "ndcg_at_50": 0.99631, "ndcg_at_100": 0.99631, "map_at_1": 0.99, "map_at_3": 0.995, "map_at_5": 0.995, "map_at_10": 0.995, "map_at_20": 0.995, "map_at_50": 0.995, "map_at_100": 0.995, "recall_at_1": 0.99, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.99, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.99, "mrr_at_3": 0.995, "mrr_at_5": 0.995, "mrr_at_10": 0.995, "mrr_at_20": 0.995, "mrr_at_50": 0.995, "mrr_at_100": 0.995, "naucs_at_1_max": 0.7222222222222201, "naucs_at_1_std": 0.7222222222222201, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_energy_test": { "ndcg_at_1": 0.94, "ndcg_at_3": 0.95131, "ndcg_at_5": 0.95992, "ndcg_at_10": 0.96641, "ndcg_at_20": 0.96641, "ndcg_at_50": 0.96641, "ndcg_at_100": 0.96641, "map_at_1": 0.94, "map_at_3": 0.94833, "map_at_5": 0.95333, "map_at_10": 0.95601, "map_at_20": 0.95601, "map_at_50": 0.95601, "map_at_100": 0.95601, "recall_at_1": 0.94, "recall_at_3": 0.96, "recall_at_5": 0.98, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.94, "precision_at_3": 0.32, "precision_at_5": 0.196, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.94, "mrr_at_3": 0.9533333333333333, "mrr_at_5": 0.9558333333333333, "mrr_at_10": 0.9589285714285716, "mrr_at_20": 0.9589285714285716, "mrr_at_50": 0.9589285714285716, "mrr_at_100": 0.9589285714285716, "naucs_at_1_max": 0.6961562402738896, "naucs_at_1_std": -0.4341736694677865, "naucs_at_1_diff1": 0.9782135076252712, "naucs_at_3_max": 0.73739495798319, "naucs_at_3_std": -0.9213352007469611, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.6136788048552745, "naucs_at_5_std": -1.7399626517273414, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.88, "ndcg_at_3": 0.95178, "ndcg_at_5": 0.95178, "ndcg_at_10": 0.95178, "ndcg_at_20": 0.95178, "ndcg_at_50": 0.95178, "ndcg_at_100": 0.95178, "map_at_1": 0.88, "map_at_3": 0.935, "map_at_5": 0.935, "map_at_10": 0.935, "map_at_20": 0.935, "map_at_50": 0.935, "map_at_100": 0.935, "recall_at_1": 0.88, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.88, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.89, "mrr_at_3": 0.9416666666666668, "mrr_at_5": 0.9416666666666668, "mrr_at_10": 0.9416666666666668, "mrr_at_20": 0.9416666666666668, "mrr_at_50": 0.9416666666666668, "mrr_at_100": 0.9416666666666668, "naucs_at_1_max": 0.42968501431753087, "naucs_at_1_std": 0.40972001272669406, "naucs_at_1_diff1": 0.8544384346166086, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.97, "ndcg_at_3": 0.98893, "ndcg_at_5": 0.98893, "ndcg_at_10": 0.98893, "ndcg_at_20": 0.98893, "ndcg_at_50": 0.98893, "ndcg_at_100": 0.98893, "map_at_1": 0.97, "map_at_3": 0.985, "map_at_5": 0.985, "map_at_10": 0.985, "map_at_20": 0.985, "map_at_50": 0.985, "map_at_100": 0.985, "recall_at_1": 0.97, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.97, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.98, "mrr_at_3": 0.99, "mrr_at_5": 0.99, "mrr_at_10": 0.99, "mrr_at_20": 0.99, "mrr_at_50": 0.99, "mrr_at_100": 0.99, "naucs_at_1_max": 0.5448179271708694, "naucs_at_1_std": 0.044817927170871553, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null } }