results.json

{"../colpali/data_dir/eval_vidore/tatdqa_test": {"ndcg_at_1": 0.48785, "ndcg_at_3": 0.60169, "ndcg_at_5": 0.63222, "ndcg_at_10": 0.66645, "ndcg_at_20": 0.68089, "ndcg_at_50": 0.69003, "ndcg_at_100": 0.69296, "map_at_1": 0.48785, "map_at_3": 0.5729, "map_at_5": 0.58985, "map_at_10": 0.60425, "map_at_20": 0.60829, "map_at_50": 0.60985, "map_at_100": 0.61013, "recall_at_1": 0.48785, "recall_at_3": 0.6853, "recall_at_5": 0.75942, "recall_at_10": 0.86391, "recall_at_20": 0.92041, "recall_at_50": 0.96537, "recall_at_100": 0.98299, "precision_at_1": 0.48785, "precision_at_3": 0.22843, "precision_at_5": 0.15188, "precision_at_10": 0.08639, "precision_at_20": 0.04602, "precision_at_50": 0.01931, "precision_at_100": 0.00983, "mrr_at_1": 0.48784933171324424, "mrr_at_3": 0.5724989874443095, "mrr_at_5": 0.5894187930336168, "mrr_at_10": 0.6039697004763833, "mrr_at_20": 0.6079520768118514, "mrr_at_50": 0.6095075712839939, "mrr_at_100": 0.6097804580564442, "naucs_at_1_max": 0.40293585061615184, "naucs_at_1_std": -0.14673937992939107, "naucs_at_1_diff1": 0.6364029867248311, "naucs_at_3_max": 0.43383381323457326, "naucs_at_3_std": -0.1446341731272394, "naucs_at_3_diff1": 0.5093861333318099, "naucs_at_5_max": 0.46417719409885344, "naucs_at_5_std": -0.08528721830449848, "naucs_at_5_diff1": 0.4952237467750283, "naucs_at_10_max": 0.5880062669352942, "naucs_at_10_std": 0.13616624027776184, "naucs_at_10_diff1": 0.48265963315950905, "naucs_at_20_max": 0.6895003871659199, "naucs_at_20_std": 0.3016095373862108, "naucs_at_20_diff1": 0.4561474185018229, "naucs_at_50_max": 0.7578149218378609, "naucs_at_50_std": 0.43006554566079686, "naucs_at_50_diff1": 0.46564615650984836, "naucs_at_100_max": 0.9117025007506158, "naucs_at_100_std": 0.7993594556523079, "naucs_at_100_diff1": 0.5807330491484243}, "../colpali/data_dir/eval_vidore/shiftproject_test": {"ndcg_at_1": 0.55, "ndcg_at_3": 0.69488, "ndcg_at_5": 0.7151, "ndcg_at_10": 0.73829, "ndcg_at_20": 0.75371, "ndcg_at_50": 0.75595, "ndcg_at_100": 0.75928, "map_at_1": 0.55, "map_at_3": 0.66167, "map_at_5": 0.67267, "map_at_10": 0.68257, "map_at_20": 0.68694, "map_at_50": 0.68742, "map_at_100": 0.68773, "recall_at_1": 0.55, "recall_at_3": 0.79, "recall_at_5": 0.84, "recall_at_10": 0.91, "recall_at_20": 0.97, "recall_at_50": 0.98, "recall_at_100": 1.0, "precision_at_1": 0.55, "precision_at_3": 0.26333, "precision_at_5": 0.168, "precision_at_10": 0.091, "precision_at_20": 0.0485, "precision_at_50": 0.0196, "precision_at_100": 0.01, "mrr_at_1": 0.56, "mrr_at_3": 0.6666666666666667, "mrr_at_5": 0.6776666666666666, "mrr_at_10": 0.6877460317460318, "mrr_at_20": 0.6921937039568619, "mrr_at_50": 0.6926698944330524, "mrr_at_100": 0.6929850459482039, "naucs_at_1_max": 0.1796250789861685, "naucs_at_1_std": -0.11289756371550948, "naucs_at_1_diff1": 0.5852278312153335, "naucs_at_3_max": 0.480847380190925, "naucs_at_3_std": 0.1570683146175488, "naucs_at_3_diff1": 0.3128141005602716, "naucs_at_5_max": 0.6464776632302403, "naucs_at_5_std": 0.15878129602356322, "naucs_at_5_diff1": 0.25552282768777623, "naucs_at_10_max": 0.6763668430335094, "naucs_at_10_std": 0.10908807967631515, "naucs_at_10_diff1": 0.33909119203236976, "naucs_at_20_max": 0.7152194211017747, "naucs_at_20_std": -0.07329598506069111, "naucs_at_20_diff1": 0.5224089635854351, "naucs_at_50_max": 0.638188608776847, "naucs_at_50_std": -0.024276377217548, "naucs_at_50_diff1": 0.3489729225023405, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "../colpali/data_dir/eval_vidore/syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.8, "ndcg_at_3": 0.85786, "ndcg_at_5": 0.8699, "ndcg_at_10": 0.88707, "ndcg_at_20": 0.88707, "ndcg_at_50": 0.89115, "ndcg_at_100": 0.89115, "map_at_1": 0.8, "map_at_3": 0.84333, "map_at_5": 0.84983, "map_at_10": 0.85751, "map_at_20": 0.85751, "map_at_50": 0.85822, "map_at_100": 0.85822, "recall_at_1": 0.8, "recall_at_3": 0.9, "recall_at_5": 0.93, "recall_at_10": 0.98, "recall_at_20": 0.98, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.8, "precision_at_3": 0.3, "precision_at_5": 0.186, "precision_at_10": 0.098, "precision_at_20": 0.049, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.8, "mrr_at_3": 0.8466666666666667, "mrr_at_5": 0.8506666666666667, "mrr_at_10": 0.8583452380952381, "mrr_at_20": 0.8583452380952381, "mrr_at_50": 0.8590497835497836, "mrr_at_100": 0.8590497835497836, "naucs_at_1_max": 0.21123876123876134, "naucs_at_1_std": -0.6175074925074933, "naucs_at_1_diff1": 0.8485514485514482, "naucs_at_3_max": 0.40303454715219317, "naucs_at_3_std": -0.6123716153127936, "naucs_at_3_diff1": 0.9052287581699348, "naucs_at_5_max": 0.7246231826063763, "naucs_at_5_std": -0.2971855408830157, "naucs_at_5_diff1": 0.9042950513538754, "naucs_at_10_max": 1.0, "naucs_at_10_std": 0.3384687208216692, "naucs_at_10_diff1": 0.861111111111116, "naucs_at_20_max": 1.0, "naucs_at_20_std": 0.3384687208216692, "naucs_at_20_diff1": 0.861111111111116, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "../colpali/data_dir/eval_vidore/syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.78, "ndcg_at_3": 0.88071, "ndcg_at_5": 0.89276, "ndcg_at_10": 0.89632, "ndcg_at_20": 0.89882, "ndcg_at_50": 0.89882, "ndcg_at_100": 0.89882, "map_at_1": 0.78, "map_at_3": 0.85667, "map_at_5": 0.86317, "map_at_10": 0.86483, "map_at_20": 0.8655, "map_at_50": 0.8655, "map_at_100": 0.8655, "recall_at_1": 0.78, "recall_at_3": 0.95, "recall_at_5": 0.98, "recall_at_10": 0.99, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.78, "precision_at_3": 0.31667, "precision_at_5": 0.196, "precision_at_10": 0.099, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.78, "mrr_at_3": 0.8583333333333333, "mrr_at_5": 0.8648333333333333, "mrr_at_10": 0.8665, "mrr_at_20": 0.8671666666666666, "mrr_at_50": 0.8671666666666666, "mrr_at_100": 0.8671666666666666, "naucs_at_1_max": 0.06732507998330828, "naucs_at_1_std": -0.20429359669866004, "naucs_at_1_diff1": 0.8101729494134546, "naucs_at_3_max": 0.24948646125116392, "naucs_at_3_std": -0.35957049486461135, "naucs_at_3_diff1": 0.7461251167133489, "naucs_at_5_max": 0.861111111111116, "naucs_at_5_std": -0.8576097105508901, "naucs_at_5_diff1": 0.8692810457516353, "naucs_at_10_max": 1.0, "naucs_at_10_std": -0.5634920634920767, "naucs_at_10_diff1": 0.8692810457516413, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "../colpali/data_dir/eval_vidore/arxivqa_test_subsampled": {"ndcg_at_1": 0.726, "ndcg_at_3": 0.77655, "ndcg_at_5": 0.79402, "ndcg_at_10": 0.80677, "ndcg_at_20": 0.81877, "ndcg_at_50": 0.82442, "ndcg_at_100": 0.82743, "map_at_1": 0.726, "map_at_3": 0.76367, "map_at_5": 0.77347, "map_at_10": 0.779, "map_at_20": 0.78222, "map_at_50": 0.78301, "map_at_100": 0.7833, "recall_at_1": 0.726, "recall_at_3": 0.814, "recall_at_5": 0.856, "recall_at_10": 0.894, "recall_at_20": 0.942, "recall_at_50": 0.972, "recall_at_100": 0.99, "precision_at_1": 0.726, "precision_at_3": 0.27133, "precision_at_5": 0.1712, "precision_at_10": 0.0894, "precision_at_20": 0.0471, "precision_at_50": 0.01944, "precision_at_100": 0.0099, "mrr_at_1": 0.728, "mrr_at_3": 0.7646666666666666, "mrr_at_5": 0.7744666666666667, "mrr_at_10": 0.7799857142857143, "mrr_at_20": 0.7832047095835021, "mrr_at_50": 0.7839945166838324, "mrr_at_100": 0.784283961872529, "naucs_at_1_max": 0.7014088774428114, "naucs_at_1_std": -0.6174744308360424, "naucs_at_1_diff1": 0.859912445278299, "naucs_at_3_max": 0.6805349262152334, "naucs_at_3_std": -0.6296656444012255, "naucs_at_3_diff1": 0.8071337742966144, "naucs_at_5_max": 0.6996433974497519, "naucs_at_5_std": -0.64988788631943, "naucs_at_5_diff1": 0.8060568402852826, "naucs_at_10_max": 0.7226186039082052, "naucs_at_10_std": -0.6212306763928108, "naucs_at_10_diff1": 0.8039419271249308, "naucs_at_20_max": 0.741153932837504, "naucs_at_20_std": -0.64313081554461, "naucs_at_20_diff1": 0.8130332592807251, "naucs_at_50_max": 0.7690742963852164, "naucs_at_50_std": -0.6326197145524973, "naucs_at_50_diff1": 0.8149259703881475, "naucs_at_100_max": 0.8846872082166138, "naucs_at_100_std": 0.16451914098974052, "naucs_at_100_diff1": 0.8627450980391941}, "../colpali/data_dir/eval_vidore/docvqa_test_subsampled": {"ndcg_at_1": 0.43459, "ndcg_at_3": 0.50969, "ndcg_at_5": 0.53421, "ndcg_at_10": 0.56029, "ndcg_at_20": 0.5753, "ndcg_at_50": 0.59035, "ndcg_at_100": 0.59717, "map_at_1": 0.43459, "map_at_3": 0.49187, "map_at_5": 0.5054, "map_at_10": 0.5159, "map_at_20": 0.51996, "map_at_50": 0.52223, "map_at_100": 0.52283, "recall_at_1": 0.43459, "recall_at_3": 0.56098, "recall_at_5": 0.62084, "recall_at_10": 0.70288, "recall_at_20": 0.76275, "recall_at_50": 0.84035, "recall_at_100": 0.88248, "precision_at_1": 0.43459, "precision_at_3": 0.18699, "precision_at_5": 0.12417, "precision_at_10": 0.07029, "precision_at_20": 0.03814, "precision_at_50": 0.01681, "precision_at_100": 0.00882, "mrr_at_1": 0.43237250554323725, "mrr_at_3": 0.4911308203991131, "mrr_at_5": 0.5046563192904656, "mrr_at_10": 0.5152931756590293, "mrr_at_20": 0.5194420968465403, "mrr_at_50": 0.5215265841990322, "mrr_at_100": 0.522211311235842, "naucs_at_1_max": 0.6008031881783024, "naucs_at_1_std": -0.09915523740473231, "naucs_at_1_diff1": 0.7803922141939246, "naucs_at_3_max": 0.639214533337205, "naucs_at_3_std": -0.07404132301892366, "naucs_at_3_diff1": 0.7109656625406306, "naucs_at_5_max": 0.7074111678732542, "naucs_at_5_std": -0.01449191167341412, "naucs_at_5_diff1": 0.671407014388343, "naucs_at_10_max": 0.7402751856218066, "naucs_at_10_std": 0.10068157371562529, "naucs_at_10_diff1": 0.6337761338157893, "naucs_at_20_max": 0.7376501645053887, "naucs_at_20_std": 0.09127011416728051, "naucs_at_20_diff1": 0.6078716660074082, "naucs_at_50_max": 0.8092108780646098, "naucs_at_50_std": 0.3451449422197839, "naucs_at_50_diff1": 0.6315909011307576, "naucs_at_100_max": 0.8710369384806684, "naucs_at_100_std": 0.5368523184971837, "naucs_at_100_diff1": 0.6283299092042248}, "../colpali/data_dir/eval_vidore/syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.83, "ndcg_at_3": 0.90547, "ndcg_at_5": 0.92139, "ndcg_at_10": 0.92139, "ndcg_at_20": 0.92139, "ndcg_at_50": 0.92139, "ndcg_at_100": 0.92139, "map_at_1": 0.83, "map_at_3": 0.88667, "map_at_5": 0.89517, "map_at_10": 0.89517, "map_at_20": 0.89517, "map_at_50": 0.89517, "map_at_100": 0.89517, "recall_at_1": 0.83, "recall_at_3": 0.96, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.83, "precision_at_3": 0.32, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.83, "mrr_at_3": 0.8866666666666667, "mrr_at_5": 0.8956666666666666, "mrr_at_10": 0.8956666666666666, "mrr_at_20": 0.8956666666666666, "mrr_at_50": 0.8956666666666666, "mrr_at_100": 0.8956666666666666, "naucs_at_1_max": 0.4146978580135842, "naucs_at_1_std": -0.42267951471527254, "naucs_at_1_diff1": 0.8203691879027112, "naucs_at_3_max": 0.29470121381886233, "naucs_at_3_std": -1.445845004668526, "naucs_at_3_diff1": 0.7741596638655437, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "../colpali/data_dir/eval_vidore/infovqa_test_subsampled": {"ndcg_at_1": 0.74291, "ndcg_at_3": 0.81482, "ndcg_at_5": 0.82789, "ndcg_at_10": 0.83977, "ndcg_at_20": 0.84406, "ndcg_at_50": 0.84819, "ndcg_at_100": 0.85019, "map_at_1": 0.74291, "map_at_3": 0.79825, "map_at_5": 0.80533, "map_at_10": 0.8103, "map_at_20": 0.81158, "map_at_50": 0.81229, "map_at_100": 0.81247, "recall_at_1": 0.74291, "recall_at_3": 0.86235, "recall_at_5": 0.89474, "recall_at_10": 0.93117, "recall_at_20": 0.94737, "recall_at_50": 0.96761, "recall_at_100": 0.97976, "precision_at_1": 0.74291, "precision_at_3": 0.28745, "precision_at_5": 0.17895, "precision_at_10": 0.09312, "precision_at_20": 0.04737, "precision_at_50": 0.01935, "precision_at_100": 0.0098, "mrr_at_1": 0.7449392712550608, "mrr_at_3": 0.7992577597840755, "mrr_at_5": 0.8067476383265858, "mrr_at_10": 0.8114243943191312, "mrr_at_20": 0.8127099508678456, "mrr_at_50": 0.8134235679335868, "mrr_at_100": 0.8136062278140098, "naucs_at_1_max": 0.5002467702621618, "naucs_at_1_std": -0.20909101479602227, "naucs_at_1_diff1": 0.8714372281197089, "naucs_at_3_max": 0.6172807720131959, "naucs_at_3_std": -0.021551366586077075, "naucs_at_3_diff1": 0.8172311850781752, "naucs_at_5_max": 0.6887509615454283, "naucs_at_5_std": 0.10606714055077916, "naucs_at_5_diff1": 0.8036156324337533, "naucs_at_10_max": 0.7608998815782839, "naucs_at_10_std": 0.2666131185414366, "naucs_at_10_diff1": 0.7894986142095037, "naucs_at_20_max": 0.8100322793839043, "naucs_at_20_std": 0.4913858915960957, "naucs_at_20_diff1": 0.771844671484287, "naucs_at_50_max": 0.9571205669011994, "naucs_at_50_std": 0.7950997422328927, "naucs_at_50_diff1": 0.7186442960252689, "naucs_at_100_max": 1.0, "naucs_at_100_std": 0.8861454756034981, "naucs_at_100_diff1": 0.7767047634631392}, "../colpali/data_dir/eval_vidore/syntheticDocQA_energy_test": {"ndcg_at_1": 0.76, "ndcg_at_3": 0.84202, "ndcg_at_5": 0.85793, "ndcg_at_10": 0.86383, "ndcg_at_20": 0.86889, "ndcg_at_50": 0.87325, "ndcg_at_100": 0.87486, "map_at_1": 0.76, "map_at_3": 0.825, "map_at_5": 0.8335, "map_at_10": 0.83561, "map_at_20": 0.837, "map_at_50": 0.83787, "map_at_100": 0.83801, "recall_at_1": 0.76, "recall_at_3": 0.89, "recall_at_5": 0.93, "recall_at_10": 0.95, "recall_at_20": 0.97, "recall_at_50": 0.99, "recall_at_100": 1.0, "precision_at_1": 0.76, "precision_at_3": 0.29667, "precision_at_5": 0.186, "precision_at_10": 0.095, "precision_at_20": 0.0485, "precision_at_50": 0.0198, "precision_at_100": 0.01, "mrr_at_1": 0.77, "mrr_at_3": 0.83, "mrr_at_5": 0.8384999999999999, "mrr_at_10": 0.8406111111111112, "mrr_at_20": 0.8420757575757576, "mrr_at_50": 0.8429650856389986, "mrr_at_100": 0.8431002207741338, "naucs_at_1_max": 0.565696114908713, "naucs_at_1_std": -0.47849788007268307, "naucs_at_1_diff1": 0.8048152634766809, "naucs_at_3_max": 0.567605875783866, "naucs_at_3_std": -0.5079031011081521, "naucs_at_3_diff1": 0.7622626922085713, "naucs_at_5_max": 0.652260904361747, "naucs_at_5_std": -0.48626117113511635, "naucs_at_5_diff1": 0.7225556889422438, "naucs_at_10_max": 0.5948646125116731, "naucs_at_10_std": -0.8161531279178166, "naucs_at_10_diff1": 0.7661064425770331, "naucs_at_20_max": 0.4609399315281665, "naucs_at_20_std": -0.7231559290382794, "naucs_at_20_diff1": 0.7587924058512269, "naucs_at_50_max": 1.0, "naucs_at_50_std": 0.7222222222222041, "naucs_at_50_diff1": 0.7222222222222041, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "../colpali/data_dir/eval_vidore/tabfquad_test_subsampled": {"ndcg_at_1": 0.83929, "ndcg_at_3": 0.90098, "ndcg_at_5": 0.91297, "ndcg_at_10": 0.92095, "ndcg_at_20": 0.92194, "ndcg_at_50": 0.92194, "ndcg_at_100": 0.92255, "map_at_1": 0.83929, "map_at_3": 0.8875, "map_at_5": 0.89429, "map_at_10": 0.89752, "map_at_20": 0.89784, "map_at_50": 0.89784, "map_at_100": 0.8979, "recall_at_1": 0.83929, "recall_at_3": 0.93929, "recall_at_5": 0.96786, "recall_at_10": 0.99286, "recall_at_20": 0.99643, "recall_at_50": 0.99643, "recall_at_100": 1.0, "precision_at_1": 0.83929, "precision_at_3": 0.3131, "precision_at_5": 0.19357, "precision_at_10": 0.09929, "precision_at_20": 0.04982, "precision_at_50": 0.01993, "precision_at_100": 0.01, "mrr_at_1": 0.8428571428571429, "mrr_at_3": 0.8892857142857142, "mrr_at_5": 0.8960714285714286, "mrr_at_10": 0.8993055555555555, "mrr_at_20": 0.8996302308802309, "mrr_at_50": 0.8996302308802309, "mrr_at_100": 0.8996887788895985, "naucs_at_1_max": 0.6253192185965297, "naucs_at_1_std": 0.2798646731419841, "naucs_at_1_diff1": 0.8628287678707856, "naucs_at_3_max": 0.8949579831932787, "naucs_at_3_std": 0.6754270335585236, "naucs_at_3_diff1": 0.8747734387872811, "naucs_at_5_max": 0.9214130096483005, "naucs_at_5_std": 0.7217553688141943, "naucs_at_5_diff1": 0.9255628177196795, "naucs_at_10_max": 1.0, "naucs_at_10_std": 0.6790382819794653, "naucs_at_10_diff1": 0.9346405228758269, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 0.8692810457516478, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 0.8692810457515607, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0}}