Upload 161 files

.gitattributes

@@ -32,3 +32,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+qa_output/nq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/squad1-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/trivia-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/webq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text

beir_output/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.18563,0.29758,0.35363,0.41208,0.44156,0.46053,0.46686,0.46938,0.47284
+map,0.18563,0.26885,0.3,0.32402,0.3322,0.33532,0.33588,0.33601,0.3361
+precision,0.18563,0.12707,0.10341,0.06984,0.04072,0.01818,0.00948,0.00483,0.001
+recall,0.18563,0.38122,0.51707,0.69844,0.81437,0.90896,0.94808,0.96586,0.99502
+mrr,0.19061,0.27039,0.30211,0.32598,0.33442,0.33755,0.3381,0.33824,0.33832
+recall_cap,0.19061,0.37838,0.51636,0.69559,0.81437,0.90896,0.94808,0.96586,0.99502
+hole,0.66927,0.65813,0.65946,0.67205,0.71277,0.79707,0.83748,0.86072,0.87458

beir_output/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.18563,
+            "NDCG@3": 0.29758,
+            "NDCG@5": 0.35363,
+            "NDCG@10": 0.41208,
+            "NDCG@20": 0.44156,
+            "NDCG@50": 0.46053,
+            "NDCG@100": 0.46686,
+            "NDCG@200": 0.46938,
+            "NDCG@1000": 0.47284
+        },
+        "map": {
+            "MAP@1": 0.18563,
+            "MAP@3": 0.26885,
+            "MAP@5": 0.3,
+            "MAP@10": 0.32402,
+            "MAP@20": 0.3322,
+            "MAP@50": 0.33532,
+            "MAP@100": 0.33588,
+            "MAP@200": 0.33601,
+            "MAP@1000": 0.3361
+        },
+        "precision": {
+            "P@1": 0.18563,
+            "P@3": 0.12707,
+            "P@5": 0.10341,
+            "P@10": 0.06984,
+            "P@20": 0.04072,
+            "P@50": 0.01818,
+            "P@100": 0.00948,
+            "P@200": 0.00483,
+            "P@1000": 0.001
+        },
+        "recall": {
+            "Recall@1": 0.18563,
+            "Recall@3": 0.38122,
+            "Recall@5": 0.51707,
+            "Recall@10": 0.69844,
+            "Recall@20": 0.81437,
+            "Recall@50": 0.90896,
+            "Recall@100": 0.94808,
+            "Recall@200": 0.96586,
+            "Recall@1000": 0.99502
+        },
+        "mrr": {
+            "MRR@1": 0.19061,
+            "MRR@3": 0.27039,
+            "MRR@5": 0.30211,
+            "MRR@10": 0.32598,
+            "MRR@20": 0.33442,
+            "MRR@50": 0.33755,
+            "MRR@100": 0.3381,
+            "MRR@200": 0.33824,
+            "MRR@1000": 0.33832
+        },
+        "recall_cap": {
+            "R_cap@1": 0.19061,
+            "R_cap@3": 0.37838,
+            "R_cap@5": 0.51636,
+            "R_cap@10": 0.69559,
+            "R_cap@20": 0.81437,
+            "R_cap@50": 0.90896,
+            "R_cap@100": 0.94808,
+            "R_cap@200": 0.96586,
+            "R_cap@1000": 0.99502
+        },
+        "hole": {
+            "Hole@1": 0.66927,
+            "Hole@3": 0.65813,
+            "Hole@5": 0.65946,
+            "Hole@10": 0.67205,
+            "Hole@20": 0.71277,
+            "Hole@50": 0.79707,
+            "Hole@100": 0.83748,
+            "Hole@200": 0.86072,
+            "Hole@1000": 0.87458
+        }
+    }
+}

beir_output/beir.json

@@ -0,0 +1,22 @@
+{
+    "eval_beir-robust04_ndcg@10": 0.37855,
+    "eval_beir-robust04_recall@10": 0.12422,
+    "eval_beir-robust04_recall@20": 0.17174,
+    "eval_beir-robust04_recall@100": 0.33149,
+    "eval_beir-bioasq_ndcg@10": 0.29947,
+    "eval_beir-bioasq_recall@10": 0.33049,
+    "eval_beir-bioasq_recall@20": 0.39463,
+    "eval_beir-bioasq_recall@100": 0.54332,
+    "eval_beir-signal1m_ndcg@10": 0.27524,
+    "eval_beir-signal1m_recall@10": 0.13187,
+    "eval_beir-signal1m_recall@20": 0.17823,
+    "eval_beir-signal1m_recall@100": 0.3144,
+    "eval_beir-trec-news_ndcg@10": 0.38759,
+    "eval_beir-trec-news_recall@10": 0.13518,
+    "eval_beir-trec-news_recall@20": 0.22371,
+    "eval_beir-trec-news_recall@100": 0.47626,
+    "eval_beir-avg_ndcg@10": 0.3352125,
+    "eval_beir-avg_recall@10": 0.18044,
+    "eval_beir-avg_recall@20": 0.2420775,
+    "eval_beir-avg_recall@100": 0.4163675
+}

beir_output/bioasq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.294,0.29212,0.29336,0.29947,0.31743,0.34556,0.36084,0.37302,0.39934
+map,0.13737,0.18785,0.20572,0.22287,0.23419,0.24232,0.24483,0.24599,0.24727
+precision,0.294,0.19867,0.1612,0.114,0.0751,0.03944,0.02292,0.01289,0.00335
+recall,0.13737,0.22748,0.27118,0.33049,0.39463,0.48602,0.54332,0.59748,0.73819
+mrr,0.294,0.36433,0.37563,0.38714,0.39182,0.39462,0.39535,0.39565,0.39586
+recall_cap,0.294,0.30267,0.31557,0.34215,0.39515,0.48602,0.54332,0.59748,0.73819
+hole,0.678,0.778,0.8188,0.8684,0.9108,0.95136,0.97018,0.98205,0.99426

beir_output/bioasq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "bioasq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.294,
+            "NDCG@3": 0.29212,
+            "NDCG@5": 0.29336,
+            "NDCG@10": 0.29947,
+            "NDCG@20": 0.31743,
+            "NDCG@50": 0.34556,
+            "NDCG@100": 0.36084,
+            "NDCG@200": 0.37302,
+            "NDCG@1000": 0.39934
+        },
+        "map": {
+            "MAP@1": 0.13737,
+            "MAP@3": 0.18785,
+            "MAP@5": 0.20572,
+            "MAP@10": 0.22287,
+            "MAP@20": 0.23419,
+            "MAP@50": 0.24232,
+            "MAP@100": 0.24483,
+            "MAP@200": 0.24599,
+            "MAP@1000": 0.24727
+        },
+        "precision": {
+            "P@1": 0.294,
+            "P@3": 0.19867,
+            "P@5": 0.1612,
+            "P@10": 0.114,
+            "P@20": 0.0751,
+            "P@50": 0.03944,
+            "P@100": 0.02292,
+            "P@200": 0.01289,
+            "P@1000": 0.00335
+        },
+        "recall": {
+            "Recall@1": 0.13737,
+            "Recall@3": 0.22748,
+            "Recall@5": 0.27118,
+            "Recall@10": 0.33049,
+            "Recall@20": 0.39463,
+            "Recall@50": 0.48602,
+            "Recall@100": 0.54332,
+            "Recall@200": 0.59748,
+            "Recall@1000": 0.73819
+        },
+        "mrr": {
+            "MRR@1": 0.294,
+            "MRR@3": 0.36433,
+            "MRR@5": 0.37563,
+            "MRR@10": 0.38714,
+            "MRR@20": 0.39182,
+            "MRR@50": 0.39462,
+            "MRR@100": 0.39535,
+            "MRR@200": 0.39565,
+            "MRR@1000": 0.39586
+        },
+        "recall_cap": {
+            "R_cap@1": 0.294,
+            "R_cap@3": 0.30267,
+            "R_cap@5": 0.31557,
+            "R_cap@10": 0.34215,
+            "R_cap@20": 0.39515,
+            "R_cap@50": 0.48602,
+            "R_cap@100": 0.54332,
+            "R_cap@200": 0.59748,
+            "R_cap@1000": 0.73819
+        },
+        "hole": {
+            "Hole@1": 0.678,
+            "Hole@3": 0.778,
+            "Hole@5": 0.8188,
+            "Hole@10": 0.8684,
+            "Hole@20": 0.9108,
+            "Hole@50": 0.95136,
+            "Hole@100": 0.97018,
+            "Hole@200": 0.98205,
+            "Hole@1000": 0.99426
+        }
+    }
+}

beir_output/climate-fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.1342,0.12634,0.14021,0.16356,0.18216,0.20743,0.22619,0.24529,0.2759
+map,0.06265,0.09114,0.10141,0.11091,0.11626,0.12085,0.12299,0.12437,0.12546
+precision,0.1342,0.09251,0.07466,0.05192,0.03401,0.01906,0.012,0.0075,0.0021
+recall,0.06265,0.12128,0.15644,0.2108,0.2629,0.35528,0.43891,0.53597,0.72163
+mrr,0.1342,0.18795,0.20342,0.21686,0.22328,0.22738,0.22865,0.22929,0.22958
+recall_cap,0.1342,0.12888,0.15644,0.2108,0.2629,0.35528,0.43891,0.53597,0.72163
+hole,0.64039,0.69251,0.71635,0.74782,0.77902,0.81636,0.84205,0.86648,0.92002

beir_output/climate-fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "climate-fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.1342,
+            "NDCG@3": 0.12634,
+            "NDCG@5": 0.14021,
+            "NDCG@10": 0.16356,
+            "NDCG@20": 0.18216,
+            "NDCG@50": 0.20743,
+            "NDCG@100": 0.22619,
+            "NDCG@200": 0.24529,
+            "NDCG@1000": 0.2759
+        },
+        "map": {
+            "MAP@1": 0.06265,
+            "MAP@3": 0.09114,
+            "MAP@5": 0.10141,
+            "MAP@10": 0.11091,
+            "MAP@20": 0.11626,
+            "MAP@50": 0.12085,
+            "MAP@100": 0.12299,
+            "MAP@200": 0.12437,
+            "MAP@1000": 0.12546
+        },
+        "precision": {
+            "P@1": 0.1342,
+            "P@3": 0.09251,
+            "P@5": 0.07466,
+            "P@10": 0.05192,
+            "P@20": 0.03401,
+            "P@50": 0.01906,
+            "P@100": 0.012,
+            "P@200": 0.0075,
+            "P@1000": 0.0021
+        },
+        "recall": {
+            "Recall@1": 0.06265,
+            "Recall@3": 0.12128,
+            "Recall@5": 0.15644,
+            "Recall@10": 0.2108,
+            "Recall@20": 0.2629,
+            "Recall@50": 0.35528,
+            "Recall@100": 0.43891,
+            "Recall@200": 0.53597,
+            "Recall@1000": 0.72163
+        },
+        "mrr": {
+            "MRR@1": 0.1342,
+            "MRR@3": 0.18795,
+            "MRR@5": 0.20342,
+            "MRR@10": 0.21686,
+            "MRR@20": 0.22328,
+            "MRR@50": 0.22738,
+            "MRR@100": 0.22865,
+            "MRR@200": 0.22929,
+            "MRR@1000": 0.22958
+        },
+        "recall_cap": {
+            "R_cap@1": 0.1342,
+            "R_cap@3": 0.12888,
+            "R_cap@5": 0.15644,
+            "R_cap@10": 0.2108,
+            "R_cap@20": 0.2629,
+            "R_cap@50": 0.35528,
+            "R_cap@100": 0.43891,
+            "R_cap@200": 0.53597,
+            "R_cap@1000": 0.72163
+        },
+        "hole": {
+            "Hole@1": 0.64039,
+            "Hole@3": 0.69251,
+            "Hole@5": 0.71635,
+            "Hole@10": 0.74782,
+            "Hole@20": 0.77902,
+            "Hole@50": 0.81636,
+            "Hole@100": 0.84205,
+            "Hole@200": 0.86648,
+            "Hole@1000": 0.92002
+        }
+    }
+}

beir_output/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.21213666666666667,0.2506633333333333,0.26989500000000005,0.29382749999999996,0.3128033333333334,0.3338566666666667,0.3470849999999999,0.35768749999999994,0.37589500000000003
+map,0.18000166666666664,0.2254708333333333,0.23794,0.24887666666666663,0.2547658333333333,0.2586833333333333,0.2600891666666666,0.26079916666666664,0.2613725
+precision,0.21213666666666667,0.11628833333333333,0.08396166666666667,0.05308916666666666,0.03235833333333333,0.016341666666666664,0.009485833333333334,0.005423333333333333,0.0013875000000000005
+recall,0.18000166666666664,0.27587249999999996,0.32479166666666665,0.3953508333333333,0.46499000000000007,0.5617300000000001,0.6354808333333334,0.702255,0.8406408333333335
+mrr,0.21213666666666667,0.2595566666666667,0.27212,0.28255916666666664,0.28760416666666666,0.29065583333333334,0.29164999999999996,0.2920924999999999,0.2924066666666667
+recall_cap,0.21213666666666667,0.2817866666666667,0.3272833333333333,0.3964641666666666,0.4654858333333334,0.5619025,0.6355466666666667,0.7022624999999999,0.8406408333333335
+hole,0.7135958333333333,0.8054299999999999,0.8364533333333332,0.8659000000000002,0.8876591666666668,0.9045275,0.9129241666666666,0.9189291666666667,0.9273433333333334

beir_output/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.21213666666666667,
+            "NDCG@3": 0.2506633333333333,
+            "NDCG@5": 0.26989500000000005,
+            "NDCG@10": 0.29382749999999996,
+            "NDCG@20": 0.3128033333333334,
+            "NDCG@50": 0.3338566666666667,
+            "NDCG@100": 0.3470849999999999,
+            "NDCG@200": 0.35768749999999994,
+            "NDCG@1000": 0.37589500000000003
+        },
+        "map": {
+            "MAP@1": 0.18000166666666664,
+            "MAP@3": 0.2254708333333333,
+            "MAP@5": 0.23794,
+            "MAP@10": 0.24887666666666663,
+            "MAP@20": 0.2547658333333333,
+            "MAP@50": 0.2586833333333333,
+            "MAP@100": 0.2600891666666666,
+            "MAP@200": 0.26079916666666664,
+            "MAP@1000": 0.2613725
+        },
+        "precision": {
+            "P@1": 0.21213666666666667,
+            "P@3": 0.11628833333333333,
+            "P@5": 0.08396166666666667,
+            "P@10": 0.05308916666666666,
+            "P@20": 0.03235833333333333,
+            "P@50": 0.016341666666666664,
+            "P@100": 0.009485833333333334,
+            "P@200": 0.005423333333333333,
+            "P@1000": 0.0013875000000000005
+        },
+        "recall": {
+            "Recall@1": 0.18000166666666664,
+            "Recall@3": 0.27587249999999996,
+            "Recall@5": 0.32479166666666665,
+            "Recall@10": 0.3953508333333333,
+            "Recall@20": 0.46499000000000007,
+            "Recall@50": 0.5617300000000001,
+            "Recall@100": 0.6354808333333334,
+            "Recall@200": 0.702255,
+            "Recall@1000": 0.8406408333333335
+        },
+        "mrr": {
+            "MRR@1": 0.21213666666666667,
+            "MRR@3": 0.2595566666666667,
+            "MRR@5": 0.27212,
+            "MRR@10": 0.28255916666666664,
+            "MRR@20": 0.28760416666666666,
+            "MRR@50": 0.29065583333333334,
+            "MRR@100": 0.29164999999999996,
+            "MRR@200": 0.2920924999999999,
+            "MRR@1000": 0.2924066666666667
+        },
+        "recall_cap": {
+            "R_cap@1": 0.21213666666666667,
+            "R_cap@3": 0.2817866666666667,
+            "R_cap@5": 0.3272833333333333,
+            "R_cap@10": 0.3964641666666666,
+            "R_cap@20": 0.4654858333333334,
+            "R_cap@50": 0.5619025,
+            "R_cap@100": 0.6355466666666667,
+            "R_cap@200": 0.7022624999999999,
+            "R_cap@1000": 0.8406408333333335
+        },
+        "hole": {
+            "Hole@1": 0.7135958333333333,
+            "Hole@3": 0.8054299999999999,
+            "Hole@5": 0.8364533333333332,
+            "Hole@10": 0.8659000000000002,
+            "Hole@20": 0.8876591666666668,
+            "Hole@50": 0.9045275,
+            "Hole@100": 0.9129241666666666,
+            "Hole@200": 0.9189291666666667,
+            "Hole@1000": 0.9273433333333334
+        }
+    }
+}

beir_output/dbpedia-entity.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.40625,0.35352,0.33821,0.32611,0.32899,0.35634,0.38856,0.41818,0.46704
+map,0.06583,0.1071,0.12381,0.14843,0.17621,0.2061,0.22039,0.22836,0.2364
+precision,0.5125,0.38833,0.3315,0.26525,0.20787,0.1376,0.09293,0.05956,0.02021
+recall,0.06583,0.12667,0.15641,0.20661,0.27924,0.39405,0.4804,0.57379,0.73626
+mrr,0.5125,0.60792,0.62417,0.63022,0.63283,0.63493,0.63516,0.63535,0.63538
+recall_cap,0.5125,0.40875,0.37146,0.34235,0.34363,0.40639,0.48564,0.57683,0.73688
+hole,0.29,0.39167,0.4405,0.51275,0.59562,0.70785,0.78465,0.84662,0.93298

beir_output/dbpedia-entity.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "dbpedia-entity",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.40625,
+            "NDCG@3": 0.35352,
+            "NDCG@5": 0.33821,
+            "NDCG@10": 0.32611,
+            "NDCG@20": 0.32899,
+            "NDCG@50": 0.35634,
+            "NDCG@100": 0.38856,
+            "NDCG@200": 0.41818,
+            "NDCG@1000": 0.46704
+        },
+        "map": {
+            "MAP@1": 0.06583,
+            "MAP@3": 0.1071,
+            "MAP@5": 0.12381,
+            "MAP@10": 0.14843,
+            "MAP@20": 0.17621,
+            "MAP@50": 0.2061,
+            "MAP@100": 0.22039,
+            "MAP@200": 0.22836,
+            "MAP@1000": 0.2364
+        },
+        "precision": {
+            "P@1": 0.5125,
+            "P@3": 0.38833,
+            "P@5": 0.3315,
+            "P@10": 0.26525,
+            "P@20": 0.20787,
+            "P@50": 0.1376,
+            "P@100": 0.09293,
+            "P@200": 0.05956,
+            "P@1000": 0.02021
+        },
+        "recall": {
+            "Recall@1": 0.06583,
+            "Recall@3": 0.12667,
+            "Recall@5": 0.15641,
+            "Recall@10": 0.20661,
+            "Recall@20": 0.27924,
+            "Recall@50": 0.39405,
+            "Recall@100": 0.4804,
+            "Recall@200": 0.57379,
+            "Recall@1000": 0.73626
+        },
+        "mrr": {
+            "MRR@1": 0.5125,
+            "MRR@3": 0.60792,
+            "MRR@5": 0.62417,
+            "MRR@10": 0.63022,
+            "MRR@20": 0.63283,
+            "MRR@50": 0.63493,
+            "MRR@100": 0.63516,
+            "MRR@200": 0.63535,
+            "MRR@1000": 0.63538
+        },
+        "recall_cap": {
+            "R_cap@1": 0.5125,
+            "R_cap@3": 0.40875,
+            "R_cap@5": 0.37146,
+            "R_cap@10": 0.34235,
+            "R_cap@20": 0.34363,
+            "R_cap@50": 0.40639,
+            "R_cap@100": 0.48564,
+            "R_cap@200": 0.57683,
+            "R_cap@1000": 0.73688
+        },
+        "hole": {
+            "Hole@1": 0.29,
+            "Hole@3": 0.39167,
+            "Hole@5": 0.4405,
+            "Hole@10": 0.51275,
+            "Hole@20": 0.59562,
+            "Hole@50": 0.70785,
+            "Hole@100": 0.78465,
+            "Hole@200": 0.84662,
+            "Hole@1000": 0.93298
+        }
+    }
+}

beir_output/fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.49265,0.60046,0.63064,0.65477,0.66695,0.67488,0.67802,0.67987,0.68296
+map,0.45973,0.55971,0.57727,0.5878,0.59138,0.59285,0.59319,0.59331,0.5934
+precision,0.49265,0.24532,0.16376,0.09013,0.04781,0.02009,0.01031,0.00525,0.00109
+recall,0.45973,0.68017,0.75305,0.82563,0.8717,0.90861,0.92616,0.93789,0.96096
+mrr,0.49265,0.59718,0.61444,0.62441,0.62756,0.62873,0.62893,0.62899,0.62904
+recall_cap,0.49265,0.68154,0.75336,0.82564,0.8717,0.90861,0.92616,0.93789,0.96096
+hole,0.4832,0.72397,0.80711,0.88801,0.93701,0.97049,0.98319,0.99022,0.99656

beir_output/fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.49265,
+            "NDCG@3": 0.60046,
+            "NDCG@5": 0.63064,
+            "NDCG@10": 0.65477,
+            "NDCG@20": 0.66695,
+            "NDCG@50": 0.67488,
+            "NDCG@100": 0.67802,
+            "NDCG@200": 0.67987,
+            "NDCG@1000": 0.68296
+        },
+        "map": {
+            "MAP@1": 0.45973,
+            "MAP@3": 0.55971,
+            "MAP@5": 0.57727,
+            "MAP@10": 0.5878,
+            "MAP@20": 0.59138,
+            "MAP@50": 0.59285,
+            "MAP@100": 0.59319,
+            "MAP@200": 0.59331,
+            "MAP@1000": 0.5934
+        },
+        "precision": {
+            "P@1": 0.49265,
+            "P@3": 0.24532,
+            "P@5": 0.16376,
+            "P@10": 0.09013,
+            "P@20": 0.04781,
+            "P@50": 0.02009,
+            "P@100": 0.01031,
+            "P@200": 0.00525,
+            "P@1000": 0.00109
+        },
+        "recall": {
+            "Recall@1": 0.45973,
+            "Recall@3": 0.68017,
+            "Recall@5": 0.75305,
+            "Recall@10": 0.82563,
+            "Recall@20": 0.8717,
+            "Recall@50": 0.90861,
+            "Recall@100": 0.92616,
+            "Recall@200": 0.93789,
+            "Recall@1000": 0.96096
+        },
+        "mrr": {
+            "MRR@1": 0.49265,
+            "MRR@3": 0.59718,
+            "MRR@5": 0.61444,
+            "MRR@10": 0.62441,
+            "MRR@20": 0.62756,
+            "MRR@50": 0.62873,
+            "MRR@100": 0.62893,
+            "MRR@200": 0.62899,
+            "MRR@1000": 0.62904
+        },
+        "recall_cap": {
+            "R_cap@1": 0.49265,
+            "R_cap@3": 0.68154,
+            "R_cap@5": 0.75336,
+            "R_cap@10": 0.82564,
+            "R_cap@20": 0.8717,
+            "R_cap@50": 0.90861,
+            "R_cap@100": 0.92616,
+            "R_cap@200": 0.93789,
+            "R_cap@1000": 0.96096
+        },
+        "hole": {
+            "Hole@1": 0.4832,
+            "Hole@3": 0.72397,
+            "Hole@5": 0.80711,
+            "Hole@10": 0.88801,
+            "Hole@20": 0.93701,
+            "Hole@50": 0.97049,
+            "Hole@100": 0.98319,
+            "Hole@200": 0.99022,
+            "Hole@1000": 0.99656
+        }
+    }
+}

beir_output/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.24537,0.23302,0.2435,0.26892,0.29455,0.32539,0.34274,0.35766,0.38375
+map,0.12068,0.17467,0.18946,0.20371,0.21286,0.21942,0.22154,0.22271,0.22367
+precision,0.24537,0.15586,0.11605,0.07608,0.04884,0.02559,0.01508,0.00869,0.00223
+recall,0.12068,0.21508,0.26244,0.33482,0.41326,0.5356,0.61544,0.69464,0.8658
+mrr,0.24537,0.30247,0.31798,0.33187,0.33702,0.34058,0.34167,0.34208,0.34234
+recall_cap,0.24537,0.23997,0.26908,0.33524,0.41326,0.5356,0.61544,0.69464,0.8658
+hole,0.70216,0.79012,0.82562,0.86713,0.89074,0.9121,0.92125,0.92792,0.9366

beir_output/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.24537,
+            "NDCG@3": 0.23302,
+            "NDCG@5": 0.2435,
+            "NDCG@10": 0.26892,
+            "NDCG@20": 0.29455,
+            "NDCG@50": 0.32539,
+            "NDCG@100": 0.34274,
+            "NDCG@200": 0.35766,
+            "NDCG@1000": 0.38375
+        },
+        "map": {
+            "MAP@1": 0.12068,
+            "MAP@3": 0.17467,
+            "MAP@5": 0.18946,
+            "MAP@10": 0.20371,
+            "MAP@20": 0.21286,
+            "MAP@50": 0.21942,
+            "MAP@100": 0.22154,
+            "MAP@200": 0.22271,
+            "MAP@1000": 0.22367
+        },
+        "precision": {
+            "P@1": 0.24537,
+            "P@3": 0.15586,
+            "P@5": 0.11605,
+            "P@10": 0.07608,
+            "P@20": 0.04884,
+            "P@50": 0.02559,
+            "P@100": 0.01508,
+            "P@200": 0.00869,
+            "P@1000": 0.00223
+        },
+        "recall": {
+            "Recall@1": 0.12068,
+            "Recall@3": 0.21508,
+            "Recall@5": 0.26244,
+            "Recall@10": 0.33482,
+            "Recall@20": 0.41326,
+            "Recall@50": 0.5356,
+            "Recall@100": 0.61544,
+            "Recall@200": 0.69464,
+            "Recall@1000": 0.8658
+        },
+        "mrr": {
+            "MRR@1": 0.24537,
+            "MRR@3": 0.30247,
+            "MRR@5": 0.31798,
+            "MRR@10": 0.33187,
+            "MRR@20": 0.33702,
+            "MRR@50": 0.34058,
+            "MRR@100": 0.34167,
+            "MRR@200": 0.34208,
+            "MRR@1000": 0.34234
+        },
+        "recall_cap": {
+            "R_cap@1": 0.24537,
+            "R_cap@3": 0.23997,
+            "R_cap@5": 0.26908,
+            "R_cap@10": 0.33524,
+            "R_cap@20": 0.41326,
+            "R_cap@50": 0.5356,
+            "R_cap@100": 0.61544,
+            "R_cap@200": 0.69464,
+            "R_cap@1000": 0.8658
+        },
+        "hole": {
+            "Hole@1": 0.70216,
+            "Hole@3": 0.79012,
+            "Hole@5": 0.82562,
+            "Hole@10": 0.86713,
+            "Hole@20": 0.89074,
+            "Hole@50": 0.9121,
+            "Hole@100": 0.92125,
+            "Hole@200": 0.92792,
+            "Hole@1000": 0.9366
+        }
+    }
+}

beir_output/hotpotqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.67468,0.52395,0.54891,0.57114,0.58641,0.59991,0.60679,0.61274,0.62304
+map,0.33734,0.4429,0.46103,0.47356,0.47938,0.48253,0.48341,0.48386,0.4842
+precision,0.67468,0.32636,0.2155,0.11893,0.06438,0.02796,0.01467,0.00768,0.00168
+recall,0.33734,0.48953,0.53876,0.59467,0.64382,0.69905,0.73363,0.76833,0.84193
+mrr,0.67468,0.73689,0.74597,0.75098,0.75304,0.75392,0.75411,0.75421,0.75426
+recall_cap,0.67468,0.48953,0.53876,0.59467,0.64382,0.69905,0.73363,0.76833,0.84193
+hole,0.31978,0.66289,0.77264,0.86887,0.92441,0.96173,0.97564,0.98306,0.99056

beir_output/hotpotqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "hotpotqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.67468,
+            "NDCG@3": 0.52395,
+            "NDCG@5": 0.54891,
+            "NDCG@10": 0.57114,
+            "NDCG@20": 0.58641,
+            "NDCG@50": 0.59991,
+            "NDCG@100": 0.60679,
+            "NDCG@200": 0.61274,
+            "NDCG@1000": 0.62304
+        },
+        "map": {
+            "MAP@1": 0.33734,
+            "MAP@3": 0.4429,
+            "MAP@5": 0.46103,
+            "MAP@10": 0.47356,
+            "MAP@20": 0.47938,
+            "MAP@50": 0.48253,
+            "MAP@100": 0.48341,
+            "MAP@200": 0.48386,
+            "MAP@1000": 0.4842
+        },
+        "precision": {
+            "P@1": 0.67468,
+            "P@3": 0.32636,
+            "P@5": 0.2155,
+            "P@10": 0.11893,
+            "P@20": 0.06438,
+            "P@50": 0.02796,
+            "P@100": 0.01467,
+            "P@200": 0.00768,
+            "P@1000": 0.00168
+        },
+        "recall": {
+            "Recall@1": 0.33734,
+            "Recall@3": 0.48953,
+            "Recall@5": 0.53876,
+            "Recall@10": 0.59467,
+            "Recall@20": 0.64382,
+            "Recall@50": 0.69905,
+            "Recall@100": 0.73363,
+            "Recall@200": 0.76833,
+            "Recall@1000": 0.84193
+        },
+        "mrr": {
+            "MRR@1": 0.67468,
+            "MRR@3": 0.73689,
+            "MRR@5": 0.74597,
+            "MRR@10": 0.75098,
+            "MRR@20": 0.75304,
+            "MRR@50": 0.75392,
+            "MRR@100": 0.75411,
+            "MRR@200": 0.75421,
+            "MRR@1000": 0.75426
+        },
+        "recall_cap": {
+            "R_cap@1": 0.67468,
+            "R_cap@3": 0.48953,
+            "R_cap@5": 0.53876,
+            "R_cap@10": 0.59467,
+            "R_cap@20": 0.64382,
+            "R_cap@50": 0.69905,
+            "R_cap@100": 0.73363,
+            "R_cap@200": 0.76833,
+            "R_cap@1000": 0.84193
+        },
+        "hole": {
+            "Hole@1": 0.31978,
+            "Hole@3": 0.66289,
+            "Hole@5": 0.77264,
+            "Hole@10": 0.86887,
+            "Hole@20": 0.92441,
+            "Hole@50": 0.96173,
+            "Hole@100": 0.97564,
+            "Hole@200": 0.98306,
+            "Hole@1000": 0.99056
+        }
+    }
+}

beir_output/msmarco.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.10516,0.17812,0.20873,0.2455,0.27229,0.2994,0.31286,0.32264,0.33635
+map,0.10287,0.15838,0.17546,0.19078,0.19812,0.20263,0.20386,0.20438,0.20471
+precision,0.10516,0.07989,0.06335,0.04342,0.02729,0.01378,0.00778,0.00426,0.00098
+recall,0.10287,0.2319,0.30528,0.41785,0.52273,0.65634,0.7377,0.80634,0.92242
+mrr,0.10501,0.16177,0.17899,0.19409,0.20144,0.20578,0.20693,0.20742,0.20773
+recall_cap,0.10501,0.2319,0.30542,0.41785,0.52273,0.65634,0.7377,0.80634,0.92242
+hole,0.89384,0.91901,0.93547,0.95554,0.9716,0.98513,0.99115,0.99467,0.99799

beir_output/msmarco.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "msmarco",
+    "split": "dev",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.10516,
+            "NDCG@3": 0.17812,
+            "NDCG@5": 0.20873,
+            "NDCG@10": 0.2455,
+            "NDCG@20": 0.27229,
+            "NDCG@50": 0.2994,
+            "NDCG@100": 0.31286,
+            "NDCG@200": 0.32264,
+            "NDCG@1000": 0.33635
+        },
+        "map": {
+            "MAP@1": 0.10287,
+            "MAP@3": 0.15838,
+            "MAP@5": 0.17546,
+            "MAP@10": 0.19078,
+            "MAP@20": 0.19812,
+            "MAP@50": 0.20263,
+            "MAP@100": 0.20386,
+            "MAP@200": 0.20438,
+            "MAP@1000": 0.20471
+        },
+        "precision": {
+            "P@1": 0.10516,
+            "P@3": 0.07989,
+            "P@5": 0.06335,
+            "P@10": 0.04342,
+            "P@20": 0.02729,
+            "P@50": 0.01378,
+            "P@100": 0.00778,
+            "P@200": 0.00426,
+            "P@1000": 0.00098
+        },
+        "recall": {
+            "Recall@1": 0.10287,
+            "Recall@3": 0.2319,
+            "Recall@5": 0.30528,
+            "Recall@10": 0.41785,
+            "Recall@20": 0.52273,
+            "Recall@50": 0.65634,
+            "Recall@100": 0.7377,
+            "Recall@200": 0.80634,
+            "Recall@1000": 0.92242
+        },
+        "mrr": {
+            "MRR@1": 0.10501,
+            "MRR@3": 0.16177,
+            "MRR@5": 0.17899,
+            "MRR@10": 0.19409,
+            "MRR@20": 0.20144,
+            "MRR@50": 0.20578,
+            "MRR@100": 0.20693,
+            "MRR@200": 0.20742,
+            "MRR@1000": 0.20773
+        },
+        "recall_cap": {
+            "R_cap@1": 0.10501,
+            "R_cap@3": 0.2319,
+            "R_cap@5": 0.30542,
+            "R_cap@10": 0.41785,
+            "R_cap@20": 0.52273,
+            "R_cap@50": 0.65634,
+            "R_cap@100": 0.7377,
+            "R_cap@200": 0.80634,
+            "R_cap@1000": 0.92242
+        },
+        "hole": {
+            "Hole@1": 0.89384,
+            "Hole@3": 0.91901,
+            "Hole@5": 0.93547,
+            "Hole@10": 0.95554,
+            "Hole@20": 0.9716,
+            "Hole@50": 0.98513,
+            "Hole@100": 0.99115,
+            "Hole@200": 0.99467,
+            "Hole@1000": 0.99799
+        }
+    }
+}

beir_output/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.40557,0.37862,0.3564,0.32723,0.31218,0.29953,0.30844,0.32584,0.39796
+map,0.05448,0.0925,0.10796,0.12297,0.1365,0.14822,0.15502,0.16044,0.16975
+precision,0.42415,0.3612,0.31084,0.24458,0.18498,0.11616,0.08015,0.05354,0.02104
+recall,0.05448,0.1037,0.13215,0.16013,0.20658,0.26201,0.31693,0.38592,0.64366
+mrr,0.43034,0.49794,0.51388,0.522,0.52698,0.52815,0.52866,0.52898,0.52914
+recall_cap,0.43034,0.38184,0.35093,0.30793,0.29445,0.29567,0.33243,0.39111,0.64366
+hole,0.0805,0.0805,0.08854,0.10031,0.11471,0.11969,0.12464,0.12772,0.13141

beir_output/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.40557,
+            "NDCG@3": 0.37862,
+            "NDCG@5": 0.3564,
+            "NDCG@10": 0.32723,
+            "NDCG@20": 0.31218,
+            "NDCG@50": 0.29953,
+            "NDCG@100": 0.30844,
+            "NDCG@200": 0.32584,
+            "NDCG@1000": 0.39796
+        },
+        "map": {
+            "MAP@1": 0.05448,
+            "MAP@3": 0.0925,
+            "MAP@5": 0.10796,
+            "MAP@10": 0.12297,
+            "MAP@20": 0.1365,
+            "MAP@50": 0.14822,
+            "MAP@100": 0.15502,
+            "MAP@200": 0.16044,
+            "MAP@1000": 0.16975
+        },
+        "precision": {
+            "P@1": 0.42415,
+            "P@3": 0.3612,
+            "P@5": 0.31084,
+            "P@10": 0.24458,
+            "P@20": 0.18498,
+            "P@50": 0.11616,
+            "P@100": 0.08015,
+            "P@200": 0.05354,
+            "P@1000": 0.02104
+        },
+        "recall": {
+            "Recall@1": 0.05448,
+            "Recall@3": 0.1037,
+            "Recall@5": 0.13215,
+            "Recall@10": 0.16013,
+            "Recall@20": 0.20658,
+            "Recall@50": 0.26201,
+            "Recall@100": 0.31693,
+            "Recall@200": 0.38592,
+            "Recall@1000": 0.64366
+        },
+        "mrr": {
+            "MRR@1": 0.43034,
+            "MRR@3": 0.49794,
+            "MRR@5": 0.51388,
+            "MRR@10": 0.522,
+            "MRR@20": 0.52698,
+            "MRR@50": 0.52815,
+            "MRR@100": 0.52866,
+            "MRR@200": 0.52898,
+            "MRR@1000": 0.52914
+        },
+        "recall_cap": {
+            "R_cap@1": 0.43034,
+            "R_cap@3": 0.38184,
+            "R_cap@5": 0.35093,
+            "R_cap@10": 0.30793,
+            "R_cap@20": 0.29445,
+            "R_cap@50": 0.29567,
+            "R_cap@100": 0.33243,
+            "R_cap@200": 0.39111,
+            "R_cap@1000": 0.64366
+        },
+        "hole": {
+            "Hole@1": 0.0805,
+            "Hole@3": 0.0805,
+            "Hole@5": 0.08854,
+            "Hole@10": 0.10031,
+            "Hole@20": 0.11471,
+            "Hole@50": 0.11969,
+            "Hole@100": 0.12464,
+            "Hole@200": 0.12772,
+            "Hole@1000": 0.13141
+        }
+    }
+}

beir_output/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.16541,0.24441,0.28439,0.32677,0.36009,0.38753,0.39991,0.40832,0.41645
+map,0.14675,0.21676,0.23957,0.25792,0.26785,0.27278,0.27397,0.27446,0.27469
+precision,0.16541,0.11713,0.09264,0.06162,0.03856,0.01869,0.01026,0.00549,0.00118
+recall,0.14675,0.30342,0.39661,0.52134,0.64569,0.77617,0.84849,0.9051,0.9705
+mrr,0.1657,0.23793,0.26022,0.2772,0.28566,0.28955,0.2905,0.29088,0.29105
+recall_cap,0.1657,0.3035,0.39661,0.52134,0.64569,0.77617,0.84849,0.9051,0.9705
+hole,0.83256,0.88026,0.90463,0.9354,0.95818,0.97823,0.98669,0.99148,0.99603

beir_output/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.16541,
+            "NDCG@3": 0.24441,
+            "NDCG@5": 0.28439,
+            "NDCG@10": 0.32677,
+            "NDCG@20": 0.36009,
+            "NDCG@50": 0.38753,
+            "NDCG@100": 0.39991,
+            "NDCG@200": 0.40832,
+            "NDCG@1000": 0.41645
+        },
+        "map": {
+            "MAP@1": 0.14675,
+            "MAP@3": 0.21676,
+            "MAP@5": 0.23957,
+            "MAP@10": 0.25792,
+            "MAP@20": 0.26785,
+            "MAP@50": 0.27278,
+            "MAP@100": 0.27397,
+            "MAP@200": 0.27446,
+            "MAP@1000": 0.27469
+        },
+        "precision": {
+            "P@1": 0.16541,
+            "P@3": 0.11713,
+            "P@5": 0.09264,
+            "P@10": 0.06162,
+            "P@20": 0.03856,
+            "P@50": 0.01869,
+            "P@100": 0.01026,
+            "P@200": 0.00549,
+            "P@1000": 0.00118
+        },
+        "recall": {
+            "Recall@1": 0.14675,
+            "Recall@3": 0.30342,
+            "Recall@5": 0.39661,
+            "Recall@10": 0.52134,
+            "Recall@20": 0.64569,
+            "Recall@50": 0.77617,
+            "Recall@100": 0.84849,
+            "Recall@200": 0.9051,
+            "Recall@1000": 0.9705
+        },
+        "mrr": {
+            "MRR@1": 0.1657,
+            "MRR@3": 0.23793,
+            "MRR@5": 0.26022,
+            "MRR@10": 0.2772,
+            "MRR@20": 0.28566,
+            "MRR@50": 0.28955,
+            "MRR@100": 0.2905,
+            "MRR@200": 0.29088,
+            "MRR@1000": 0.29105
+        },
+        "recall_cap": {
+            "R_cap@1": 0.1657,
+            "R_cap@3": 0.3035,
+            "R_cap@5": 0.39661,
+            "R_cap@10": 0.52134,
+            "R_cap@20": 0.64569,
+            "R_cap@50": 0.77617,
+            "R_cap@100": 0.84849,
+            "R_cap@200": 0.9051,
+            "R_cap@1000": 0.9705
+        },
+        "hole": {
+            "Hole@1": 0.83256,
+            "Hole@3": 0.88026,
+            "Hole@5": 0.90463,
+            "Hole@10": 0.9354,
+            "Hole@20": 0.95818,
+            "Hole@50": 0.97823,
+            "Hole@100": 0.98669,
+            "Hole@200": 0.99148,
+            "Hole@1000": 0.99603
+        }
+    }
+}

beir_output/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.6939,0.75303,0.77367,0.79323,0.80502,0.81293,0.81587,0.81726,0.81869
+map,0.6044,0.7115,0.731,0.7439,0.74933,0.75179,0.75233,0.7525,0.7526
+precision,0.6939,0.3306,0.22074,0.12313,0.06668,0.02863,0.01479,0.00755,0.00155
+recall,0.6044,0.78684,0.84108,0.8979,0.93776,0.96826,0.98182,0.98893,0.99729
+mrr,0.6936,0.76262,0.77241,0.77827,0.78036,0.78101,0.78113,0.78115,0.78117
+recall_cap,0.6936,0.79985,0.84621,0.8993,0.93803,0.96827,0.98182,0.98893,0.99729
+hole,0.2856,0.63617,0.74152,0.83694,0.89235,0.93037,0.94497,0.95363,0.96218

beir_output/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.6939,
+            "NDCG@3": 0.75303,
+            "NDCG@5": 0.77367,
+            "NDCG@10": 0.79323,
+            "NDCG@20": 0.80502,
+            "NDCG@50": 0.81293,
+            "NDCG@100": 0.81587,
+            "NDCG@200": 0.81726,
+            "NDCG@1000": 0.81869
+        },
+        "map": {
+            "MAP@1": 0.6044,
+            "MAP@3": 0.7115,
+            "MAP@5": 0.731,
+            "MAP@10": 0.7439,
+            "MAP@20": 0.74933,
+            "MAP@50": 0.75179,
+            "MAP@100": 0.75233,
+            "MAP@200": 0.7525,
+            "MAP@1000": 0.7526
+        },
+        "precision": {
+            "P@1": 0.6939,
+            "P@3": 0.3306,
+            "P@5": 0.22074,
+            "P@10": 0.12313,
+            "P@20": 0.06668,
+            "P@50": 0.02863,
+            "P@100": 0.01479,
+            "P@200": 0.00755,
+            "P@1000": 0.00155
+        },
+        "recall": {
+            "Recall@1": 0.6044,
+            "Recall@3": 0.78684,
+            "Recall@5": 0.84108,
+            "Recall@10": 0.8979,
+            "Recall@20": 0.93776,
+            "Recall@50": 0.96826,
+            "Recall@100": 0.98182,
+            "Recall@200": 0.98893,
+            "Recall@1000": 0.99729
+        },
+        "mrr": {
+            "MRR@1": 0.6936,
+            "MRR@3": 0.76262,
+            "MRR@5": 0.77241,
+            "MRR@10": 0.77827,
+            "MRR@20": 0.78036,
+            "MRR@50": 0.78101,
+            "MRR@100": 0.78113,
+            "MRR@200": 0.78115,
+            "MRR@1000": 0.78117
+        },
+        "recall_cap": {
+            "R_cap@1": 0.6936,
+            "R_cap@3": 0.79985,
+            "R_cap@5": 0.84621,
+            "R_cap@10": 0.8993,
+            "R_cap@20": 0.93803,
+            "R_cap@50": 0.96827,
+            "R_cap@100": 0.98182,
+            "R_cap@200": 0.98893,
+            "R_cap@1000": 0.99729
+        },
+        "hole": {
+            "Hole@1": 0.2856,
+            "Hole@3": 0.63617,
+            "Hole@5": 0.74152,
+            "Hole@10": 0.83694,
+            "Hole@20": 0.89235,
+            "Hole@50": 0.93037,
+            "Hole@100": 0.94497,
+            "Hole@200": 0.95363,
+            "Hole@1000": 0.96218
+        }
+    }
+}

beir_output/robust04.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.46988,0.42346,0.40392,0.37855,0.34462,0.3295,0.33976,0.36572,0.44216
+map,0.02261,0.04667,0.06384,0.09032,0.11263,0.14149,0.15873,0.17061,0.18399
+precision,0.52209,0.44578,0.41847,0.36064,0.28233,0.19968,0.14466,0.09775,0.03329
+recall,0.02261,0.05176,0.07773,0.12422,0.17174,0.25892,0.33149,0.41266,0.59615
+mrr,0.52209,0.60977,0.63307,0.64538,0.64839,0.64886,0.64891,0.64901,0.64907
+recall_cap,0.52209,0.44578,0.41988,0.3744,0.32335,0.31799,0.35197,0.41597,0.59615
+hole,0.03614,0.04953,0.06345,0.07952,0.11145,0.16137,0.2047,0.25353,0.36401

beir_output/robust04.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "robust04",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.46988,
+            "NDCG@3": 0.42346,
+            "NDCG@5": 0.40392,
+            "NDCG@10": 0.37855,
+            "NDCG@20": 0.34462,
+            "NDCG@50": 0.3295,
+            "NDCG@100": 0.33976,
+            "NDCG@200": 0.36572,
+            "NDCG@1000": 0.44216
+        },
+        "map": {
+            "MAP@1": 0.02261,
+            "MAP@3": 0.04667,
+            "MAP@5": 0.06384,
+            "MAP@10": 0.09032,
+            "MAP@20": 0.11263,
+            "MAP@50": 0.14149,
+            "MAP@100": 0.15873,
+            "MAP@200": 0.17061,
+            "MAP@1000": 0.18399
+        },
+        "precision": {
+            "P@1": 0.52209,
+            "P@3": 0.44578,
+            "P@5": 0.41847,
+            "P@10": 0.36064,
+            "P@20": 0.28233,
+            "P@50": 0.19968,
+            "P@100": 0.14466,
+            "P@200": 0.09775,
+            "P@1000": 0.03329
+        },
+        "recall": {
+            "Recall@1": 0.02261,
+            "Recall@3": 0.05176,
+            "Recall@5": 0.07773,
+            "Recall@10": 0.12422,
+            "Recall@20": 0.17174,
+            "Recall@50": 0.25892,
+            "Recall@100": 0.33149,
+            "Recall@200": 0.41266,
+            "Recall@1000": 0.59615
+        },
+        "mrr": {
+            "MRR@1": 0.52209,
+            "MRR@3": 0.60977,
+            "MRR@5": 0.63307,
+            "MRR@10": 0.64538,
+            "MRR@20": 0.64839,
+            "MRR@50": 0.64886,
+            "MRR@100": 0.64891,
+            "MRR@200": 0.64901,
+            "MRR@1000": 0.64907
+        },
+        "recall_cap": {
+            "R_cap@1": 0.52209,
+            "R_cap@3": 0.44578,
+            "R_cap@5": 0.41988,
+            "R_cap@10": 0.3744,
+            "R_cap@20": 0.32335,
+            "R_cap@50": 0.31799,
+            "R_cap@100": 0.35197,
+            "R_cap@200": 0.41597,
+            "R_cap@1000": 0.59615
+        },
+        "hole": {
+            "Hole@1": 0.03614,
+            "Hole@3": 0.04953,
+            "Hole@5": 0.06345,
+            "Hole@10": 0.07952,
+            "Hole@20": 0.11145,
+            "Hole@50": 0.16137,
+            "Hole@100": 0.2047,
+            "Hole@200": 0.25353,
+            "Hole@1000": 0.36401
+        }
+    }
+}

beir_output/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.199,0.15864,0.13784,0.16768,0.19301,0.22352,0.24115,0.25867,0.29535
+map,0.04028,0.07092,0.0832,0.09697,0.10584,0.1126,0.11505,0.11651,0.11794
+precision,0.199,0.14733,0.12,0.0871,0.05825,0.0323,0.01932,0.01149,0.00324
+recall,0.04028,0.08968,0.12163,0.17673,0.2363,0.32737,0.3918,0.4662,0.6573
+mrr,0.199,0.26117,0.27917,0.29335,0.29962,0.30393,0.30479,0.30523,0.30549
+recall_cap,0.199,0.14733,0.12163,0.17673,0.2363,0.32737,0.3918,0.4662,0.6573
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

beir_output/scidocs.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scidocs",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.199,
+            "NDCG@3": 0.15864,
+            "NDCG@5": 0.13784,
+            "NDCG@10": 0.16768,
+            "NDCG@20": 0.19301,
+            "NDCG@50": 0.22352,
+            "NDCG@100": 0.24115,
+            "NDCG@200": 0.25867,
+            "NDCG@1000": 0.29535
+        },
+        "map": {
+            "MAP@1": 0.04028,
+            "MAP@3": 0.07092,
+            "MAP@5": 0.0832,
+            "MAP@10": 0.09697,
+            "MAP@20": 0.10584,
+            "MAP@50": 0.1126,
+            "MAP@100": 0.11505,
+            "MAP@200": 0.11651,
+            "MAP@1000": 0.11794
+        },
+        "precision": {
+            "P@1": 0.199,
+            "P@3": 0.14733,
+            "P@5": 0.12,
+            "P@10": 0.0871,
+            "P@20": 0.05825,
+            "P@50": 0.0323,
+            "P@100": 0.01932,
+            "P@200": 0.01149,
+            "P@1000": 0.00324
+        },
+        "recall": {
+            "Recall@1": 0.04028,
+            "Recall@3": 0.08968,
+            "Recall@5": 0.12163,
+            "Recall@10": 0.17673,
+            "Recall@20": 0.2363,
+            "Recall@50": 0.32737,
+            "Recall@100": 0.3918,
+            "Recall@200": 0.4662,
+            "Recall@1000": 0.6573
+        },
+        "mrr": {
+            "MRR@1": 0.199,
+            "MRR@3": 0.26117,
+            "MRR@5": 0.27917,
+            "MRR@10": 0.29335,
+            "MRR@20": 0.29962,
+            "MRR@50": 0.30393,
+            "MRR@100": 0.30479,
+            "MRR@200": 0.30523,
+            "MRR@1000": 0.30549
+        },
+        "recall_cap": {
+            "R_cap@1": 0.199,
+            "R_cap@3": 0.14733,
+            "R_cap@5": 0.12163,
+            "R_cap@10": 0.17673,
+            "R_cap@20": 0.2363,
+            "R_cap@50": 0.32737,
+            "R_cap@100": 0.3918,
+            "R_cap@200": 0.4662,
+            "R_cap@1000": 0.6573
+        },
+        "hole": {
+            "Hole@1": 0.0,
+            "Hole@3": 0.0,
+            "Hole@5": 0.0,
+            "Hole@10": 0.0,
+            "Hole@20": 0.0,
+            "Hole@50": 0.0,
+            "Hole@100": 0.0,
+            "Hole@200": 0.0,
+            "Hole@1000": 0.0
+        }
+    }
+}

beir_output/scifact.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.55,0.64298,0.66559,0.67807,0.69009,0.70116,0.70619,0.70914,0.71429
+map,0.52261,0.61074,0.62728,0.63222,0.63601,0.63824,0.63872,0.63886,0.63901
+precision,0.55,0.25667,0.17067,0.08967,0.04783,0.0204,0.01053,0.00538,0.00112
+recall,0.52261,0.70856,0.76333,0.804,0.84811,0.89989,0.92989,0.951,0.99333
+mrr,0.55,0.63,0.6415,0.64599,0.64876,0.65026,0.6507,0.65083,0.65095
+recall_cap,0.55,0.71167,0.76333,0.804,0.84811,0.89989,0.92989,0.951,0.99333
+hole,0.43667,0.71222,0.79067,0.859,0.895,0.91947,0.93027,0.93642,0.94127

beir_output/scifact.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scifact",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.55,
+            "NDCG@3": 0.64298,
+            "NDCG@5": 0.66559,
+            "NDCG@10": 0.67807,
+            "NDCG@20": 0.69009,
+            "NDCG@50": 0.70116,
+            "NDCG@100": 0.70619,
+            "NDCG@200": 0.70914,
+            "NDCG@1000": 0.71429
+        },
+        "map": {
+            "MAP@1": 0.52261,
+            "MAP@3": 0.61074,
+            "MAP@5": 0.62728,
+            "MAP@10": 0.63222,
+            "MAP@20": 0.63601,
+            "MAP@50": 0.63824,
+            "MAP@100": 0.63872,
+            "MAP@200": 0.63886,
+            "MAP@1000": 0.63901
+        },
+        "precision": {
+            "P@1": 0.55,
+            "P@3": 0.25667,
+            "P@5": 0.17067,
+            "P@10": 0.08967,
+            "P@20": 0.04783,
+            "P@50": 0.0204,
+            "P@100": 0.01053,
+            "P@200": 0.00538,
+            "P@1000": 0.00112
+        },
+        "recall": {
+            "Recall@1": 0.52261,
+            "Recall@3": 0.70856,
+            "Recall@5": 0.76333,
+            "Recall@10": 0.804,
+            "Recall@20": 0.84811,
+            "Recall@50": 0.89989,
+            "Recall@100": 0.92989,
+            "Recall@200": 0.951,
+            "Recall@1000": 0.99333
+        },
+        "mrr": {
+            "MRR@1": 0.55,
+            "MRR@3": 0.63,
+            "MRR@5": 0.6415,
+            "MRR@10": 0.64599,
+            "MRR@20": 0.64876,
+            "MRR@50": 0.65026,
+            "MRR@100": 0.6507,
+            "MRR@200": 0.65083,
+            "MRR@1000": 0.65095
+        },
+        "recall_cap": {
+            "R_cap@1": 0.55,
+            "R_cap@3": 0.71167,
+            "R_cap@5": 0.76333,
+            "R_cap@10": 0.804,
+            "R_cap@20": 0.84811,
+            "R_cap@50": 0.89989,
+            "R_cap@100": 0.92989,
+            "R_cap@200": 0.951,
+            "R_cap@1000": 0.99333
+        },
+        "hole": {
+            "Hole@1": 0.43667,
+            "Hole@3": 0.71222,
+            "Hole@5": 0.79067,
+            "Hole@10": 0.859,
+            "Hole@20": 0.895,
+            "Hole@50": 0.91947,
+            "Hole@100": 0.93027,
+            "Hole@200": 0.93642,
+            "Hole@1000": 0.94127
+        }
+    }
+}

beir_output/senteval-core.json

@@ -0,0 +1,5 @@
+{
+    "eval_senteval-stsb_spearman": 0.7379237505860065,
+    "eval_senteval-sickr_spearman": 0.7253075282545358,
+    "eval_senteval-avg_sts": 0.7316156394202711
+}

beir_output/senteval.json

@@ -0,0 +1,99 @@
+{
+    "STSBenchmark": {
+        "train": {
+            "pearson": [
+                0.7078537910606651,
+                0.0
+            ],
+            "spearman": [
+                0.7005999583866984,
+                0.0
+            ],
+            "nsamples": 5749
+        },
+        "dev": {
+            "pearson": [
+                0.7192800747133679,
+                3.077700364928066e-239
+            ],
+            "spearman": [
+                0.7379237505860065,
+                4.277474410967731e-258
+            ],
+            "nsamples": 1500
+        },
+        "test": {
+            "pearson": [
+                0.6983884458054606,
+                2.940291533445568e-202
+            ],
+            "spearman": [
+                0.7136101793848104,
+                4.3889157567246634e-215
+            ],
+            "nsamples": 1379
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7084868825859301,
+                "mean": 0.7085074371931644,
+                "wmean": 0.7083274482665213
+            },
+            "spearman": {
+                "all": 0.7134156978222834,
+                "mean": 0.7173779627858385,
+                "wmean": 0.7091681993527809
+            }
+        }
+    },
+    "SICKRelatedness": {
+        "train": {
+            "pearson": [
+                0.7474648285643389,
+                0.0
+            ],
+            "spearman": [
+                0.6940199688267912,
+                0.0
+            ],
+            "nsamples": 4500
+        },
+        "dev": {
+            "pearson": [
+                0.7666627901165247,
+                6.873435561530908e-98
+            ],
+            "spearman": [
+                0.7253075282545358,
+                8.79855875263332e-83
+            ],
+            "nsamples": 500
+        },
+        "test": {
+            "pearson": [
+                0.7471456909275643,
+                0.0
+            ],
+            "spearman": [
+                0.6877954280401497,
+                0.0
+            ],
+            "nsamples": 4927
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7481870904024901,
+                "mean": 0.753757769869476,
+                "wmean": 0.7482733900269867
+            },
+            "spearman": {
+                "all": 0.6925290217821645,
+                "mean": 0.7023743083738255,
+                "wmean": 0.6925064669891857
+            }
+        }
+    },
+    "eval_senteval-stsb_spearman": 0.7379237505860065,
+    "eval_senteval-sickr_spearman": 0.7253075282545358,
+    "eval_senteval-avg_sts": 0.7316156394202711
+}

beir_output/signal1m.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.39691,0.34354,0.30792,0.27524,0.25363,0.26803,0.2964,0.31951,0.36544
+map,0.03023,0.06227,0.07502,0.10121,0.12173,0.139,0.14765,0.1521,0.15657
+precision,0.4433,0.34364,0.28866,0.2268,0.16495,0.09773,0.06268,0.03814,0.01109
+recall,0.03023,0.067,0.08481,0.13187,0.17823,0.2445,0.3144,0.37994,0.53162
+mrr,0.45361,0.49828,0.51529,0.52335,0.52792,0.52908,0.52954,0.52977,0.52987
+recall_cap,0.45361,0.34364,0.29278,0.24325,0.21636,0.24354,0.3144,0.37994,0.53162
+hole,0.53608,0.64948,0.70103,0.76701,0.82835,0.89856,0.93474,0.96026,0.98773

beir_output/signal1m.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "signal1m",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.39691,
+            "NDCG@3": 0.34354,
+            "NDCG@5": 0.30792,
+            "NDCG@10": 0.27524,
+            "NDCG@20": 0.25363,
+            "NDCG@50": 0.26803,
+            "NDCG@100": 0.2964,
+            "NDCG@200": 0.31951,
+            "NDCG@1000": 0.36544
+        },
+        "map": {
+            "MAP@1": 0.03023,
+            "MAP@3": 0.06227,
+            "MAP@5": 0.07502,
+            "MAP@10": 0.10121,
+            "MAP@20": 0.12173,
+            "MAP@50": 0.139,
+            "MAP@100": 0.14765,
+            "MAP@200": 0.1521,
+            "MAP@1000": 0.15657
+        },
+        "precision": {
+            "P@1": 0.4433,
+            "P@3": 0.34364,
+            "P@5": 0.28866,
+            "P@10": 0.2268,
+            "P@20": 0.16495,
+            "P@50": 0.09773,
+            "P@100": 0.06268,
+            "P@200": 0.03814,
+            "P@1000": 0.01109
+        },
+        "recall": {
+            "Recall@1": 0.03023,
+            "Recall@3": 0.067,
+            "Recall@5": 0.08481,
+            "Recall@10": 0.13187,
+            "Recall@20": 0.17823,
+            "Recall@50": 0.2445,
+            "Recall@100": 0.3144,
+            "Recall@200": 0.37994,
+            "Recall@1000": 0.53162
+        },
+        "mrr": {
+            "MRR@1": 0.45361,
+            "MRR@3": 0.49828,
+            "MRR@5": 0.51529,
+            "MRR@10": 0.52335,
+            "MRR@20": 0.52792,
+            "MRR@50": 0.52908,
+            "MRR@100": 0.52954,
+            "MRR@200": 0.52977,
+            "MRR@1000": 0.52987
+        },
+        "recall_cap": {
+            "R_cap@1": 0.45361,
+            "R_cap@3": 0.34364,
+            "R_cap@5": 0.29278,
+            "R_cap@10": 0.24325,
+            "R_cap@20": 0.21636,
+            "R_cap@50": 0.24354,
+            "R_cap@100": 0.3144,
+            "R_cap@200": 0.37994,
+            "R_cap@1000": 0.53162
+        },
+        "hole": {
+            "Hole@1": 0.53608,
+            "Hole@3": 0.64948,
+            "Hole@5": 0.70103,
+            "Hole@10": 0.76701,
+            "Hole@20": 0.82835,
+            "Hole@50": 0.89856,
+            "Hole@100": 0.93474,
+            "Hole@200": 0.96026,
+            "Hole@1000": 0.98773
+        }
+    }
+}

beir_output/trec-covid.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.7,0.64112,0.62447,0.5952,0.56185,0.51107,0.44272,0.38211,0.41141
+map,0.00201,0.00525,0.00836,0.01472,0.02487,0.04932,0.0767,0.11208,0.1907
+precision,0.74,0.68667,0.68,0.642,0.6,0.5432,0.4574,0.3799,0.1902
+recall,0.00201,0.00559,0.00921,0.01686,0.02999,0.06511,0.10661,0.17003,0.39328
+mrr,0.74,0.81,0.81,0.81667,0.81833,0.81987,0.81987,0.81987,0.81987
+recall_cap,0.74,0.68667,0.68,0.642,0.6,0.5432,0.4574,0.38332,0.39489
+hole,0.06,0.10667,0.096,0.112,0.142,0.1736,0.2218,0.2779,0.40066

beir_output/trec-covid.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "trec-covid",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.7,
+            "NDCG@3": 0.64112,
+            "NDCG@5": 0.62447,
+            "NDCG@10": 0.5952,
+            "NDCG@20": 0.56185,
+            "NDCG@50": 0.51107,
+            "NDCG@100": 0.44272,
+            "NDCG@200": 0.38211,
+            "NDCG@1000": 0.41141
+        },
+        "map": {
+            "MAP@1": 0.00201,
+            "MAP@3": 0.00525,
+            "MAP@5": 0.00836,
+            "MAP@10": 0.01472,
+            "MAP@20": 0.02487,
+            "MAP@50": 0.04932,
+            "MAP@100": 0.0767,
+            "MAP@200": 0.11208,
+            "MAP@1000": 0.1907
+        },
+        "precision": {
+            "P@1": 0.74,
+            "P@3": 0.68667,
+            "P@5": 0.68,
+            "P@10": 0.642,
+            "P@20": 0.6,
+            "P@50": 0.5432,
+            "P@100": 0.4574,
+            "P@200": 0.3799,
+            "P@1000": 0.1902
+        },
+        "recall": {
+            "Recall@1": 0.00201,
+            "Recall@3": 0.00559,
+            "Recall@5": 0.00921,
+            "Recall@10": 0.01686,
+            "Recall@20": 0.02999,
+            "Recall@50": 0.06511,
+            "Recall@100": 0.10661,
+            "Recall@200": 0.17003,
+            "Recall@1000": 0.39328
+        },
+        "mrr": {
+            "MRR@1": 0.74,
+            "MRR@3": 0.81,
+            "MRR@5": 0.81,
+            "MRR@10": 0.81667,
+            "MRR@20": 0.81833,
+            "MRR@50": 0.81987,
+            "MRR@100": 0.81987,
+            "MRR@200": 0.81987,
+            "MRR@1000": 0.81987
+        },
+        "recall_cap": {
+            "R_cap@1": 0.74,
+            "R_cap@3": 0.68667,
+            "R_cap@5": 0.68,
+            "R_cap@10": 0.642,
+            "R_cap@20": 0.6,
+            "R_cap@50": 0.5432,
+            "R_cap@100": 0.4574,
+            "R_cap@200": 0.38332,
+            "R_cap@1000": 0.39489
+        },
+        "hole": {
+            "Hole@1": 0.06,
+            "Hole@3": 0.10667,
+            "Hole@5": 0.096,
+            "Hole@10": 0.112,
+            "Hole@20": 0.142,
+            "Hole@50": 0.1736,
+            "Hole@100": 0.2218,
+            "Hole@200": 0.2779,
+            "Hole@1000": 0.40066
+        }
+    }
+}

beir_output/trec-news.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.46637,0.43713,0.41289,0.38759,0.38587,0.40476,0.43898,0.4869,0.56665
+map,0.01676,0.04315,0.06103,0.09603,0.14065,0.19425,0.22906,0.25123,0.2717
+precision,0.5614,0.5614,0.50175,0.44386,0.39211,0.27825,0.20316,0.13377,0.04098
+recall,0.01676,0.05255,0.07592,0.13518,0.22371,0.36683,0.47626,0.59333,0.80739
+mrr,0.5614,0.68129,0.68129,0.68891,0.69272,0.69413,0.69413,0.69413,0.69419
+recall_cap,0.5614,0.5614,0.50175,0.44827,0.43106,0.42833,0.48689,0.59385,0.80739
+hole,0.17544,0.15789,0.22105,0.26316,0.31404,0.42175,0.52298,0.62807,0.82446

beir_output/trec-news.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "trec-news",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.46637,
+            "NDCG@3": 0.43713,
+            "NDCG@5": 0.41289,
+            "NDCG@10": 0.38759,
+            "NDCG@20": 0.38587,
+            "NDCG@50": 0.40476,
+            "NDCG@100": 0.43898,
+            "NDCG@200": 0.4869,
+            "NDCG@1000": 0.56665
+        },
+        "map": {
+            "MAP@1": 0.01676,
+            "MAP@3": 0.04315,
+            "MAP@5": 0.06103,
+            "MAP@10": 0.09603,
+            "MAP@20": 0.14065,
+            "MAP@50": 0.19425,
+            "MAP@100": 0.22906,
+            "MAP@200": 0.25123,
+            "MAP@1000": 0.2717
+        },
+        "precision": {
+            "P@1": 0.5614,
+            "P@3": 0.5614,
+            "P@5": 0.50175,
+            "P@10": 0.44386,
+            "P@20": 0.39211,
+            "P@50": 0.27825,
+            "P@100": 0.20316,
+            "P@200": 0.13377,
+            "P@1000": 0.04098
+        },
+        "recall": {
+            "Recall@1": 0.01676,
+            "Recall@3": 0.05255,
+            "Recall@5": 0.07592,
+            "Recall@10": 0.13518,
+            "Recall@20": 0.22371,
+            "Recall@50": 0.36683,
+            "Recall@100": 0.47626,
+            "Recall@200": 0.59333,
+            "Recall@1000": 0.80739
+        },
+        "mrr": {
+            "MRR@1": 0.5614,
+            "MRR@3": 0.68129,
+            "MRR@5": 0.68129,
+            "MRR@10": 0.68891,
+            "MRR@20": 0.69272,
+            "MRR@50": 0.69413,
+            "MRR@100": 0.69413,
+            "MRR@200": 0.69413,
+            "MRR@1000": 0.69419
+        },
+        "recall_cap": {
+            "R_cap@1": 0.5614,
+            "R_cap@3": 0.5614,
+            "R_cap@5": 0.50175,
+            "R_cap@10": 0.44827,
+            "R_cap@20": 0.43106,
+            "R_cap@50": 0.42833,
+            "R_cap@100": 0.48689,
+            "R_cap@200": 0.59385,
+            "R_cap@1000": 0.80739
+        },
+        "hole": {
+            "Hole@1": 0.17544,
+            "Hole@3": 0.15789,
+            "Hole@5": 0.22105,
+            "Hole@10": 0.26316,
+            "Hole@20": 0.31404,
+            "Hole@50": 0.42175,
+            "Hole@100": 0.52298,
+            "Hole@200": 0.62807,
+            "Hole@1000": 0.82446
+        }
+    }
+}

beir_output/webis-touche2020.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.16327,0.16309,0.17653,0.17834,0.20454,0.2582,0.29732,0.34312,0.42448
+map,0.01566,0.02856,0.04376,0.06659,0.09003,0.11035,0.11912,0.12706,0.13481
+precision,0.16327,0.17007,0.18776,0.16939,0.14184,0.09469,0.06429,0.04378,0.01461
+recall,0.01566,0.04171,0.07486,0.13628,0.21481,0.32919,0.41812,0.54051,0.79752
+mrr,0.16327,0.28571,0.31531,0.33861,0.34451,0.34736,0.34758,0.34758,0.3478
+recall_cap,0.16327,0.17007,0.18776,0.18882,0.21937,0.32919,0.41812,0.54051,0.79752
+hole,0.79592,0.78231,0.77551,0.78776,0.81429,0.86163,0.90224,0.93,0.96904

beir_output/webis-touche2020.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "webis-touche2020",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.16327,
+            "NDCG@3": 0.16309,
+            "NDCG@5": 0.17653,
+            "NDCG@10": 0.17834,
+            "NDCG@20": 0.20454,
+            "NDCG@50": 0.2582,
+            "NDCG@100": 0.29732,
+            "NDCG@200": 0.34312,
+            "NDCG@1000": 0.42448
+        },
+        "map": {
+            "MAP@1": 0.01566,
+            "MAP@3": 0.02856,
+            "MAP@5": 0.04376,
+            "MAP@10": 0.06659,
+            "MAP@20": 0.09003,
+            "MAP@50": 0.11035,
+            "MAP@100": 0.11912,
+            "MAP@200": 0.12706,
+            "MAP@1000": 0.13481
+        },
+        "precision": {
+            "P@1": 0.16327,
+            "P@3": 0.17007,
+            "P@5": 0.18776,
+            "P@10": 0.16939,
+            "P@20": 0.14184,
+            "P@50": 0.09469,
+            "P@100": 0.06429,
+            "P@200": 0.04378,
+            "P@1000": 0.01461
+        },
+        "recall": {
+            "Recall@1": 0.01566,
+            "Recall@3": 0.04171,
+            "Recall@5": 0.07486,
+            "Recall@10": 0.13628,
+            "Recall@20": 0.21481,
+            "Recall@50": 0.32919,
+            "Recall@100": 0.41812,
+            "Recall@200": 0.54051,
+            "Recall@1000": 0.79752
+        },
+        "mrr": {
+            "MRR@1": 0.16327,
+            "MRR@3": 0.28571,
+            "MRR@5": 0.31531,
+            "MRR@10": 0.33861,
+            "MRR@20": 0.34451,
+            "MRR@50": 0.34736,
+            "MRR@100": 0.34758,
+            "MRR@200": 0.34758,
+            "MRR@1000": 0.3478
+        },
+        "recall_cap": {
+            "R_cap@1": 0.16327,
+            "R_cap@3": 0.17007,
+            "R_cap@5": 0.18776,
+            "R_cap@10": 0.18882,
+            "R_cap@20": 0.21937,
+            "R_cap@50": 0.32919,
+            "R_cap@100": 0.41812,
+            "R_cap@200": 0.54051,
+            "R_cap@1000": 0.79752
+        },
+        "hole": {
+            "Hole@1": 0.79592,
+            "Hole@3": 0.78231,
+            "Hole@5": 0.77551,
+            "Hole@10": 0.78776,
+            "Hole@20": 0.81429,
+            "Hole@50": 0.86163,
+            "Hole@100": 0.90224,
+            "Hole@200": 0.93,
+            "Hole@1000": 0.96904
+        }
+    }
+}

cc.moco.RC20+T0gen.bs2048.gpu8.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export LOCAL_RANK=0
+export WORLD_SIZE=8
+
+export TOKENIZERS_PARALLELISM=false
+export NUM_WORKER=8
+export MAX_STEPS=200000
+
+export CUDA_LAUNCH_BLOCKING=1
+export NCCL_DEBUG=INFO
+
+export EXP_NAME=cc-hybrid.RC20+T0gen80.seed477.moco-2e14.contriever256-special50.bert-base-uncased.avg.dot.q128d256.step200k.bs2048.lr5e5
+export PROJECT_DIR=output_dir/$EXP_NAME
+mkdir -p $PROJECT_DIR
+cp "$0" $PROJECT_DIR  # copy bash to project dir
+echo $PROJECT_DIR
+
+export WANDB_NAME=$EXP_NAME
+export WANDB_PROJECT=unsup_retrieval_cc
+export WANDB_DIR=$PROJECT_DIR
+mkdir -p $WANDB_DIR/wandb
+
+nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=3112 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type moco --train_file /export/home/data/search/upr/cc/T03B_PileCC_title.json:/export/home/data/search/upr/cc/T03B_PileCC_topic.json:/export/home/data/search/upr/cc/T03B_PileCC_exsum.json --dev_file /export/home/data/pretrain/wiki2021_structure/wiki_psgs_w100.dev.tail2e13.tsv --data_type hf --data_pipeline_name contriever256-special80% --remove_unused_columns False --sim_type dot --queue_size 16384 --momentum 0.9995 --output_dir $PROJECT_DIR --overwrite_output_dir --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --warmup_steps 10000 --logging_steps 100 --eval_steps 10000 --save_steps 100000000 --per_device_train_batch_size 256 --per_device_eval_batch_size 256 --beir_batch_size 256 --dataloader_num_workers $NUM_WORKER --learning_rate 5e-5 --max_q_tokens 128 --max_d_tokens 256 --evaluation_strategy steps --load_best_model_at_end --do_train --do_eval --run_name $EXP_NAME --fp16 --seed 477 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup.log 2>&1 & echo $! > run.pid
+
+# export WANDB_RUN_ID=10zycdpe
+# nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=3112 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type inbatch --reload_model_from $PROJECT_DIR --remove_unused_columns False --sim_type dot --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --do_eval --skip_senteval --skip_qaeval --beir_datasets robust04 bioasq signal1m trec-news --run_name $EXP_NAME --fp16 --seed 477 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup-eval.log 2>&1 & echo $! > run.pid
+

eval_output/checkpoint-100000/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.1707,0.27989,0.33083,0.38844,0.42211,0.44099,0.44859,0.45193,0.45599
+map,0.1707,0.25213,0.28055,0.30452,0.31397,0.31705,0.31771,0.3179,0.31799
+precision,0.1707,0.1202,0.09673,0.06607,0.03962,0.01774,0.00934,0.00479,0.00099
+recall,0.1707,0.3606,0.48364,0.66074,0.79232,0.88691,0.93385,0.95733,0.99218
+mrr,0.17496,0.25296,0.28312,0.30639,0.31591,0.31897,0.31965,0.31983,0.31993
+recall_cap,0.17496,0.35633,0.4872,0.66003,0.79232,0.8862,0.93385,0.95733,0.99218
+hole,0.68137,0.67259,0.67838,0.68741,0.72639,0.80312,0.84001,0.86271,0.87658

eval_output/checkpoint-100000/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.1707,
+            "NDCG@3": 0.27989,
+            "NDCG@5": 0.33083,
+            "NDCG@10": 0.38844,
+            "NDCG@20": 0.42211,
+            "NDCG@50": 0.44099,
+            "NDCG@100": 0.44859,
+            "NDCG@200": 0.45193,
+            "NDCG@1000": 0.45599
+        },
+        "map": {
+            "MAP@1": 0.1707,
+            "MAP@3": 0.25213,
+            "MAP@5": 0.28055,
+            "MAP@10": 0.30452,
+            "MAP@20": 0.31397,
+            "MAP@50": 0.31705,
+            "MAP@100": 0.31771,
+            "MAP@200": 0.3179,
+            "MAP@1000": 0.31799
+        },
+        "precision": {
+            "P@1": 0.1707,
+            "P@3": 0.1202,
+            "P@5": 0.09673,
+            "P@10": 0.06607,
+            "P@20": 0.03962,
+            "P@50": 0.01774,
+            "P@100": 0.00934,
+            "P@200": 0.00479,
+            "P@1000": 0.00099
+        },
+        "recall": {
+            "Recall@1": 0.1707,
+            "Recall@3": 0.3606,
+            "Recall@5": 0.48364,
+            "Recall@10": 0.66074,
+            "Recall@20": 0.79232,
+            "Recall@50": 0.88691,
+            "Recall@100": 0.93385,
+            "Recall@200": 0.95733,
+            "Recall@1000": 0.99218
+        },
+        "mrr": {
+            "MRR@1": 0.17496,
+            "MRR@3": 0.25296,
+            "MRR@5": 0.28312,
+            "MRR@10": 0.30639,
+            "MRR@20": 0.31591,
+            "MRR@50": 0.31897,
+            "MRR@100": 0.31965,
+            "MRR@200": 0.31983,
+            "MRR@1000": 0.31993
+        },
+        "recall_cap": {
+            "R_cap@1": 0.17496,
+            "R_cap@3": 0.35633,
+            "R_cap@5": 0.4872,
+            "R_cap@10": 0.66003,
+            "R_cap@20": 0.79232,
+            "R_cap@50": 0.8862,
+            "R_cap@100": 0.93385,
+            "R_cap@200": 0.95733,
+            "R_cap@1000": 0.99218
+        },
+        "hole": {
+            "Hole@1": 0.68137,
+            "Hole@3": 0.67259,
+            "Hole@5": 0.67838,
+            "Hole@10": 0.68741,
+            "Hole@20": 0.72639,
+            "Hole@50": 0.80312,
+            "Hole@100": 0.84001,
+            "Hole@200": 0.86271,
+            "Hole@1000": 0.87658
+        }
+    }
+}

eval_output/checkpoint-100000/beir.json

@@ -0,0 +1,46 @@
+{
+    "eval_beir-nfcorpus_ndcg@10": 0.31588,
+    "eval_beir-nfcorpus_recall@10": 0.15306,
+    "eval_beir-nfcorpus_recall@20": 0.18707,
+    "eval_beir-nfcorpus_recall@100": 0.30013,
+    "eval_beir-fiqa_ndcg@10": 0.26042,
+    "eval_beir-fiqa_recall@10": 0.32002,
+    "eval_beir-fiqa_recall@20": 0.40158,
+    "eval_beir-fiqa_recall@100": 0.60288,
+    "eval_beir-arguana_ndcg@10": 0.38844,
+    "eval_beir-arguana_recall@10": 0.66074,
+    "eval_beir-arguana_recall@20": 0.79232,
+    "eval_beir-arguana_recall@100": 0.93385,
+    "eval_beir-scidocs_ndcg@10": 0.16274,
+    "eval_beir-scidocs_recall@10": 0.17032,
+    "eval_beir-scidocs_recall@20": 0.22802,
+    "eval_beir-scidocs_recall@100": 0.38188,
+    "eval_beir-scifact_ndcg@10": 0.64579,
+    "eval_beir-scifact_recall@10": 0.78133,
+    "eval_beir-scifact_recall@20": 0.82478,
+    "eval_beir-scifact_recall@100": 0.91656,
+    "eval_beir-webis-touche2020_ndcg@10": 0.18165,
+    "eval_beir-webis-touche2020_recall@10": 0.12815,
+    "eval_beir-webis-touche2020_recall@20": 0.2125,
+    "eval_beir-webis-touche2020_recall@100": 0.41576,
+    "eval_beir-cqadupstack_ndcg@10": 0.2861625,
+    "eval_beir-cqadupstack_recall@10": 0.3857458333333333,
+    "eval_beir-cqadupstack_recall@20": 0.4529275000000001,
+    "eval_beir-cqadupstack_recall@100": 0.6233916666666667,
+    "eval_beir-trec-covid_ndcg@10": 0.62084,
+    "eval_beir-trec-covid_recall@10": 0.654,
+    "eval_beir-trec-covid_recall@20": 0.597,
+    "eval_beir-trec-covid_recall@100": 0.4618,
+    "eval_beir-quora_ndcg@10": 0.78828,
+    "eval_beir-quora_recall@10": 0.89395,
+    "eval_beir-quora_recall@20": 0.93398,
+    "eval_beir-quora_recall@100": 0.97984,
+    "eval_beir-nq_ndcg@10": 0.31474,
+    "eval_beir-nq_recall@10": 0.50365,
+    "eval_beir-nq_recall@20": 0.62739,
+    "eval_beir-nq_recall@100": 0.82966,
+    "eval_beir-avg_ndcg@10": 0.39649425,
+    "eval_beir-avg_recall@10": 0.4650965833333334,
+    "eval_beir-avg_recall@20": 0.52575675,
+    "eval_beir-avg_recall@100": 0.6445751666666666
+}

eval_output/checkpoint-100000/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.20413333333333336,0.24472916666666666,0.26334583333333333,0.2861625,0.304375,0.3253958333333334,0.3388866666666667,0.34939833333333326,0.3678083333333333
+map,0.17258750000000003,0.21921333333333337,0.23139583333333333,0.24185083333333335,0.2474525,0.2514625,0.2529033333333333,0.2535908333333334,0.254165
+precision,0.20413333333333336,0.11410833333333331,0.08259083333333334,0.051921666666666665,0.03155166666666667,0.01598583333333333,0.009346666666666665,0.005321666666666666,0.0013741666666666665
+recall,0.17258750000000003,0.27087249999999996,0.31813,0.3857458333333333,0.4529275000000001,0.5486483333333335,0.6233916666666667,0.6902508333333334,0.8302958333333335
+mrr,0.20413333333333336,0.2531466666666667,0.2652733333333333,0.2751208333333333,0.27999999999999997,0.28312916666666665,0.28416416666666666,0.28461333333333333,0.28493
+recall_cap,0.20413333333333336,0.2768041666666667,0.320735,0.38687999999999995,0.45344333333333325,0.5488241666666667,0.6234591666666668,0.6902591666666668,0.8302958333333335
+hole,0.7244833333333333,0.80658,0.8358816666666665,0.86501,0.8862483333333334,0.9035175000000001,0.911515,0.9175591666666666,0.926325

eval_output/checkpoint-100000/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.20413333333333336,
+            "NDCG@3": 0.24472916666666666,
+            "NDCG@5": 0.26334583333333333,
+            "NDCG@10": 0.2861625,
+            "NDCG@20": 0.304375,
+            "NDCG@50": 0.3253958333333334,
+            "NDCG@100": 0.3388866666666667,
+            "NDCG@200": 0.34939833333333326,
+            "NDCG@1000": 0.3678083333333333
+        },
+        "map": {
+            "MAP@1": 0.17258750000000003,
+            "MAP@3": 0.21921333333333337,
+            "MAP@5": 0.23139583333333333,
+            "MAP@10": 0.24185083333333335,
+            "MAP@20": 0.2474525,
+            "MAP@50": 0.2514625,
+            "MAP@100": 0.2529033333333333,
+            "MAP@200": 0.2535908333333334,
+            "MAP@1000": 0.254165
+        },
+        "precision": {
+            "P@1": 0.20413333333333336,
+            "P@3": 0.11410833333333331,
+            "P@5": 0.08259083333333334,
+            "P@10": 0.051921666666666665,
+            "P@20": 0.03155166666666667,
+            "P@50": 0.01598583333333333,
+            "P@100": 0.009346666666666665,
+            "P@200": 0.005321666666666666,
+            "P@1000": 0.0013741666666666665
+        },
+        "recall": {
+            "Recall@1": 0.17258750000000003,
+            "Recall@3": 0.27087249999999996,
+            "Recall@5": 0.31813,
+            "Recall@10": 0.3857458333333333,
+            "Recall@20": 0.4529275000000001,
+            "Recall@50": 0.5486483333333335,
+            "Recall@100": 0.6233916666666667,
+            "Recall@200": 0.6902508333333334,
+            "Recall@1000": 0.8302958333333335
+        },
+        "mrr": {
+            "MRR@1": 0.20413333333333336,
+            "MRR@3": 0.2531466666666667,
+            "MRR@5": 0.2652733333333333,
+            "MRR@10": 0.2751208333333333,
+            "MRR@20": 0.27999999999999997,
+            "MRR@50": 0.28312916666666665,
+            "MRR@100": 0.28416416666666666,
+            "MRR@200": 0.28461333333333333,
+            "MRR@1000": 0.28493
+        },
+        "recall_cap": {
+            "R_cap@1": 0.20413333333333336,
+            "R_cap@3": 0.2768041666666667,
+            "R_cap@5": 0.320735,
+            "R_cap@10": 0.38687999999999995,
+            "R_cap@20": 0.45344333333333325,
+            "R_cap@50": 0.5488241666666667,
+            "R_cap@100": 0.6234591666666668,
+            "R_cap@200": 0.6902591666666668,
+            "R_cap@1000": 0.8302958333333335
+        },
+        "hole": {
+            "Hole@1": 0.7244833333333333,
+            "Hole@3": 0.80658,
+            "Hole@5": 0.8358816666666665,
+            "Hole@10": 0.86501,
+            "Hole@20": 0.8862483333333334,
+            "Hole@50": 0.9035175000000001,
+            "Hole@100": 0.911515,
+            "Hole@200": 0.9175591666666666,
+            "Hole@1000": 0.926325
+        }
+    }
+}

eval_output/checkpoint-100000/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.24074,0.22723,0.23517,0.26042,0.28694,0.31535,0.33445,0.35137,0.37621
+map,0.12737,0.17314,0.18492,0.19913,0.20835,0.21438,0.21667,0.21799,0.21893
+precision,0.24074,0.14969,0.10988,0.07284,0.04676,0.02438,0.01461,0.0086,0.0022
+recall,0.12737,0.20708,0.24682,0.32002,0.40158,0.51349,0.60288,0.69438,0.85589
+mrr,0.24074,0.29552,0.30918,0.32099,0.32759,0.33113,0.33227,0.33269,0.3329
+recall_cap,0.24074,0.23071,0.25342,0.32053,0.40158,0.51349,0.60288,0.69438,0.85589
+hole,0.71759,0.79733,0.83333,0.86991,0.8946,0.91623,0.92454,0.93005,0.93751