Upload 130 files

.gitattributes

@@ -32,3 +32,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+qa_output/nq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/squad1-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/trivia-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/webq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text

beir_output/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.1707,0.28031,0.32773,0.37881,0.41118,0.43359,0.44097,0.44518,0.45009
+map,0.1707,0.25202,0.27847,0.29956,0.30864,0.31241,0.31309,0.31331,0.31343
+precision,0.1707,0.12091,0.09545,0.06351,0.03809,0.01745,0.00917,0.00474,0.00099
+recall,0.1707,0.36273,0.47724,0.63514,0.76174,0.87269,0.9175,0.94737,0.98933
+mrr,0.17639,0.25356,0.28126,0.3021,0.31107,0.31481,0.31549,0.31572,0.31583
+recall_cap,0.17639,0.35917,0.47937,0.63656,0.76245,0.87269,0.9175,0.94737,0.98933
+hole,0.67141,0.67686,0.68293,0.69417,0.73674,0.81273,0.85085,0.87242,0.88471

beir_output/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.1707,
+            "NDCG@3": 0.28031,
+            "NDCG@5": 0.32773,
+            "NDCG@10": 0.37881,
+            "NDCG@20": 0.41118,
+            "NDCG@50": 0.43359,
+            "NDCG@100": 0.44097,
+            "NDCG@200": 0.44518,
+            "NDCG@1000": 0.45009
+        },
+        "map": {
+            "MAP@1": 0.1707,
+            "MAP@3": 0.25202,
+            "MAP@5": 0.27847,
+            "MAP@10": 0.29956,
+            "MAP@20": 0.30864,
+            "MAP@50": 0.31241,
+            "MAP@100": 0.31309,
+            "MAP@200": 0.31331,
+            "MAP@1000": 0.31343
+        },
+        "precision": {
+            "P@1": 0.1707,
+            "P@3": 0.12091,
+            "P@5": 0.09545,
+            "P@10": 0.06351,
+            "P@20": 0.03809,
+            "P@50": 0.01745,
+            "P@100": 0.00917,
+            "P@200": 0.00474,
+            "P@1000": 0.00099
+        },
+        "recall": {
+            "Recall@1": 0.1707,
+            "Recall@3": 0.36273,
+            "Recall@5": 0.47724,
+            "Recall@10": 0.63514,
+            "Recall@20": 0.76174,
+            "Recall@50": 0.87269,
+            "Recall@100": 0.9175,
+            "Recall@200": 0.94737,
+            "Recall@1000": 0.98933
+        },
+        "mrr": {
+            "MRR@1": 0.17639,
+            "MRR@3": 0.25356,
+            "MRR@5": 0.28126,
+            "MRR@10": 0.3021,
+            "MRR@20": 0.31107,
+            "MRR@50": 0.31481,
+            "MRR@100": 0.31549,
+            "MRR@200": 0.31572,
+            "MRR@1000": 0.31583
+        },
+        "recall_cap": {
+            "R_cap@1": 0.17639,
+            "R_cap@3": 0.35917,
+            "R_cap@5": 0.47937,
+            "R_cap@10": 0.63656,
+            "R_cap@20": 0.76245,
+            "R_cap@50": 0.87269,
+            "R_cap@100": 0.9175,
+            "R_cap@200": 0.94737,
+            "R_cap@1000": 0.98933
+        },
+        "hole": {
+            "Hole@1": 0.67141,
+            "Hole@3": 0.67686,
+            "Hole@5": 0.68293,
+            "Hole@10": 0.69417,
+            "Hole@20": 0.73674,
+            "Hole@50": 0.81273,
+            "Hole@100": 0.85085,
+            "Hole@200": 0.87242,
+            "Hole@1000": 0.88471
+        }
+    }
+}

beir_output/beir.json

@@ -0,0 +1,26 @@
+{
+    "eval_beir-msmarco_ndcg@10": 0.21755,
+    "eval_beir-msmarco_recall@10": 0.37253,
+    "eval_beir-msmarco_recall@20": 0.47955,
+    "eval_beir-msmarco_recall@100": 0.69947,
+    "eval_beir-dbpedia-entity_ndcg@10": 0.30952,
+    "eval_beir-dbpedia-entity_recall@10": 0.21,
+    "eval_beir-dbpedia-entity_recall@20": 0.2695,
+    "eval_beir-dbpedia-entity_recall@100": 0.47187,
+    "eval_beir-fever_ndcg@10": 0.59083,
+    "eval_beir-fever_recall@10": 0.77263,
+    "eval_beir-fever_recall@20": 0.82815,
+    "eval_beir-fever_recall@100": 0.90212,
+    "eval_beir-climate-fever_ndcg@10": 0.17148,
+    "eval_beir-climate-fever_recall@10": 0.21584,
+    "eval_beir-climate-fever_recall@20": 0.28686,
+    "eval_beir-climate-fever_recall@100": 0.47004,
+    "eval_beir-hotpotqa_ndcg@10": 0.51242,
+    "eval_beir-hotpotqa_recall@10": 0.54774,
+    "eval_beir-hotpotqa_recall@20": 0.60459,
+    "eval_beir-hotpotqa_recall@100": 0.71269,
+    "eval_beir-avg_ndcg@10": 0.36036,
+    "eval_beir-avg_recall@10": 0.42374800000000007,
+    "eval_beir-avg_recall@20": 0.49372999999999995,
+    "eval_beir-avg_recall@100": 0.651238
+}

beir_output/climate-fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.14788,0.13613,0.14839,0.17148,0.19627,0.22294,0.24274,0.26068,0.28832
+map,0.06935,0.09841,0.10767,0.11697,0.12408,0.12928,0.13159,0.13289,0.1339
+precision,0.14788,0.09859,0.07739,0.05362,0.0373,0.0207,0.01303,0.00793,0.00213
+recall,0.06935,0.12893,0.16257,0.21584,0.28686,0.38308,0.47004,0.56089,0.72887
+mrr,0.14788,0.20304,0.21975,0.23202,0.24052,0.24439,0.24562,0.24621,0.24645
+recall_cap,0.14788,0.13757,0.16257,0.21584,0.28686,0.38308,0.47004,0.56089,0.72887
+hole,0.63192,0.6823,0.71218,0.75036,0.77935,0.81696,0.84354,0.86921,0.92224

beir_output/climate-fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "climate-fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.14788,
+            "NDCG@3": 0.13613,
+            "NDCG@5": 0.14839,
+            "NDCG@10": 0.17148,
+            "NDCG@20": 0.19627,
+            "NDCG@50": 0.22294,
+            "NDCG@100": 0.24274,
+            "NDCG@200": 0.26068,
+            "NDCG@1000": 0.28832
+        },
+        "map": {
+            "MAP@1": 0.06935,
+            "MAP@3": 0.09841,
+            "MAP@5": 0.10767,
+            "MAP@10": 0.11697,
+            "MAP@20": 0.12408,
+            "MAP@50": 0.12928,
+            "MAP@100": 0.13159,
+            "MAP@200": 0.13289,
+            "MAP@1000": 0.1339
+        },
+        "precision": {
+            "P@1": 0.14788,
+            "P@3": 0.09859,
+            "P@5": 0.07739,
+            "P@10": 0.05362,
+            "P@20": 0.0373,
+            "P@50": 0.0207,
+            "P@100": 0.01303,
+            "P@200": 0.00793,
+            "P@1000": 0.00213
+        },
+        "recall": {
+            "Recall@1": 0.06935,
+            "Recall@3": 0.12893,
+            "Recall@5": 0.16257,
+            "Recall@10": 0.21584,
+            "Recall@20": 0.28686,
+            "Recall@50": 0.38308,
+            "Recall@100": 0.47004,
+            "Recall@200": 0.56089,
+            "Recall@1000": 0.72887
+        },
+        "mrr": {
+            "MRR@1": 0.14788,
+            "MRR@3": 0.20304,
+            "MRR@5": 0.21975,
+            "MRR@10": 0.23202,
+            "MRR@20": 0.24052,
+            "MRR@50": 0.24439,
+            "MRR@100": 0.24562,
+            "MRR@200": 0.24621,
+            "MRR@1000": 0.24645
+        },
+        "recall_cap": {
+            "R_cap@1": 0.14788,
+            "R_cap@3": 0.13757,
+            "R_cap@5": 0.16257,
+            "R_cap@10": 0.21584,
+            "R_cap@20": 0.28686,
+            "R_cap@50": 0.38308,
+            "R_cap@100": 0.47004,
+            "R_cap@200": 0.56089,
+            "R_cap@1000": 0.72887
+        },
+        "hole": {
+            "Hole@1": 0.63192,
+            "Hole@3": 0.6823,
+            "Hole@5": 0.71218,
+            "Hole@10": 0.75036,
+            "Hole@20": 0.77935,
+            "Hole@50": 0.81696,
+            "Hole@100": 0.84354,
+            "Hole@200": 0.86921,
+            "Hole@1000": 0.92224
+        }
+    }
+}

beir_output/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.20244416666666668,0.23995333333333335,0.25859583333333336,0.2803316666666667,0.2988066666666666,0.31907,0.33133583333333333,0.3424008333333333,0.36126166666666665
+map,0.17069,0.21522,0.22750083333333335,0.2373575,0.24307416666666667,0.24681416666666667,0.24814583333333337,0.24888,0.24945166666666663
+precision,0.20244416666666668,0.1111775,0.08060833333333334,0.050455,0.030930833333333324,0.015574166666666667,0.00905,0.0052200000000000015,0.0013491666666666667
+recall,0.17069,0.26443249999999996,0.31205999999999995,0.3765525,0.4442291666666666,0.5372158333333333,0.604765,0.6749175,0.8195983333333334
+mrr,0.20244416666666668,0.2489716666666666,0.2607483333333334,0.27058250000000006,0.27553999999999995,0.2785458333333333,0.27949583333333333,0.27997916666666667,0.28030416666666663
+recall_cap,0.20244416666666668,0.27020916666666667,0.31458749999999996,0.3776725000000001,0.44470750000000003,0.5373941666666667,0.6048366666666666,0.674925,0.8195983333333334
+hole,0.7220733333333333,0.8039649999999999,0.8346633333333333,0.86442,0.8839733333333335,0.9019533333333333,0.9109491666666666,0.9171174999999999,0.9264724999999999

beir_output/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.20244416666666668,
+            "NDCG@3": 0.23995333333333335,
+            "NDCG@5": 0.25859583333333336,
+            "NDCG@10": 0.2803316666666667,
+            "NDCG@20": 0.2988066666666666,
+            "NDCG@50": 0.31907,
+            "NDCG@100": 0.33133583333333333,
+            "NDCG@200": 0.3424008333333333,
+            "NDCG@1000": 0.36126166666666665
+        },
+        "map": {
+            "MAP@1": 0.17069,
+            "MAP@3": 0.21522,
+            "MAP@5": 0.22750083333333335,
+            "MAP@10": 0.2373575,
+            "MAP@20": 0.24307416666666667,
+            "MAP@50": 0.24681416666666667,
+            "MAP@100": 0.24814583333333337,
+            "MAP@200": 0.24888,
+            "MAP@1000": 0.24945166666666663
+        },
+        "precision": {
+            "P@1": 0.20244416666666668,
+            "P@3": 0.1111775,
+            "P@5": 0.08060833333333334,
+            "P@10": 0.050455,
+            "P@20": 0.030930833333333324,
+            "P@50": 0.015574166666666667,
+            "P@100": 0.00905,
+            "P@200": 0.0052200000000000015,
+            "P@1000": 0.0013491666666666667
+        },
+        "recall": {
+            "Recall@1": 0.17069,
+            "Recall@3": 0.26443249999999996,
+            "Recall@5": 0.31205999999999995,
+            "Recall@10": 0.3765525,
+            "Recall@20": 0.4442291666666666,
+            "Recall@50": 0.5372158333333333,
+            "Recall@100": 0.604765,
+            "Recall@200": 0.6749175,
+            "Recall@1000": 0.8195983333333334
+        },
+        "mrr": {
+            "MRR@1": 0.20244416666666668,
+            "MRR@3": 0.2489716666666666,
+            "MRR@5": 0.2607483333333334,
+            "MRR@10": 0.27058250000000006,
+            "MRR@20": 0.27553999999999995,
+            "MRR@50": 0.2785458333333333,
+            "MRR@100": 0.27949583333333333,
+            "MRR@200": 0.27997916666666667,
+            "MRR@1000": 0.28030416666666663
+        },
+        "recall_cap": {
+            "R_cap@1": 0.20244416666666668,
+            "R_cap@3": 0.27020916666666667,
+            "R_cap@5": 0.31458749999999996,
+            "R_cap@10": 0.3776725000000001,
+            "R_cap@20": 0.44470750000000003,
+            "R_cap@50": 0.5373941666666667,
+            "R_cap@100": 0.6048366666666666,
+            "R_cap@200": 0.674925,
+            "R_cap@1000": 0.8195983333333334
+        },
+        "hole": {
+            "Hole@1": 0.7220733333333333,
+            "Hole@3": 0.8039649999999999,
+            "Hole@5": 0.8346633333333333,
+            "Hole@10": 0.86442,
+            "Hole@20": 0.8839733333333335,
+            "Hole@50": 0.9019533333333333,
+            "Hole@100": 0.9109491666666666,
+            "Hole@200": 0.9171174999999999,
+            "Hole@1000": 0.9264724999999999
+        }
+    }
+}

beir_output/dbpedia-entity.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.3475,0.31619,0.31187,0.30952,0.3111,0.34062,0.3723,0.4003,0.4491
+map,0.06104,0.09687,0.11544,0.14131,0.16544,0.19505,0.2093,0.21699,0.22446
+precision,0.455,0.36417,0.3225,0.262,0.20325,0.1362,0.09175,0.05899,0.01999
+recall,0.06104,0.11425,0.15092,0.21,0.2695,0.38462,0.47187,0.55369,0.72474
+mrr,0.455,0.55458,0.57408,0.58363,0.58628,0.5883,0.58854,0.58867,0.58871
+recall_cap,0.455,0.38167,0.36079,0.34234,0.33385,0.39734,0.47763,0.55729,0.72533
+hole,0.34,0.42,0.462,0.525,0.60413,0.71385,0.78948,0.8499,0.9336

beir_output/dbpedia-entity.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "dbpedia-entity",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.3475,
+            "NDCG@3": 0.31619,
+            "NDCG@5": 0.31187,
+            "NDCG@10": 0.30952,
+            "NDCG@20": 0.3111,
+            "NDCG@50": 0.34062,
+            "NDCG@100": 0.3723,
+            "NDCG@200": 0.4003,
+            "NDCG@1000": 0.4491
+        },
+        "map": {
+            "MAP@1": 0.06104,
+            "MAP@3": 0.09687,
+            "MAP@5": 0.11544,
+            "MAP@10": 0.14131,
+            "MAP@20": 0.16544,
+            "MAP@50": 0.19505,
+            "MAP@100": 0.2093,
+            "MAP@200": 0.21699,
+            "MAP@1000": 0.22446
+        },
+        "precision": {
+            "P@1": 0.455,
+            "P@3": 0.36417,
+            "P@5": 0.3225,
+            "P@10": 0.262,
+            "P@20": 0.20325,
+            "P@50": 0.1362,
+            "P@100": 0.09175,
+            "P@200": 0.05899,
+            "P@1000": 0.01999
+        },
+        "recall": {
+            "Recall@1": 0.06104,
+            "Recall@3": 0.11425,
+            "Recall@5": 0.15092,
+            "Recall@10": 0.21,
+            "Recall@20": 0.2695,
+            "Recall@50": 0.38462,
+            "Recall@100": 0.47187,
+            "Recall@200": 0.55369,
+            "Recall@1000": 0.72474
+        },
+        "mrr": {
+            "MRR@1": 0.455,
+            "MRR@3": 0.55458,
+            "MRR@5": 0.57408,
+            "MRR@10": 0.58363,
+            "MRR@20": 0.58628,
+            "MRR@50": 0.5883,
+            "MRR@100": 0.58854,
+            "MRR@200": 0.58867,
+            "MRR@1000": 0.58871
+        },
+        "recall_cap": {
+            "R_cap@1": 0.455,
+            "R_cap@3": 0.38167,
+            "R_cap@5": 0.36079,
+            "R_cap@10": 0.34234,
+            "R_cap@20": 0.33385,
+            "R_cap@50": 0.39734,
+            "R_cap@100": 0.47763,
+            "R_cap@200": 0.55729,
+            "R_cap@1000": 0.72533
+        },
+        "hole": {
+            "Hole@1": 0.34,
+            "Hole@3": 0.42,
+            "Hole@5": 0.462,
+            "Hole@10": 0.525,
+            "Hole@20": 0.60413,
+            "Hole@50": 0.71385,
+            "Hole@100": 0.78948,
+            "Hole@200": 0.8499,
+            "Hole@1000": 0.9336
+        }
+    }
+}

beir_output/fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.41629,0.53428,0.56371,0.59083,0.60547,0.61614,0.62033,0.62315,0.62703
+map,0.38911,0.49325,0.51003,0.52171,0.52596,0.5279,0.52832,0.5285,0.5286
+precision,0.41629,0.22352,0.15014,0.0842,0.04534,0.01937,0.01001,0.00514,0.00108
+recall,0.38911,0.62028,0.69116,0.77263,0.82815,0.87828,0.90212,0.9205,0.9508
+mrr,0.41629,0.52513,0.54248,0.55383,0.55774,0.55943,0.55975,0.55986,0.55993
+recall_cap,0.41629,0.62141,0.69141,0.77264,0.82815,0.87828,0.90212,0.9205,0.9508
+hole,0.55821,0.74917,0.82514,0.89647,0.94084,0.97236,0.98419,0.99072,0.99674

beir_output/fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.41629,
+            "NDCG@3": 0.53428,
+            "NDCG@5": 0.56371,
+            "NDCG@10": 0.59083,
+            "NDCG@20": 0.60547,
+            "NDCG@50": 0.61614,
+            "NDCG@100": 0.62033,
+            "NDCG@200": 0.62315,
+            "NDCG@1000": 0.62703
+        },
+        "map": {
+            "MAP@1": 0.38911,
+            "MAP@3": 0.49325,
+            "MAP@5": 0.51003,
+            "MAP@10": 0.52171,
+            "MAP@20": 0.52596,
+            "MAP@50": 0.5279,
+            "MAP@100": 0.52832,
+            "MAP@200": 0.5285,
+            "MAP@1000": 0.5286
+        },
+        "precision": {
+            "P@1": 0.41629,
+            "P@3": 0.22352,
+            "P@5": 0.15014,
+            "P@10": 0.0842,
+            "P@20": 0.04534,
+            "P@50": 0.01937,
+            "P@100": 0.01001,
+            "P@200": 0.00514,
+            "P@1000": 0.00108
+        },
+        "recall": {
+            "Recall@1": 0.38911,
+            "Recall@3": 0.62028,
+            "Recall@5": 0.69116,
+            "Recall@10": 0.77263,
+            "Recall@20": 0.82815,
+            "Recall@50": 0.87828,
+            "Recall@100": 0.90212,
+            "Recall@200": 0.9205,
+            "Recall@1000": 0.9508
+        },
+        "mrr": {
+            "MRR@1": 0.41629,
+            "MRR@3": 0.52513,
+            "MRR@5": 0.54248,
+            "MRR@10": 0.55383,
+            "MRR@20": 0.55774,
+            "MRR@50": 0.55943,
+            "MRR@100": 0.55975,
+            "MRR@200": 0.55986,
+            "MRR@1000": 0.55993
+        },
+        "recall_cap": {
+            "R_cap@1": 0.41629,
+            "R_cap@3": 0.62141,
+            "R_cap@5": 0.69141,
+            "R_cap@10": 0.77264,
+            "R_cap@20": 0.82815,
+            "R_cap@50": 0.87828,
+            "R_cap@100": 0.90212,
+            "R_cap@200": 0.9205,
+            "R_cap@1000": 0.9508
+        },
+        "hole": {
+            "Hole@1": 0.55821,
+            "Hole@3": 0.74917,
+            "Hole@5": 0.82514,
+            "Hole@10": 0.89647,
+            "Hole@20": 0.94084,
+            "Hole@50": 0.97236,
+            "Hole@100": 0.98419,
+            "Hole@200": 0.99072,
+            "Hole@1000": 0.99674
+        }
+    }
+}

beir_output/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.23302,0.2131,0.22737,0.25271,0.2769,0.30444,0.32137,0.33759,0.36433
+map,0.12405,0.16458,0.17971,0.19266,0.20052,0.20609,0.20812,0.2094,0.21039
+precision,0.23302,0.1358,0.10463,0.07068,0.04437,0.02346,0.01398,0.00821,0.00216
+recall,0.12405,0.19635,0.24497,0.31547,0.39221,0.49942,0.57736,0.6645,0.83618
+mrr,0.23302,0.27829,0.29473,0.30777,0.3144,0.31786,0.31885,0.31931,0.31958
+recall_cap,0.23302,0.21579,0.25041,0.3158,0.39221,0.49942,0.57736,0.6645,0.83618
+hole,0.70833,0.8035,0.83364,0.86867,0.89244,0.9113,0.92069,0.92654,0.93518

beir_output/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.23302,
+            "NDCG@3": 0.2131,
+            "NDCG@5": 0.22737,
+            "NDCG@10": 0.25271,
+            "NDCG@20": 0.2769,
+            "NDCG@50": 0.30444,
+            "NDCG@100": 0.32137,
+            "NDCG@200": 0.33759,
+            "NDCG@1000": 0.36433
+        },
+        "map": {
+            "MAP@1": 0.12405,
+            "MAP@3": 0.16458,
+            "MAP@5": 0.17971,
+            "MAP@10": 0.19266,
+            "MAP@20": 0.20052,
+            "MAP@50": 0.20609,
+            "MAP@100": 0.20812,
+            "MAP@200": 0.2094,
+            "MAP@1000": 0.21039
+        },
+        "precision": {
+            "P@1": 0.23302,
+            "P@3": 0.1358,
+            "P@5": 0.10463,
+            "P@10": 0.07068,
+            "P@20": 0.04437,
+            "P@50": 0.02346,
+            "P@100": 0.01398,
+            "P@200": 0.00821,
+            "P@1000": 0.00216
+        },
+        "recall": {
+            "Recall@1": 0.12405,
+            "Recall@3": 0.19635,
+            "Recall@5": 0.24497,
+            "Recall@10": 0.31547,
+            "Recall@20": 0.39221,
+            "Recall@50": 0.49942,
+            "Recall@100": 0.57736,
+            "Recall@200": 0.6645,
+            "Recall@1000": 0.83618
+        },
+        "mrr": {
+            "MRR@1": 0.23302,
+            "MRR@3": 0.27829,
+            "MRR@5": 0.29473,
+            "MRR@10": 0.30777,
+            "MRR@20": 0.3144,
+            "MRR@50": 0.31786,
+            "MRR@100": 0.31885,
+            "MRR@200": 0.31931,
+            "MRR@1000": 0.31958
+        },
+        "recall_cap": {
+            "R_cap@1": 0.23302,
+            "R_cap@3": 0.21579,
+            "R_cap@5": 0.25041,
+            "R_cap@10": 0.3158,
+            "R_cap@20": 0.39221,
+            "R_cap@50": 0.49942,
+            "R_cap@100": 0.57736,
+            "R_cap@200": 0.6645,
+            "R_cap@1000": 0.83618
+        },
+        "hole": {
+            "Hole@1": 0.70833,
+            "Hole@3": 0.8035,
+            "Hole@5": 0.83364,
+            "Hole@10": 0.86867,
+            "Hole@20": 0.89244,
+            "Hole@50": 0.9113,
+            "Hole@100": 0.92069,
+            "Hole@200": 0.92654,
+            "Hole@1000": 0.93518
+        }
+    }
+}

beir_output/hotpotqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.58488,0.46291,0.48756,0.51242,0.53009,0.54678,0.55471,0.5614,0.57309
+map,0.29244,0.38755,0.40461,0.41781,0.42428,0.42802,0.42902,0.42953,0.42991
+precision,0.58488,0.29124,0.19425,0.10955,0.06046,0.02692,0.01425,0.00752,0.00167
+recall,0.29244,0.43687,0.48562,0.54774,0.60459,0.67292,0.71269,0.75165,0.83471
+mrr,0.58488,0.65258,0.66312,0.67061,0.67359,0.67491,0.67521,0.67533,0.67541
+recall_cap,0.58488,0.43687,0.48562,0.54774,0.60459,0.67292,0.71269,0.75165,0.83471
+hole,0.4077,0.69714,0.79327,0.87796,0.92708,0.96156,0.97484,0.98229,0.98984

beir_output/hotpotqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "hotpotqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.58488,
+            "NDCG@3": 0.46291,
+            "NDCG@5": 0.48756,
+            "NDCG@10": 0.51242,
+            "NDCG@20": 0.53009,
+            "NDCG@50": 0.54678,
+            "NDCG@100": 0.55471,
+            "NDCG@200": 0.5614,
+            "NDCG@1000": 0.57309
+        },
+        "map": {
+            "MAP@1": 0.29244,
+            "MAP@3": 0.38755,
+            "MAP@5": 0.40461,
+            "MAP@10": 0.41781,
+            "MAP@20": 0.42428,
+            "MAP@50": 0.42802,
+            "MAP@100": 0.42902,
+            "MAP@200": 0.42953,
+            "MAP@1000": 0.42991
+        },
+        "precision": {
+            "P@1": 0.58488,
+            "P@3": 0.29124,
+            "P@5": 0.19425,
+            "P@10": 0.10955,
+            "P@20": 0.06046,
+            "P@50": 0.02692,
+            "P@100": 0.01425,
+            "P@200": 0.00752,
+            "P@1000": 0.00167
+        },
+        "recall": {
+            "Recall@1": 0.29244,
+            "Recall@3": 0.43687,
+            "Recall@5": 0.48562,
+            "Recall@10": 0.54774,
+            "Recall@20": 0.60459,
+            "Recall@50": 0.67292,
+            "Recall@100": 0.71269,
+            "Recall@200": 0.75165,
+            "Recall@1000": 0.83471
+        },
+        "mrr": {
+            "MRR@1": 0.58488,
+            "MRR@3": 0.65258,
+            "MRR@5": 0.66312,
+            "MRR@10": 0.67061,
+            "MRR@20": 0.67359,
+            "MRR@50": 0.67491,
+            "MRR@100": 0.67521,
+            "MRR@200": 0.67533,
+            "MRR@1000": 0.67541
+        },
+        "recall_cap": {
+            "R_cap@1": 0.58488,
+            "R_cap@3": 0.43687,
+            "R_cap@5": 0.48562,
+            "R_cap@10": 0.54774,
+            "R_cap@20": 0.60459,
+            "R_cap@50": 0.67292,
+            "R_cap@100": 0.71269,
+            "R_cap@200": 0.75165,
+            "R_cap@1000": 0.83471
+        },
+        "hole": {
+            "Hole@1": 0.4077,
+            "Hole@3": 0.69714,
+            "Hole@5": 0.79327,
+            "Hole@10": 0.87796,
+            "Hole@20": 0.92708,
+            "Hole@50": 0.96156,
+            "Hole@100": 0.97484,
+            "Hole@200": 0.98229,
+            "Hole@1000": 0.98984
+        }
+    }
+}

beir_output/msmarco.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.09384,0.15457,0.18401,0.21755,0.24496,0.27133,0.28615,0.29677,0.31206
+map,0.09148,0.13806,0.15451,0.16842,0.17604,0.18041,0.18175,0.18232,0.18269
+precision,0.09384,0.06848,0.05599,0.03872,0.02496,0.01276,0.00734,0.00408,0.00096
+recall,0.09148,0.19936,0.26987,0.37253,0.47955,0.60978,0.69947,0.77351,0.90289
+mrr,0.09384,0.14124,0.15766,0.17162,0.17917,0.18342,0.18472,0.18526,0.1856
+recall_cap,0.09384,0.19936,0.26972,0.37253,0.47955,0.60978,0.69947,0.77351,0.90289
+hole,0.9053,0.93066,0.94301,0.9601,0.97397,0.98613,0.99159,0.99482,0.99798

beir_output/msmarco.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "msmarco",
+    "split": "dev",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.09384,
+            "NDCG@3": 0.15457,
+            "NDCG@5": 0.18401,
+            "NDCG@10": 0.21755,
+            "NDCG@20": 0.24496,
+            "NDCG@50": 0.27133,
+            "NDCG@100": 0.28615,
+            "NDCG@200": 0.29677,
+            "NDCG@1000": 0.31206
+        },
+        "map": {
+            "MAP@1": 0.09148,
+            "MAP@3": 0.13806,
+            "MAP@5": 0.15451,
+            "MAP@10": 0.16842,
+            "MAP@20": 0.17604,
+            "MAP@50": 0.18041,
+            "MAP@100": 0.18175,
+            "MAP@200": 0.18232,
+            "MAP@1000": 0.18269
+        },
+        "precision": {
+            "P@1": 0.09384,
+            "P@3": 0.06848,
+            "P@5": 0.05599,
+            "P@10": 0.03872,
+            "P@20": 0.02496,
+            "P@50": 0.01276,
+            "P@100": 0.00734,
+            "P@200": 0.00408,
+            "P@1000": 0.00096
+        },
+        "recall": {
+            "Recall@1": 0.09148,
+            "Recall@3": 0.19936,
+            "Recall@5": 0.26987,
+            "Recall@10": 0.37253,
+            "Recall@20": 0.47955,
+            "Recall@50": 0.60978,
+            "Recall@100": 0.69947,
+            "Recall@200": 0.77351,
+            "Recall@1000": 0.90289
+        },
+        "mrr": {
+            "MRR@1": 0.09384,
+            "MRR@3": 0.14124,
+            "MRR@5": 0.15766,
+            "MRR@10": 0.17162,
+            "MRR@20": 0.17917,
+            "MRR@50": 0.18342,
+            "MRR@100": 0.18472,
+            "MRR@200": 0.18526,
+            "MRR@1000": 0.1856
+        },
+        "recall_cap": {
+            "R_cap@1": 0.09384,
+            "R_cap@3": 0.19936,
+            "R_cap@5": 0.26972,
+            "R_cap@10": 0.37253,
+            "R_cap@20": 0.47955,
+            "R_cap@50": 0.60978,
+            "R_cap@100": 0.69947,
+            "R_cap@200": 0.77351,
+            "R_cap@1000": 0.90289
+        },
+        "hole": {
+            "Hole@1": 0.9053,
+            "Hole@3": 0.93066,
+            "Hole@5": 0.94301,
+            "Hole@10": 0.9601,
+            "Hole@20": 0.97397,
+            "Hole@50": 0.98613,
+            "Hole@100": 0.99159,
+            "Hole@200": 0.99482,
+            "Hole@1000": 0.99798
+        }
+    }
+}

beir_output/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.37152,0.34236,0.32119,0.29721,0.27552,0.26519,0.27338,0.29114,0.36249
+map,0.04398,0.08002,0.09218,0.10681,0.11679,0.12707,0.13314,0.13805,0.14671
+precision,0.39319,0.32921,0.28173,0.22415,0.16409,0.10526,0.0731,0.04941,0.02017
+recall,0.04398,0.0957,0.11594,0.1453,0.17684,0.22559,0.27314,0.34803,0.59701
+mrr,0.39938,0.46491,0.47668,0.48612,0.49194,0.49302,0.4934,0.49383,0.49407
+recall_cap,0.39938,0.34985,0.31765,0.28258,0.25444,0.25623,0.28778,0.35318,0.59701
+hole,0.0774,0.08359,0.09659,0.10402,0.11517,0.1208,0.12399,0.12703,0.13305

beir_output/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.37152,
+            "NDCG@3": 0.34236,
+            "NDCG@5": 0.32119,
+            "NDCG@10": 0.29721,
+            "NDCG@20": 0.27552,
+            "NDCG@50": 0.26519,
+            "NDCG@100": 0.27338,
+            "NDCG@200": 0.29114,
+            "NDCG@1000": 0.36249
+        },
+        "map": {
+            "MAP@1": 0.04398,
+            "MAP@3": 0.08002,
+            "MAP@5": 0.09218,
+            "MAP@10": 0.10681,
+            "MAP@20": 0.11679,
+            "MAP@50": 0.12707,
+            "MAP@100": 0.13314,
+            "MAP@200": 0.13805,
+            "MAP@1000": 0.14671
+        },
+        "precision": {
+            "P@1": 0.39319,
+            "P@3": 0.32921,
+            "P@5": 0.28173,
+            "P@10": 0.22415,
+            "P@20": 0.16409,
+            "P@50": 0.10526,
+            "P@100": 0.0731,
+            "P@200": 0.04941,
+            "P@1000": 0.02017
+        },
+        "recall": {
+            "Recall@1": 0.04398,
+            "Recall@3": 0.0957,
+            "Recall@5": 0.11594,
+            "Recall@10": 0.1453,
+            "Recall@20": 0.17684,
+            "Recall@50": 0.22559,
+            "Recall@100": 0.27314,
+            "Recall@200": 0.34803,
+            "Recall@1000": 0.59701
+        },
+        "mrr": {
+            "MRR@1": 0.39938,
+            "MRR@3": 0.46491,
+            "MRR@5": 0.47668,
+            "MRR@10": 0.48612,
+            "MRR@20": 0.49194,
+            "MRR@50": 0.49302,
+            "MRR@100": 0.4934,
+            "MRR@200": 0.49383,
+            "MRR@1000": 0.49407
+        },
+        "recall_cap": {
+            "R_cap@1": 0.39938,
+            "R_cap@3": 0.34985,
+            "R_cap@5": 0.31765,
+            "R_cap@10": 0.28258,
+            "R_cap@20": 0.25444,
+            "R_cap@50": 0.25623,
+            "R_cap@100": 0.28778,
+            "R_cap@200": 0.35318,
+            "R_cap@1000": 0.59701
+        },
+        "hole": {
+            "Hole@1": 0.0774,
+            "Hole@3": 0.08359,
+            "Hole@5": 0.09659,
+            "Hole@10": 0.10402,
+            "Hole@20": 0.11517,
+            "Hole@50": 0.1208,
+            "Hole@100": 0.12399,
+            "Hole@200": 0.12703,
+            "Hole@1000": 0.13305
+        }
+    }
+}

beir_output/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.13789,0.20795,0.24521,0.28932,0.32042,0.3491,0.36179,0.37019,0.38166
+map,0.12189,0.18352,0.20487,0.22404,0.23326,0.2383,0.23955,0.24003,0.24034
+precision,0.13789,0.09985,0.08076,0.05588,0.03511,0.01744,0.00965,0.00518,0.00116
+recall,0.12189,0.25937,0.34598,0.47579,0.59195,0.72796,0.80159,0.85837,0.95119
+mrr,0.13818,0.20288,0.22363,0.24159,0.24972,0.25407,0.25507,0.25544,0.25567
+recall_cap,0.13818,0.25946,0.34598,0.47579,0.59195,0.72796,0.80159,0.85837,0.95119
+hole,0.85892,0.89677,0.91582,0.94056,0.96133,0.97926,0.98697,0.9916,0.99595

beir_output/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.13789,
+            "NDCG@3": 0.20795,
+            "NDCG@5": 0.24521,
+            "NDCG@10": 0.28932,
+            "NDCG@20": 0.32042,
+            "NDCG@50": 0.3491,
+            "NDCG@100": 0.36179,
+            "NDCG@200": 0.37019,
+            "NDCG@1000": 0.38166
+        },
+        "map": {
+            "MAP@1": 0.12189,
+            "MAP@3": 0.18352,
+            "MAP@5": 0.20487,
+            "MAP@10": 0.22404,
+            "MAP@20": 0.23326,
+            "MAP@50": 0.2383,
+            "MAP@100": 0.23955,
+            "MAP@200": 0.24003,
+            "MAP@1000": 0.24034
+        },
+        "precision": {
+            "P@1": 0.13789,
+            "P@3": 0.09985,
+            "P@5": 0.08076,
+            "P@10": 0.05588,
+            "P@20": 0.03511,
+            "P@50": 0.01744,
+            "P@100": 0.00965,
+            "P@200": 0.00518,
+            "P@1000": 0.00116
+        },
+        "recall": {
+            "Recall@1": 0.12189,
+            "Recall@3": 0.25937,
+            "Recall@5": 0.34598,
+            "Recall@10": 0.47579,
+            "Recall@20": 0.59195,
+            "Recall@50": 0.72796,
+            "Recall@100": 0.80159,
+            "Recall@200": 0.85837,
+            "Recall@1000": 0.95119
+        },
+        "mrr": {
+            "MRR@1": 0.13818,
+            "MRR@3": 0.20288,
+            "MRR@5": 0.22363,
+            "MRR@10": 0.24159,
+            "MRR@20": 0.24972,
+            "MRR@50": 0.25407,
+            "MRR@100": 0.25507,
+            "MRR@200": 0.25544,
+            "MRR@1000": 0.25567
+        },
+        "recall_cap": {
+            "R_cap@1": 0.13818,
+            "R_cap@3": 0.25946,
+            "R_cap@5": 0.34598,
+            "R_cap@10": 0.47579,
+            "R_cap@20": 0.59195,
+            "R_cap@50": 0.72796,
+            "R_cap@100": 0.80159,
+            "R_cap@200": 0.85837,
+            "R_cap@1000": 0.95119
+        },
+        "hole": {
+            "Hole@1": 0.85892,
+            "Hole@3": 0.89677,
+            "Hole@5": 0.91582,
+            "Hole@10": 0.94056,
+            "Hole@20": 0.96133,
+            "Hole@50": 0.97926,
+            "Hole@100": 0.98697,
+            "Hole@200": 0.9916,
+            "Hole@1000": 0.99595
+        }
+    }
+}

beir_output/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.723,0.77381,0.79417,0.81279,0.82301,0.82993,0.83269,0.83416,0.83543
+map,0.62985,0.73335,0.75278,0.76549,0.77056,0.77274,0.77326,0.77346,0.77355
+precision,0.723,0.3382,0.2252,0.12512,0.06743,0.02872,0.01482,0.00758,0.00156
+recall,0.62985,0.80077,0.85534,0.9098,0.94365,0.97077,0.98333,0.9906,0.99787
+mrr,0.7227,0.78562,0.79552,0.80088,0.80249,0.80305,0.80315,0.80317,0.80319
+recall_cap,0.7227,0.8147,0.8608,0.91133,0.94397,0.9708,0.98333,0.9906,0.99787
+hole,0.2578,0.6254,0.73292,0.8304,0.88665,0.92629,0.94129,0.95034,0.95968

beir_output/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.723,
+            "NDCG@3": 0.77381,
+            "NDCG@5": 0.79417,
+            "NDCG@10": 0.81279,
+            "NDCG@20": 0.82301,
+            "NDCG@50": 0.82993,
+            "NDCG@100": 0.83269,
+            "NDCG@200": 0.83416,
+            "NDCG@1000": 0.83543
+        },
+        "map": {
+            "MAP@1": 0.62985,
+            "MAP@3": 0.73335,
+            "MAP@5": 0.75278,
+            "MAP@10": 0.76549,
+            "MAP@20": 0.77056,
+            "MAP@50": 0.77274,
+            "MAP@100": 0.77326,
+            "MAP@200": 0.77346,
+            "MAP@1000": 0.77355
+        },
+        "precision": {
+            "P@1": 0.723,
+            "P@3": 0.3382,
+            "P@5": 0.2252,
+            "P@10": 0.12512,
+            "P@20": 0.06743,
+            "P@50": 0.02872,
+            "P@100": 0.01482,
+            "P@200": 0.00758,
+            "P@1000": 0.00156
+        },
+        "recall": {
+            "Recall@1": 0.62985,
+            "Recall@3": 0.80077,
+            "Recall@5": 0.85534,
+            "Recall@10": 0.9098,
+            "Recall@20": 0.94365,
+            "Recall@50": 0.97077,
+            "Recall@100": 0.98333,
+            "Recall@200": 0.9906,
+            "Recall@1000": 0.99787
+        },
+        "mrr": {
+            "MRR@1": 0.7227,
+            "MRR@3": 0.78562,
+            "MRR@5": 0.79552,
+            "MRR@10": 0.80088,
+            "MRR@20": 0.80249,
+            "MRR@50": 0.80305,
+            "MRR@100": 0.80315,
+            "MRR@200": 0.80317,
+            "MRR@1000": 0.80319
+        },
+        "recall_cap": {
+            "R_cap@1": 0.7227,
+            "R_cap@3": 0.8147,
+            "R_cap@5": 0.8608,
+            "R_cap@10": 0.91133,
+            "R_cap@20": 0.94397,
+            "R_cap@50": 0.9708,
+            "R_cap@100": 0.98333,
+            "R_cap@200": 0.9906,
+            "R_cap@1000": 0.99787
+        },
+        "hole": {
+            "Hole@1": 0.2578,
+            "Hole@3": 0.6254,
+            "Hole@5": 0.73292,
+            "Hole@10": 0.8304,
+            "Hole@20": 0.88665,
+            "Hole@50": 0.92629,
+            "Hole@100": 0.94129,
+            "Hole@200": 0.95034,
+            "Hole@1000": 0.95968
+        }
+    }
+}

beir_output/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.193,0.14845,0.13019,0.1582,0.18256,0.21015,0.22813,0.24603,0.28037
+map,0.03923,0.06632,0.07798,0.09057,0.09851,0.10477,0.10709,0.10857,0.1099
+precision,0.193,0.136,0.1128,0.0818,0.05505,0.03018,0.01833,0.01104,0.00308
+recall,0.03923,0.08313,0.11458,0.16598,0.22317,0.3058,0.37162,0.4477,0.62633
+mrr,0.193,0.24833,0.26838,0.28265,0.29015,0.29325,0.29423,0.29472,0.29499
+recall_cap,0.193,0.136,0.11458,0.16598,0.22317,0.3058,0.37162,0.4477,0.62633
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

beir_output/scidocs.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scidocs",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.193,
+            "NDCG@3": 0.14845,
+            "NDCG@5": 0.13019,
+            "NDCG@10": 0.1582,
+            "NDCG@20": 0.18256,
+            "NDCG@50": 0.21015,
+            "NDCG@100": 0.22813,
+            "NDCG@200": 0.24603,
+            "NDCG@1000": 0.28037
+        },
+        "map": {
+            "MAP@1": 0.03923,
+            "MAP@3": 0.06632,
+            "MAP@5": 0.07798,
+            "MAP@10": 0.09057,
+            "MAP@20": 0.09851,
+            "MAP@50": 0.10477,
+            "MAP@100": 0.10709,
+            "MAP@200": 0.10857,
+            "MAP@1000": 0.1099
+        },
+        "precision": {
+            "P@1": 0.193,
+            "P@3": 0.136,
+            "P@5": 0.1128,
+            "P@10": 0.0818,
+            "P@20": 0.05505,
+            "P@50": 0.03018,
+            "P@100": 0.01833,
+            "P@200": 0.01104,
+            "P@1000": 0.00308
+        },
+        "recall": {
+            "Recall@1": 0.03923,
+            "Recall@3": 0.08313,
+            "Recall@5": 0.11458,
+            "Recall@10": 0.16598,
+            "Recall@20": 0.22317,
+            "Recall@50": 0.3058,
+            "Recall@100": 0.37162,
+            "Recall@200": 0.4477,
+            "Recall@1000": 0.62633
+        },
+        "mrr": {
+            "MRR@1": 0.193,
+            "MRR@3": 0.24833,
+            "MRR@5": 0.26838,
+            "MRR@10": 0.28265,
+            "MRR@20": 0.29015,
+            "MRR@50": 0.29325,
+            "MRR@100": 0.29423,
+            "MRR@200": 0.29472,
+            "MRR@1000": 0.29499
+        },
+        "recall_cap": {
+            "R_cap@1": 0.193,
+            "R_cap@3": 0.136,
+            "R_cap@5": 0.11458,
+            "R_cap@10": 0.16598,
+            "R_cap@20": 0.22317,
+            "R_cap@50": 0.3058,
+            "R_cap@100": 0.37162,
+            "R_cap@200": 0.4477,
+            "R_cap@1000": 0.62633
+        },
+        "hole": {
+            "Hole@1": 0.0,
+            "Hole@3": 0.0,
+            "Hole@5": 0.0,
+            "Hole@10": 0.0,
+            "Hole@20": 0.0,
+            "Hole@50": 0.0,
+            "Hole@100": 0.0,
+            "Hole@200": 0.0,
+            "Hole@1000": 0.0
+        }
+    }
+}

beir_output/scifact.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.50333,0.58579,0.60892,0.63462,0.65224,0.66278,0.66455,0.66831,0.67604
+map,0.47817,0.55652,0.57248,0.58476,0.58989,0.59173,0.59191,0.59214,0.59234
+precision,0.50333,0.23444,0.15467,0.087,0.04717,0.02013,0.0102,0.00527,0.00112
+recall,0.47817,0.64044,0.70044,0.77344,0.84122,0.89256,0.90322,0.92833,0.99333
+mrr,0.50333,0.57556,0.58822,0.59809,0.60285,0.60428,0.6044,0.60457,0.60474
+recall_cap,0.50333,0.64444,0.70044,0.77344,0.84122,0.89256,0.90322,0.92833,0.99333
+hole,0.47,0.73,0.79733,0.86467,0.89583,0.9228,0.9329,0.93887,0.94298

beir_output/scifact.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scifact",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.50333,
+            "NDCG@3": 0.58579,
+            "NDCG@5": 0.60892,
+            "NDCG@10": 0.63462,
+            "NDCG@20": 0.65224,
+            "NDCG@50": 0.66278,
+            "NDCG@100": 0.66455,
+            "NDCG@200": 0.66831,
+            "NDCG@1000": 0.67604
+        },
+        "map": {
+            "MAP@1": 0.47817,
+            "MAP@3": 0.55652,
+            "MAP@5": 0.57248,
+            "MAP@10": 0.58476,
+            "MAP@20": 0.58989,
+            "MAP@50": 0.59173,
+            "MAP@100": 0.59191,
+            "MAP@200": 0.59214,
+            "MAP@1000": 0.59234
+        },
+        "precision": {
+            "P@1": 0.50333,
+            "P@3": 0.23444,
+            "P@5": 0.15467,
+            "P@10": 0.087,
+            "P@20": 0.04717,
+            "P@50": 0.02013,
+            "P@100": 0.0102,
+            "P@200": 0.00527,
+            "P@1000": 0.00112
+        },
+        "recall": {
+            "Recall@1": 0.47817,
+            "Recall@3": 0.64044,
+            "Recall@5": 0.70044,
+            "Recall@10": 0.77344,
+            "Recall@20": 0.84122,
+            "Recall@50": 0.89256,
+            "Recall@100": 0.90322,
+            "Recall@200": 0.92833,
+            "Recall@1000": 0.99333
+        },
+        "mrr": {
+            "MRR@1": 0.50333,
+            "MRR@3": 0.57556,
+            "MRR@5": 0.58822,
+            "MRR@10": 0.59809,
+            "MRR@20": 0.60285,
+            "MRR@50": 0.60428,
+            "MRR@100": 0.6044,
+            "MRR@200": 0.60457,
+            "MRR@1000": 0.60474
+        },
+        "recall_cap": {
+            "R_cap@1": 0.50333,
+            "R_cap@3": 0.64444,
+            "R_cap@5": 0.70044,
+            "R_cap@10": 0.77344,
+            "R_cap@20": 0.84122,
+            "R_cap@50": 0.89256,
+            "R_cap@100": 0.90322,
+            "R_cap@200": 0.92833,
+            "R_cap@1000": 0.99333
+        },
+        "hole": {
+            "Hole@1": 0.47,
+            "Hole@3": 0.73,
+            "Hole@5": 0.79733,
+            "Hole@10": 0.86467,
+            "Hole@20": 0.89583,
+            "Hole@50": 0.9228,
+            "Hole@100": 0.9329,
+            "Hole@200": 0.93887,
+            "Hole@1000": 0.94298
+        }
+    }
+}

beir_output/senteval-core.json

@@ -0,0 +1,5 @@
+{
+    "eval_senteval-stsb_spearman": 0.8010173901569266,
+    "eval_senteval-sickr_spearman": 0.7309801454276812,
+    "eval_senteval-avg_sts": 0.7659987677923039
+}

beir_output/senteval.json

@@ -0,0 +1,99 @@
+{
+    "STSBenchmark": {
+        "train": {
+            "pearson": [
+                0.7868313677903059,
+                0.0
+            ],
+            "spearman": [
+                0.7533061602723176,
+                0.0
+            ],
+            "nsamples": 5749
+        },
+        "dev": {
+            "pearson": [
+                0.8010234738701788,
+                0.0
+            ],
+            "spearman": [
+                0.8010173901569266,
+                0.0
+            ],
+            "nsamples": 1500
+        },
+        "test": {
+            "pearson": [
+                0.7766524771374954,
+                1.1662374252806932e-278
+            ],
+            "spearman": [
+                0.7638460069862814,
+                3.8220116833205965e-264
+            ],
+            "nsamples": 1379
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7882517084559206,
+                "mean": 0.7881691062659933,
+                "wmean": 0.7876718254757005
+            },
+            "spearman": {
+                "all": 0.7668425138024548,
+                "mean": 0.7727231858051752,
+                "wmean": 0.7632854478761041
+            }
+        }
+    },
+    "SICKRelatedness": {
+        "train": {
+            "pearson": [
+                0.7853064744370001,
+                0.0
+            ],
+            "spearman": [
+                0.7045889188784015,
+                0.0
+            ],
+            "nsamples": 4500
+        },
+        "dev": {
+            "pearson": [
+                0.7952406911697713,
+                2.6811654216760936e-110
+            ],
+            "spearman": [
+                0.7309801454276812,
+                1.0951577254305346e-84
+            ],
+            "nsamples": 500
+        },
+        "test": {
+            "pearson": [
+                0.7794843403240526,
+                0.0
+            ],
+            "spearman": [
+                0.6977349592492605,
+                0.0
+            ],
+            "nsamples": 4927
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7828650051750712,
+                "mean": 0.7866771686436079,
+                "wmean": 0.7829171779317007
+            },
+            "spearman": {
+                "all": 0.7025121676824595,
+                "mean": 0.7111013411851145,
+                "wmean": 0.702516404944873
+            }
+        }
+    },
+    "eval_senteval-stsb_spearman": 0.8010173901569266,
+    "eval_senteval-sickr_spearman": 0.7309801454276812,
+    "eval_senteval-avg_sts": 0.7659987677923039
+}

beir_output/trec-covid.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.56,0.57235,0.58672,0.5527,0.53857,0.47697,0.42291,0.3714,0.39631
+map,0.00184,0.00467,0.00739,0.01297,0.02303,0.04425,0.07042,0.10452,0.17663
+precision,0.64,0.63333,0.648,0.6,0.581,0.5036,0.4402,0.3737,0.1817
+recall,0.00184,0.00501,0.00847,0.01535,0.02911,0.06047,0.10295,0.16819,0.37887
+mrr,0.62,0.73333,0.75633,0.75633,0.75758,0.75758,0.75798,0.75798,0.75798
+recall_cap,0.62,0.63333,0.648,0.6,0.581,0.5036,0.4402,0.3769,0.3802
+hole,0.18,0.12,0.128,0.156,0.172,0.2236,0.2614,0.2987,0.41988

beir_output/trec-covid.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "trec-covid",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.56,
+            "NDCG@3": 0.57235,
+            "NDCG@5": 0.58672,
+            "NDCG@10": 0.5527,
+            "NDCG@20": 0.53857,
+            "NDCG@50": 0.47697,
+            "NDCG@100": 0.42291,
+            "NDCG@200": 0.3714,
+            "NDCG@1000": 0.39631
+        },
+        "map": {
+            "MAP@1": 0.00184,
+            "MAP@3": 0.00467,
+            "MAP@5": 0.00739,
+            "MAP@10": 0.01297,
+            "MAP@20": 0.02303,
+            "MAP@50": 0.04425,
+            "MAP@100": 0.07042,
+            "MAP@200": 0.10452,
+            "MAP@1000": 0.17663
+        },
+        "precision": {
+            "P@1": 0.64,
+            "P@3": 0.63333,
+            "P@5": 0.648,
+            "P@10": 0.6,
+            "P@20": 0.581,
+            "P@50": 0.5036,
+            "P@100": 0.4402,
+            "P@200": 0.3737,
+            "P@1000": 0.1817
+        },
+        "recall": {
+            "Recall@1": 0.00184,
+            "Recall@3": 0.00501,
+            "Recall@5": 0.00847,
+            "Recall@10": 0.01535,
+            "Recall@20": 0.02911,
+            "Recall@50": 0.06047,
+            "Recall@100": 0.10295,
+            "Recall@200": 0.16819,
+            "Recall@1000": 0.37887
+        },
+        "mrr": {
+            "MRR@1": 0.62,
+            "MRR@3": 0.73333,
+            "MRR@5": 0.75633,
+            "MRR@10": 0.75633,
+            "MRR@20": 0.75758,
+            "MRR@50": 0.75758,
+            "MRR@100": 0.75798,
+            "MRR@200": 0.75798,
+            "MRR@1000": 0.75798
+        },
+        "recall_cap": {
+            "R_cap@1": 0.62,
+            "R_cap@3": 0.63333,
+            "R_cap@5": 0.648,
+            "R_cap@10": 0.6,
+            "R_cap@20": 0.581,
+            "R_cap@50": 0.5036,
+            "R_cap@100": 0.4402,
+            "R_cap@200": 0.3769,
+            "R_cap@1000": 0.3802
+        },
+        "hole": {
+            "Hole@1": 0.18,
+            "Hole@3": 0.12,
+            "Hole@5": 0.128,
+            "Hole@10": 0.156,
+            "Hole@20": 0.172,
+            "Hole@50": 0.2236,
+            "Hole@100": 0.2614,
+            "Hole@200": 0.2987,
+            "Hole@1000": 0.41988
+        }
+    }
+}

beir_output/webis-touche2020.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.16327,0.16264,0.18057,0.18394,0.19381,0.254,0.30616,0.34753,0.42673
+map,0.01449,0.02801,0.04614,0.06951,0.08346,0.10513,0.1164,0.12353,0.13188
+precision,0.16327,0.18367,0.2,0.17143,0.12653,0.09306,0.06898,0.0452,0.01484
+recall,0.01449,0.04138,0.07967,0.13306,0.18963,0.31867,0.44207,0.5494,0.8077
+mrr,0.16327,0.26871,0.3085,0.32207,0.33106,0.33521,0.33553,0.33553,0.3356
+recall_cap,0.16327,0.18367,0.2,0.18954,0.19453,0.31867,0.44207,0.5494,0.8077
+hole,0.7551,0.7415,0.75102,0.78571,0.83571,0.86735,0.89653,0.92694,0.96731

beir_output/webis-touche2020.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "webis-touche2020",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.16327,
+            "NDCG@3": 0.16264,
+            "NDCG@5": 0.18057,
+            "NDCG@10": 0.18394,
+            "NDCG@20": 0.19381,
+            "NDCG@50": 0.254,
+            "NDCG@100": 0.30616,
+            "NDCG@200": 0.34753,
+            "NDCG@1000": 0.42673
+        },
+        "map": {
+            "MAP@1": 0.01449,
+            "MAP@3": 0.02801,
+            "MAP@5": 0.04614,
+            "MAP@10": 0.06951,
+            "MAP@20": 0.08346,
+            "MAP@50": 0.10513,
+            "MAP@100": 0.1164,
+            "MAP@200": 0.12353,
+            "MAP@1000": 0.13188
+        },
+        "precision": {
+            "P@1": 0.16327,
+            "P@3": 0.18367,
+            "P@5": 0.2,
+            "P@10": 0.17143,
+            "P@20": 0.12653,
+            "P@50": 0.09306,
+            "P@100": 0.06898,
+            "P@200": 0.0452,
+            "P@1000": 0.01484
+        },
+        "recall": {
+            "Recall@1": 0.01449,
+            "Recall@3": 0.04138,
+            "Recall@5": 0.07967,
+            "Recall@10": 0.13306,
+            "Recall@20": 0.18963,
+            "Recall@50": 0.31867,
+            "Recall@100": 0.44207,
+            "Recall@200": 0.5494,
+            "Recall@1000": 0.8077
+        },
+        "mrr": {
+            "MRR@1": 0.16327,
+            "MRR@3": 0.26871,
+            "MRR@5": 0.3085,
+            "MRR@10": 0.32207,
+            "MRR@20": 0.33106,
+            "MRR@50": 0.33521,
+            "MRR@100": 0.33553,
+            "MRR@200": 0.33553,
+            "MRR@1000": 0.3356
+        },
+        "recall_cap": {
+            "R_cap@1": 0.16327,
+            "R_cap@3": 0.18367,
+            "R_cap@5": 0.2,
+            "R_cap@10": 0.18954,
+            "R_cap@20": 0.19453,
+            "R_cap@50": 0.31867,
+            "R_cap@100": 0.44207,
+            "R_cap@200": 0.5494,
+            "R_cap@1000": 0.8077
+        },
+        "hole": {
+            "Hole@1": 0.7551,
+            "Hole@3": 0.7415,
+            "Hole@5": 0.75102,
+            "Hole@10": 0.78571,
+            "Hole@20": 0.83571,
+            "Hole@50": 0.86735,
+            "Hole@100": 0.89653,
+            "Hole@200": 0.92694,
+            "Hole@1000": 0.96731
+        }
+    }
+}

cc.moco.2e14.title05.bs1024.gpu8.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export LOCAL_RANK=0
+export WORLD_SIZE=8
+
+export TOKENIZERS_PARALLELISM=true
+export NUM_WORKER=8
+export MAX_STEPS=100000
+
+export CUDA_LAUNCH_BLOCKING=1
+export NCCL_DEBUG=INFO
+
+export EXP_NAME=cc.moco-2e14.contriever256-Qtitle50.bert-base-uncased.avg.dot.q128d256.step100k.bs1024.lr5e5
+export PROJECT_DIR=output_dir/$EXP_NAME
+mkdir -p $PROJECT_DIR
+cp "$0" $PROJECT_DIR  # copy bash to project dir
+echo $PROJECT_DIR
+
+export WANDB_NAME=$EXP_NAME
+export WANDB_PROJECT=unsup_retrieval_cc
+export WANDB_DIR=$PROJECT_DIR
+mkdir -p $WANDB_DIR/wandb
+
+nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=31133 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type moco --train_file pile_Pile-CC --dev_file /export/home/data/pretrain/wiki2021_structure/wiki_psgs_w100.dev.tail2e13.tsv --data_type hf --data_pipeline_name contriever256-Qtitle50% --remove_unused_columns False --sim_type dot --queue_size 16384 --momentum 0.9995 --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --warmup_steps 10000 --logging_steps 100 --eval_steps 10000 --save_steps 1000000 --per_device_train_batch_size 128 --per_device_eval_batch_size 128 --dataloader_num_workers $NUM_WORKER --learning_rate 5e-5 --max_q_tokens 128 --max_d_tokens 256 --evaluation_strategy steps --load_best_model_at_end --overwrite_output_dir --do_train --do_eval --run_name $EXP_NAME --fp16 --seed 42 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup.log 2>&1 & echo $! > run.pid
+
+#export WANDB_RUN_ID=1553gyex
+#nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=3112 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type inbatch --reload_model_from $PROJECT_DIR --remove_unused_columns False --sim_type dot --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --do_eval --run_name $EXP_NAME --fp16 --seed 477 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup-eval.log 2>&1 & echo $! > run.pid
+

eval_output/checkpoint-100000/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.1707,0.28031,0.32773,0.37881,0.41118,0.43359,0.44097,0.44518,0.45009
+map,0.1707,0.25202,0.27847,0.29956,0.30864,0.31241,0.31309,0.31331,0.31343
+precision,0.1707,0.12091,0.09545,0.06351,0.03809,0.01745,0.00917,0.00474,0.00099
+recall,0.1707,0.36273,0.47724,0.63514,0.76174,0.87269,0.9175,0.94737,0.98933
+mrr,0.17639,0.25356,0.28126,0.3021,0.31107,0.31481,0.31549,0.31572,0.31583
+recall_cap,0.17639,0.35917,0.47937,0.63656,0.76245,0.87269,0.9175,0.94737,0.98933
+hole,0.67141,0.67686,0.68293,0.69417,0.73674,0.81273,0.85085,0.87242,0.88471

eval_output/checkpoint-100000/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.1707,
+            "NDCG@3": 0.28031,
+            "NDCG@5": 0.32773,
+            "NDCG@10": 0.37881,
+            "NDCG@20": 0.41118,
+            "NDCG@50": 0.43359,
+            "NDCG@100": 0.44097,
+            "NDCG@200": 0.44518,
+            "NDCG@1000": 0.45009
+        },
+        "map": {
+            "MAP@1": 0.1707,
+            "MAP@3": 0.25202,
+            "MAP@5": 0.27847,
+            "MAP@10": 0.29956,
+            "MAP@20": 0.30864,
+            "MAP@50": 0.31241,
+            "MAP@100": 0.31309,
+            "MAP@200": 0.31331,
+            "MAP@1000": 0.31343
+        },
+        "precision": {
+            "P@1": 0.1707,
+            "P@3": 0.12091,
+            "P@5": 0.09545,
+            "P@10": 0.06351,
+            "P@20": 0.03809,
+            "P@50": 0.01745,
+            "P@100": 0.00917,
+            "P@200": 0.00474,
+            "P@1000": 0.00099
+        },
+        "recall": {
+            "Recall@1": 0.1707,
+            "Recall@3": 0.36273,
+            "Recall@5": 0.47724,
+            "Recall@10": 0.63514,
+            "Recall@20": 0.76174,
+            "Recall@50": 0.87269,
+            "Recall@100": 0.9175,
+            "Recall@200": 0.94737,
+            "Recall@1000": 0.98933
+        },
+        "mrr": {
+            "MRR@1": 0.17639,
+            "MRR@3": 0.25356,
+            "MRR@5": 0.28126,
+            "MRR@10": 0.3021,
+            "MRR@20": 0.31107,
+            "MRR@50": 0.31481,
+            "MRR@100": 0.31549,
+            "MRR@200": 0.31572,
+            "MRR@1000": 0.31583
+        },
+        "recall_cap": {
+            "R_cap@1": 0.17639,
+            "R_cap@3": 0.35917,
+            "R_cap@5": 0.47937,
+            "R_cap@10": 0.63656,
+            "R_cap@20": 0.76245,
+            "R_cap@50": 0.87269,
+            "R_cap@100": 0.9175,
+            "R_cap@200": 0.94737,
+            "R_cap@1000": 0.98933
+        },
+        "hole": {
+            "Hole@1": 0.67141,
+            "Hole@3": 0.67686,
+            "Hole@5": 0.68293,
+            "Hole@10": 0.69417,
+            "Hole@20": 0.73674,
+            "Hole@50": 0.81273,
+            "Hole@100": 0.85085,
+            "Hole@200": 0.87242,
+            "Hole@1000": 0.88471
+        }
+    }
+}

eval_output/checkpoint-100000/beir.json

@@ -0,0 +1,46 @@
+{
+    "eval_beir-nfcorpus_ndcg@10": 0.29721,
+    "eval_beir-nfcorpus_recall@10": 0.1453,
+    "eval_beir-nfcorpus_recall@20": 0.17684,
+    "eval_beir-nfcorpus_recall@100": 0.27314,
+    "eval_beir-fiqa_ndcg@10": 0.25271,
+    "eval_beir-fiqa_recall@10": 0.31547,
+    "eval_beir-fiqa_recall@20": 0.39221,
+    "eval_beir-fiqa_recall@100": 0.57736,
+    "eval_beir-arguana_ndcg@10": 0.37881,
+    "eval_beir-arguana_recall@10": 0.63514,
+    "eval_beir-arguana_recall@20": 0.76174,
+    "eval_beir-arguana_recall@100": 0.9175,
+    "eval_beir-scidocs_ndcg@10": 0.1582,
+    "eval_beir-scidocs_recall@10": 0.16598,
+    "eval_beir-scidocs_recall@20": 0.22317,
+    "eval_beir-scidocs_recall@100": 0.37162,
+    "eval_beir-scifact_ndcg@10": 0.63462,
+    "eval_beir-scifact_recall@10": 0.77344,
+    "eval_beir-scifact_recall@20": 0.84122,
+    "eval_beir-scifact_recall@100": 0.90322,
+    "eval_beir-webis-touche2020_ndcg@10": 0.18394,
+    "eval_beir-webis-touche2020_recall@10": 0.13306,
+    "eval_beir-webis-touche2020_recall@20": 0.18963,
+    "eval_beir-webis-touche2020_recall@100": 0.44207,
+    "eval_beir-cqadupstack_ndcg@10": 0.2803316666666667,
+    "eval_beir-cqadupstack_recall@10": 0.3765525,
+    "eval_beir-cqadupstack_recall@20": 0.4442291666666666,
+    "eval_beir-cqadupstack_recall@100": 0.604765,
+    "eval_beir-trec-covid_ndcg@10": 0.5527,
+    "eval_beir-trec-covid_recall@10": 0.6,
+    "eval_beir-trec-covid_recall@20": 0.581,
+    "eval_beir-trec-covid_recall@100": 0.4402,
+    "eval_beir-quora_ndcg@10": 0.81279,
+    "eval_beir-quora_recall@10": 0.9098,
+    "eval_beir-quora_recall@20": 0.94365,
+    "eval_beir-quora_recall@100": 0.98333,
+    "eval_beir-nq_ndcg@10": 0.28932,
+    "eval_beir-nq_recall@10": 0.47579,
+    "eval_beir-nq_recall@20": 0.59195,
+    "eval_beir-nq_recall@100": 0.80159,
+    "eval_beir-avg_ndcg@10": 0.3840631666666666,
+    "eval_beir-avg_recall@10": 0.45305324999999996,
+    "eval_beir-avg_recall@20": 0.5145639166666667,
+    "eval_beir-avg_recall@100": 0.6314795
+}

eval_output/checkpoint-100000/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.20244416666666668,0.23995333333333335,0.25859583333333336,0.2803316666666667,0.2988066666666666,0.31907,0.33133583333333333,0.3424008333333333,0.36126166666666665
+map,0.17069,0.21522,0.22750083333333335,0.2373575,0.24307416666666667,0.24681416666666667,0.24814583333333337,0.24888,0.24945166666666663
+precision,0.20244416666666668,0.1111775,0.08060833333333334,0.050455,0.030930833333333324,0.015574166666666667,0.00905,0.0052200000000000015,0.0013491666666666667
+recall,0.17069,0.26443249999999996,0.31205999999999995,0.3765525,0.4442291666666666,0.5372158333333333,0.604765,0.6749175,0.8195983333333334
+mrr,0.20244416666666668,0.2489716666666666,0.2607483333333334,0.27058250000000006,0.27553999999999995,0.2785458333333333,0.27949583333333333,0.27997916666666667,0.28030416666666663
+recall_cap,0.20244416666666668,0.27020916666666667,0.31458749999999996,0.3776725000000001,0.44470750000000003,0.5373941666666667,0.6048366666666666,0.674925,0.8195983333333334
+hole,0.7220733333333333,0.8039649999999999,0.8346633333333333,0.86442,0.8839733333333335,0.9019533333333333,0.9109491666666666,0.9171174999999999,0.9264724999999999

eval_output/checkpoint-100000/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.20244416666666668,
+            "NDCG@3": 0.23995333333333335,
+            "NDCG@5": 0.25859583333333336,
+            "NDCG@10": 0.2803316666666667,
+            "NDCG@20": 0.2988066666666666,
+            "NDCG@50": 0.31907,
+            "NDCG@100": 0.33133583333333333,
+            "NDCG@200": 0.3424008333333333,
+            "NDCG@1000": 0.36126166666666665
+        },
+        "map": {
+            "MAP@1": 0.17069,
+            "MAP@3": 0.21522,
+            "MAP@5": 0.22750083333333335,
+            "MAP@10": 0.2373575,
+            "MAP@20": 0.24307416666666667,
+            "MAP@50": 0.24681416666666667,
+            "MAP@100": 0.24814583333333337,
+            "MAP@200": 0.24888,
+            "MAP@1000": 0.24945166666666663
+        },
+        "precision": {
+            "P@1": 0.20244416666666668,
+            "P@3": 0.1111775,
+            "P@5": 0.08060833333333334,
+            "P@10": 0.050455,
+            "P@20": 0.030930833333333324,
+            "P@50": 0.015574166666666667,
+            "P@100": 0.00905,
+            "P@200": 0.0052200000000000015,
+            "P@1000": 0.0013491666666666667
+        },
+        "recall": {
+            "Recall@1": 0.17069,
+            "Recall@3": 0.26443249999999996,
+            "Recall@5": 0.31205999999999995,
+            "Recall@10": 0.3765525,
+            "Recall@20": 0.4442291666666666,
+            "Recall@50": 0.5372158333333333,
+            "Recall@100": 0.604765,
+            "Recall@200": 0.6749175,
+            "Recall@1000": 0.8195983333333334
+        },
+        "mrr": {
+            "MRR@1": 0.20244416666666668,
+            "MRR@3": 0.2489716666666666,
+            "MRR@5": 0.2607483333333334,
+            "MRR@10": 0.27058250000000006,
+            "MRR@20": 0.27553999999999995,
+            "MRR@50": 0.2785458333333333,
+            "MRR@100": 0.27949583333333333,
+            "MRR@200": 0.27997916666666667,
+            "MRR@1000": 0.28030416666666663
+        },
+        "recall_cap": {
+            "R_cap@1": 0.20244416666666668,
+            "R_cap@3": 0.27020916666666667,
+            "R_cap@5": 0.31458749999999996,
+            "R_cap@10": 0.3776725000000001,
+            "R_cap@20": 0.44470750000000003,
+            "R_cap@50": 0.5373941666666667,
+            "R_cap@100": 0.6048366666666666,
+            "R_cap@200": 0.674925,
+            "R_cap@1000": 0.8195983333333334
+        },
+        "hole": {
+            "Hole@1": 0.7220733333333333,
+            "Hole@3": 0.8039649999999999,
+            "Hole@5": 0.8346633333333333,
+            "Hole@10": 0.86442,
+            "Hole@20": 0.8839733333333335,
+            "Hole@50": 0.9019533333333333,
+            "Hole@100": 0.9109491666666666,
+            "Hole@200": 0.9171174999999999,
+            "Hole@1000": 0.9264724999999999
+        }
+    }
+}

eval_output/checkpoint-100000/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.23302,0.2131,0.22737,0.25271,0.2769,0.30444,0.32137,0.33759,0.36433
+map,0.12405,0.16458,0.17971,0.19266,0.20052,0.20609,0.20812,0.2094,0.21039
+precision,0.23302,0.1358,0.10463,0.07068,0.04437,0.02346,0.01398,0.00821,0.00216
+recall,0.12405,0.19635,0.24497,0.31547,0.39221,0.49942,0.57736,0.6645,0.83618
+mrr,0.23302,0.27829,0.29473,0.30777,0.3144,0.31786,0.31885,0.31931,0.31958
+recall_cap,0.23302,0.21579,0.25041,0.3158,0.39221,0.49942,0.57736,0.6645,0.83618
+hole,0.70833,0.8035,0.83364,0.86867,0.89244,0.9113,0.92069,0.92654,0.93518

eval_output/checkpoint-100000/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.23302,
+            "NDCG@3": 0.2131,
+            "NDCG@5": 0.22737,
+            "NDCG@10": 0.25271,
+            "NDCG@20": 0.2769,
+            "NDCG@50": 0.30444,
+            "NDCG@100": 0.32137,
+            "NDCG@200": 0.33759,
+            "NDCG@1000": 0.36433
+        },
+        "map": {
+            "MAP@1": 0.12405,
+            "MAP@3": 0.16458,
+            "MAP@5": 0.17971,
+            "MAP@10": 0.19266,
+            "MAP@20": 0.20052,
+            "MAP@50": 0.20609,
+            "MAP@100": 0.20812,
+            "MAP@200": 0.2094,
+            "MAP@1000": 0.21039
+        },
+        "precision": {
+            "P@1": 0.23302,
+            "P@3": 0.1358,
+            "P@5": 0.10463,
+            "P@10": 0.07068,
+            "P@20": 0.04437,
+            "P@50": 0.02346,
+            "P@100": 0.01398,
+            "P@200": 0.00821,
+            "P@1000": 0.00216
+        },
+        "recall": {
+            "Recall@1": 0.12405,
+            "Recall@3": 0.19635,
+            "Recall@5": 0.24497,
+            "Recall@10": 0.31547,
+            "Recall@20": 0.39221,
+            "Recall@50": 0.49942,
+            "Recall@100": 0.57736,
+            "Recall@200": 0.6645,
+            "Recall@1000": 0.83618
+        },
+        "mrr": {
+            "MRR@1": 0.23302,
+            "MRR@3": 0.27829,
+            "MRR@5": 0.29473,
+            "MRR@10": 0.30777,
+            "MRR@20": 0.3144,
+            "MRR@50": 0.31786,
+            "MRR@100": 0.31885,
+            "MRR@200": 0.31931,
+            "MRR@1000": 0.31958
+        },
+        "recall_cap": {
+            "R_cap@1": 0.23302,
+            "R_cap@3": 0.21579,
+            "R_cap@5": 0.25041,
+            "R_cap@10": 0.3158,
+            "R_cap@20": 0.39221,
+            "R_cap@50": 0.49942,
+            "R_cap@100": 0.57736,
+            "R_cap@200": 0.6645,
+            "R_cap@1000": 0.83618
+        },
+        "hole": {
+            "Hole@1": 0.70833,
+            "Hole@3": 0.8035,
+            "Hole@5": 0.83364,
+            "Hole@10": 0.86867,
+            "Hole@20": 0.89244,
+            "Hole@50": 0.9113,
+            "Hole@100": 0.92069,
+            "Hole@200": 0.92654,
+            "Hole@1000": 0.93518
+        }
+    }
+}

eval_output/checkpoint-100000/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.37152,0.34236,0.32119,0.29721,0.27552,0.26519,0.27338,0.29114,0.36249
+map,0.04398,0.08002,0.09218,0.10681,0.11679,0.12707,0.13314,0.13805,0.14671
+precision,0.39319,0.32921,0.28173,0.22415,0.16409,0.10526,0.0731,0.04941,0.02017
+recall,0.04398,0.0957,0.11594,0.1453,0.17684,0.22559,0.27314,0.34803,0.59701
+mrr,0.39938,0.46491,0.47668,0.48612,0.49194,0.49302,0.4934,0.49383,0.49407
+recall_cap,0.39938,0.34985,0.31765,0.28258,0.25444,0.25623,0.28778,0.35318,0.59701
+hole,0.0774,0.08359,0.09659,0.10402,0.11517,0.1208,0.12399,0.12703,0.13305

eval_output/checkpoint-100000/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.37152,
+            "NDCG@3": 0.34236,
+            "NDCG@5": 0.32119,
+            "NDCG@10": 0.29721,
+            "NDCG@20": 0.27552,
+            "NDCG@50": 0.26519,
+            "NDCG@100": 0.27338,
+            "NDCG@200": 0.29114,
+            "NDCG@1000": 0.36249
+        },
+        "map": {
+            "MAP@1": 0.04398,
+            "MAP@3": 0.08002,
+            "MAP@5": 0.09218,
+            "MAP@10": 0.10681,
+            "MAP@20": 0.11679,
+            "MAP@50": 0.12707,
+            "MAP@100": 0.13314,
+            "MAP@200": 0.13805,
+            "MAP@1000": 0.14671
+        },
+        "precision": {
+            "P@1": 0.39319,
+            "P@3": 0.32921,
+            "P@5": 0.28173,
+            "P@10": 0.22415,
+            "P@20": 0.16409,
+            "P@50": 0.10526,
+            "P@100": 0.0731,
+            "P@200": 0.04941,
+            "P@1000": 0.02017
+        },
+        "recall": {
+            "Recall@1": 0.04398,
+            "Recall@3": 0.0957,
+            "Recall@5": 0.11594,
+            "Recall@10": 0.1453,
+            "Recall@20": 0.17684,
+            "Recall@50": 0.22559,
+            "Recall@100": 0.27314,
+            "Recall@200": 0.34803,
+            "Recall@1000": 0.59701
+        },
+        "mrr": {
+            "MRR@1": 0.39938,
+            "MRR@3": 0.46491,
+            "MRR@5": 0.47668,
+            "MRR@10": 0.48612,
+            "MRR@20": 0.49194,
+            "MRR@50": 0.49302,
+            "MRR@100": 0.4934,
+            "MRR@200": 0.49383,
+            "MRR@1000": 0.49407
+        },
+        "recall_cap": {
+            "R_cap@1": 0.39938,
+            "R_cap@3": 0.34985,
+            "R_cap@5": 0.31765,
+            "R_cap@10": 0.28258,
+            "R_cap@20": 0.25444,
+            "R_cap@50": 0.25623,
+            "R_cap@100": 0.28778,
+            "R_cap@200": 0.35318,
+            "R_cap@1000": 0.59701
+        },
+        "hole": {
+            "Hole@1": 0.0774,
+            "Hole@3": 0.08359,
+            "Hole@5": 0.09659,
+            "Hole@10": 0.10402,
+            "Hole@20": 0.11517,
+            "Hole@50": 0.1208,
+            "Hole@100": 0.12399,
+            "Hole@200": 0.12703,
+            "Hole@1000": 0.13305
+        }
+    }
+}

eval_output/checkpoint-100000/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.13789,0.20795,0.24521,0.28932,0.32042,0.3491,0.36179,0.37019,0.38166
+map,0.12189,0.18352,0.20487,0.22404,0.23326,0.2383,0.23955,0.24003,0.24034
+precision,0.13789,0.09985,0.08076,0.05588,0.03511,0.01744,0.00965,0.00518,0.00116
+recall,0.12189,0.25937,0.34598,0.47579,0.59195,0.72796,0.80159,0.85837,0.95119
+mrr,0.13818,0.20288,0.22363,0.24159,0.24972,0.25407,0.25507,0.25544,0.25567
+recall_cap,0.13818,0.25946,0.34598,0.47579,0.59195,0.72796,0.80159,0.85837,0.95119
+hole,0.85892,0.89677,0.91582,0.94056,0.96133,0.97926,0.98697,0.9916,0.99595

eval_output/checkpoint-100000/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.13789,
+            "NDCG@3": 0.20795,
+            "NDCG@5": 0.24521,
+            "NDCG@10": 0.28932,
+            "NDCG@20": 0.32042,
+            "NDCG@50": 0.3491,
+            "NDCG@100": 0.36179,
+            "NDCG@200": 0.37019,
+            "NDCG@1000": 0.38166
+        },
+        "map": {
+            "MAP@1": 0.12189,
+            "MAP@3": 0.18352,
+            "MAP@5": 0.20487,
+            "MAP@10": 0.22404,
+            "MAP@20": 0.23326,
+            "MAP@50": 0.2383,
+            "MAP@100": 0.23955,
+            "MAP@200": 0.24003,
+            "MAP@1000": 0.24034
+        },
+        "precision": {
+            "P@1": 0.13789,
+            "P@3": 0.09985,
+            "P@5": 0.08076,
+            "P@10": 0.05588,
+            "P@20": 0.03511,
+            "P@50": 0.01744,
+            "P@100": 0.00965,
+            "P@200": 0.00518,
+            "P@1000": 0.00116
+        },
+        "recall": {
+            "Recall@1": 0.12189,
+            "Recall@3": 0.25937,
+            "Recall@5": 0.34598,
+            "Recall@10": 0.47579,
+            "Recall@20": 0.59195,
+            "Recall@50": 0.72796,
+            "Recall@100": 0.80159,
+            "Recall@200": 0.85837,
+            "Recall@1000": 0.95119
+        },
+        "mrr": {
+            "MRR@1": 0.13818,
+            "MRR@3": 0.20288,
+            "MRR@5": 0.22363,
+            "MRR@10": 0.24159,
+            "MRR@20": 0.24972,
+            "MRR@50": 0.25407,
+            "MRR@100": 0.25507,
+            "MRR@200": 0.25544,
+            "MRR@1000": 0.25567
+        },
+        "recall_cap": {
+            "R_cap@1": 0.13818,
+            "R_cap@3": 0.25946,
+            "R_cap@5": 0.34598,
+            "R_cap@10": 0.47579,
+            "R_cap@20": 0.59195,
+            "R_cap@50": 0.72796,
+            "R_cap@100": 0.80159,
+            "R_cap@200": 0.85837,
+            "R_cap@1000": 0.95119
+        },
+        "hole": {
+            "Hole@1": 0.85892,
+            "Hole@3": 0.89677,
+            "Hole@5": 0.91582,
+            "Hole@10": 0.94056,
+            "Hole@20": 0.96133,
+            "Hole@50": 0.97926,
+            "Hole@100": 0.98697,
+            "Hole@200": 0.9916,
+            "Hole@1000": 0.99595
+        }
+    }
+}

eval_output/checkpoint-100000/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.723,0.77381,0.79417,0.81279,0.82301,0.82993,0.83269,0.83416,0.83543
+map,0.62985,0.73335,0.75278,0.76549,0.77056,0.77274,0.77326,0.77346,0.77355
+precision,0.723,0.3382,0.2252,0.12512,0.06743,0.02872,0.01482,0.00758,0.00156
+recall,0.62985,0.80077,0.85534,0.9098,0.94365,0.97077,0.98333,0.9906,0.99787
+mrr,0.7227,0.78562,0.79552,0.80088,0.80249,0.80305,0.80315,0.80317,0.80319
+recall_cap,0.7227,0.8147,0.8608,0.91133,0.94397,0.9708,0.98333,0.9906,0.99787
+hole,0.2578,0.6254,0.73292,0.8304,0.88665,0.92629,0.94129,0.95034,0.95968

eval_output/checkpoint-100000/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.723,
+            "NDCG@3": 0.77381,
+            "NDCG@5": 0.79417,
+            "NDCG@10": 0.81279,
+            "NDCG@20": 0.82301,
+            "NDCG@50": 0.82993,
+            "NDCG@100": 0.83269,
+            "NDCG@200": 0.83416,
+            "NDCG@1000": 0.83543
+        },
+        "map": {
+            "MAP@1": 0.62985,
+            "MAP@3": 0.73335,
+            "MAP@5": 0.75278,
+            "MAP@10": 0.76549,
+            "MAP@20": 0.77056,
+            "MAP@50": 0.77274,
+            "MAP@100": 0.77326,
+            "MAP@200": 0.77346,
+            "MAP@1000": 0.77355
+        },
+        "precision": {
+            "P@1": 0.723,
+            "P@3": 0.3382,
+            "P@5": 0.2252,
+            "P@10": 0.12512,
+            "P@20": 0.06743,
+            "P@50": 0.02872,
+            "P@100": 0.01482,
+            "P@200": 0.00758,
+            "P@1000": 0.00156
+        },
+        "recall": {
+            "Recall@1": 0.62985,
+            "Recall@3": 0.80077,
+            "Recall@5": 0.85534,
+            "Recall@10": 0.9098,
+            "Recall@20": 0.94365,
+            "Recall@50": 0.97077,
+            "Recall@100": 0.98333,
+            "Recall@200": 0.9906,
+            "Recall@1000": 0.99787
+        },
+        "mrr": {
+            "MRR@1": 0.7227,
+            "MRR@3": 0.78562,
+            "MRR@5": 0.79552,
+            "MRR@10": 0.80088,
+            "MRR@20": 0.80249,
+            "MRR@50": 0.80305,
+            "MRR@100": 0.80315,
+            "MRR@200": 0.80317,
+            "MRR@1000": 0.80319
+        },
+        "recall_cap": {
+            "R_cap@1": 0.7227,
+            "R_cap@3": 0.8147,
+            "R_cap@5": 0.8608,
+            "R_cap@10": 0.91133,
+            "R_cap@20": 0.94397,
+            "R_cap@50": 0.9708,
+            "R_cap@100": 0.98333,
+            "R_cap@200": 0.9906,
+            "R_cap@1000": 0.99787
+        },
+        "hole": {
+            "Hole@1": 0.2578,
+            "Hole@3": 0.6254,
+            "Hole@5": 0.73292,
+            "Hole@10": 0.8304,
+            "Hole@20": 0.88665,
+            "Hole@50": 0.92629,
+            "Hole@100": 0.94129,
+            "Hole@200": 0.95034,
+            "Hole@1000": 0.95968
+        }
+    }
+}

eval_output/checkpoint-100000/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.193,0.14845,0.13019,0.1582,0.18256,0.21015,0.22813,0.24603,0.28037
+map,0.03923,0.06632,0.07798,0.09057,0.09851,0.10477,0.10709,0.10857,0.1099
+precision,0.193,0.136,0.1128,0.0818,0.05505,0.03018,0.01833,0.01104,0.00308
+recall,0.03923,0.08313,0.11458,0.16598,0.22317,0.3058,0.37162,0.4477,0.62633
+mrr,0.193,0.24833,0.26838,0.28265,0.29015,0.29325,0.29423,0.29472,0.29499
+recall_cap,0.193,0.136,0.11458,0.16598,0.22317,0.3058,0.37162,0.4477,0.62633
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0