Upload 130 files

.gitattributes

@@ -32,3 +32,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+qa_output/nq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/squad1-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/trivia-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/webq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text

beir_output/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.17994,0.29655,0.35002,0.40399,0.43672,0.45789,0.46332,0.46683,0.46953
+map,0.17994,0.26671,0.2963,0.31853,0.32771,0.33129,0.33177,0.33195,0.33202
+precision,0.17994,0.12779,0.1027,0.06807,0.04043,0.01826,0.00947,0.00486,0.00099
+recall,0.17994,0.38336,0.51351,0.68065,0.80868,0.91323,0.94666,0.97155,0.99431
+mrr,0.18492,0.2679,0.29859,0.32078,0.32996,0.33354,0.33402,0.3342,0.33427
+recall_cap,0.18492,0.37909,0.51351,0.68065,0.80868,0.91323,0.94666,0.97155,0.99431
+hole,0.66572,0.66761,0.66302,0.66216,0.70932,0.78587,0.8261,0.84968,0.86683

beir_output/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.17994,
+            "NDCG@3": 0.29655,
+            "NDCG@5": 0.35002,
+            "NDCG@10": 0.40399,
+            "NDCG@20": 0.43672,
+            "NDCG@50": 0.45789,
+            "NDCG@100": 0.46332,
+            "NDCG@200": 0.46683,
+            "NDCG@1000": 0.46953
+        },
+        "map": {
+            "MAP@1": 0.17994,
+            "MAP@3": 0.26671,
+            "MAP@5": 0.2963,
+            "MAP@10": 0.31853,
+            "MAP@20": 0.32771,
+            "MAP@50": 0.33129,
+            "MAP@100": 0.33177,
+            "MAP@200": 0.33195,
+            "MAP@1000": 0.33202
+        },
+        "precision": {
+            "P@1": 0.17994,
+            "P@3": 0.12779,
+            "P@5": 0.1027,
+            "P@10": 0.06807,
+            "P@20": 0.04043,
+            "P@50": 0.01826,
+            "P@100": 0.00947,
+            "P@200": 0.00486,
+            "P@1000": 0.00099
+        },
+        "recall": {
+            "Recall@1": 0.17994,
+            "Recall@3": 0.38336,
+            "Recall@5": 0.51351,
+            "Recall@10": 0.68065,
+            "Recall@20": 0.80868,
+            "Recall@50": 0.91323,
+            "Recall@100": 0.94666,
+            "Recall@200": 0.97155,
+            "Recall@1000": 0.99431
+        },
+        "mrr": {
+            "MRR@1": 0.18492,
+            "MRR@3": 0.2679,
+            "MRR@5": 0.29859,
+            "MRR@10": 0.32078,
+            "MRR@20": 0.32996,
+            "MRR@50": 0.33354,
+            "MRR@100": 0.33402,
+            "MRR@200": 0.3342,
+            "MRR@1000": 0.33427
+        },
+        "recall_cap": {
+            "R_cap@1": 0.18492,
+            "R_cap@3": 0.37909,
+            "R_cap@5": 0.51351,
+            "R_cap@10": 0.68065,
+            "R_cap@20": 0.80868,
+            "R_cap@50": 0.91323,
+            "R_cap@100": 0.94666,
+            "R_cap@200": 0.97155,
+            "R_cap@1000": 0.99431
+        },
+        "hole": {
+            "Hole@1": 0.66572,
+            "Hole@3": 0.66761,
+            "Hole@5": 0.66302,
+            "Hole@10": 0.66216,
+            "Hole@20": 0.70932,
+            "Hole@50": 0.78587,
+            "Hole@100": 0.8261,
+            "Hole@200": 0.84968,
+            "Hole@1000": 0.86683
+        }
+    }
+}

beir_output/beir.json

@@ -0,0 +1,26 @@
+{
+    "eval_beir-msmarco_ndcg@10": 0.23068,
+    "eval_beir-msmarco_recall@10": 0.39509,
+    "eval_beir-msmarco_recall@20": 0.49656,
+    "eval_beir-msmarco_recall@100": 0.70484,
+    "eval_beir-dbpedia-entity_ndcg@10": 0.30373,
+    "eval_beir-dbpedia-entity_recall@10": 0.20251,
+    "eval_beir-dbpedia-entity_recall@20": 0.25943,
+    "eval_beir-dbpedia-entity_recall@100": 0.43711,
+    "eval_beir-fever_ndcg@10": 0.60393,
+    "eval_beir-fever_recall@10": 0.77489,
+    "eval_beir-fever_recall@20": 0.82755,
+    "eval_beir-fever_recall@100": 0.90348,
+    "eval_beir-climate-fever_ndcg@10": 0.1608,
+    "eval_beir-climate-fever_recall@10": 0.19355,
+    "eval_beir-climate-fever_recall@20": 0.25216,
+    "eval_beir-climate-fever_recall@100": 0.4308,
+    "eval_beir-hotpotqa_ndcg@10": 0.53065,
+    "eval_beir-hotpotqa_recall@10": 0.5607,
+    "eval_beir-hotpotqa_recall@20": 0.60614,
+    "eval_beir-hotpotqa_recall@100": 0.70459,
+    "eval_beir-avg_ndcg@10": 0.365958,
+    "eval_beir-avg_recall@10": 0.42534799999999995,
+    "eval_beir-avg_recall@20": 0.488368,
+    "eval_beir-avg_recall@100": 0.6361640000000001
+}

beir_output/climate-fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.14919,0.13288,0.14334,0.1608,0.18118,0.20782,0.22595,0.24323,0.27279
+map,0.0721,0.09691,0.10536,0.11248,0.11814,0.12293,0.12496,0.12622,0.12722
+precision,0.14919,0.09446,0.07283,0.04782,0.03251,0.01876,0.01176,0.00723,0.00202
+recall,0.0721,0.12433,0.15404,0.19355,0.25216,0.34971,0.4308,0.51824,0.70022
+mrr,0.14919,0.19891,0.21233,0.22212,0.22949,0.23383,0.23512,0.2357,0.23598
+recall_cap,0.14919,0.13236,0.15404,0.19355,0.25216,0.34971,0.4308,0.51824,0.70022
+hole,0.6228,0.69055,0.72326,0.75759,0.7885,0.82274,0.84891,0.87401,0.92612

beir_output/climate-fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "climate-fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.14919,
+            "NDCG@3": 0.13288,
+            "NDCG@5": 0.14334,
+            "NDCG@10": 0.1608,
+            "NDCG@20": 0.18118,
+            "NDCG@50": 0.20782,
+            "NDCG@100": 0.22595,
+            "NDCG@200": 0.24323,
+            "NDCG@1000": 0.27279
+        },
+        "map": {
+            "MAP@1": 0.0721,
+            "MAP@3": 0.09691,
+            "MAP@5": 0.10536,
+            "MAP@10": 0.11248,
+            "MAP@20": 0.11814,
+            "MAP@50": 0.12293,
+            "MAP@100": 0.12496,
+            "MAP@200": 0.12622,
+            "MAP@1000": 0.12722
+        },
+        "precision": {
+            "P@1": 0.14919,
+            "P@3": 0.09446,
+            "P@5": 0.07283,
+            "P@10": 0.04782,
+            "P@20": 0.03251,
+            "P@50": 0.01876,
+            "P@100": 0.01176,
+            "P@200": 0.00723,
+            "P@1000": 0.00202
+        },
+        "recall": {
+            "Recall@1": 0.0721,
+            "Recall@3": 0.12433,
+            "Recall@5": 0.15404,
+            "Recall@10": 0.19355,
+            "Recall@20": 0.25216,
+            "Recall@50": 0.34971,
+            "Recall@100": 0.4308,
+            "Recall@200": 0.51824,
+            "Recall@1000": 0.70022
+        },
+        "mrr": {
+            "MRR@1": 0.14919,
+            "MRR@3": 0.19891,
+            "MRR@5": 0.21233,
+            "MRR@10": 0.22212,
+            "MRR@20": 0.22949,
+            "MRR@50": 0.23383,
+            "MRR@100": 0.23512,
+            "MRR@200": 0.2357,
+            "MRR@1000": 0.23598
+        },
+        "recall_cap": {
+            "R_cap@1": 0.14919,
+            "R_cap@3": 0.13236,
+            "R_cap@5": 0.15404,
+            "R_cap@10": 0.19355,
+            "R_cap@20": 0.25216,
+            "R_cap@50": 0.34971,
+            "R_cap@100": 0.4308,
+            "R_cap@200": 0.51824,
+            "R_cap@1000": 0.70022
+        },
+        "hole": {
+            "Hole@1": 0.6228,
+            "Hole@3": 0.69055,
+            "Hole@5": 0.72326,
+            "Hole@10": 0.75759,
+            "Hole@20": 0.7885,
+            "Hole@50": 0.82274,
+            "Hole@100": 0.84891,
+            "Hole@200": 0.87401,
+            "Hole@1000": 0.92612
+        }
+    }
+}

beir_output/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.192585,0.23012000000000002,0.24853083333333334,0.27109750000000005,0.28913,0.3094608333333333,0.32246,0.33299416666666665,0.3518408333333333
+map,0.16343416666666669,0.20673,0.2185975,0.22862166666666664,0.23417583333333333,0.23797916666666666,0.2393291666666667,0.24002916666666665,0.24060999999999996
+precision,0.192585,0.10611916666666665,0.07762750000000002,0.04901916666666666,0.029904166666666666,0.01523916666666667,0.008928333333333332,0.005132499999999999,0.0013341666666666667
+recall,0.16343416666666669,0.25509166666666666,0.3009616666666667,0.3678975,0.43421249999999995,0.5266183333333333,0.5988908333333334,0.6657075,0.8091266666666668
+mrr,0.192585,0.23851833333333336,0.2500325,0.26011166666666663,0.26493666666666665,0.2679816666666666,0.26897666666666664,0.26943,0.2697675
+recall_cap,0.192585,0.2600166666666667,0.30309416666666666,0.3688741666666666,0.43466750000000004,0.526785,0.59896,0.6657150000000001,0.8091266666666668
+hole,0.7308800000000001,0.8147808333333336,0.8420441666666667,0.8696600000000001,0.8884650000000002,0.9042541666666666,0.9118466666666666,0.9178058333333333,0.9268700000000001

beir_output/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.192585,
+            "NDCG@3": 0.23012000000000002,
+            "NDCG@5": 0.24853083333333334,
+            "NDCG@10": 0.27109750000000005,
+            "NDCG@20": 0.28913,
+            "NDCG@50": 0.3094608333333333,
+            "NDCG@100": 0.32246,
+            "NDCG@200": 0.33299416666666665,
+            "NDCG@1000": 0.3518408333333333
+        },
+        "map": {
+            "MAP@1": 0.16343416666666669,
+            "MAP@3": 0.20673,
+            "MAP@5": 0.2185975,
+            "MAP@10": 0.22862166666666664,
+            "MAP@20": 0.23417583333333333,
+            "MAP@50": 0.23797916666666666,
+            "MAP@100": 0.2393291666666667,
+            "MAP@200": 0.24002916666666665,
+            "MAP@1000": 0.24060999999999996
+        },
+        "precision": {
+            "P@1": 0.192585,
+            "P@3": 0.10611916666666665,
+            "P@5": 0.07762750000000002,
+            "P@10": 0.04901916666666666,
+            "P@20": 0.029904166666666666,
+            "P@50": 0.01523916666666667,
+            "P@100": 0.008928333333333332,
+            "P@200": 0.005132499999999999,
+            "P@1000": 0.0013341666666666667
+        },
+        "recall": {
+            "Recall@1": 0.16343416666666669,
+            "Recall@3": 0.25509166666666666,
+            "Recall@5": 0.3009616666666667,
+            "Recall@10": 0.3678975,
+            "Recall@20": 0.43421249999999995,
+            "Recall@50": 0.5266183333333333,
+            "Recall@100": 0.5988908333333334,
+            "Recall@200": 0.6657075,
+            "Recall@1000": 0.8091266666666668
+        },
+        "mrr": {
+            "MRR@1": 0.192585,
+            "MRR@3": 0.23851833333333336,
+            "MRR@5": 0.2500325,
+            "MRR@10": 0.26011166666666663,
+            "MRR@20": 0.26493666666666665,
+            "MRR@50": 0.2679816666666666,
+            "MRR@100": 0.26897666666666664,
+            "MRR@200": 0.26943,
+            "MRR@1000": 0.2697675
+        },
+        "recall_cap": {
+            "R_cap@1": 0.192585,
+            "R_cap@3": 0.2600166666666667,
+            "R_cap@5": 0.30309416666666666,
+            "R_cap@10": 0.3688741666666666,
+            "R_cap@20": 0.43466750000000004,
+            "R_cap@50": 0.526785,
+            "R_cap@100": 0.59896,
+            "R_cap@200": 0.6657150000000001,
+            "R_cap@1000": 0.8091266666666668
+        },
+        "hole": {
+            "Hole@1": 0.7308800000000001,
+            "Hole@3": 0.8147808333333336,
+            "Hole@5": 0.8420441666666667,
+            "Hole@10": 0.8696600000000001,
+            "Hole@20": 0.8884650000000002,
+            "Hole@50": 0.9042541666666666,
+            "Hole@100": 0.9118466666666666,
+            "Hole@200": 0.9178058333333333,
+            "Hole@1000": 0.9268700000000001
+        }
+    }
+}

beir_output/dbpedia-entity.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.36875,0.32874,0.31294,0.30373,0.30402,0.32911,0.35849,0.38684,0.43573
+map,0.06926,0.10575,0.12215,0.14409,0.16563,0.19148,0.20372,0.21097,0.21851
+precision,0.475,0.365,0.3075,0.24,0.18462,0.12395,0.0841,0.05481,0.01894
+recall,0.06926,0.12583,0.15695,0.20251,0.25943,0.35666,0.43711,0.52176,0.69639
+mrr,0.475,0.57,0.586,0.5932,0.5973,0.59923,0.59968,0.59979,0.59983
+recall_cap,0.475,0.38833,0.35217,0.32183,0.31643,0.36807,0.44235,0.52523,0.69703
+hole,0.345,0.41917,0.477,0.55175,0.62375,0.728,0.79747,0.85571,0.93543

beir_output/dbpedia-entity.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "dbpedia-entity",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.36875,
+            "NDCG@3": 0.32874,
+            "NDCG@5": 0.31294,
+            "NDCG@10": 0.30373,
+            "NDCG@20": 0.30402,
+            "NDCG@50": 0.32911,
+            "NDCG@100": 0.35849,
+            "NDCG@200": 0.38684,
+            "NDCG@1000": 0.43573
+        },
+        "map": {
+            "MAP@1": 0.06926,
+            "MAP@3": 0.10575,
+            "MAP@5": 0.12215,
+            "MAP@10": 0.14409,
+            "MAP@20": 0.16563,
+            "MAP@50": 0.19148,
+            "MAP@100": 0.20372,
+            "MAP@200": 0.21097,
+            "MAP@1000": 0.21851
+        },
+        "precision": {
+            "P@1": 0.475,
+            "P@3": 0.365,
+            "P@5": 0.3075,
+            "P@10": 0.24,
+            "P@20": 0.18462,
+            "P@50": 0.12395,
+            "P@100": 0.0841,
+            "P@200": 0.05481,
+            "P@1000": 0.01894
+        },
+        "recall": {
+            "Recall@1": 0.06926,
+            "Recall@3": 0.12583,
+            "Recall@5": 0.15695,
+            "Recall@10": 0.20251,
+            "Recall@20": 0.25943,
+            "Recall@50": 0.35666,
+            "Recall@100": 0.43711,
+            "Recall@200": 0.52176,
+            "Recall@1000": 0.69639
+        },
+        "mrr": {
+            "MRR@1": 0.475,
+            "MRR@3": 0.57,
+            "MRR@5": 0.586,
+            "MRR@10": 0.5932,
+            "MRR@20": 0.5973,
+            "MRR@50": 0.59923,
+            "MRR@100": 0.59968,
+            "MRR@200": 0.59979,
+            "MRR@1000": 0.59983
+        },
+        "recall_cap": {
+            "R_cap@1": 0.475,
+            "R_cap@3": 0.38833,
+            "R_cap@5": 0.35217,
+            "R_cap@10": 0.32183,
+            "R_cap@20": 0.31643,
+            "R_cap@50": 0.36807,
+            "R_cap@100": 0.44235,
+            "R_cap@200": 0.52523,
+            "R_cap@1000": 0.69703
+        },
+        "hole": {
+            "Hole@1": 0.345,
+            "Hole@3": 0.41917,
+            "Hole@5": 0.477,
+            "Hole@10": 0.55175,
+            "Hole@20": 0.62375,
+            "Hole@50": 0.728,
+            "Hole@100": 0.79747,
+            "Hole@200": 0.85571,
+            "Hole@1000": 0.93543
+        }
+    }
+}

beir_output/fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.44299,0.54757,0.58121,0.60393,0.61781,0.6287,0.63281,0.63544,0.63902
+map,0.41347,0.50886,0.52803,0.53779,0.54183,0.54371,0.54411,0.54427,0.54437
+precision,0.44299,0.22472,0.15302,0.08438,0.04527,0.01935,0.00999,0.00512,0.00107
+recall,0.41347,0.62531,0.70642,0.77489,0.82755,0.87977,0.90348,0.92077,0.94834
+mrr,0.44299,0.5428,0.56232,0.5718,0.57551,0.57723,0.57753,0.57764,0.57769
+recall_cap,0.44299,0.62636,0.7067,0.7749,0.82755,0.87977,0.90348,0.92077,0.94834
+hole,0.5282,0.74772,0.82244,0.89775,0.94215,0.97327,0.98506,0.99146,0.99722

beir_output/fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.44299,
+            "NDCG@3": 0.54757,
+            "NDCG@5": 0.58121,
+            "NDCG@10": 0.60393,
+            "NDCG@20": 0.61781,
+            "NDCG@50": 0.6287,
+            "NDCG@100": 0.63281,
+            "NDCG@200": 0.63544,
+            "NDCG@1000": 0.63902
+        },
+        "map": {
+            "MAP@1": 0.41347,
+            "MAP@3": 0.50886,
+            "MAP@5": 0.52803,
+            "MAP@10": 0.53779,
+            "MAP@20": 0.54183,
+            "MAP@50": 0.54371,
+            "MAP@100": 0.54411,
+            "MAP@200": 0.54427,
+            "MAP@1000": 0.54437
+        },
+        "precision": {
+            "P@1": 0.44299,
+            "P@3": 0.22472,
+            "P@5": 0.15302,
+            "P@10": 0.08438,
+            "P@20": 0.04527,
+            "P@50": 0.01935,
+            "P@100": 0.00999,
+            "P@200": 0.00512,
+            "P@1000": 0.00107
+        },
+        "recall": {
+            "Recall@1": 0.41347,
+            "Recall@3": 0.62531,
+            "Recall@5": 0.70642,
+            "Recall@10": 0.77489,
+            "Recall@20": 0.82755,
+            "Recall@50": 0.87977,
+            "Recall@100": 0.90348,
+            "Recall@200": 0.92077,
+            "Recall@1000": 0.94834
+        },
+        "mrr": {
+            "MRR@1": 0.44299,
+            "MRR@3": 0.5428,
+            "MRR@5": 0.56232,
+            "MRR@10": 0.5718,
+            "MRR@20": 0.57551,
+            "MRR@50": 0.57723,
+            "MRR@100": 0.57753,
+            "MRR@200": 0.57764,
+            "MRR@1000": 0.57769
+        },
+        "recall_cap": {
+            "R_cap@1": 0.44299,
+            "R_cap@3": 0.62636,
+            "R_cap@5": 0.7067,
+            "R_cap@10": 0.7749,
+            "R_cap@20": 0.82755,
+            "R_cap@50": 0.87977,
+            "R_cap@100": 0.90348,
+            "R_cap@200": 0.92077,
+            "R_cap@1000": 0.94834
+        },
+        "hole": {
+            "Hole@1": 0.5282,
+            "Hole@3": 0.74772,
+            "Hole@5": 0.82244,
+            "Hole@10": 0.89775,
+            "Hole@20": 0.94215,
+            "Hole@50": 0.97327,
+            "Hole@100": 0.98506,
+            "Hole@200": 0.99146,
+            "Hole@1000": 0.99722
+        }
+    }
+}

beir_output/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.25,0.22799,0.23651,0.25834,0.28319,0.3095,0.32969,0.34363,0.36984
+map,0.13137,0.17689,0.1892,0.20107,0.20928,0.21457,0.21703,0.21807,0.21903
+precision,0.25,0.14815,0.10864,0.0696,0.04414,0.02309,0.01418,0.00817,0.00213
+recall,0.13137,0.20853,0.247,0.31154,0.39001,0.49282,0.58671,0.66108,0.83076
+mrr,0.25,0.29707,0.3081,0.31926,0.32579,0.32932,0.33053,0.33097,0.33125
+recall_cap,0.25,0.22942,0.25298,0.31193,0.39001,0.49282,0.58671,0.66108,0.83076
+hole,0.69599,0.78909,0.82901,0.86728,0.89352,0.91475,0.92202,0.92806,0.93701

beir_output/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.25,
+            "NDCG@3": 0.22799,
+            "NDCG@5": 0.23651,
+            "NDCG@10": 0.25834,
+            "NDCG@20": 0.28319,
+            "NDCG@50": 0.3095,
+            "NDCG@100": 0.32969,
+            "NDCG@200": 0.34363,
+            "NDCG@1000": 0.36984
+        },
+        "map": {
+            "MAP@1": 0.13137,
+            "MAP@3": 0.17689,
+            "MAP@5": 0.1892,
+            "MAP@10": 0.20107,
+            "MAP@20": 0.20928,
+            "MAP@50": 0.21457,
+            "MAP@100": 0.21703,
+            "MAP@200": 0.21807,
+            "MAP@1000": 0.21903
+        },
+        "precision": {
+            "P@1": 0.25,
+            "P@3": 0.14815,
+            "P@5": 0.10864,
+            "P@10": 0.0696,
+            "P@20": 0.04414,
+            "P@50": 0.02309,
+            "P@100": 0.01418,
+            "P@200": 0.00817,
+            "P@1000": 0.00213
+        },
+        "recall": {
+            "Recall@1": 0.13137,
+            "Recall@3": 0.20853,
+            "Recall@5": 0.247,
+            "Recall@10": 0.31154,
+            "Recall@20": 0.39001,
+            "Recall@50": 0.49282,
+            "Recall@100": 0.58671,
+            "Recall@200": 0.66108,
+            "Recall@1000": 0.83076
+        },
+        "mrr": {
+            "MRR@1": 0.25,
+            "MRR@3": 0.29707,
+            "MRR@5": 0.3081,
+            "MRR@10": 0.31926,
+            "MRR@20": 0.32579,
+            "MRR@50": 0.32932,
+            "MRR@100": 0.33053,
+            "MRR@200": 0.33097,
+            "MRR@1000": 0.33125
+        },
+        "recall_cap": {
+            "R_cap@1": 0.25,
+            "R_cap@3": 0.22942,
+            "R_cap@5": 0.25298,
+            "R_cap@10": 0.31193,
+            "R_cap@20": 0.39001,
+            "R_cap@50": 0.49282,
+            "R_cap@100": 0.58671,
+            "R_cap@200": 0.66108,
+            "R_cap@1000": 0.83076
+        },
+        "hole": {
+            "Hole@1": 0.69599,
+            "Hole@3": 0.78909,
+            "Hole@5": 0.82901,
+            "Hole@10": 0.86728,
+            "Hole@20": 0.89352,
+            "Hole@50": 0.91475,
+            "Hole@100": 0.92202,
+            "Hole@200": 0.92806,
+            "Hole@1000": 0.93701
+        }
+    }
+}

beir_output/hotpotqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.6185,0.48136,0.50696,0.53065,0.54475,0.55921,0.5671,0.57347,0.58466
+map,0.30925,0.40225,0.4199,0.43304,0.43822,0.44152,0.44254,0.44302,0.44339
+precision,0.6185,0.30056,0.20059,0.11214,0.06061,0.02661,0.01409,0.00742,0.00164
+recall,0.30925,0.45084,0.50149,0.5607,0.60614,0.66516,0.70459,0.74153,0.82134
+mrr,0.6185,0.68485,0.69592,0.70193,0.70423,0.70532,0.70561,0.70573,0.70579
+recall_cap,0.6185,0.45084,0.50149,0.5607,0.60614,0.66516,0.70459,0.74153,0.82134
+hole,0.37367,0.68868,0.78806,0.87621,0.92849,0.96324,0.97643,0.98373,0.99089

beir_output/hotpotqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "hotpotqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.6185,
+            "NDCG@3": 0.48136,
+            "NDCG@5": 0.50696,
+            "NDCG@10": 0.53065,
+            "NDCG@20": 0.54475,
+            "NDCG@50": 0.55921,
+            "NDCG@100": 0.5671,
+            "NDCG@200": 0.57347,
+            "NDCG@1000": 0.58466
+        },
+        "map": {
+            "MAP@1": 0.30925,
+            "MAP@3": 0.40225,
+            "MAP@5": 0.4199,
+            "MAP@10": 0.43304,
+            "MAP@20": 0.43822,
+            "MAP@50": 0.44152,
+            "MAP@100": 0.44254,
+            "MAP@200": 0.44302,
+            "MAP@1000": 0.44339
+        },
+        "precision": {
+            "P@1": 0.6185,
+            "P@3": 0.30056,
+            "P@5": 0.20059,
+            "P@10": 0.11214,
+            "P@20": 0.06061,
+            "P@50": 0.02661,
+            "P@100": 0.01409,
+            "P@200": 0.00742,
+            "P@1000": 0.00164
+        },
+        "recall": {
+            "Recall@1": 0.30925,
+            "Recall@3": 0.45084,
+            "Recall@5": 0.50149,
+            "Recall@10": 0.5607,
+            "Recall@20": 0.60614,
+            "Recall@50": 0.66516,
+            "Recall@100": 0.70459,
+            "Recall@200": 0.74153,
+            "Recall@1000": 0.82134
+        },
+        "mrr": {
+            "MRR@1": 0.6185,
+            "MRR@3": 0.68485,
+            "MRR@5": 0.69592,
+            "MRR@10": 0.70193,
+            "MRR@20": 0.70423,
+            "MRR@50": 0.70532,
+            "MRR@100": 0.70561,
+            "MRR@200": 0.70573,
+            "MRR@1000": 0.70579
+        },
+        "recall_cap": {
+            "R_cap@1": 0.6185,
+            "R_cap@3": 0.45084,
+            "R_cap@5": 0.50149,
+            "R_cap@10": 0.5607,
+            "R_cap@20": 0.60614,
+            "R_cap@50": 0.66516,
+            "R_cap@100": 0.70459,
+            "R_cap@200": 0.74153,
+            "R_cap@1000": 0.82134
+        },
+        "hole": {
+            "Hole@1": 0.37367,
+            "Hole@3": 0.68868,
+            "Hole@5": 0.78806,
+            "Hole@10": 0.87621,
+            "Hole@20": 0.92849,
+            "Hole@50": 0.96324,
+            "Hole@100": 0.97643,
+            "Hole@200": 0.98373,
+            "Hole@1000": 0.99089
+        }
+    }
+}

beir_output/msmarco.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.09971,0.16486,0.19507,0.23068,0.25659,0.28198,0.29564,0.30623,0.32102
+map,0.09735,0.14693,0.16373,0.17864,0.18576,0.18994,0.19116,0.19173,0.19208
+precision,0.09971,0.0734,0.05926,0.041,0.02589,0.01303,0.00741,0.0041,0.00096
+recall,0.09735,0.21353,0.28659,0.39509,0.49656,0.622,0.70484,0.77908,0.90429
+mrr,0.09957,0.15033,0.16702,0.18175,0.18884,0.19295,0.19414,0.19468,0.195
+recall_cap,0.09957,0.21354,0.28674,0.39509,0.4967,0.622,0.70484,0.77908,0.90429
+hole,0.89943,0.9255,0.93946,0.95784,0.97292,0.98583,0.99143,0.9948,0.99799

beir_output/msmarco.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "msmarco",
+    "split": "dev",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.09971,
+            "NDCG@3": 0.16486,
+            "NDCG@5": 0.19507,
+            "NDCG@10": 0.23068,
+            "NDCG@20": 0.25659,
+            "NDCG@50": 0.28198,
+            "NDCG@100": 0.29564,
+            "NDCG@200": 0.30623,
+            "NDCG@1000": 0.32102
+        },
+        "map": {
+            "MAP@1": 0.09735,
+            "MAP@3": 0.14693,
+            "MAP@5": 0.16373,
+            "MAP@10": 0.17864,
+            "MAP@20": 0.18576,
+            "MAP@50": 0.18994,
+            "MAP@100": 0.19116,
+            "MAP@200": 0.19173,
+            "MAP@1000": 0.19208
+        },
+        "precision": {
+            "P@1": 0.09971,
+            "P@3": 0.0734,
+            "P@5": 0.05926,
+            "P@10": 0.041,
+            "P@20": 0.02589,
+            "P@50": 0.01303,
+            "P@100": 0.00741,
+            "P@200": 0.0041,
+            "P@1000": 0.00096
+        },
+        "recall": {
+            "Recall@1": 0.09735,
+            "Recall@3": 0.21353,
+            "Recall@5": 0.28659,
+            "Recall@10": 0.39509,
+            "Recall@20": 0.49656,
+            "Recall@50": 0.622,
+            "Recall@100": 0.70484,
+            "Recall@200": 0.77908,
+            "Recall@1000": 0.90429
+        },
+        "mrr": {
+            "MRR@1": 0.09957,
+            "MRR@3": 0.15033,
+            "MRR@5": 0.16702,
+            "MRR@10": 0.18175,
+            "MRR@20": 0.18884,
+            "MRR@50": 0.19295,
+            "MRR@100": 0.19414,
+            "MRR@200": 0.19468,
+            "MRR@1000": 0.195
+        },
+        "recall_cap": {
+            "R_cap@1": 0.09957,
+            "R_cap@3": 0.21354,
+            "R_cap@5": 0.28674,
+            "R_cap@10": 0.39509,
+            "R_cap@20": 0.4967,
+            "R_cap@50": 0.622,
+            "R_cap@100": 0.70484,
+            "R_cap@200": 0.77908,
+            "R_cap@1000": 0.90429
+        },
+        "hole": {
+            "Hole@1": 0.89943,
+            "Hole@3": 0.9255,
+            "Hole@5": 0.93946,
+            "Hole@10": 0.95784,
+            "Hole@20": 0.97292,
+            "Hole@50": 0.98583,
+            "Hole@100": 0.99143,
+            "Hole@200": 0.9948,
+            "Hole@1000": 0.99799
+        }
+    }
+}

beir_output/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.39009,0.34526,0.32945,0.29872,0.27827,0.26819,0.27359,0.28971,0.36245
+map,0.05024,0.08179,0.09597,0.10993,0.12002,0.12968,0.13509,0.13948,0.14787
+precision,0.40867,0.32508,0.28607,0.21889,0.16223,0.10334,0.06969,0.04724,0.01958
+recall,0.05024,0.09062,0.11787,0.1448,0.17552,0.22738,0.27142,0.33549,0.5963
+mrr,0.41486,0.47781,0.49267,0.49864,0.50229,0.50472,0.50511,0.50536,0.50562
+recall_cap,0.41486,0.34314,0.32337,0.27771,0.25297,0.25746,0.28514,0.34028,0.5963
+hole,0.09907,0.09494,0.09969,0.10681,0.11502,0.12638,0.13077,0.13211,0.13479

beir_output/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.39009,
+            "NDCG@3": 0.34526,
+            "NDCG@5": 0.32945,
+            "NDCG@10": 0.29872,
+            "NDCG@20": 0.27827,
+            "NDCG@50": 0.26819,
+            "NDCG@100": 0.27359,
+            "NDCG@200": 0.28971,
+            "NDCG@1000": 0.36245
+        },
+        "map": {
+            "MAP@1": 0.05024,
+            "MAP@3": 0.08179,
+            "MAP@5": 0.09597,
+            "MAP@10": 0.10993,
+            "MAP@20": 0.12002,
+            "MAP@50": 0.12968,
+            "MAP@100": 0.13509,
+            "MAP@200": 0.13948,
+            "MAP@1000": 0.14787
+        },
+        "precision": {
+            "P@1": 0.40867,
+            "P@3": 0.32508,
+            "P@5": 0.28607,
+            "P@10": 0.21889,
+            "P@20": 0.16223,
+            "P@50": 0.10334,
+            "P@100": 0.06969,
+            "P@200": 0.04724,
+            "P@1000": 0.01958
+        },
+        "recall": {
+            "Recall@1": 0.05024,
+            "Recall@3": 0.09062,
+            "Recall@5": 0.11787,
+            "Recall@10": 0.1448,
+            "Recall@20": 0.17552,
+            "Recall@50": 0.22738,
+            "Recall@100": 0.27142,
+            "Recall@200": 0.33549,
+            "Recall@1000": 0.5963
+        },
+        "mrr": {
+            "MRR@1": 0.41486,
+            "MRR@3": 0.47781,
+            "MRR@5": 0.49267,
+            "MRR@10": 0.49864,
+            "MRR@20": 0.50229,
+            "MRR@50": 0.50472,
+            "MRR@100": 0.50511,
+            "MRR@200": 0.50536,
+            "MRR@1000": 0.50562
+        },
+        "recall_cap": {
+            "R_cap@1": 0.41486,
+            "R_cap@3": 0.34314,
+            "R_cap@5": 0.32337,
+            "R_cap@10": 0.27771,
+            "R_cap@20": 0.25297,
+            "R_cap@50": 0.25746,
+            "R_cap@100": 0.28514,
+            "R_cap@200": 0.34028,
+            "R_cap@1000": 0.5963
+        },
+        "hole": {
+            "Hole@1": 0.09907,
+            "Hole@3": 0.09494,
+            "Hole@5": 0.09969,
+            "Hole@10": 0.10681,
+            "Hole@20": 0.11502,
+            "Hole@50": 0.12638,
+            "Hole@100": 0.13077,
+            "Hole@200": 0.13211,
+            "Hole@1000": 0.13479
+        }
+    }
+}

beir_output/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.12746,0.19636,0.23196,0.27193,0.30539,0.33566,0.34947,0.35837,0.36951
+map,0.11233,0.17216,0.19235,0.20986,0.21949,0.22493,0.22631,0.22683,0.22712
+precision,0.12746,0.09511,0.07735,0.05319,0.03417,0.01722,0.00961,0.00518,0.00115
+recall,0.11233,0.24722,0.32964,0.4465,0.57305,0.7163,0.79637,0.85605,0.94699
+mrr,0.12775,0.19168,0.21203,0.22789,0.2368,0.24129,0.24237,0.24279,0.24301
+recall_cap,0.12775,0.2473,0.32964,0.4465,0.57305,0.7163,0.79637,0.85605,0.94699
+hole,0.8708,0.90276,0.92034,0.94426,0.96302,0.98001,0.98772,0.99227,0.99649

beir_output/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.12746,
+            "NDCG@3": 0.19636,
+            "NDCG@5": 0.23196,
+            "NDCG@10": 0.27193,
+            "NDCG@20": 0.30539,
+            "NDCG@50": 0.33566,
+            "NDCG@100": 0.34947,
+            "NDCG@200": 0.35837,
+            "NDCG@1000": 0.36951
+        },
+        "map": {
+            "MAP@1": 0.11233,
+            "MAP@3": 0.17216,
+            "MAP@5": 0.19235,
+            "MAP@10": 0.20986,
+            "MAP@20": 0.21949,
+            "MAP@50": 0.22493,
+            "MAP@100": 0.22631,
+            "MAP@200": 0.22683,
+            "MAP@1000": 0.22712
+        },
+        "precision": {
+            "P@1": 0.12746,
+            "P@3": 0.09511,
+            "P@5": 0.07735,
+            "P@10": 0.05319,
+            "P@20": 0.03417,
+            "P@50": 0.01722,
+            "P@100": 0.00961,
+            "P@200": 0.00518,
+            "P@1000": 0.00115
+        },
+        "recall": {
+            "Recall@1": 0.11233,
+            "Recall@3": 0.24722,
+            "Recall@5": 0.32964,
+            "Recall@10": 0.4465,
+            "Recall@20": 0.57305,
+            "Recall@50": 0.7163,
+            "Recall@100": 0.79637,
+            "Recall@200": 0.85605,
+            "Recall@1000": 0.94699
+        },
+        "mrr": {
+            "MRR@1": 0.12775,
+            "MRR@3": 0.19168,
+            "MRR@5": 0.21203,
+            "MRR@10": 0.22789,
+            "MRR@20": 0.2368,
+            "MRR@50": 0.24129,
+            "MRR@100": 0.24237,
+            "MRR@200": 0.24279,
+            "MRR@1000": 0.24301
+        },
+        "recall_cap": {
+            "R_cap@1": 0.12775,
+            "R_cap@3": 0.2473,
+            "R_cap@5": 0.32964,
+            "R_cap@10": 0.4465,
+            "R_cap@20": 0.57305,
+            "R_cap@50": 0.7163,
+            "R_cap@100": 0.79637,
+            "R_cap@200": 0.85605,
+            "R_cap@1000": 0.94699
+        },
+        "hole": {
+            "Hole@1": 0.8708,
+            "Hole@3": 0.90276,
+            "Hole@5": 0.92034,
+            "Hole@10": 0.94426,
+            "Hole@20": 0.96302,
+            "Hole@50": 0.98001,
+            "Hole@100": 0.98772,
+            "Hole@200": 0.99227,
+            "Hole@1000": 0.99649
+        }
+    }
+}

beir_output/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.685,0.7395,0.76137,0.78168,0.79418,0.80255,0.80571,0.80755,0.80936
+map,0.59804,0.69932,0.71913,0.73214,0.73769,0.74018,0.74075,0.74096,0.74108
+precision,0.685,0.32323,0.2163,0.12095,0.06584,0.02832,0.01467,0.00752,0.00155
+recall,0.59804,0.77072,0.82798,0.887,0.92894,0.9619,0.97651,0.9862,0.99706
+mrr,0.6846,0.75002,0.76074,0.76723,0.76944,0.77018,0.77031,0.77036,0.77037
+recall_cap,0.6846,0.78343,0.83279,0.8883,0.92921,0.96192,0.97651,0.9862,0.99706
+hole,0.2952,0.64243,0.74542,0.83701,0.89174,0.9305,0.94534,0.95402,0.9628

beir_output/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.685,
+            "NDCG@3": 0.7395,
+            "NDCG@5": 0.76137,
+            "NDCG@10": 0.78168,
+            "NDCG@20": 0.79418,
+            "NDCG@50": 0.80255,
+            "NDCG@100": 0.80571,
+            "NDCG@200": 0.80755,
+            "NDCG@1000": 0.80936
+        },
+        "map": {
+            "MAP@1": 0.59804,
+            "MAP@3": 0.69932,
+            "MAP@5": 0.71913,
+            "MAP@10": 0.73214,
+            "MAP@20": 0.73769,
+            "MAP@50": 0.74018,
+            "MAP@100": 0.74075,
+            "MAP@200": 0.74096,
+            "MAP@1000": 0.74108
+        },
+        "precision": {
+            "P@1": 0.685,
+            "P@3": 0.32323,
+            "P@5": 0.2163,
+            "P@10": 0.12095,
+            "P@20": 0.06584,
+            "P@50": 0.02832,
+            "P@100": 0.01467,
+            "P@200": 0.00752,
+            "P@1000": 0.00155
+        },
+        "recall": {
+            "Recall@1": 0.59804,
+            "Recall@3": 0.77072,
+            "Recall@5": 0.82798,
+            "Recall@10": 0.887,
+            "Recall@20": 0.92894,
+            "Recall@50": 0.9619,
+            "Recall@100": 0.97651,
+            "Recall@200": 0.9862,
+            "Recall@1000": 0.99706
+        },
+        "mrr": {
+            "MRR@1": 0.6846,
+            "MRR@3": 0.75002,
+            "MRR@5": 0.76074,
+            "MRR@10": 0.76723,
+            "MRR@20": 0.76944,
+            "MRR@50": 0.77018,
+            "MRR@100": 0.77031,
+            "MRR@200": 0.77036,
+            "MRR@1000": 0.77037
+        },
+        "recall_cap": {
+            "R_cap@1": 0.6846,
+            "R_cap@3": 0.78343,
+            "R_cap@5": 0.83279,
+            "R_cap@10": 0.8883,
+            "R_cap@20": 0.92921,
+            "R_cap@50": 0.96192,
+            "R_cap@100": 0.97651,
+            "R_cap@200": 0.9862,
+            "R_cap@1000": 0.99706
+        },
+        "hole": {
+            "Hole@1": 0.2952,
+            "Hole@3": 0.64243,
+            "Hole@5": 0.74542,
+            "Hole@10": 0.83701,
+            "Hole@20": 0.89174,
+            "Hole@50": 0.9305,
+            "Hole@100": 0.94534,
+            "Hole@200": 0.95402,
+            "Hole@1000": 0.9628
+        }
+    }
+}

beir_output/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.185,0.14848,0.12947,0.15698,0.1812,0.20964,0.22858,0.24364,0.27844
+map,0.03768,0.06572,0.07756,0.09023,0.09798,0.10442,0.10697,0.10817,0.10949
+precision,0.185,0.138,0.113,0.0818,0.05495,0.03034,0.01859,0.01087,0.00306
+recall,0.03768,0.08418,0.11458,0.16563,0.2229,0.30743,0.37647,0.44063,0.62183
+mrr,0.185,0.24667,0.26352,0.27664,0.28379,0.28738,0.2884,0.28884,0.28913
+recall_cap,0.185,0.138,0.11458,0.16563,0.2229,0.30743,0.37647,0.44063,0.62183
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

beir_output/scidocs.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scidocs",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.185,
+            "NDCG@3": 0.14848,
+            "NDCG@5": 0.12947,
+            "NDCG@10": 0.15698,
+            "NDCG@20": 0.1812,
+            "NDCG@50": 0.20964,
+            "NDCG@100": 0.22858,
+            "NDCG@200": 0.24364,
+            "NDCG@1000": 0.27844
+        },
+        "map": {
+            "MAP@1": 0.03768,
+            "MAP@3": 0.06572,
+            "MAP@5": 0.07756,
+            "MAP@10": 0.09023,
+            "MAP@20": 0.09798,
+            "MAP@50": 0.10442,
+            "MAP@100": 0.10697,
+            "MAP@200": 0.10817,
+            "MAP@1000": 0.10949
+        },
+        "precision": {
+            "P@1": 0.185,
+            "P@3": 0.138,
+            "P@5": 0.113,
+            "P@10": 0.0818,
+            "P@20": 0.05495,
+            "P@50": 0.03034,
+            "P@100": 0.01859,
+            "P@200": 0.01087,
+            "P@1000": 0.00306
+        },
+        "recall": {
+            "Recall@1": 0.03768,
+            "Recall@3": 0.08418,
+            "Recall@5": 0.11458,
+            "Recall@10": 0.16563,
+            "Recall@20": 0.2229,
+            "Recall@50": 0.30743,
+            "Recall@100": 0.37647,
+            "Recall@200": 0.44063,
+            "Recall@1000": 0.62183
+        },
+        "mrr": {
+            "MRR@1": 0.185,
+            "MRR@3": 0.24667,
+            "MRR@5": 0.26352,
+            "MRR@10": 0.27664,
+            "MRR@20": 0.28379,
+            "MRR@50": 0.28738,
+            "MRR@100": 0.2884,
+            "MRR@200": 0.28884,
+            "MRR@1000": 0.28913
+        },
+        "recall_cap": {
+            "R_cap@1": 0.185,
+            "R_cap@3": 0.138,
+            "R_cap@5": 0.11458,
+            "R_cap@10": 0.16563,
+            "R_cap@20": 0.2229,
+            "R_cap@50": 0.30743,
+            "R_cap@100": 0.37647,
+            "R_cap@200": 0.44063,
+            "R_cap@1000": 0.62183
+        },
+        "hole": {
+            "Hole@1": 0.0,
+            "Hole@3": 0.0,
+            "Hole@5": 0.0,
+            "Hole@10": 0.0,
+            "Hole@20": 0.0,
+            "Hole@50": 0.0,
+            "Hole@100": 0.0,
+            "Hole@200": 0.0,
+            "Hole@1000": 0.0
+        }
+    }
+}

beir_output/scifact.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.48,0.58722,0.6208,0.6428,0.65414,0.66505,0.67137,0.67517,0.67826
+map,0.45789,0.55272,0.57472,0.58576,0.58927,0.59133,0.59198,0.5922,0.59228
+precision,0.48,0.23667,0.162,0.08933,0.0475,0.0202,0.0105,0.00542,0.00111
+recall,0.45789,0.65811,0.73989,0.80067,0.84244,0.89422,0.93156,0.95722,0.98333
+mrr,0.48,0.57389,0.59106,0.59949,0.60203,0.60372,0.60431,0.60449,0.60454
+recall_cap,0.48,0.66111,0.73989,0.80067,0.84244,0.89422,0.93156,0.95722,0.98333
+hole,0.49667,0.73778,0.79933,0.857,0.89617,0.92093,0.9304,0.93572,0.9419

beir_output/scifact.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scifact",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.48,
+            "NDCG@3": 0.58722,
+            "NDCG@5": 0.6208,
+            "NDCG@10": 0.6428,
+            "NDCG@20": 0.65414,
+            "NDCG@50": 0.66505,
+            "NDCG@100": 0.67137,
+            "NDCG@200": 0.67517,
+            "NDCG@1000": 0.67826
+        },
+        "map": {
+            "MAP@1": 0.45789,
+            "MAP@3": 0.55272,
+            "MAP@5": 0.57472,
+            "MAP@10": 0.58576,
+            "MAP@20": 0.58927,
+            "MAP@50": 0.59133,
+            "MAP@100": 0.59198,
+            "MAP@200": 0.5922,
+            "MAP@1000": 0.59228
+        },
+        "precision": {
+            "P@1": 0.48,
+            "P@3": 0.23667,
+            "P@5": 0.162,
+            "P@10": 0.08933,
+            "P@20": 0.0475,
+            "P@50": 0.0202,
+            "P@100": 0.0105,
+            "P@200": 0.00542,
+            "P@1000": 0.00111
+        },
+        "recall": {
+            "Recall@1": 0.45789,
+            "Recall@3": 0.65811,
+            "Recall@5": 0.73989,
+            "Recall@10": 0.80067,
+            "Recall@20": 0.84244,
+            "Recall@50": 0.89422,
+            "Recall@100": 0.93156,
+            "Recall@200": 0.95722,
+            "Recall@1000": 0.98333
+        },
+        "mrr": {
+            "MRR@1": 0.48,
+            "MRR@3": 0.57389,
+            "MRR@5": 0.59106,
+            "MRR@10": 0.59949,
+            "MRR@20": 0.60203,
+            "MRR@50": 0.60372,
+            "MRR@100": 0.60431,
+            "MRR@200": 0.60449,
+            "MRR@1000": 0.60454
+        },
+        "recall_cap": {
+            "R_cap@1": 0.48,
+            "R_cap@3": 0.66111,
+            "R_cap@5": 0.73989,
+            "R_cap@10": 0.80067,
+            "R_cap@20": 0.84244,
+            "R_cap@50": 0.89422,
+            "R_cap@100": 0.93156,
+            "R_cap@200": 0.95722,
+            "R_cap@1000": 0.98333
+        },
+        "hole": {
+            "Hole@1": 0.49667,
+            "Hole@3": 0.73778,
+            "Hole@5": 0.79933,
+            "Hole@10": 0.857,
+            "Hole@20": 0.89617,
+            "Hole@50": 0.92093,
+            "Hole@100": 0.9304,
+            "Hole@200": 0.93572,
+            "Hole@1000": 0.9419
+        }
+    }
+}

beir_output/senteval-core.json

@@ -0,0 +1,5 @@
+{
+    "eval_senteval-stsb_spearman": 0.7800424504336374,
+    "eval_senteval-sickr_spearman": 0.724076491124677,
+    "eval_senteval-avg_sts": 0.7520594707791572
+}

beir_output/senteval.json

@@ -0,0 +1,99 @@
+{
+    "STSBenchmark": {
+        "train": {
+            "pearson": [
+                0.759143306453737,
+                0.0
+            ],
+            "spearman": [
+                0.732317209733238,
+                0.0
+            ],
+            "nsamples": 5749
+        },
+        "dev": {
+            "pearson": [
+                0.7746081783379711,
+                2.2556430824992415e-300
+            ],
+            "spearman": [
+                0.7800424504336374,
+                2.5466091811087085e-307
+            ],
+            "nsamples": 1500
+        },
+        "test": {
+            "pearson": [
+                0.7580850707783013,
+                6.51859239939566e-258
+            ],
+            "spearman": [
+                0.7560754511600052,
+                8.850873074119353e-256
+            ],
+            "nsamples": 1379
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7620304519086235,
+                "mean": 0.7639455185233365,
+                "wmean": 0.7616627780381048
+            },
+            "spearman": {
+                "all": 0.7488342650348635,
+                "mean": 0.7561450371089601,
+                "wmean": 0.7444116088962087
+            }
+        }
+    },
+    "SICKRelatedness": {
+        "train": {
+            "pearson": [
+                0.7660762263543586,
+                0.0
+            ],
+            "spearman": [
+                0.6943387839868075,
+                0.0
+            ],
+            "nsamples": 4500
+        },
+        "dev": {
+            "pearson": [
+                0.7787463535407387,
+                6.565381473913046e-103
+            ],
+            "spearman": [
+                0.724076491124677,
+                2.2464753888818782e-82
+            ],
+            "nsamples": 500
+        },
+        "test": {
+            "pearson": [
+                0.7585953923013815,
+                0.0
+            ],
+            "spearman": [
+                0.6847050750070173,
+                0.0
+            ],
+            "nsamples": 4927
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7629449380408607,
+                "mean": 0.7678059907321595,
+                "wmean": 0.7630014801283258
+            },
+            "spearman": {
+                "all": 0.6909722716792152,
+                "mean": 0.7010401167061673,
+                "wmean": 0.6910551705512789
+            }
+        }
+    },
+    "eval_senteval-stsb_spearman": 0.7800424504336374,
+    "eval_senteval-sickr_spearman": 0.724076491124677,
+    "eval_senteval-avg_sts": 0.7520594707791572
+}

beir_output/trec-covid.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.6,0.56408,0.54106,0.54138,0.51862,0.47223,0.42514,0.36662,0.38826
+map,0.00165,0.00428,0.00652,0.01197,0.02139,0.04282,0.07024,0.10296,0.17186
+precision,0.62,0.61333,0.58,0.586,0.558,0.5024,0.447,0.3686,0.17696
+recall,0.00165,0.00479,0.00773,0.01504,0.02779,0.06046,0.10381,0.16614,0.37011
+mrr,0.6,0.72,0.729,0.73519,0.7386,0.7386,0.7386,0.7386,0.7386
+recall_cap,0.6,0.61333,0.58,0.586,0.558,0.5024,0.447,0.37201,0.37157
+hole,0.12,0.16667,0.18,0.17,0.193,0.2268,0.2568,0.3038,0.43114

beir_output/trec-covid.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "trec-covid",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.6,
+            "NDCG@3": 0.56408,
+            "NDCG@5": 0.54106,
+            "NDCG@10": 0.54138,
+            "NDCG@20": 0.51862,
+            "NDCG@50": 0.47223,
+            "NDCG@100": 0.42514,
+            "NDCG@200": 0.36662,
+            "NDCG@1000": 0.38826
+        },
+        "map": {
+            "MAP@1": 0.00165,
+            "MAP@3": 0.00428,
+            "MAP@5": 0.00652,
+            "MAP@10": 0.01197,
+            "MAP@20": 0.02139,
+            "MAP@50": 0.04282,
+            "MAP@100": 0.07024,
+            "MAP@200": 0.10296,
+            "MAP@1000": 0.17186
+        },
+        "precision": {
+            "P@1": 0.62,
+            "P@3": 0.61333,
+            "P@5": 0.58,
+            "P@10": 0.586,
+            "P@20": 0.558,
+            "P@50": 0.5024,
+            "P@100": 0.447,
+            "P@200": 0.3686,
+            "P@1000": 0.17696
+        },
+        "recall": {
+            "Recall@1": 0.00165,
+            "Recall@3": 0.00479,
+            "Recall@5": 0.00773,
+            "Recall@10": 0.01504,
+            "Recall@20": 0.02779,
+            "Recall@50": 0.06046,
+            "Recall@100": 0.10381,
+            "Recall@200": 0.16614,
+            "Recall@1000": 0.37011
+        },
+        "mrr": {
+            "MRR@1": 0.6,
+            "MRR@3": 0.72,
+            "MRR@5": 0.729,
+            "MRR@10": 0.73519,
+            "MRR@20": 0.7386,
+            "MRR@50": 0.7386,
+            "MRR@100": 0.7386,
+            "MRR@200": 0.7386,
+            "MRR@1000": 0.7386
+        },
+        "recall_cap": {
+            "R_cap@1": 0.6,
+            "R_cap@3": 0.61333,
+            "R_cap@5": 0.58,
+            "R_cap@10": 0.586,
+            "R_cap@20": 0.558,
+            "R_cap@50": 0.5024,
+            "R_cap@100": 0.447,
+            "R_cap@200": 0.37201,
+            "R_cap@1000": 0.37157
+        },
+        "hole": {
+            "Hole@1": 0.12,
+            "Hole@3": 0.16667,
+            "Hole@5": 0.18,
+            "Hole@10": 0.17,
+            "Hole@20": 0.193,
+            "Hole@50": 0.2268,
+            "Hole@100": 0.2568,
+            "Hole@200": 0.3038,
+            "Hole@1000": 0.43114
+        }
+    }
+}

beir_output/webis-touche2020.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.15306,0.15147,0.1453,0.1662,0.1872,0.24458,0.29519,0.33534,0.41391
+map,0.01896,0.03952,0.05251,0.07494,0.09136,0.10936,0.12184,0.12886,0.13662
+precision,0.18367,0.17007,0.1551,0.16122,0.12959,0.09061,0.06714,0.04398,0.01449
+recall,0.01896,0.04765,0.07211,0.13396,0.19766,0.31034,0.43201,0.53473,0.78761
+mrr,0.18367,0.2585,0.27177,0.30632,0.31326,0.31895,0.31895,0.31929,0.31937
+recall_cap,0.18367,0.17007,0.1551,0.18293,0.20282,0.31034,0.43201,0.53473,0.78761
+hole,0.7551,0.7619,0.78367,0.79184,0.83163,0.87184,0.89796,0.92867,0.96863

beir_output/webis-touche2020.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "webis-touche2020",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.15306,
+            "NDCG@3": 0.15147,
+            "NDCG@5": 0.1453,
+            "NDCG@10": 0.1662,
+            "NDCG@20": 0.1872,
+            "NDCG@50": 0.24458,
+            "NDCG@100": 0.29519,
+            "NDCG@200": 0.33534,
+            "NDCG@1000": 0.41391
+        },
+        "map": {
+            "MAP@1": 0.01896,
+            "MAP@3": 0.03952,
+            "MAP@5": 0.05251,
+            "MAP@10": 0.07494,
+            "MAP@20": 0.09136,
+            "MAP@50": 0.10936,
+            "MAP@100": 0.12184,
+            "MAP@200": 0.12886,
+            "MAP@1000": 0.13662
+        },
+        "precision": {
+            "P@1": 0.18367,
+            "P@3": 0.17007,
+            "P@5": 0.1551,
+            "P@10": 0.16122,
+            "P@20": 0.12959,
+            "P@50": 0.09061,
+            "P@100": 0.06714,
+            "P@200": 0.04398,
+            "P@1000": 0.01449
+        },
+        "recall": {
+            "Recall@1": 0.01896,
+            "Recall@3": 0.04765,
+            "Recall@5": 0.07211,
+            "Recall@10": 0.13396,
+            "Recall@20": 0.19766,
+            "Recall@50": 0.31034,
+            "Recall@100": 0.43201,
+            "Recall@200": 0.53473,
+            "Recall@1000": 0.78761
+        },
+        "mrr": {
+            "MRR@1": 0.18367,
+            "MRR@3": 0.2585,
+            "MRR@5": 0.27177,
+            "MRR@10": 0.30632,
+            "MRR@20": 0.31326,
+            "MRR@50": 0.31895,
+            "MRR@100": 0.31895,
+            "MRR@200": 0.31929,
+            "MRR@1000": 0.31937
+        },
+        "recall_cap": {
+            "R_cap@1": 0.18367,
+            "R_cap@3": 0.17007,
+            "R_cap@5": 0.1551,
+            "R_cap@10": 0.18293,
+            "R_cap@20": 0.20282,
+            "R_cap@50": 0.31034,
+            "R_cap@100": 0.43201,
+            "R_cap@200": 0.53473,
+            "R_cap@1000": 0.78761
+        },
+        "hole": {
+            "Hole@1": 0.7551,
+            "Hole@3": 0.7619,
+            "Hole@5": 0.78367,
+            "Hole@10": 0.79184,
+            "Hole@20": 0.83163,
+            "Hole@50": 0.87184,
+            "Hole@100": 0.89796,
+            "Hole@200": 0.92867,
+            "Hole@1000": 0.96863
+        }
+    }
+}

cc.moco.exsum50.bs1024.gpu8.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export LOCAL_RANK=0
+export WORLD_SIZE=8
+
+export TOKENIZERS_PARALLELISM=true
+export NUM_WORKER=6
+export MAX_STEPS=100000
+
+export CUDA_LAUNCH_BLOCKING=1
+export NCCL_DEBUG=INFO
+
+export EXP_NAME=cc.T03b_exsum50.moco-2e14.contriever256-special50.bert-base-uncased.avg.dot.q128d256.step100k.bs1024.lr5e5
+export PROJECT_DIR=output_dir/$EXP_NAME
+mkdir -p $PROJECT_DIR
+cp "$0" $PROJECT_DIR  # copy bash to project dir
+echo $PROJECT_DIR
+
+export WANDB_NAME=$EXP_NAME
+export WANDB_PROJECT=unsup_retrieval_cc
+export WANDB_DIR=$PROJECT_DIR
+mkdir -p $WANDB_DIR/wandb
+
+nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=31133 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type moco --train_file /export/home/data/search/upr/cc/T03B_PileCC_exsum.json --dev_file /export/home/data/pretrain/wiki2021_structure/wiki_psgs_w100.dev.tail2e13.tsv --data_type hf --data_pipeline_name contriever256-special50% --remove_unused_columns False --sim_type dot --queue_size 16384 --momentum 0.9995 --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --warmup_steps 10000 --logging_steps 100 --eval_steps 10000 --save_steps 1000000 --per_device_train_batch_size 128 --per_device_eval_batch_size 128 --dataloader_num_workers $NUM_WORKER --learning_rate 5e-5 --max_q_tokens 128 --max_d_tokens 256 --evaluation_strategy steps --load_best_model_at_end --overwrite_output_dir --do_train --do_eval --run_name $EXP_NAME --fp16 --seed 42 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup.log 2>&1 & echo $! > run.pid
+
+#export WANDB_RUN_ID=13h4vrrq
+#nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=3112 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type inbatch --reload_model_from $PROJECT_DIR --remove_unused_columns False --sim_type dot --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --do_eval --run_name $EXP_NAME --fp16 --seed 477 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup-eval.log 2>&1 & echo $! > run.pid
+

eval_output/checkpoint-100000/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.17994,0.29655,0.35002,0.40399,0.43672,0.45789,0.46332,0.46683,0.46953
+map,0.17994,0.26671,0.2963,0.31853,0.32771,0.33129,0.33177,0.33195,0.33202
+precision,0.17994,0.12779,0.1027,0.06807,0.04043,0.01826,0.00947,0.00486,0.00099
+recall,0.17994,0.38336,0.51351,0.68065,0.80868,0.91323,0.94666,0.97155,0.99431
+mrr,0.18492,0.2679,0.29859,0.32078,0.32996,0.33354,0.33402,0.3342,0.33427
+recall_cap,0.18492,0.37909,0.51351,0.68065,0.80868,0.91323,0.94666,0.97155,0.99431
+hole,0.66572,0.66761,0.66302,0.66216,0.70932,0.78587,0.8261,0.84968,0.86683

eval_output/checkpoint-100000/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.17994,
+            "NDCG@3": 0.29655,
+            "NDCG@5": 0.35002,
+            "NDCG@10": 0.40399,
+            "NDCG@20": 0.43672,
+            "NDCG@50": 0.45789,
+            "NDCG@100": 0.46332,
+            "NDCG@200": 0.46683,
+            "NDCG@1000": 0.46953
+        },
+        "map": {
+            "MAP@1": 0.17994,
+            "MAP@3": 0.26671,
+            "MAP@5": 0.2963,
+            "MAP@10": 0.31853,
+            "MAP@20": 0.32771,
+            "MAP@50": 0.33129,
+            "MAP@100": 0.33177,
+            "MAP@200": 0.33195,
+            "MAP@1000": 0.33202
+        },
+        "precision": {
+            "P@1": 0.17994,
+            "P@3": 0.12779,
+            "P@5": 0.1027,
+            "P@10": 0.06807,
+            "P@20": 0.04043,
+            "P@50": 0.01826,
+            "P@100": 0.00947,
+            "P@200": 0.00486,
+            "P@1000": 0.00099
+        },
+        "recall": {
+            "Recall@1": 0.17994,
+            "Recall@3": 0.38336,
+            "Recall@5": 0.51351,
+            "Recall@10": 0.68065,
+            "Recall@20": 0.80868,
+            "Recall@50": 0.91323,
+            "Recall@100": 0.94666,
+            "Recall@200": 0.97155,
+            "Recall@1000": 0.99431
+        },
+        "mrr": {
+            "MRR@1": 0.18492,
+            "MRR@3": 0.2679,
+            "MRR@5": 0.29859,
+            "MRR@10": 0.32078,
+            "MRR@20": 0.32996,
+            "MRR@50": 0.33354,
+            "MRR@100": 0.33402,
+            "MRR@200": 0.3342,
+            "MRR@1000": 0.33427
+        },
+        "recall_cap": {
+            "R_cap@1": 0.18492,
+            "R_cap@3": 0.37909,
+            "R_cap@5": 0.51351,
+            "R_cap@10": 0.68065,
+            "R_cap@20": 0.80868,
+            "R_cap@50": 0.91323,
+            "R_cap@100": 0.94666,
+            "R_cap@200": 0.97155,
+            "R_cap@1000": 0.99431
+        },
+        "hole": {
+            "Hole@1": 0.66572,
+            "Hole@3": 0.66761,
+            "Hole@5": 0.66302,
+            "Hole@10": 0.66216,
+            "Hole@20": 0.70932,
+            "Hole@50": 0.78587,
+            "Hole@100": 0.8261,
+            "Hole@200": 0.84968,
+            "Hole@1000": 0.86683
+        }
+    }
+}

eval_output/checkpoint-100000/beir.json

@@ -0,0 +1,46 @@
+{
+    "eval_beir-nfcorpus_ndcg@10": 0.29872,
+    "eval_beir-nfcorpus_recall@10": 0.1448,
+    "eval_beir-nfcorpus_recall@20": 0.17552,
+    "eval_beir-nfcorpus_recall@100": 0.27142,
+    "eval_beir-fiqa_ndcg@10": 0.25834,
+    "eval_beir-fiqa_recall@10": 0.31154,
+    "eval_beir-fiqa_recall@20": 0.39001,
+    "eval_beir-fiqa_recall@100": 0.58671,
+    "eval_beir-arguana_ndcg@10": 0.40399,
+    "eval_beir-arguana_recall@10": 0.68065,
+    "eval_beir-arguana_recall@20": 0.80868,
+    "eval_beir-arguana_recall@100": 0.94666,
+    "eval_beir-scidocs_ndcg@10": 0.15698,
+    "eval_beir-scidocs_recall@10": 0.16563,
+    "eval_beir-scidocs_recall@20": 0.2229,
+    "eval_beir-scidocs_recall@100": 0.37647,
+    "eval_beir-scifact_ndcg@10": 0.6428,
+    "eval_beir-scifact_recall@10": 0.80067,
+    "eval_beir-scifact_recall@20": 0.84244,
+    "eval_beir-scifact_recall@100": 0.93156,
+    "eval_beir-webis-touche2020_ndcg@10": 0.1662,
+    "eval_beir-webis-touche2020_recall@10": 0.13396,
+    "eval_beir-webis-touche2020_recall@20": 0.19766,
+    "eval_beir-webis-touche2020_recall@100": 0.43201,
+    "eval_beir-cqadupstack_ndcg@10": 0.27109750000000005,
+    "eval_beir-cqadupstack_recall@10": 0.3678975,
+    "eval_beir-cqadupstack_recall@20": 0.43421249999999995,
+    "eval_beir-cqadupstack_recall@100": 0.5988908333333334,
+    "eval_beir-trec-covid_ndcg@10": 0.54138,
+    "eval_beir-trec-covid_recall@10": 0.586,
+    "eval_beir-trec-covid_recall@20": 0.558,
+    "eval_beir-trec-covid_recall@100": 0.447,
+    "eval_beir-quora_ndcg@10": 0.78168,
+    "eval_beir-quora_recall@10": 0.887,
+    "eval_beir-quora_recall@20": 0.92894,
+    "eval_beir-quora_recall@100": 0.97651,
+    "eval_beir-nq_ndcg@10": 0.27193,
+    "eval_beir-nq_recall@10": 0.4465,
+    "eval_beir-nq_recall@20": 0.57305,
+    "eval_beir-nq_recall@100": 0.79637,
+    "eval_beir-avg_ndcg@10": 0.37931175,
+    "eval_beir-avg_recall@10": 0.45246475,
+    "eval_beir-avg_recall@20": 0.51314125,
+    "eval_beir-avg_recall@100": 0.6363600833333335
+}

eval_output/checkpoint-100000/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.192585,0.23012000000000002,0.24853083333333334,0.27109750000000005,0.28913,0.3094608333333333,0.32246,0.33299416666666665,0.3518408333333333
+map,0.16343416666666669,0.20673,0.2185975,0.22862166666666664,0.23417583333333333,0.23797916666666666,0.2393291666666667,0.24002916666666665,0.24060999999999996
+precision,0.192585,0.10611916666666665,0.07762750000000002,0.04901916666666666,0.029904166666666666,0.01523916666666667,0.008928333333333332,0.005132499999999999,0.0013341666666666667
+recall,0.16343416666666669,0.25509166666666666,0.3009616666666667,0.3678975,0.43421249999999995,0.5266183333333333,0.5988908333333334,0.6657075,0.8091266666666668
+mrr,0.192585,0.23851833333333336,0.2500325,0.26011166666666663,0.26493666666666665,0.2679816666666666,0.26897666666666664,0.26943,0.2697675
+recall_cap,0.192585,0.2600166666666667,0.30309416666666666,0.3688741666666666,0.43466750000000004,0.526785,0.59896,0.6657150000000001,0.8091266666666668
+hole,0.7308800000000001,0.8147808333333336,0.8420441666666667,0.8696600000000001,0.8884650000000002,0.9042541666666666,0.9118466666666666,0.9178058333333333,0.9268700000000001

eval_output/checkpoint-100000/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.192585,
+            "NDCG@3": 0.23012000000000002,
+            "NDCG@5": 0.24853083333333334,
+            "NDCG@10": 0.27109750000000005,
+            "NDCG@20": 0.28913,
+            "NDCG@50": 0.3094608333333333,
+            "NDCG@100": 0.32246,
+            "NDCG@200": 0.33299416666666665,
+            "NDCG@1000": 0.3518408333333333
+        },
+        "map": {
+            "MAP@1": 0.16343416666666669,
+            "MAP@3": 0.20673,
+            "MAP@5": 0.2185975,
+            "MAP@10": 0.22862166666666664,
+            "MAP@20": 0.23417583333333333,
+            "MAP@50": 0.23797916666666666,
+            "MAP@100": 0.2393291666666667,
+            "MAP@200": 0.24002916666666665,
+            "MAP@1000": 0.24060999999999996
+        },
+        "precision": {
+            "P@1": 0.192585,
+            "P@3": 0.10611916666666665,
+            "P@5": 0.07762750000000002,
+            "P@10": 0.04901916666666666,
+            "P@20": 0.029904166666666666,
+            "P@50": 0.01523916666666667,
+            "P@100": 0.008928333333333332,
+            "P@200": 0.005132499999999999,
+            "P@1000": 0.0013341666666666667
+        },
+        "recall": {
+            "Recall@1": 0.16343416666666669,
+            "Recall@3": 0.25509166666666666,
+            "Recall@5": 0.3009616666666667,
+            "Recall@10": 0.3678975,
+            "Recall@20": 0.43421249999999995,
+            "Recall@50": 0.5266183333333333,
+            "Recall@100": 0.5988908333333334,
+            "Recall@200": 0.6657075,
+            "Recall@1000": 0.8091266666666668
+        },
+        "mrr": {
+            "MRR@1": 0.192585,
+            "MRR@3": 0.23851833333333336,
+            "MRR@5": 0.2500325,
+            "MRR@10": 0.26011166666666663,
+            "MRR@20": 0.26493666666666665,
+            "MRR@50": 0.2679816666666666,
+            "MRR@100": 0.26897666666666664,
+            "MRR@200": 0.26943,
+            "MRR@1000": 0.2697675
+        },
+        "recall_cap": {
+            "R_cap@1": 0.192585,
+            "R_cap@3": 0.2600166666666667,
+            "R_cap@5": 0.30309416666666666,
+            "R_cap@10": 0.3688741666666666,
+            "R_cap@20": 0.43466750000000004,
+            "R_cap@50": 0.526785,
+            "R_cap@100": 0.59896,
+            "R_cap@200": 0.6657150000000001,
+            "R_cap@1000": 0.8091266666666668
+        },
+        "hole": {
+            "Hole@1": 0.7308800000000001,
+            "Hole@3": 0.8147808333333336,
+            "Hole@5": 0.8420441666666667,
+            "Hole@10": 0.8696600000000001,
+            "Hole@20": 0.8884650000000002,
+            "Hole@50": 0.9042541666666666,
+            "Hole@100": 0.9118466666666666,
+            "Hole@200": 0.9178058333333333,
+            "Hole@1000": 0.9268700000000001
+        }
+    }
+}

eval_output/checkpoint-100000/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.25,0.22799,0.23651,0.25834,0.28319,0.3095,0.32969,0.34363,0.36984
+map,0.13137,0.17689,0.1892,0.20107,0.20928,0.21457,0.21703,0.21807,0.21903
+precision,0.25,0.14815,0.10864,0.0696,0.04414,0.02309,0.01418,0.00817,0.00213
+recall,0.13137,0.20853,0.247,0.31154,0.39001,0.49282,0.58671,0.66108,0.83076
+mrr,0.25,0.29707,0.3081,0.31926,0.32579,0.32932,0.33053,0.33097,0.33125
+recall_cap,0.25,0.22942,0.25298,0.31193,0.39001,0.49282,0.58671,0.66108,0.83076
+hole,0.69599,0.78909,0.82901,0.86728,0.89352,0.91475,0.92202,0.92806,0.93701

eval_output/checkpoint-100000/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.25,
+            "NDCG@3": 0.22799,
+            "NDCG@5": 0.23651,
+            "NDCG@10": 0.25834,
+            "NDCG@20": 0.28319,
+            "NDCG@50": 0.3095,
+            "NDCG@100": 0.32969,
+            "NDCG@200": 0.34363,
+            "NDCG@1000": 0.36984
+        },
+        "map": {
+            "MAP@1": 0.13137,
+            "MAP@3": 0.17689,
+            "MAP@5": 0.1892,
+            "MAP@10": 0.20107,
+            "MAP@20": 0.20928,
+            "MAP@50": 0.21457,
+            "MAP@100": 0.21703,
+            "MAP@200": 0.21807,
+            "MAP@1000": 0.21903
+        },
+        "precision": {
+            "P@1": 0.25,
+            "P@3": 0.14815,
+            "P@5": 0.10864,
+            "P@10": 0.0696,
+            "P@20": 0.04414,
+            "P@50": 0.02309,
+            "P@100": 0.01418,
+            "P@200": 0.00817,
+            "P@1000": 0.00213
+        },
+        "recall": {
+            "Recall@1": 0.13137,
+            "Recall@3": 0.20853,
+            "Recall@5": 0.247,
+            "Recall@10": 0.31154,
+            "Recall@20": 0.39001,
+            "Recall@50": 0.49282,
+            "Recall@100": 0.58671,
+            "Recall@200": 0.66108,
+            "Recall@1000": 0.83076
+        },
+        "mrr": {
+            "MRR@1": 0.25,
+            "MRR@3": 0.29707,
+            "MRR@5": 0.3081,
+            "MRR@10": 0.31926,
+            "MRR@20": 0.32579,
+            "MRR@50": 0.32932,
+            "MRR@100": 0.33053,
+            "MRR@200": 0.33097,
+            "MRR@1000": 0.33125
+        },
+        "recall_cap": {
+            "R_cap@1": 0.25,
+            "R_cap@3": 0.22942,
+            "R_cap@5": 0.25298,
+            "R_cap@10": 0.31193,
+            "R_cap@20": 0.39001,
+            "R_cap@50": 0.49282,
+            "R_cap@100": 0.58671,
+            "R_cap@200": 0.66108,
+            "R_cap@1000": 0.83076
+        },
+        "hole": {
+            "Hole@1": 0.69599,
+            "Hole@3": 0.78909,
+            "Hole@5": 0.82901,
+            "Hole@10": 0.86728,
+            "Hole@20": 0.89352,
+            "Hole@50": 0.91475,
+            "Hole@100": 0.92202,
+            "Hole@200": 0.92806,
+            "Hole@1000": 0.93701
+        }
+    }
+}

eval_output/checkpoint-100000/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.39009,0.34526,0.32945,0.29872,0.27827,0.26819,0.27359,0.28971,0.36245
+map,0.05024,0.08179,0.09597,0.10993,0.12002,0.12968,0.13509,0.13948,0.14787
+precision,0.40867,0.32508,0.28607,0.21889,0.16223,0.10334,0.06969,0.04724,0.01958
+recall,0.05024,0.09062,0.11787,0.1448,0.17552,0.22738,0.27142,0.33549,0.5963
+mrr,0.41486,0.47781,0.49267,0.49864,0.50229,0.50472,0.50511,0.50536,0.50562
+recall_cap,0.41486,0.34314,0.32337,0.27771,0.25297,0.25746,0.28514,0.34028,0.5963
+hole,0.09907,0.09494,0.09969,0.10681,0.11502,0.12638,0.13077,0.13211,0.13479

eval_output/checkpoint-100000/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.39009,
+            "NDCG@3": 0.34526,
+            "NDCG@5": 0.32945,
+            "NDCG@10": 0.29872,
+            "NDCG@20": 0.27827,
+            "NDCG@50": 0.26819,
+            "NDCG@100": 0.27359,
+            "NDCG@200": 0.28971,
+            "NDCG@1000": 0.36245
+        },
+        "map": {
+            "MAP@1": 0.05024,
+            "MAP@3": 0.08179,
+            "MAP@5": 0.09597,
+            "MAP@10": 0.10993,
+            "MAP@20": 0.12002,
+            "MAP@50": 0.12968,
+            "MAP@100": 0.13509,
+            "MAP@200": 0.13948,
+            "MAP@1000": 0.14787
+        },
+        "precision": {
+            "P@1": 0.40867,
+            "P@3": 0.32508,
+            "P@5": 0.28607,
+            "P@10": 0.21889,
+            "P@20": 0.16223,
+            "P@50": 0.10334,
+            "P@100": 0.06969,
+            "P@200": 0.04724,
+            "P@1000": 0.01958
+        },
+        "recall": {
+            "Recall@1": 0.05024,
+            "Recall@3": 0.09062,
+            "Recall@5": 0.11787,
+            "Recall@10": 0.1448,
+            "Recall@20": 0.17552,
+            "Recall@50": 0.22738,
+            "Recall@100": 0.27142,
+            "Recall@200": 0.33549,
+            "Recall@1000": 0.5963
+        },
+        "mrr": {
+            "MRR@1": 0.41486,
+            "MRR@3": 0.47781,
+            "MRR@5": 0.49267,
+            "MRR@10": 0.49864,
+            "MRR@20": 0.50229,
+            "MRR@50": 0.50472,
+            "MRR@100": 0.50511,
+            "MRR@200": 0.50536,
+            "MRR@1000": 0.50562
+        },
+        "recall_cap": {
+            "R_cap@1": 0.41486,
+            "R_cap@3": 0.34314,
+            "R_cap@5": 0.32337,
+            "R_cap@10": 0.27771,
+            "R_cap@20": 0.25297,
+            "R_cap@50": 0.25746,
+            "R_cap@100": 0.28514,
+            "R_cap@200": 0.34028,
+            "R_cap@1000": 0.5963
+        },
+        "hole": {
+            "Hole@1": 0.09907,
+            "Hole@3": 0.09494,
+            "Hole@5": 0.09969,
+            "Hole@10": 0.10681,
+            "Hole@20": 0.11502,
+            "Hole@50": 0.12638,
+            "Hole@100": 0.13077,
+            "Hole@200": 0.13211,
+            "Hole@1000": 0.13479
+        }
+    }
+}

eval_output/checkpoint-100000/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.12746,0.19636,0.23196,0.27193,0.30539,0.33566,0.34947,0.35837,0.36951
+map,0.11233,0.17216,0.19235,0.20986,0.21949,0.22493,0.22631,0.22683,0.22712
+precision,0.12746,0.09511,0.07735,0.05319,0.03417,0.01722,0.00961,0.00518,0.00115
+recall,0.11233,0.24722,0.32964,0.4465,0.57305,0.7163,0.79637,0.85605,0.94699
+mrr,0.12775,0.19168,0.21203,0.22789,0.2368,0.24129,0.24237,0.24279,0.24301
+recall_cap,0.12775,0.2473,0.32964,0.4465,0.57305,0.7163,0.79637,0.85605,0.94699
+hole,0.8708,0.90276,0.92034,0.94426,0.96302,0.98001,0.98772,0.99227,0.99649

eval_output/checkpoint-100000/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.12746,
+            "NDCG@3": 0.19636,
+            "NDCG@5": 0.23196,
+            "NDCG@10": 0.27193,
+            "NDCG@20": 0.30539,
+            "NDCG@50": 0.33566,
+            "NDCG@100": 0.34947,
+            "NDCG@200": 0.35837,
+            "NDCG@1000": 0.36951
+        },
+        "map": {
+            "MAP@1": 0.11233,
+            "MAP@3": 0.17216,
+            "MAP@5": 0.19235,
+            "MAP@10": 0.20986,
+            "MAP@20": 0.21949,
+            "MAP@50": 0.22493,
+            "MAP@100": 0.22631,
+            "MAP@200": 0.22683,
+            "MAP@1000": 0.22712
+        },
+        "precision": {
+            "P@1": 0.12746,
+            "P@3": 0.09511,
+            "P@5": 0.07735,
+            "P@10": 0.05319,
+            "P@20": 0.03417,
+            "P@50": 0.01722,
+            "P@100": 0.00961,
+            "P@200": 0.00518,
+            "P@1000": 0.00115
+        },
+        "recall": {
+            "Recall@1": 0.11233,
+            "Recall@3": 0.24722,
+            "Recall@5": 0.32964,
+            "Recall@10": 0.4465,
+            "Recall@20": 0.57305,
+            "Recall@50": 0.7163,
+            "Recall@100": 0.79637,
+            "Recall@200": 0.85605,
+            "Recall@1000": 0.94699
+        },
+        "mrr": {
+            "MRR@1": 0.12775,
+            "MRR@3": 0.19168,
+            "MRR@5": 0.21203,
+            "MRR@10": 0.22789,
+            "MRR@20": 0.2368,
+            "MRR@50": 0.24129,
+            "MRR@100": 0.24237,
+            "MRR@200": 0.24279,
+            "MRR@1000": 0.24301
+        },
+        "recall_cap": {
+            "R_cap@1": 0.12775,
+            "R_cap@3": 0.2473,
+            "R_cap@5": 0.32964,
+            "R_cap@10": 0.4465,
+            "R_cap@20": 0.57305,
+            "R_cap@50": 0.7163,
+            "R_cap@100": 0.79637,
+            "R_cap@200": 0.85605,
+            "R_cap@1000": 0.94699
+        },
+        "hole": {
+            "Hole@1": 0.8708,
+            "Hole@3": 0.90276,
+            "Hole@5": 0.92034,
+            "Hole@10": 0.94426,
+            "Hole@20": 0.96302,
+            "Hole@50": 0.98001,
+            "Hole@100": 0.98772,
+            "Hole@200": 0.99227,
+            "Hole@1000": 0.99649
+        }
+    }
+}

eval_output/checkpoint-100000/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.685,0.7395,0.76137,0.78168,0.79418,0.80255,0.80571,0.80755,0.80936
+map,0.59804,0.69932,0.71913,0.73214,0.73769,0.74018,0.74075,0.74096,0.74108
+precision,0.685,0.32323,0.2163,0.12095,0.06584,0.02832,0.01467,0.00752,0.00155
+recall,0.59804,0.77072,0.82798,0.887,0.92894,0.9619,0.97651,0.9862,0.99706
+mrr,0.6846,0.75002,0.76074,0.76723,0.76944,0.77018,0.77031,0.77036,0.77037
+recall_cap,0.6846,0.78343,0.83279,0.8883,0.92921,0.96192,0.97651,0.9862,0.99706
+hole,0.2952,0.64243,0.74542,0.83701,0.89174,0.9305,0.94534,0.95402,0.9628

eval_output/checkpoint-100000/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.685,
+            "NDCG@3": 0.7395,
+            "NDCG@5": 0.76137,
+            "NDCG@10": 0.78168,
+            "NDCG@20": 0.79418,
+            "NDCG@50": 0.80255,
+            "NDCG@100": 0.80571,
+            "NDCG@200": 0.80755,
+            "NDCG@1000": 0.80936
+        },
+        "map": {
+            "MAP@1": 0.59804,
+            "MAP@3": 0.69932,
+            "MAP@5": 0.71913,
+            "MAP@10": 0.73214,
+            "MAP@20": 0.73769,
+            "MAP@50": 0.74018,
+            "MAP@100": 0.74075,
+            "MAP@200": 0.74096,
+            "MAP@1000": 0.74108
+        },
+        "precision": {
+            "P@1": 0.685,
+            "P@3": 0.32323,
+            "P@5": 0.2163,
+            "P@10": 0.12095,
+            "P@20": 0.06584,
+            "P@50": 0.02832,
+            "P@100": 0.01467,
+            "P@200": 0.00752,
+            "P@1000": 0.00155
+        },
+        "recall": {
+            "Recall@1": 0.59804,
+            "Recall@3": 0.77072,
+            "Recall@5": 0.82798,
+            "Recall@10": 0.887,
+            "Recall@20": 0.92894,
+            "Recall@50": 0.9619,
+            "Recall@100": 0.97651,
+            "Recall@200": 0.9862,
+            "Recall@1000": 0.99706
+        },
+        "mrr": {
+            "MRR@1": 0.6846,
+            "MRR@3": 0.75002,
+            "MRR@5": 0.76074,
+            "MRR@10": 0.76723,
+            "MRR@20": 0.76944,
+            "MRR@50": 0.77018,
+            "MRR@100": 0.77031,
+            "MRR@200": 0.77036,
+            "MRR@1000": 0.77037
+        },
+        "recall_cap": {
+            "R_cap@1": 0.6846,
+            "R_cap@3": 0.78343,
+            "R_cap@5": 0.83279,
+            "R_cap@10": 0.8883,
+            "R_cap@20": 0.92921,
+            "R_cap@50": 0.96192,
+            "R_cap@100": 0.97651,
+            "R_cap@200": 0.9862,
+            "R_cap@1000": 0.99706
+        },
+        "hole": {
+            "Hole@1": 0.2952,
+            "Hole@3": 0.64243,
+            "Hole@5": 0.74542,
+            "Hole@10": 0.83701,
+            "Hole@20": 0.89174,
+            "Hole@50": 0.9305,
+            "Hole@100": 0.94534,
+            "Hole@200": 0.95402,
+            "Hole@1000": 0.9628
+        }
+    }
+}

eval_output/checkpoint-100000/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.185,0.14848,0.12947,0.15698,0.1812,0.20964,0.22858,0.24364,0.27844
+map,0.03768,0.06572,0.07756,0.09023,0.09798,0.10442,0.10697,0.10817,0.10949
+precision,0.185,0.138,0.113,0.0818,0.05495,0.03034,0.01859,0.01087,0.00306
+recall,0.03768,0.08418,0.11458,0.16563,0.2229,0.30743,0.37647,0.44063,0.62183
+mrr,0.185,0.24667,0.26352,0.27664,0.28379,0.28738,0.2884,0.28884,0.28913
+recall_cap,0.185,0.138,0.11458,0.16563,0.2229,0.30743,0.37647,0.44063,0.62183
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0