Add MTEB evaluation

evaluation/mteb/AmazonCounterfactualClassification.json

@@ -0,0 +1,16 @@
+{
+    "test": {
+        "en": {
+            "accuracy": 0.6922388059701492,
+            "accuracy_stderr": 0.05577698009153047,
+            "ap": 0.3204724673950256,
+            "ap_stderr": 0.030758638728823475,
+            "f1": 0.6325719825770428,
+            "f1_stderr": 0.04382866443453507,
+            "main_score": 0.3204724673950256
+        },
+        "evaluation_time": 234.06
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/AmazonPolarityClassification.json

@@ -0,0 +1,14 @@
+{
+    "test": {
+        "accuracy": 0.7126109999999999,
+        "accuracy_stderr": 0.0371266464483395,
+        "ap": 0.6616336378255403,
+        "ap_stderr": 0.038282983973039475,
+        "evaluation_time": 330535.51,
+        "f1": 0.7089719145825304,
+        "f1_stderr": 0.03949505400938336,
+        "main_score": 0.7126109999999999
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/AmazonReviewsClassification.json

@@ -0,0 +1,14 @@
+{
+    "test": {
+        "en": {
+            "accuracy": 0.39191999999999994,
+            "accuracy_stderr": 0.023273538622220733,
+            "f1": 0.38580766731113825,
+            "f1_stderr": 0.018793905233795604,
+            "main_score": 0.39191999999999994
+        },
+        "evaluation_time": 2060.82
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/ArguAna.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 477.21,
+    "map_at_1": 0.27312,
+    "map_at_10": 0.4262,
+    "map_at_100": 0.43707,
+    "map_at_1000": 0.43715,
+    "map_at_3": 0.37624,
+    "map_at_5": 0.40498,
+    "mrr_at_1": 0.27667,
+    "mrr_at_10": 0.42737,
+    "mrr_at_100": 0.43823,
+    "mrr_at_1000": 0.43831,
+    "mrr_at_3": 0.37743,
+    "mrr_at_5": 0.40616,
+    "ndcg_at_1": 0.27312,
+    "ndcg_at_10": 0.51375,
+    "ndcg_at_100": 0.55778,
+    "ndcg_at_1000": 0.55966,
+    "ndcg_at_3": 0.41087,
+    "ndcg_at_5": 0.46269,
+    "precision_at_1": 0.27312,
+    "precision_at_10": 0.07945,
+    "precision_at_100": 0.00982,
+    "precision_at_1000": 0.001,
+    "precision_at_3": 0.17046,
+    "precision_at_5": 0.12745,
+    "recall_at_1": 0.27312,
+    "recall_at_10": 0.79445,
+    "recall_at_100": 0.98151,
+    "recall_at_1000": 0.99573,
+    "recall_at_3": 0.51138,
+    "recall_at_5": 0.63727
+  }
+}

evaluation/mteb/ArxivClusteringP2P.json

@@ -0,0 +1,9 @@
+{
+    "test": {
+        "evaluation_time": 122260.97,
+        "v_measure": 0.4559037428592033,
+        "v_measure_std": 0.13565379029776853
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/ArxivClusteringS2S.json

@@ -0,0 +1,9 @@
+{
+    "test": {
+        "evaluation_time": 15111.26,
+        "v_measure": 0.3886371701986363,
+        "v_measure_std": 0.13693041258907623
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/AskUbuntuDupQuestions.json

@@ -0,0 +1,9 @@
+{
+    "test": {
+        "evaluation_time": 129.15,
+        "map": 0.6162556869142777,
+        "mrr": 0.7583256386580486
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/BIOSSES.json

@@ -0,0 +1,17 @@
+{
+  "test": {
+    "cos_sim": {
+      "pearson": 0.8996074355094802,
+      "spearman": 0.862501580394454
+    },
+    "euclidean": {
+      "pearson": 0.8218427440380462,
+      "spearman": 0.8014760935017947
+    },
+    "evaluation_time": 13.54,
+    "manhattan": {
+      "pearson": 0.8224621578156392,
+      "spearman": 0.8000363016590163
+    }
+  }
+}

evaluation/mteb/Banking77Classification.json

@@ -0,0 +1,12 @@
+{
+    "test": {
+        "accuracy": 0.8449350649350649,
+        "accuracy_stderr": 0.005092747481478234,
+        "evaluation_time": 568.26,
+        "f1": 0.8442493432337361,
+        "f1_stderr": 0.005291617674599615,
+        "main_score": 0.8449350649350649
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/BiorxivClusteringP2P.json

@@ -0,0 +1,9 @@
+{
+    "test": {
+        "evaluation_time": 14349.06,
+        "v_measure": 0.3655145972298938,
+        "v_measure_std": 0.01180536429604858
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/BiorxivClusteringS2S.json

@@ -0,0 +1,9 @@
+{
+    "test": {
+        "evaluation_time": 1826.36,
+        "v_measure": 0.3369901851846774,
+        "v_measure_std": 0.011822688018810827
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/CQADupstackAndroidRetrieval.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 2419.91,
+    "map_at_1": 0.30499,
+    "map_at_10": 0.41208,
+    "map_at_100": 0.42638,
+    "map_at_1000": 0.42754,
+    "map_at_3": 0.37506,
+    "map_at_5": 0.39422,
+    "mrr_at_1": 0.37339,
+    "mrr_at_10": 0.47051,
+    "mrr_at_100": 0.47745,
+    "mrr_at_1000": 0.47786,
+    "mrr_at_3": 0.44087,
+    "mrr_at_5": 0.45711,
+    "ndcg_at_1": 0.37339,
+    "ndcg_at_10": 0.47666,
+    "ndcg_at_100": 0.52994,
+    "ndcg_at_1000": 0.54929,
+    "ndcg_at_3": 0.41982,
+    "ndcg_at_5": 0.4442,
+    "precision_at_1": 0.37339,
+    "precision_at_10": 0.09127,
+    "precision_at_100": 0.01475,
+    "precision_at_1000": 0.00194,
+    "precision_at_3": 0.20076,
+    "precision_at_5": 0.14449,
+    "recall_at_1": 0.30499,
+    "recall_at_10": 0.60328,
+    "recall_at_100": 0.82579,
+    "recall_at_1000": 0.95074,
+    "recall_at_3": 0.4417,
+    "recall_at_5": 0.5094
+  }
+}

evaluation/mteb/CQADupstackEnglishRetrieval.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 3724.38,
+    "map_at_1": 0.30613,
+    "map_at_10": 0.40781,
+    "map_at_100": 0.42018,
+    "map_at_1000": 0.42133,
+    "map_at_3": 0.37816,
+    "map_at_5": 0.39389,
+    "mrr_at_1": 0.38408,
+    "mrr_at_10": 0.46631,
+    "mrr_at_100": 0.47332,
+    "mrr_at_1000": 0.47368,
+    "mrr_at_3": 0.44384,
+    "mrr_at_5": 0.45661,
+    "ndcg_at_1": 0.38408,
+    "ndcg_at_10": 0.4638,
+    "ndcg_at_100": 0.5081,
+    "ndcg_at_1000": 0.52663,
+    "ndcg_at_3": 0.4218,
+    "ndcg_at_5": 0.43974,
+    "precision_at_1": 0.38408,
+    "precision_at_10": 0.08656,
+    "precision_at_100": 0.01386,
+    "precision_at_1000": 0.00184,
+    "precision_at_3": 0.20276,
+    "precision_at_5": 0.14242,
+    "recall_at_1": 0.30613,
+    "recall_at_10": 0.5644,
+    "recall_at_100": 0.75044,
+    "recall_at_1000": 0.86426,
+    "recall_at_3": 0.43766,
+    "recall_at_5": 0.48998
+  }
+}

evaluation/mteb/CQADupstackGamingRetrieval.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 3971.77,
+    "map_at_1": 0.37371,
+    "map_at_10": 0.49718,
+    "map_at_100": 0.50737,
+    "map_at_1000": 0.5079,
+    "map_at_3": 0.46231,
+    "map_at_5": 0.48329,
+    "mrr_at_1": 0.42884,
+    "mrr_at_10": 0.53176,
+    "mrr_at_100": 0.53817,
+    "mrr_at_1000": 0.53845,
+    "mrr_at_3": 0.50199,
+    "mrr_at_5": 0.5213,
+    "ndcg_at_1": 0.42884,
+    "ndcg_at_10": 0.55826,
+    "ndcg_at_100": 0.5993,
+    "ndcg_at_1000": 0.61013,
+    "ndcg_at_3": 0.49764,
+    "ndcg_at_5": 0.53026,
+    "precision_at_1": 0.42884,
+    "precision_at_10": 0.09047,
+    "precision_at_100": 0.01212,
+    "precision_at_1000": 0.00135,
+    "precision_at_3": 0.22132,
+    "precision_at_5": 0.15524,
+    "recall_at_1": 0.37371,
+    "recall_at_10": 0.70482,
+    "recall_at_100": 0.88425,
+    "recall_at_1000": 0.96034,
+    "recall_at_3": 0.5443,
+    "recall_at_5": 0.62328
+  }
+}

evaluation/mteb/CQADupstackGisRetrieval.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 5450.64,
+    "map_at_1": 0.22876,
+    "map_at_10": 0.31715,
+    "map_at_100": 0.32847,
+    "map_at_1000": 0.32922,
+    "map_at_3": 0.2905,
+    "map_at_5": 0.30396,
+    "mrr_at_1": 0.2452,
+    "mrr_at_10": 0.33497,
+    "mrr_at_100": 0.34455,
+    "mrr_at_1000": 0.3451,
+    "mrr_at_3": 0.30791,
+    "mrr_at_5": 0.32175,
+    "ndcg_at_1": 0.2452,
+    "ndcg_at_10": 0.3695,
+    "ndcg_at_100": 0.42238,
+    "ndcg_at_1000": 0.44148,
+    "ndcg_at_3": 0.31435,
+    "ndcg_at_5": 0.33839,
+    "precision_at_1": 0.2452,
+    "precision_at_10": 0.05932,
+    "precision_at_100": 0.00901,
+    "precision_at_1000": 0.0011,
+    "precision_at_3": 0.13446,
+    "precision_at_5": 0.09469,
+    "recall_at_1": 0.22876,
+    "recall_at_10": 0.5138,
+    "recall_at_100": 0.75311,
+    "recall_at_1000": 0.89718,
+    "recall_at_3": 0.3626,
+    "recall_at_5": 0.42249
+  }
+}

evaluation/mteb/CQADupstackMathematicaRetrieval.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 2931.62,
+    "map_at_1": 0.14984,
+    "map_at_10": 0.23457,
+    "map_at_100": 0.24723,
+    "map_at_1000": 0.24846,
+    "map_at_3": 0.20873,
+    "map_at_5": 0.22357,
+    "mrr_at_1": 0.18159,
+    "mrr_at_10": 0.27431,
+    "mrr_at_100": 0.28449,
+    "mrr_at_1000": 0.2852,
+    "mrr_at_3": 0.24979,
+    "mrr_at_5": 0.26447,
+    "ndcg_at_1": 0.18159,
+    "ndcg_at_10": 0.28628,
+    "ndcg_at_100": 0.34741,
+    "ndcg_at_1000": 0.37516,
+    "ndcg_at_3": 0.23902,
+    "ndcg_at_5": 0.26294,
+    "precision_at_1": 0.18159,
+    "precision_at_10": 0.05485,
+    "precision_at_100": 0.00985,
+    "precision_at_1000": 0.00136,
+    "precision_at_3": 0.11774,
+    "precision_at_5": 0.08731,
+    "recall_at_1": 0.14984,
+    "recall_at_10": 0.40198,
+    "recall_at_100": 0.67115,
+    "recall_at_1000": 0.86497,
+    "recall_at_3": 0.27639,
+    "recall_at_5": 0.33595
+  }
+}

evaluation/mteb/CQADupstackPhysicsRetrieval.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 5338.65,
+    "map_at_1": 0.29067,
+    "map_at_10": 0.39457,
+    "map_at_100": 0.4083,
+    "map_at_1000": 0.4094,
+    "map_at_3": 0.35995,
+    "map_at_5": 0.38159,
+    "mrr_at_1": 0.34937,
+    "mrr_at_10": 0.44755,
+    "mrr_at_100": 0.45549,
+    "mrr_at_1000": 0.45589,
+    "mrr_at_3": 0.41947,
+    "mrr_at_5": 0.43733,
+    "ndcg_at_1": 0.34937,
+    "ndcg_at_10": 0.45573,
+    "ndcg_at_100": 0.51267,
+    "ndcg_at_1000": 0.53184,
+    "ndcg_at_3": 0.39962,
+    "ndcg_at_5": 0.4302,
+    "precision_at_1": 0.34937,
+    "precision_at_10": 0.08296,
+    "precision_at_100": 0.0132,
+    "precision_at_1000": 0.00167,
+    "precision_at_3": 0.188,
+    "precision_at_5": 0.13763,
+    "recall_at_1": 0.29067,
+    "recall_at_10": 0.58298,
+    "recall_at_100": 0.82251,
+    "recall_at_1000": 0.94476,
+    "recall_at_3": 0.42984,
+    "recall_at_5": 0.50658
+  }
+}

evaluation/mteb/CQADupstackProgrammersRetrieval.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 4979.86,
+    "map_at_1": 0.25986,
+    "map_at_10": 0.35746,
+    "map_at_100": 0.37067,
+    "map_at_1000": 0.37191,
+    "map_at_3": 0.32599,
+    "map_at_5": 0.34239,
+    "mrr_at_1": 0.31735,
+    "mrr_at_10": 0.40515,
+    "mrr_at_100": 0.41459,
+    "mrr_at_1000": 0.41516,
+    "mrr_at_3": 0.37938,
+    "mrr_at_5": 0.3925,
+    "ndcg_at_1": 0.31735,
+    "ndcg_at_10": 0.41484,
+    "ndcg_at_100": 0.47047,
+    "ndcg_at_1000": 0.49427,
+    "ndcg_at_3": 0.36255,
+    "ndcg_at_5": 0.38375,
+    "precision_at_1": 0.31735,
+    "precision_at_10": 0.0766,
+    "precision_at_100": 0.01234,
+    "precision_at_1000": 0.0016,
+    "precision_at_3": 0.17428,
+    "precision_at_5": 0.12329,
+    "recall_at_1": 0.25986,
+    "recall_at_10": 0.53761,
+    "recall_at_100": 0.77149,
+    "recall_at_1000": 0.93342,
+    "recall_at_3": 0.39068,
+    "recall_at_5": 0.44693
+  }
+}

evaluation/mteb/CQADupstackRetrieval.json

@@ -0,0 +1,13 @@
+{
+    "dataset_version": null,
+    "mteb_version": "0.0.2",
+    "test": {
+        "evaluation_time": 7822.83,
+        "ndcg_at_1": 0.2821458333333333,
+        "ndcg_at_3": 0.3325866666666667,
+        "ndcg_at_5": 0.35453333333333326,
+        "ndcg_at_10": 0.381195,
+        "ndcg_at_100": 0.43441,
+        "ndcg_at_1000": 0.45826083333333334
+    }
+}

evaluation/mteb/CQADupstackStatsRetrieval.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 6837.54,
+    "map_at_1": 0.22082,
+    "map_at_10": 0.29216,
+    "map_at_100": 0.30163,
+    "map_at_1000": 0.30269,
+    "map_at_3": 0.26942,
+    "map_at_5": 0.28236,
+    "mrr_at_1": 0.24847,
+    "mrr_at_10": 0.31919,
+    "mrr_at_100": 0.32817,
+    "mrr_at_1000": 0.32897,
+    "mrr_at_3": 0.29831,
+    "mrr_at_5": 0.3102,
+    "ndcg_at_1": 0.24847,
+    "ndcg_at_10": 0.334,
+    "ndcg_at_100": 0.38354,
+    "ndcg_at_1000": 0.41045,
+    "ndcg_at_3": 0.29236,
+    "ndcg_at_5": 0.31258,
+    "precision_at_1": 0.24847,
+    "precision_at_10": 0.05353,
+    "precision_at_100": 0.00853,
+    "precision_at_1000": 0.00116,
+    "precision_at_3": 0.12679,
+    "precision_at_5": 0.08988,
+    "recall_at_1": 0.22082,
+    "recall_at_10": 0.43505,
+    "recall_at_100": 0.66454,
+    "recall_at_1000": 0.86378,
+    "recall_at_3": 0.32163,
+    "recall_at_5": 0.3706
+  }
+}

evaluation/mteb/CQADupstackTexRetrieval.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 11622.05,
+    "map_at_1": 0.1554,
+    "map_at_10": 0.22362,
+    "map_at_100": 0.23435,
+    "map_at_1000": 0.23564,
+    "map_at_3": 0.20143,
+    "map_at_5": 0.21324,
+    "mrr_at_1": 0.18892,
+    "mrr_at_10": 0.25943,
+    "mrr_at_100": 0.26883,
+    "mrr_at_1000": 0.26969,
+    "mrr_at_3": 0.23727,
+    "mrr_at_5": 0.24923,
+    "ndcg_at_1": 0.18892,
+    "ndcg_at_10": 0.26811,
+    "ndcg_at_100": 0.32066,
+    "ndcg_at_1000": 0.35166,
+    "ndcg_at_3": 0.22706,
+    "ndcg_at_5": 0.24508,
+    "precision_at_1": 0.18892,
+    "precision_at_10": 0.04942,
+    "precision_at_100": 0.00878,
+    "precision_at_1000": 0.00131,
+    "precision_at_3": 0.10748,
+    "precision_at_5": 0.07784,
+    "recall_at_1": 0.1554,
+    "recall_at_10": 0.36743,
+    "recall_at_100": 0.60525,
+    "recall_at_1000": 0.82576,
+    "recall_at_3": 0.25252,
+    "recall_at_5": 0.29872
+  }
+}

evaluation/mteb/CQADupstackUnixRetrieval.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 7165.48,
+    "map_at_1": 0.24453,
+    "map_at_10": 0.33363,
+    "map_at_100": 0.34579,
+    "map_at_1000": 0.34686,
+    "map_at_3": 0.30583,
+    "map_at_5": 0.32118,
+    "mrr_at_1": 0.28918,
+    "mrr_at_10": 0.37675,
+    "mrr_at_100": 0.38567,
+    "mrr_at_1000": 0.38632,
+    "mrr_at_3": 0.35261,
+    "mrr_at_5": 0.36576,
+    "ndcg_at_1": 0.28918,
+    "ndcg_at_10": 0.38736,
+    "ndcg_at_100": 0.44261,
+    "ndcg_at_1000": 0.4672,
+    "ndcg_at_3": 0.3381,
+    "ndcg_at_5": 0.36009,
+    "precision_at_1": 0.28918,
+    "precision_at_10": 0.06586,
+    "precision_at_100": 0.01047,
+    "precision_at_1000": 0.00137,
+    "precision_at_3": 0.15361,
+    "precision_at_5": 0.10858,
+    "recall_at_1": 0.24453,
+    "recall_at_10": 0.50886,
+    "recall_at_100": 0.7503,
+    "recall_at_1000": 0.92123,
+    "recall_at_3": 0.37138,
+    "recall_at_5": 0.42865
+  }
+}

evaluation/mteb/CQADupstackWebmastersRetrieval.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 2264.66,
+    "map_at_1": 0.2457,
+    "map_at_10": 0.33672,
+    "map_at_100": 0.35244,
+    "map_at_1000": 0.35467,
+    "map_at_3": 0.30712,
+    "map_at_5": 0.32383,
+    "mrr_at_1": 0.29644,
+    "mrr_at_10": 0.38344,
+    "mrr_at_100": 0.39219,
+    "mrr_at_1000": 0.39282,
+    "mrr_at_3": 0.35771,
+    "mrr_at_5": 0.37273,
+    "ndcg_at_1": 0.29644,
+    "ndcg_at_10": 0.39567,
+    "ndcg_at_100": 0.45097,
+    "ndcg_at_1000": 0.47923,
+    "ndcg_at_3": 0.34768,
+    "ndcg_at_5": 0.37122,
+    "precision_at_1": 0.29644,
+    "precision_at_10": 0.07589,
+    "precision_at_100": 0.01478,
+    "precision_at_1000": 0.00235,
+    "precision_at_3": 0.16337,
+    "precision_at_5": 0.12055,
+    "recall_at_1": 0.2457,
+    "recall_at_10": 0.51009,
+    "recall_at_100": 0.75423,
+    "recall_at_1000": 0.93671,
+    "recall_at_3": 0.36926,
+    "recall_at_5": 0.43245
+  }
+}

evaluation/mteb/CQADupstackWordpressRetrieval.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 7608.65,
+    "map_at_1": 0.21356,
+    "map_at_10": 0.27904,
+    "map_at_100": 0.28938,
+    "map_at_1000": 0.29036,
+    "map_at_3": 0.25726,
+    "map_at_5": 0.26935,
+    "mrr_at_1": 0.22551,
+    "mrr_at_10": 0.29259,
+    "mrr_at_100": 0.30272,
+    "mrr_at_1000": 0.30348,
+    "mrr_at_3": 0.27295,
+    "mrr_at_5": 0.28358,
+    "ndcg_at_1": 0.22551,
+    "ndcg_at_10": 0.31817,
+    "ndcg_at_100": 0.37164,
+    "ndcg_at_1000": 0.3982,
+    "ndcg_at_3": 0.27596,
+    "ndcg_at_5": 0.29568,
+    "precision_at_1": 0.22551,
+    "precision_at_10": 0.04917,
+    "precision_at_100": 0.00828,
+    "precision_at_1000": 0.00114,
+    "precision_at_3": 0.11583,
+    "precision_at_5": 0.08133,
+    "recall_at_1": 0.21356,
+    "recall_at_10": 0.42489,
+    "recall_at_100": 0.67128,
+    "recall_at_1000": 0.87441,
+    "recall_at_3": 0.31165,
+    "recall_at_5": 0.35853
+  }
+}

evaluation/mteb/ClimateFEVER.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 64667.21,
+    "map_at_1": 0.12306,
+    "map_at_10": 0.21523,
+    "map_at_100": 0.23358,
+    "map_at_1000": 0.23541,
+    "map_at_3": 0.17809,
+    "map_at_5": 0.19631,
+    "mrr_at_1": 0.27948,
+    "mrr_at_10": 0.40355,
+    "mrr_at_100": 0.41166,
+    "mrr_at_1000": 0.41203,
+    "mrr_at_3": 0.36819,
+    "mrr_at_5": 0.38959,
+    "ndcg_at_1": 0.27948,
+    "ndcg_at_10": 0.30462,
+    "ndcg_at_100": 0.37473,
+    "ndcg_at_1000": 0.40718,
+    "ndcg_at_3": 0.24646,
+    "ndcg_at_5": 0.26642,
+    "precision_at_1": 0.27948,
+    "precision_at_10": 0.09648,
+    "precision_at_100": 0.01724,
+    "precision_at_1000": 0.00232,
+    "precision_at_3": 0.1848,
+    "precision_at_5": 0.14293,
+    "recall_at_1": 0.12306,
+    "recall_at_10": 0.37181,
+    "recall_at_100": 0.61148,
+    "recall_at_1000": 0.79401,
+    "recall_at_3": 0.22883,
+    "recall_at_5": 0.2859
+  }
+}

evaluation/mteb/DBPedia.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 43004.49,
+    "map_at_1": 0.09357,
+    "map_at_10": 0.18849,
+    "map_at_100": 0.25369,
+    "map_at_1000": 0.2695,
+    "map_at_3": 0.13625,
+    "map_at_5": 0.15957,
+    "mrr_at_1": 0.6775,
+    "mrr_at_10": 0.74734,
+    "mrr_at_100": 0.751,
+    "mrr_at_1000": 0.75109,
+    "mrr_at_3": 0.73542,
+    "mrr_at_5": 0.74167,
+    "ndcg_at_1": 0.55375,
+    "ndcg_at_10": 0.39874,
+    "ndcg_at_100": 0.43098,
+    "ndcg_at_1000": 0.50692,
+    "ndcg_at_3": 0.44856,
+    "ndcg_at_5": 0.42139,
+    "precision_at_1": 0.6775,
+    "precision_at_10": 0.311,
+    "precision_at_100": 0.09303,
+    "precision_at_1000": 0.02006,
+    "precision_at_3": 0.4825,
+    "precision_at_5": 0.4095,
+    "recall_at_1": 0.09357,
+    "recall_at_10": 0.23832,
+    "recall_at_100": 0.47906,
+    "recall_at_1000": 0.71309,
+    "recall_at_3": 0.14512,
+    "recall_at_5": 0.183
+  }
+}

evaluation/mteb/EmotionClassification.json

@@ -0,0 +1,12 @@
+{
+    "test": {
+        "accuracy": 0.49655000000000005,
+        "accuracy_stderr": 0.023384236143179877,
+        "evaluation_time": 408.2,
+        "f1": 0.45519761909389506,
+        "f1_stderr": 0.017373883846959346,
+        "main_score": 0.49655000000000005
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/FEVER.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 65632.31,
+    "map_at_1": 0.6274,
+    "map_at_10": 0.7307,
+    "map_at_100": 0.73398,
+    "map_at_1000": 0.7341,
+    "map_at_3": 0.71338,
+    "map_at_5": 0.72423,
+    "mrr_at_1": 0.67777,
+    "mrr_at_10": 0.77873,
+    "mrr_at_100": 0.78091,
+    "mrr_at_1000": 0.78094,
+    "mrr_at_3": 0.76375,
+    "mrr_at_5": 0.77316,
+    "ndcg_at_1": 0.67777,
+    "ndcg_at_10": 0.7824,
+    "ndcg_at_100": 0.79557,
+    "ndcg_at_1000": 0.79814,
+    "ndcg_at_3": 0.75125,
+    "ndcg_at_5": 0.76834,
+    "precision_at_1": 0.67777,
+    "precision_at_10": 0.09832,
+    "precision_at_100": 0.01061,
+    "precision_at_1000": 0.0011,
+    "precision_at_3": 0.29433,
+    "precision_at_5": 0.18665,
+    "recall_at_1": 0.6274,
+    "recall_at_10": 0.89505,
+    "recall_at_100": 0.95102,
+    "recall_at_1000": 0.96825,
+    "recall_at_3": 0.81028,
+    "recall_at_5": 0.85281
+  }
+}

evaluation/mteb/FiQA2018.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 951.67,
+    "map_at_1": 0.18467,
+    "map_at_10": 0.30021,
+    "map_at_100": 0.31739,
+    "map_at_1000": 0.31934,
+    "map_at_3": 0.26003,
+    "map_at_5": 0.28338,
+    "mrr_at_1": 0.3534,
+    "mrr_at_10": 0.44109,
+    "mrr_at_100": 0.44993,
+    "mrr_at_1000": 0.45042,
+    "mrr_at_3": 0.41667,
+    "mrr_at_5": 0.4314,
+    "ndcg_at_1": 0.3534,
+    "ndcg_at_10": 0.37202,
+    "ndcg_at_100": 0.43853,
+    "ndcg_at_1000": 0.47235,
+    "ndcg_at_3": 0.335,
+    "ndcg_at_5": 0.34985,
+    "precision_at_1": 0.3534,
+    "precision_at_10": 0.10247,
+    "precision_at_100": 0.01715,
+    "precision_at_1000": 0.00232,
+    "precision_at_3": 0.22222,
+    "precision_at_5": 0.16574,
+    "recall_at_1": 0.18467,
+    "recall_at_10": 0.44081,
+    "recall_at_100": 0.68722,
+    "recall_at_1000": 0.89087,
+    "recall_at_3": 0.30567,
+    "recall_at_5": 0.36982
+  }
+}

evaluation/mteb/HotpotQA.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 46237.87,
+    "map_at_1": 0.35726,
+    "map_at_10": 0.50207,
+    "map_at_100": 0.51055,
+    "map_at_1000": 0.51128,
+    "map_at_3": 0.47576,
+    "map_at_5": 0.49172,
+    "mrr_at_1": 0.71452,
+    "mrr_at_10": 0.77419,
+    "mrr_at_100": 0.77711,
+    "mrr_at_1000": 0.77723,
+    "mrr_at_3": 0.76394,
+    "mrr_at_5": 0.77001,
+    "ndcg_at_1": 0.71452,
+    "ndcg_at_10": 0.59261,
+    "ndcg_at_100": 0.62424,
+    "ndcg_at_1000": 0.63951,
+    "ndcg_at_3": 0.55327,
+    "ndcg_at_5": 0.57417,
+    "precision_at_1": 0.71452,
+    "precision_at_10": 0.12061,
+    "precision_at_100": 0.01455,
+    "precision_at_1000": 0.00166,
+    "precision_at_3": 0.3436,
+    "precision_at_5": 0.22266,
+    "recall_at_1": 0.35726,
+    "recall_at_10": 0.60304,
+    "recall_at_100": 0.72755,
+    "recall_at_1000": 0.82978,
+    "recall_at_3": 0.5154,
+    "recall_at_5": 0.55665
+  }
+}

evaluation/mteb/ImdbClassification.json

@@ -0,0 +1,14 @@
+{
+    "test": {
+        "accuracy": 0.666376,
+        "accuracy_stderr": 0.03435007633179292,
+        "ap": 0.6148938261286748,
+        "ap_stderr": 0.028266712412564548,
+        "evaluation_time": 38609.7,
+        "f1": 0.6635089269264965,
+        "f1_stderr": 0.03526614235815782,
+        "main_score": 0.666376
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/MSMARCO.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "validation": {
+    "evaluation_time": 82887.84,
+    "map_at_1": 0.20842,
+    "map_at_10": 0.32992,
+    "map_at_100": 0.34236,
+    "map_at_1000": 0.34286,
+    "map_at_3": 0.29049,
+    "map_at_5": 0.31392,
+    "mrr_at_1": 0.21375,
+    "mrr_at_10": 0.33581,
+    "mrr_at_100": 0.3476,
+    "mrr_at_1000": 0.34803,
+    "mrr_at_3": 0.29704,
+    "mrr_at_5": 0.32015,
+    "ndcg_at_1": 0.21375,
+    "ndcg_at_10": 0.39905,
+    "ndcg_at_100": 0.45843,
+    "ndcg_at_1000": 0.47084,
+    "ndcg_at_3": 0.31919,
+    "ndcg_at_5": 0.36107,
+    "precision_at_1": 0.21375,
+    "precision_at_10": 0.06393,
+    "precision_at_100": 0.00935,
+    "precision_at_1000": 0.00104,
+    "precision_at_3": 0.13663,
+    "precision_at_5": 0.10324,
+    "recall_at_1": 0.20842,
+    "recall_at_10": 0.6117,
+    "recall_at_100": 0.88518,
+    "recall_at_1000": 0.97993,
+    "recall_at_3": 0.39571,
+    "recall_at_5": 0.49654
+  }
+}

evaluation/mteb/MTOPDomainClassification.json

@@ -0,0 +1,14 @@
+{
+    "test": {
+        "en": {
+            "accuracy": 0.9346557227542178,
+            "accuracy_stderr": 0.0072510589372134955,
+            "f1": 0.9287345917772146,
+            "f1_stderr": 0.0082627627462556,
+            "main_score": 0.9346557227542178
+        },
+        "evaluation_time": 475.45
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/MTOPIntentClassification.json

@@ -0,0 +1,14 @@
+{
+    "test": {
+        "en": {
+            "accuracy": 0.7242134062927497,
+            "accuracy_stderr": 0.012571448106282981,
+            "f1": 0.5503624810959269,
+            "f1_stderr": 0.011737816802681789,
+            "main_score": 0.7242134062927497
+        },
+        "evaluation_time": 612.9
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/MassiveIntentClassification.json

@@ -0,0 +1,14 @@
+{
+    "test": {
+        "en": {
+            "accuracy": 0.703866845998655,
+            "accuracy_stderr": 0.00456616151893328,
+            "f1": 0.6896745198729209,
+            "f1_stderr": 0.006294250065644286,
+            "main_score": 0.703866845998655
+        },
+        "evaluation_time": 387.35
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/MassiveScenarioClassification.json

@@ -0,0 +1,14 @@
+{
+    "test": {
+        "en": {
+            "accuracy": 0.7627774041694687,
+            "accuracy_stderr": 0.014020552186569309,
+            "f1": 0.7672936190462792,
+            "f1_stderr": 0.01203156406861017,
+            "main_score": 0.7627774041694687
+        },
+        "evaluation_time": 333.83
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/MedrxivClusteringP2P.json

@@ -0,0 +1,9 @@
+{
+    "test": {
+        "evaluation_time": 7211.78,
+        "v_measure": 0.3151174592577334,
+        "v_measure_std": 0.012772027882021399
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/MedrxivClusteringS2S.json

@@ -0,0 +1,9 @@
+{
+    "test": {
+        "evaluation_time": 1053.59,
+        "v_measure": 0.28764235987575365,
+        "v_measure_std": 0.011424256185368544
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/MindSmallReranking.json

@@ -0,0 +1,9 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 72941.42,
+    "map": 0.32293531363866007,
+    "mrr": 0.33536774455851687
+  }
+}

evaluation/mteb/NFCorpus.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 166.79,
+    "map_at_1": 0.05702,
+    "map_at_10": 0.13642,
+    "map_at_100": 0.17503,
+    "map_at_1000": 0.19126,
+    "map_at_3": 0.09748,
+    "map_at_5": 0.11642,
+    "mrr_at_1": 0.4582,
+    "mrr_at_10": 0.54821,
+    "mrr_at_100": 0.55422,
+    "mrr_at_1000": 0.55453,
+    "mrr_at_3": 0.52374,
+    "mrr_at_5": 0.53937,
+    "ndcg_at_1": 0.44272,
+    "ndcg_at_10": 0.36213,
+    "ndcg_at_100": 0.33829,
+    "ndcg_at_1000": 0.42557,
+    "ndcg_at_3": 0.40814,
+    "ndcg_at_5": 0.39562,
+    "precision_at_1": 0.45511,
+    "precision_at_10": 0.27214,
+    "precision_at_100": 0.08941,
+    "precision_at_1000": 0.02187,
+    "precision_at_3": 0.37874,
+    "precision_at_5": 0.34489,
+    "recall_at_1": 0.05702,
+    "recall_at_10": 0.17638,
+    "recall_at_100": 0.34419,
+    "recall_at_1000": 0.6641,
+    "recall_at_3": 0.10914,
+    "recall_at_5": 0.14032
+  }
+}

evaluation/mteb/NQ.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 31038.1,
+    "map_at_1": 0.30567,
+    "map_at_10": 0.4501,
+    "map_at_100": 0.46091,
+    "map_at_1000": 0.46126,
+    "map_at_3": 0.40897,
+    "map_at_5": 0.43301,
+    "mrr_at_1": 0.3456,
+    "mrr_at_10": 0.47725,
+    "mrr_at_100": 0.48548,
+    "mrr_at_1000": 0.48572,
+    "mrr_at_3": 0.44361,
+    "mrr_at_5": 0.46351,
+    "ndcg_at_1": 0.34531,
+    "ndcg_at_10": 0.5241,
+    "ndcg_at_100": 0.56999,
+    "ndcg_at_1000": 0.57831,
+    "ndcg_at_3": 0.44734,
+    "ndcg_at_5": 0.48701,
+    "precision_at_1": 0.34531,
+    "precision_at_10": 0.08612,
+    "precision_at_100": 0.01118,
+    "precision_at_1000": 0.0012,
+    "precision_at_3": 0.20307,
+    "precision_at_5": 0.14519,
+    "recall_at_1": 0.30567,
+    "recall_at_10": 0.72238,
+    "recall_at_100": 0.92154,
+    "recall_at_1000": 0.98375,
+    "recall_at_3": 0.52438,
+    "recall_at_5": 0.61517
+  }
+}

evaluation/mteb/QuoraRetrieval.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 1922.86,
+    "map_at_1": 0.6598,
+    "map_at_10": 0.80056,
+    "map_at_100": 0.80763,
+    "map_at_1000": 0.80786,
+    "map_at_3": 0.76848,
+    "map_at_5": 0.78854,
+    "mrr_at_1": 0.7586,
+    "mrr_at_10": 0.83397,
+    "mrr_at_100": 0.83555,
+    "mrr_at_1000": 0.83557,
+    "mrr_at_3": 0.82033,
+    "mrr_at_5": 0.8297,
+    "ndcg_at_1": 0.7588,
+    "ndcg_at_10": 0.84581,
+    "ndcg_at_100": 0.86151,
+    "ndcg_at_1000": 0.86315,
+    "ndcg_at_3": 0.80902,
+    "ndcg_at_5": 0.82953,
+    "precision_at_1": 0.7588,
+    "precision_at_10": 0.12986,
+    "precision_at_100": 0.01511,
+    "precision_at_1000": 0.00156,
+    "precision_at_3": 0.35383,
+    "precision_at_5": 0.23556,
+    "recall_at_1": 0.6598,
+    "recall_at_10": 0.93716,
+    "recall_at_100": 0.99218,
+    "recall_at_1000": 0.9997,
+    "recall_at_3": 0.83551,
+    "recall_at_5": 0.88998
+  }
+}

evaluation/mteb/RedditClustering.json

@@ -0,0 +1,9 @@
+{
+    "test": {
+        "evaluation_time": 14260.04,
+        "v_measure": 0.4045148482612238,
+        "v_measure_std": 0.10798668357466681
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/RedditClusteringP2P.json

@@ -0,0 +1,9 @@
+{
+    "dataset_version": null,
+    "mteb_version": "0.0.2",
+    "test": {
+        "evaluation_time": 57925.8,
+        "v_measure": 0.5574949067303913,
+        "v_measure_std": 0.12781562321045892
+    }
+}

evaluation/mteb/SCIDOCS.json

@@ -0,0 +1,37 @@
+{
+  "dataset_version": null,
+  "mteb_version": "0.0.2",
+  "test": {
+    "evaluation_time": 626.47,
+    "map_at_1": 0.04903,
+    "map_at_10": 0.11926,
+    "map_at_100": 0.13917,
+    "map_at_1000": 0.14215,
+    "map_at_3": 0.088,
+    "map_at_5": 0.10361,
+    "mrr_at_1": 0.241,
+    "mrr_at_10": 0.34482,
+    "mrr_at_100": 0.35566,
+    "mrr_at_1000": 0.35619,
+    "mrr_at_3": 0.31433,
+    "mrr_at_5": 0.33243,
+    "ndcg_at_1": 0.241,
+    "ndcg_at_10": 0.19873,
+    "ndcg_at_100": 0.27606,
+    "ndcg_at_1000": 0.32811,
+    "ndcg_at_3": 0.19498,
+    "ndcg_at_5": 0.16813,
+    "precision_at_1": 0.241,
+    "precision_at_10": 0.1008,
+    "precision_at_100": 0.02122,
+    "precision_at_1000": 0.00337,
+    "precision_at_3": 0.182,
+    "precision_at_5": 0.1462,
+    "recall_at_1": 0.04903,
+    "recall_at_10": 0.20438,
+    "recall_at_100": 0.43043,
+    "recall_at_1000": 0.6841,
+    "recall_at_3": 0.11068,
+    "recall_at_5": 0.14818
+  }
+}

evaluation/mteb/SICK-R.json

@@ -0,0 +1,17 @@
+{
+  "test": {
+    "cos_sim": {
+      "pearson": 0.7858086597995997,
+      "spearman": 0.6963214182814992
+    },
+    "euclidean": {
+      "pearson": 0.7276175489042691,
+      "spearman": 0.6784965161872971
+    },
+    "evaluation_time": 199.7,
+    "manhattan": {
+      "pearson": 0.7273812689782593,
+      "spearman": 0.6783610439531278
+    }
+  }
+}

evaluation/mteb/STS12.json

@@ -0,0 +1,19 @@
+{
+    "test": {
+        "cos_sim": {
+            "pearson": 0.7513970861325007,
+            "spearman": 0.675020551515597
+        },
+        "euclidean": {
+            "pearson": 0.6633415412418275,
+            "spearman": 0.6682145056673268
+        },
+        "evaluation_time": 100.4,
+        "manhattan": {
+            "pearson": 0.6655489484006415,
+            "spearman": 0.6695147433279057
+        }
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/STS13.json

@@ -0,0 +1,19 @@
+{
+    "test": {
+        "cos_sim": {
+            "pearson": 0.7885850536483447,
+            "spearman": 0.7916333501772059
+        },
+        "euclidean": {
+            "pearson": 0.7274090561408476,
+            "spearman": 0.7357374448302961
+        },
+        "evaluation_time": 43.53,
+        "manhattan": {
+            "pearson": 0.7292980654233225,
+            "spearman": 0.7372777155112589
+        }
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}

evaluation/mteb/STS14.json

@@ -0,0 +1,19 @@
+{
+    "test": {
+        "cos_sim": {
+            "pearson": 0.7951125593897028,
+            "spearman": 0.7446048326701329
+        },
+        "euclidean": {
+            "pearson": 0.7087726087052986,
+            "spearman": 0.677721470654411
+        },
+        "evaluation_time": 107.32,
+        "manhattan": {
+            "pearson": 0.7105892792135637,
+            "spearman": 0.6793472619779036
+        }
+    },
+    "dataset_version": null,
+    "mteb_version": "0.0.2"
+}