diff --git a/beir.json b/evaluation/beir/beir.json similarity index 100% rename from beir.json rename to evaluation/beir/beir.json diff --git a/evaluation/mteb/AmazonCounterfactualClassification.json b/evaluation/mteb/AmazonCounterfactualClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..59d8dca041bcb951fe16425e7009328aa7b2f98f --- /dev/null +++ b/evaluation/mteb/AmazonCounterfactualClassification.json @@ -0,0 +1,16 @@ +{ + "test": { + "en": { + "accuracy": 0.6922388059701492, + "accuracy_stderr": 0.05577698009153047, + "ap": 0.3204724673950256, + "ap_stderr": 0.030758638728823475, + "f1": 0.6325719825770428, + "f1_stderr": 0.04382866443453507, + "main_score": 0.3204724673950256 + }, + "evaluation_time": 234.06 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/AmazonPolarityClassification.json b/evaluation/mteb/AmazonPolarityClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..ee9a28221e559e6e2e46358bbe04cb495a8303b8 --- /dev/null +++ b/evaluation/mteb/AmazonPolarityClassification.json @@ -0,0 +1,14 @@ +{ + "test": { + "accuracy": 0.7126109999999999, + "accuracy_stderr": 0.0371266464483395, + "ap": 0.6616336378255403, + "ap_stderr": 0.038282983973039475, + "evaluation_time": 330535.51, + "f1": 0.7089719145825304, + "f1_stderr": 0.03949505400938336, + "main_score": 0.7126109999999999 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/AmazonReviewsClassification.json b/evaluation/mteb/AmazonReviewsClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..1a5f5c20a1eb60f5f8f662b26608f654fba54e18 --- /dev/null +++ b/evaluation/mteb/AmazonReviewsClassification.json @@ -0,0 +1,14 @@ +{ + "test": { + "en": { + "accuracy": 0.39191999999999994, + "accuracy_stderr": 0.023273538622220733, + "f1": 0.38580766731113825, + "f1_stderr": 0.018793905233795604, + "main_score": 0.39191999999999994 + }, + "evaluation_time": 2060.82 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/ArguAna.json b/evaluation/mteb/ArguAna.json new file mode 100644 index 0000000000000000000000000000000000000000..3d823c40321156582506c10bcb8bb60f0208d949 --- /dev/null +++ b/evaluation/mteb/ArguAna.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 477.21, + "map_at_1": 0.27312, + "map_at_10": 0.4262, + "map_at_100": 0.43707, + "map_at_1000": 0.43715, + "map_at_3": 0.37624, + "map_at_5": 0.40498, + "mrr_at_1": 0.27667, + "mrr_at_10": 0.42737, + "mrr_at_100": 0.43823, + "mrr_at_1000": 0.43831, + "mrr_at_3": 0.37743, + "mrr_at_5": 0.40616, + "ndcg_at_1": 0.27312, + "ndcg_at_10": 0.51375, + "ndcg_at_100": 0.55778, + "ndcg_at_1000": 0.55966, + "ndcg_at_3": 0.41087, + "ndcg_at_5": 0.46269, + "precision_at_1": 0.27312, + "precision_at_10": 0.07945, + "precision_at_100": 0.00982, + "precision_at_1000": 0.001, + "precision_at_3": 0.17046, + "precision_at_5": 0.12745, + "recall_at_1": 0.27312, + "recall_at_10": 0.79445, + "recall_at_100": 0.98151, + "recall_at_1000": 0.99573, + "recall_at_3": 0.51138, + "recall_at_5": 0.63727 + } +} \ No newline at end of file diff --git a/evaluation/mteb/ArxivClusteringP2P.json b/evaluation/mteb/ArxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..3dbc777afff007aab6954750960944408c6df3e8 --- /dev/null +++ b/evaluation/mteb/ArxivClusteringP2P.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 122260.97, + "v_measure": 0.4559037428592033, + "v_measure_std": 0.13565379029776853 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/ArxivClusteringS2S.json b/evaluation/mteb/ArxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..e1b8c6c576db6d7da4ce5402b8eb1780a42a5b8e --- /dev/null +++ b/evaluation/mteb/ArxivClusteringS2S.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 15111.26, + "v_measure": 0.3886371701986363, + "v_measure_std": 0.13693041258907623 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/AskUbuntuDupQuestions.json b/evaluation/mteb/AskUbuntuDupQuestions.json new file mode 100644 index 0000000000000000000000000000000000000000..8bdd603491e7c3bf6ecb3fe104dc546bd70fb0a3 --- /dev/null +++ b/evaluation/mteb/AskUbuntuDupQuestions.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 129.15, + "map": 0.6162556869142777, + "mrr": 0.7583256386580486 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/BIOSSES.json b/evaluation/mteb/BIOSSES.json new file mode 100644 index 0000000000000000000000000000000000000000..dcacf18d20ea6724c69a97a6205e9fe9b4eea811 --- /dev/null +++ b/evaluation/mteb/BIOSSES.json @@ -0,0 +1,17 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.8996074355094802, + "spearman": 0.862501580394454 + }, + "euclidean": { + "pearson": 0.8218427440380462, + "spearman": 0.8014760935017947 + }, + "evaluation_time": 13.54, + "manhattan": { + "pearson": 0.8224621578156392, + "spearman": 0.8000363016590163 + } + } +} \ No newline at end of file diff --git a/evaluation/mteb/Banking77Classification.json b/evaluation/mteb/Banking77Classification.json new file mode 100644 index 0000000000000000000000000000000000000000..87c12cd7ec566935cd29c7cab85f10ba06c22855 --- /dev/null +++ b/evaluation/mteb/Banking77Classification.json @@ -0,0 +1,12 @@ +{ + "test": { + "accuracy": 0.8449350649350649, + "accuracy_stderr": 0.005092747481478234, + "evaluation_time": 568.26, + "f1": 0.8442493432337361, + "f1_stderr": 0.005291617674599615, + "main_score": 0.8449350649350649 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/BiorxivClusteringP2P.json b/evaluation/mteb/BiorxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..ec33e8b974e0a2dfce52d81a0ea6c0cd65056478 --- /dev/null +++ b/evaluation/mteb/BiorxivClusteringP2P.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 14349.06, + "v_measure": 0.3655145972298938, + "v_measure_std": 0.01180536429604858 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/BiorxivClusteringS2S.json b/evaluation/mteb/BiorxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..045172bb4e042bfe8b2459fa7e7d51cf2f75ce20 --- /dev/null +++ b/evaluation/mteb/BiorxivClusteringS2S.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 1826.36, + "v_measure": 0.3369901851846774, + "v_measure_std": 0.011822688018810827 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/CQADupstackAndroidRetrieval.json b/evaluation/mteb/CQADupstackAndroidRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..f92b4f338bf0accf50a189bce00e7681bf79f493 --- /dev/null +++ b/evaluation/mteb/CQADupstackAndroidRetrieval.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 2419.91, + "map_at_1": 0.30499, + "map_at_10": 0.41208, + "map_at_100": 0.42638, + "map_at_1000": 0.42754, + "map_at_3": 0.37506, + "map_at_5": 0.39422, + "mrr_at_1": 0.37339, + "mrr_at_10": 0.47051, + "mrr_at_100": 0.47745, + "mrr_at_1000": 0.47786, + "mrr_at_3": 0.44087, + "mrr_at_5": 0.45711, + "ndcg_at_1": 0.37339, + "ndcg_at_10": 0.47666, + "ndcg_at_100": 0.52994, + "ndcg_at_1000": 0.54929, + "ndcg_at_3": 0.41982, + "ndcg_at_5": 0.4442, + "precision_at_1": 0.37339, + "precision_at_10": 0.09127, + "precision_at_100": 0.01475, + "precision_at_1000": 0.00194, + "precision_at_3": 0.20076, + "precision_at_5": 0.14449, + "recall_at_1": 0.30499, + "recall_at_10": 0.60328, + "recall_at_100": 0.82579, + "recall_at_1000": 0.95074, + "recall_at_3": 0.4417, + "recall_at_5": 0.5094 + } +} \ No newline at end of file diff --git a/evaluation/mteb/CQADupstackEnglishRetrieval.json b/evaluation/mteb/CQADupstackEnglishRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..a3f61418eeda6e63a2889cbcd2f1c955b8def573 --- /dev/null +++ b/evaluation/mteb/CQADupstackEnglishRetrieval.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 3724.38, + "map_at_1": 0.30613, + "map_at_10": 0.40781, + "map_at_100": 0.42018, + "map_at_1000": 0.42133, + "map_at_3": 0.37816, + "map_at_5": 0.39389, + "mrr_at_1": 0.38408, + "mrr_at_10": 0.46631, + "mrr_at_100": 0.47332, + "mrr_at_1000": 0.47368, + "mrr_at_3": 0.44384, + "mrr_at_5": 0.45661, + "ndcg_at_1": 0.38408, + "ndcg_at_10": 0.4638, + "ndcg_at_100": 0.5081, + "ndcg_at_1000": 0.52663, + "ndcg_at_3": 0.4218, + "ndcg_at_5": 0.43974, + "precision_at_1": 0.38408, + "precision_at_10": 0.08656, + "precision_at_100": 0.01386, + "precision_at_1000": 0.00184, + "precision_at_3": 0.20276, + "precision_at_5": 0.14242, + "recall_at_1": 0.30613, + "recall_at_10": 0.5644, + "recall_at_100": 0.75044, + "recall_at_1000": 0.86426, + "recall_at_3": 0.43766, + "recall_at_5": 0.48998 + } +} \ No newline at end of file diff --git a/evaluation/mteb/CQADupstackGamingRetrieval.json b/evaluation/mteb/CQADupstackGamingRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..e6b47acee1e146d1777665db59b231c91b2c29a9 --- /dev/null +++ b/evaluation/mteb/CQADupstackGamingRetrieval.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 3971.77, + "map_at_1": 0.37371, + "map_at_10": 0.49718, + "map_at_100": 0.50737, + "map_at_1000": 0.5079, + "map_at_3": 0.46231, + "map_at_5": 0.48329, + "mrr_at_1": 0.42884, + "mrr_at_10": 0.53176, + "mrr_at_100": 0.53817, + "mrr_at_1000": 0.53845, + "mrr_at_3": 0.50199, + "mrr_at_5": 0.5213, + "ndcg_at_1": 0.42884, + "ndcg_at_10": 0.55826, + "ndcg_at_100": 0.5993, + "ndcg_at_1000": 0.61013, + "ndcg_at_3": 0.49764, + "ndcg_at_5": 0.53026, + "precision_at_1": 0.42884, + "precision_at_10": 0.09047, + "precision_at_100": 0.01212, + "precision_at_1000": 0.00135, + "precision_at_3": 0.22132, + "precision_at_5": 0.15524, + "recall_at_1": 0.37371, + "recall_at_10": 0.70482, + "recall_at_100": 0.88425, + "recall_at_1000": 0.96034, + "recall_at_3": 0.5443, + "recall_at_5": 0.62328 + } +} \ No newline at end of file diff --git a/evaluation/mteb/CQADupstackGisRetrieval.json b/evaluation/mteb/CQADupstackGisRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..0395fe304df1f930a0acebc37d9aa0f0456c9d98 --- /dev/null +++ b/evaluation/mteb/CQADupstackGisRetrieval.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 5450.64, + "map_at_1": 0.22876, + "map_at_10": 0.31715, + "map_at_100": 0.32847, + "map_at_1000": 0.32922, + "map_at_3": 0.2905, + "map_at_5": 0.30396, + "mrr_at_1": 0.2452, + "mrr_at_10": 0.33497, + "mrr_at_100": 0.34455, + "mrr_at_1000": 0.3451, + "mrr_at_3": 0.30791, + "mrr_at_5": 0.32175, + "ndcg_at_1": 0.2452, + "ndcg_at_10": 0.3695, + "ndcg_at_100": 0.42238, + "ndcg_at_1000": 0.44148, + "ndcg_at_3": 0.31435, + "ndcg_at_5": 0.33839, + "precision_at_1": 0.2452, + "precision_at_10": 0.05932, + "precision_at_100": 0.00901, + "precision_at_1000": 0.0011, + "precision_at_3": 0.13446, + "precision_at_5": 0.09469, + "recall_at_1": 0.22876, + "recall_at_10": 0.5138, + "recall_at_100": 0.75311, + "recall_at_1000": 0.89718, + "recall_at_3": 0.3626, + "recall_at_5": 0.42249 + } +} \ No newline at end of file diff --git a/evaluation/mteb/CQADupstackMathematicaRetrieval.json b/evaluation/mteb/CQADupstackMathematicaRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..39b173fbcc870875cebd7be5fc7cad23f17a10bd --- /dev/null +++ b/evaluation/mteb/CQADupstackMathematicaRetrieval.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 2931.62, + "map_at_1": 0.14984, + "map_at_10": 0.23457, + "map_at_100": 0.24723, + "map_at_1000": 0.24846, + "map_at_3": 0.20873, + "map_at_5": 0.22357, + "mrr_at_1": 0.18159, + "mrr_at_10": 0.27431, + "mrr_at_100": 0.28449, + "mrr_at_1000": 0.2852, + "mrr_at_3": 0.24979, + "mrr_at_5": 0.26447, + "ndcg_at_1": 0.18159, + "ndcg_at_10": 0.28628, + "ndcg_at_100": 0.34741, + "ndcg_at_1000": 0.37516, + "ndcg_at_3": 0.23902, + "ndcg_at_5": 0.26294, + "precision_at_1": 0.18159, + "precision_at_10": 0.05485, + "precision_at_100": 0.00985, + "precision_at_1000": 0.00136, + "precision_at_3": 0.11774, + "precision_at_5": 0.08731, + "recall_at_1": 0.14984, + "recall_at_10": 0.40198, + "recall_at_100": 0.67115, + "recall_at_1000": 0.86497, + "recall_at_3": 0.27639, + "recall_at_5": 0.33595 + } +} \ No newline at end of file diff --git a/evaluation/mteb/CQADupstackPhysicsRetrieval.json b/evaluation/mteb/CQADupstackPhysicsRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..42a6efcdbb45b6ff3b9ca727ac282d2e9613903d --- /dev/null +++ b/evaluation/mteb/CQADupstackPhysicsRetrieval.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 5338.65, + "map_at_1": 0.29067, + "map_at_10": 0.39457, + "map_at_100": 0.4083, + "map_at_1000": 0.4094, + "map_at_3": 0.35995, + "map_at_5": 0.38159, + "mrr_at_1": 0.34937, + "mrr_at_10": 0.44755, + "mrr_at_100": 0.45549, + "mrr_at_1000": 0.45589, + "mrr_at_3": 0.41947, + "mrr_at_5": 0.43733, + "ndcg_at_1": 0.34937, + "ndcg_at_10": 0.45573, + "ndcg_at_100": 0.51267, + "ndcg_at_1000": 0.53184, + "ndcg_at_3": 0.39962, + "ndcg_at_5": 0.4302, + "precision_at_1": 0.34937, + "precision_at_10": 0.08296, + "precision_at_100": 0.0132, + "precision_at_1000": 0.00167, + "precision_at_3": 0.188, + "precision_at_5": 0.13763, + "recall_at_1": 0.29067, + "recall_at_10": 0.58298, + "recall_at_100": 0.82251, + "recall_at_1000": 0.94476, + "recall_at_3": 0.42984, + "recall_at_5": 0.50658 + } +} \ No newline at end of file diff --git a/evaluation/mteb/CQADupstackProgrammersRetrieval.json b/evaluation/mteb/CQADupstackProgrammersRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..c110c28519acd4d8e555abf72d467404c1255e7e --- /dev/null +++ b/evaluation/mteb/CQADupstackProgrammersRetrieval.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 4979.86, + "map_at_1": 0.25986, + "map_at_10": 0.35746, + "map_at_100": 0.37067, + "map_at_1000": 0.37191, + "map_at_3": 0.32599, + "map_at_5": 0.34239, + "mrr_at_1": 0.31735, + "mrr_at_10": 0.40515, + "mrr_at_100": 0.41459, + "mrr_at_1000": 0.41516, + "mrr_at_3": 0.37938, + "mrr_at_5": 0.3925, + "ndcg_at_1": 0.31735, + "ndcg_at_10": 0.41484, + "ndcg_at_100": 0.47047, + "ndcg_at_1000": 0.49427, + "ndcg_at_3": 0.36255, + "ndcg_at_5": 0.38375, + "precision_at_1": 0.31735, + "precision_at_10": 0.0766, + "precision_at_100": 0.01234, + "precision_at_1000": 0.0016, + "precision_at_3": 0.17428, + "precision_at_5": 0.12329, + "recall_at_1": 0.25986, + "recall_at_10": 0.53761, + "recall_at_100": 0.77149, + "recall_at_1000": 0.93342, + "recall_at_3": 0.39068, + "recall_at_5": 0.44693 + } +} \ No newline at end of file diff --git a/evaluation/mteb/CQADupstackRetrieval.json b/evaluation/mteb/CQADupstackRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..337fd71684d278079383e7a77acdd3965b28986a --- /dev/null +++ b/evaluation/mteb/CQADupstackRetrieval.json @@ -0,0 +1,13 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 7822.83, + "ndcg_at_1": 0.2821458333333333, + "ndcg_at_3": 0.3325866666666667, + "ndcg_at_5": 0.35453333333333326, + "ndcg_at_10": 0.381195, + "ndcg_at_100": 0.43441, + "ndcg_at_1000": 0.45826083333333334 + } +} diff --git a/evaluation/mteb/CQADupstackStatsRetrieval.json b/evaluation/mteb/CQADupstackStatsRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..7a10da544568ad9e06476af85e734429dcc10f2f --- /dev/null +++ b/evaluation/mteb/CQADupstackStatsRetrieval.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 6837.54, + "map_at_1": 0.22082, + "map_at_10": 0.29216, + "map_at_100": 0.30163, + "map_at_1000": 0.30269, + "map_at_3": 0.26942, + "map_at_5": 0.28236, + "mrr_at_1": 0.24847, + "mrr_at_10": 0.31919, + "mrr_at_100": 0.32817, + "mrr_at_1000": 0.32897, + "mrr_at_3": 0.29831, + "mrr_at_5": 0.3102, + "ndcg_at_1": 0.24847, + "ndcg_at_10": 0.334, + "ndcg_at_100": 0.38354, + "ndcg_at_1000": 0.41045, + "ndcg_at_3": 0.29236, + "ndcg_at_5": 0.31258, + "precision_at_1": 0.24847, + "precision_at_10": 0.05353, + "precision_at_100": 0.00853, + "precision_at_1000": 0.00116, + "precision_at_3": 0.12679, + "precision_at_5": 0.08988, + "recall_at_1": 0.22082, + "recall_at_10": 0.43505, + "recall_at_100": 0.66454, + "recall_at_1000": 0.86378, + "recall_at_3": 0.32163, + "recall_at_5": 0.3706 + } +} \ No newline at end of file diff --git a/evaluation/mteb/CQADupstackTexRetrieval.json b/evaluation/mteb/CQADupstackTexRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..9222abd1bf720a05af96d364acff487b9cfb7282 --- /dev/null +++ b/evaluation/mteb/CQADupstackTexRetrieval.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 11622.05, + "map_at_1": 0.1554, + "map_at_10": 0.22362, + "map_at_100": 0.23435, + "map_at_1000": 0.23564, + "map_at_3": 0.20143, + "map_at_5": 0.21324, + "mrr_at_1": 0.18892, + "mrr_at_10": 0.25943, + "mrr_at_100": 0.26883, + "mrr_at_1000": 0.26969, + "mrr_at_3": 0.23727, + "mrr_at_5": 0.24923, + "ndcg_at_1": 0.18892, + "ndcg_at_10": 0.26811, + "ndcg_at_100": 0.32066, + "ndcg_at_1000": 0.35166, + "ndcg_at_3": 0.22706, + "ndcg_at_5": 0.24508, + "precision_at_1": 0.18892, + "precision_at_10": 0.04942, + "precision_at_100": 0.00878, + "precision_at_1000": 0.00131, + "precision_at_3": 0.10748, + "precision_at_5": 0.07784, + "recall_at_1": 0.1554, + "recall_at_10": 0.36743, + "recall_at_100": 0.60525, + "recall_at_1000": 0.82576, + "recall_at_3": 0.25252, + "recall_at_5": 0.29872 + } +} \ No newline at end of file diff --git a/evaluation/mteb/CQADupstackUnixRetrieval.json b/evaluation/mteb/CQADupstackUnixRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..c95174b9634563b9954cc59559653db3070caa46 --- /dev/null +++ b/evaluation/mteb/CQADupstackUnixRetrieval.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 7165.48, + "map_at_1": 0.24453, + "map_at_10": 0.33363, + "map_at_100": 0.34579, + "map_at_1000": 0.34686, + "map_at_3": 0.30583, + "map_at_5": 0.32118, + "mrr_at_1": 0.28918, + "mrr_at_10": 0.37675, + "mrr_at_100": 0.38567, + "mrr_at_1000": 0.38632, + "mrr_at_3": 0.35261, + "mrr_at_5": 0.36576, + "ndcg_at_1": 0.28918, + "ndcg_at_10": 0.38736, + "ndcg_at_100": 0.44261, + "ndcg_at_1000": 0.4672, + "ndcg_at_3": 0.3381, + "ndcg_at_5": 0.36009, + "precision_at_1": 0.28918, + "precision_at_10": 0.06586, + "precision_at_100": 0.01047, + "precision_at_1000": 0.00137, + "precision_at_3": 0.15361, + "precision_at_5": 0.10858, + "recall_at_1": 0.24453, + "recall_at_10": 0.50886, + "recall_at_100": 0.7503, + "recall_at_1000": 0.92123, + "recall_at_3": 0.37138, + "recall_at_5": 0.42865 + } +} \ No newline at end of file diff --git a/evaluation/mteb/CQADupstackWebmastersRetrieval.json b/evaluation/mteb/CQADupstackWebmastersRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..26aadc49fbbe630dcc423a6ccb23892141dec010 --- /dev/null +++ b/evaluation/mteb/CQADupstackWebmastersRetrieval.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 2264.66, + "map_at_1": 0.2457, + "map_at_10": 0.33672, + "map_at_100": 0.35244, + "map_at_1000": 0.35467, + "map_at_3": 0.30712, + "map_at_5": 0.32383, + "mrr_at_1": 0.29644, + "mrr_at_10": 0.38344, + "mrr_at_100": 0.39219, + "mrr_at_1000": 0.39282, + "mrr_at_3": 0.35771, + "mrr_at_5": 0.37273, + "ndcg_at_1": 0.29644, + "ndcg_at_10": 0.39567, + "ndcg_at_100": 0.45097, + "ndcg_at_1000": 0.47923, + "ndcg_at_3": 0.34768, + "ndcg_at_5": 0.37122, + "precision_at_1": 0.29644, + "precision_at_10": 0.07589, + "precision_at_100": 0.01478, + "precision_at_1000": 0.00235, + "precision_at_3": 0.16337, + "precision_at_5": 0.12055, + "recall_at_1": 0.2457, + "recall_at_10": 0.51009, + "recall_at_100": 0.75423, + "recall_at_1000": 0.93671, + "recall_at_3": 0.36926, + "recall_at_5": 0.43245 + } +} \ No newline at end of file diff --git a/evaluation/mteb/CQADupstackWordpressRetrieval.json b/evaluation/mteb/CQADupstackWordpressRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..a7ed6d0f5d63192c62f58c65ab69a648c41555a0 --- /dev/null +++ b/evaluation/mteb/CQADupstackWordpressRetrieval.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 7608.65, + "map_at_1": 0.21356, + "map_at_10": 0.27904, + "map_at_100": 0.28938, + "map_at_1000": 0.29036, + "map_at_3": 0.25726, + "map_at_5": 0.26935, + "mrr_at_1": 0.22551, + "mrr_at_10": 0.29259, + "mrr_at_100": 0.30272, + "mrr_at_1000": 0.30348, + "mrr_at_3": 0.27295, + "mrr_at_5": 0.28358, + "ndcg_at_1": 0.22551, + "ndcg_at_10": 0.31817, + "ndcg_at_100": 0.37164, + "ndcg_at_1000": 0.3982, + "ndcg_at_3": 0.27596, + "ndcg_at_5": 0.29568, + "precision_at_1": 0.22551, + "precision_at_10": 0.04917, + "precision_at_100": 0.00828, + "precision_at_1000": 0.00114, + "precision_at_3": 0.11583, + "precision_at_5": 0.08133, + "recall_at_1": 0.21356, + "recall_at_10": 0.42489, + "recall_at_100": 0.67128, + "recall_at_1000": 0.87441, + "recall_at_3": 0.31165, + "recall_at_5": 0.35853 + } +} \ No newline at end of file diff --git a/evaluation/mteb/ClimateFEVER.json b/evaluation/mteb/ClimateFEVER.json new file mode 100644 index 0000000000000000000000000000000000000000..9698948245c734729fdbb1b7e616659eed06915c --- /dev/null +++ b/evaluation/mteb/ClimateFEVER.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 64667.21, + "map_at_1": 0.12306, + "map_at_10": 0.21523, + "map_at_100": 0.23358, + "map_at_1000": 0.23541, + "map_at_3": 0.17809, + "map_at_5": 0.19631, + "mrr_at_1": 0.27948, + "mrr_at_10": 0.40355, + "mrr_at_100": 0.41166, + "mrr_at_1000": 0.41203, + "mrr_at_3": 0.36819, + "mrr_at_5": 0.38959, + "ndcg_at_1": 0.27948, + "ndcg_at_10": 0.30462, + "ndcg_at_100": 0.37473, + "ndcg_at_1000": 0.40718, + "ndcg_at_3": 0.24646, + "ndcg_at_5": 0.26642, + "precision_at_1": 0.27948, + "precision_at_10": 0.09648, + "precision_at_100": 0.01724, + "precision_at_1000": 0.00232, + "precision_at_3": 0.1848, + "precision_at_5": 0.14293, + "recall_at_1": 0.12306, + "recall_at_10": 0.37181, + "recall_at_100": 0.61148, + "recall_at_1000": 0.79401, + "recall_at_3": 0.22883, + "recall_at_5": 0.2859 + } +} \ No newline at end of file diff --git a/evaluation/mteb/DBPedia.json b/evaluation/mteb/DBPedia.json new file mode 100644 index 0000000000000000000000000000000000000000..cbc75c5f15abd6134b0b31f1c2b6e68bfcb27d0e --- /dev/null +++ b/evaluation/mteb/DBPedia.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 43004.49, + "map_at_1": 0.09357, + "map_at_10": 0.18849, + "map_at_100": 0.25369, + "map_at_1000": 0.2695, + "map_at_3": 0.13625, + "map_at_5": 0.15957, + "mrr_at_1": 0.6775, + "mrr_at_10": 0.74734, + "mrr_at_100": 0.751, + "mrr_at_1000": 0.75109, + "mrr_at_3": 0.73542, + "mrr_at_5": 0.74167, + "ndcg_at_1": 0.55375, + "ndcg_at_10": 0.39874, + "ndcg_at_100": 0.43098, + "ndcg_at_1000": 0.50692, + "ndcg_at_3": 0.44856, + "ndcg_at_5": 0.42139, + "precision_at_1": 0.6775, + "precision_at_10": 0.311, + "precision_at_100": 0.09303, + "precision_at_1000": 0.02006, + "precision_at_3": 0.4825, + "precision_at_5": 0.4095, + "recall_at_1": 0.09357, + "recall_at_10": 0.23832, + "recall_at_100": 0.47906, + "recall_at_1000": 0.71309, + "recall_at_3": 0.14512, + "recall_at_5": 0.183 + } +} \ No newline at end of file diff --git a/evaluation/mteb/EmotionClassification.json b/evaluation/mteb/EmotionClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..78a4d2d3f0d8aacad179ed00f9652ff4ce1fa21d --- /dev/null +++ b/evaluation/mteb/EmotionClassification.json @@ -0,0 +1,12 @@ +{ + "test": { + "accuracy": 0.49655000000000005, + "accuracy_stderr": 0.023384236143179877, + "evaluation_time": 408.2, + "f1": 0.45519761909389506, + "f1_stderr": 0.017373883846959346, + "main_score": 0.49655000000000005 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/FEVER.json b/evaluation/mteb/FEVER.json new file mode 100644 index 0000000000000000000000000000000000000000..be9d1f7b513cb3bcf4b254c6b56ed265a868ab37 --- /dev/null +++ b/evaluation/mteb/FEVER.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 65632.31, + "map_at_1": 0.6274, + "map_at_10": 0.7307, + "map_at_100": 0.73398, + "map_at_1000": 0.7341, + "map_at_3": 0.71338, + "map_at_5": 0.72423, + "mrr_at_1": 0.67777, + "mrr_at_10": 0.77873, + "mrr_at_100": 0.78091, + "mrr_at_1000": 0.78094, + "mrr_at_3": 0.76375, + "mrr_at_5": 0.77316, + "ndcg_at_1": 0.67777, + "ndcg_at_10": 0.7824, + "ndcg_at_100": 0.79557, + "ndcg_at_1000": 0.79814, + "ndcg_at_3": 0.75125, + "ndcg_at_5": 0.76834, + "precision_at_1": 0.67777, + "precision_at_10": 0.09832, + "precision_at_100": 0.01061, + "precision_at_1000": 0.0011, + "precision_at_3": 0.29433, + "precision_at_5": 0.18665, + "recall_at_1": 0.6274, + "recall_at_10": 0.89505, + "recall_at_100": 0.95102, + "recall_at_1000": 0.96825, + "recall_at_3": 0.81028, + "recall_at_5": 0.85281 + } +} \ No newline at end of file diff --git a/evaluation/mteb/FiQA2018.json b/evaluation/mteb/FiQA2018.json new file mode 100644 index 0000000000000000000000000000000000000000..60dac9163219c4bab33af7c926604cb95afe760c --- /dev/null +++ b/evaluation/mteb/FiQA2018.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 951.67, + "map_at_1": 0.18467, + "map_at_10": 0.30021, + "map_at_100": 0.31739, + "map_at_1000": 0.31934, + "map_at_3": 0.26003, + "map_at_5": 0.28338, + "mrr_at_1": 0.3534, + "mrr_at_10": 0.44109, + "mrr_at_100": 0.44993, + "mrr_at_1000": 0.45042, + "mrr_at_3": 0.41667, + "mrr_at_5": 0.4314, + "ndcg_at_1": 0.3534, + "ndcg_at_10": 0.37202, + "ndcg_at_100": 0.43853, + "ndcg_at_1000": 0.47235, + "ndcg_at_3": 0.335, + "ndcg_at_5": 0.34985, + "precision_at_1": 0.3534, + "precision_at_10": 0.10247, + "precision_at_100": 0.01715, + "precision_at_1000": 0.00232, + "precision_at_3": 0.22222, + "precision_at_5": 0.16574, + "recall_at_1": 0.18467, + "recall_at_10": 0.44081, + "recall_at_100": 0.68722, + "recall_at_1000": 0.89087, + "recall_at_3": 0.30567, + "recall_at_5": 0.36982 + } +} \ No newline at end of file diff --git a/evaluation/mteb/HotpotQA.json b/evaluation/mteb/HotpotQA.json new file mode 100644 index 0000000000000000000000000000000000000000..b88fdba1d5f9f02145165d3208f0038bdbb60967 --- /dev/null +++ b/evaluation/mteb/HotpotQA.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 46237.87, + "map_at_1": 0.35726, + "map_at_10": 0.50207, + "map_at_100": 0.51055, + "map_at_1000": 0.51128, + "map_at_3": 0.47576, + "map_at_5": 0.49172, + "mrr_at_1": 0.71452, + "mrr_at_10": 0.77419, + "mrr_at_100": 0.77711, + "mrr_at_1000": 0.77723, + "mrr_at_3": 0.76394, + "mrr_at_5": 0.77001, + "ndcg_at_1": 0.71452, + "ndcg_at_10": 0.59261, + "ndcg_at_100": 0.62424, + "ndcg_at_1000": 0.63951, + "ndcg_at_3": 0.55327, + "ndcg_at_5": 0.57417, + "precision_at_1": 0.71452, + "precision_at_10": 0.12061, + "precision_at_100": 0.01455, + "precision_at_1000": 0.00166, + "precision_at_3": 0.3436, + "precision_at_5": 0.22266, + "recall_at_1": 0.35726, + "recall_at_10": 0.60304, + "recall_at_100": 0.72755, + "recall_at_1000": 0.82978, + "recall_at_3": 0.5154, + "recall_at_5": 0.55665 + } +} \ No newline at end of file diff --git a/evaluation/mteb/ImdbClassification.json b/evaluation/mteb/ImdbClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..b04ac01ab285323583ef42c9017e33e386f27db9 --- /dev/null +++ b/evaluation/mteb/ImdbClassification.json @@ -0,0 +1,14 @@ +{ + "test": { + "accuracy": 0.666376, + "accuracy_stderr": 0.03435007633179292, + "ap": 0.6148938261286748, + "ap_stderr": 0.028266712412564548, + "evaluation_time": 38609.7, + "f1": 0.6635089269264965, + "f1_stderr": 0.03526614235815782, + "main_score": 0.666376 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/MSMARCO.json b/evaluation/mteb/MSMARCO.json new file mode 100644 index 0000000000000000000000000000000000000000..a5a2893c9797b4f765bc8d67e51c7d013201f7b0 --- /dev/null +++ b/evaluation/mteb/MSMARCO.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "validation": { + "evaluation_time": 82887.84, + "map_at_1": 0.20842, + "map_at_10": 0.32992, + "map_at_100": 0.34236, + "map_at_1000": 0.34286, + "map_at_3": 0.29049, + "map_at_5": 0.31392, + "mrr_at_1": 0.21375, + "mrr_at_10": 0.33581, + "mrr_at_100": 0.3476, + "mrr_at_1000": 0.34803, + "mrr_at_3": 0.29704, + "mrr_at_5": 0.32015, + "ndcg_at_1": 0.21375, + "ndcg_at_10": 0.39905, + "ndcg_at_100": 0.45843, + "ndcg_at_1000": 0.47084, + "ndcg_at_3": 0.31919, + "ndcg_at_5": 0.36107, + "precision_at_1": 0.21375, + "precision_at_10": 0.06393, + "precision_at_100": 0.00935, + "precision_at_1000": 0.00104, + "precision_at_3": 0.13663, + "precision_at_5": 0.10324, + "recall_at_1": 0.20842, + "recall_at_10": 0.6117, + "recall_at_100": 0.88518, + "recall_at_1000": 0.97993, + "recall_at_3": 0.39571, + "recall_at_5": 0.49654 + } +} \ No newline at end of file diff --git a/evaluation/mteb/MTOPDomainClassification.json b/evaluation/mteb/MTOPDomainClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..3c1733d4e6e7654ea367d948751daa05c069b81a --- /dev/null +++ b/evaluation/mteb/MTOPDomainClassification.json @@ -0,0 +1,14 @@ +{ + "test": { + "en": { + "accuracy": 0.9346557227542178, + "accuracy_stderr": 0.0072510589372134955, + "f1": 0.9287345917772146, + "f1_stderr": 0.0082627627462556, + "main_score": 0.9346557227542178 + }, + "evaluation_time": 475.45 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/MTOPIntentClassification.json b/evaluation/mteb/MTOPIntentClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..692232a446f1697dd09cf6e3c359decf32bc8cfd --- /dev/null +++ b/evaluation/mteb/MTOPIntentClassification.json @@ -0,0 +1,14 @@ +{ + "test": { + "en": { + "accuracy": 0.7242134062927497, + "accuracy_stderr": 0.012571448106282981, + "f1": 0.5503624810959269, + "f1_stderr": 0.011737816802681789, + "main_score": 0.7242134062927497 + }, + "evaluation_time": 612.9 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/MassiveIntentClassification.json b/evaluation/mteb/MassiveIntentClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..8f83e42ca1903d29ca6d9f1c3ae957eda97b9f9f --- /dev/null +++ b/evaluation/mteb/MassiveIntentClassification.json @@ -0,0 +1,14 @@ +{ + "test": { + "en": { + "accuracy": 0.703866845998655, + "accuracy_stderr": 0.00456616151893328, + "f1": 0.6896745198729209, + "f1_stderr": 0.006294250065644286, + "main_score": 0.703866845998655 + }, + "evaluation_time": 387.35 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/MassiveScenarioClassification.json b/evaluation/mteb/MassiveScenarioClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..f6f5337f59f4fe9ba2d35b123e0cb230852f3340 --- /dev/null +++ b/evaluation/mteb/MassiveScenarioClassification.json @@ -0,0 +1,14 @@ +{ + "test": { + "en": { + "accuracy": 0.7627774041694687, + "accuracy_stderr": 0.014020552186569309, + "f1": 0.7672936190462792, + "f1_stderr": 0.01203156406861017, + "main_score": 0.7627774041694687 + }, + "evaluation_time": 333.83 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/MedrxivClusteringP2P.json b/evaluation/mteb/MedrxivClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..1295ee4726c010a6afd87f14b511ecf2ba452674 --- /dev/null +++ b/evaluation/mteb/MedrxivClusteringP2P.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 7211.78, + "v_measure": 0.3151174592577334, + "v_measure_std": 0.012772027882021399 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/MedrxivClusteringS2S.json b/evaluation/mteb/MedrxivClusteringS2S.json new file mode 100644 index 0000000000000000000000000000000000000000..0729eb80023bc0bc8870b896448d3a55a2d3de62 --- /dev/null +++ b/evaluation/mteb/MedrxivClusteringS2S.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 1053.59, + "v_measure": 0.28764235987575365, + "v_measure_std": 0.011424256185368544 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/MindSmallReranking.json b/evaluation/mteb/MindSmallReranking.json new file mode 100644 index 0000000000000000000000000000000000000000..a5ffc4ee18b0328a2c43632060b729e20a7b9107 --- /dev/null +++ b/evaluation/mteb/MindSmallReranking.json @@ -0,0 +1,9 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 72941.42, + "map": 0.32293531363866007, + "mrr": 0.33536774455851687 + } +} \ No newline at end of file diff --git a/evaluation/mteb/NFCorpus.json b/evaluation/mteb/NFCorpus.json new file mode 100644 index 0000000000000000000000000000000000000000..bb49c78635b45bcc51655d8aca79a51ab9e5a87b --- /dev/null +++ b/evaluation/mteb/NFCorpus.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 166.79, + "map_at_1": 0.05702, + "map_at_10": 0.13642, + "map_at_100": 0.17503, + "map_at_1000": 0.19126, + "map_at_3": 0.09748, + "map_at_5": 0.11642, + "mrr_at_1": 0.4582, + "mrr_at_10": 0.54821, + "mrr_at_100": 0.55422, + "mrr_at_1000": 0.55453, + "mrr_at_3": 0.52374, + "mrr_at_5": 0.53937, + "ndcg_at_1": 0.44272, + "ndcg_at_10": 0.36213, + "ndcg_at_100": 0.33829, + "ndcg_at_1000": 0.42557, + "ndcg_at_3": 0.40814, + "ndcg_at_5": 0.39562, + "precision_at_1": 0.45511, + "precision_at_10": 0.27214, + "precision_at_100": 0.08941, + "precision_at_1000": 0.02187, + "precision_at_3": 0.37874, + "precision_at_5": 0.34489, + "recall_at_1": 0.05702, + "recall_at_10": 0.17638, + "recall_at_100": 0.34419, + "recall_at_1000": 0.6641, + "recall_at_3": 0.10914, + "recall_at_5": 0.14032 + } +} \ No newline at end of file diff --git a/evaluation/mteb/NQ.json b/evaluation/mteb/NQ.json new file mode 100644 index 0000000000000000000000000000000000000000..955b6441d16787a3562ec9b348f8605bb5c29d89 --- /dev/null +++ b/evaluation/mteb/NQ.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 31038.1, + "map_at_1": 0.30567, + "map_at_10": 0.4501, + "map_at_100": 0.46091, + "map_at_1000": 0.46126, + "map_at_3": 0.40897, + "map_at_5": 0.43301, + "mrr_at_1": 0.3456, + "mrr_at_10": 0.47725, + "mrr_at_100": 0.48548, + "mrr_at_1000": 0.48572, + "mrr_at_3": 0.44361, + "mrr_at_5": 0.46351, + "ndcg_at_1": 0.34531, + "ndcg_at_10": 0.5241, + "ndcg_at_100": 0.56999, + "ndcg_at_1000": 0.57831, + "ndcg_at_3": 0.44734, + "ndcg_at_5": 0.48701, + "precision_at_1": 0.34531, + "precision_at_10": 0.08612, + "precision_at_100": 0.01118, + "precision_at_1000": 0.0012, + "precision_at_3": 0.20307, + "precision_at_5": 0.14519, + "recall_at_1": 0.30567, + "recall_at_10": 0.72238, + "recall_at_100": 0.92154, + "recall_at_1000": 0.98375, + "recall_at_3": 0.52438, + "recall_at_5": 0.61517 + } +} \ No newline at end of file diff --git a/evaluation/mteb/QuoraRetrieval.json b/evaluation/mteb/QuoraRetrieval.json new file mode 100644 index 0000000000000000000000000000000000000000..5e4283b3d63475558854691df1d6e03888a89c00 --- /dev/null +++ b/evaluation/mteb/QuoraRetrieval.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 1922.86, + "map_at_1": 0.6598, + "map_at_10": 0.80056, + "map_at_100": 0.80763, + "map_at_1000": 0.80786, + "map_at_3": 0.76848, + "map_at_5": 0.78854, + "mrr_at_1": 0.7586, + "mrr_at_10": 0.83397, + "mrr_at_100": 0.83555, + "mrr_at_1000": 0.83557, + "mrr_at_3": 0.82033, + "mrr_at_5": 0.8297, + "ndcg_at_1": 0.7588, + "ndcg_at_10": 0.84581, + "ndcg_at_100": 0.86151, + "ndcg_at_1000": 0.86315, + "ndcg_at_3": 0.80902, + "ndcg_at_5": 0.82953, + "precision_at_1": 0.7588, + "precision_at_10": 0.12986, + "precision_at_100": 0.01511, + "precision_at_1000": 0.00156, + "precision_at_3": 0.35383, + "precision_at_5": 0.23556, + "recall_at_1": 0.6598, + "recall_at_10": 0.93716, + "recall_at_100": 0.99218, + "recall_at_1000": 0.9997, + "recall_at_3": 0.83551, + "recall_at_5": 0.88998 + } +} \ No newline at end of file diff --git a/evaluation/mteb/RedditClustering.json b/evaluation/mteb/RedditClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..f40ca692e61b5ba8a2ed3b2dff734437233e5180 --- /dev/null +++ b/evaluation/mteb/RedditClustering.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 14260.04, + "v_measure": 0.4045148482612238, + "v_measure_std": 0.10798668357466681 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/RedditClusteringP2P.json b/evaluation/mteb/RedditClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..2ddf6eb0da04e803fcf34f51b0fb94769c162a9c --- /dev/null +++ b/evaluation/mteb/RedditClusteringP2P.json @@ -0,0 +1,9 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 57925.8, + "v_measure": 0.5574949067303913, + "v_measure_std": 0.12781562321045892 + } +} \ No newline at end of file diff --git a/evaluation/mteb/SCIDOCS.json b/evaluation/mteb/SCIDOCS.json new file mode 100644 index 0000000000000000000000000000000000000000..d17ff311d06b15d0562299b5bcbac0857e3b40ac --- /dev/null +++ b/evaluation/mteb/SCIDOCS.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 626.47, + "map_at_1": 0.04903, + "map_at_10": 0.11926, + "map_at_100": 0.13917, + "map_at_1000": 0.14215, + "map_at_3": 0.088, + "map_at_5": 0.10361, + "mrr_at_1": 0.241, + "mrr_at_10": 0.34482, + "mrr_at_100": 0.35566, + "mrr_at_1000": 0.35619, + "mrr_at_3": 0.31433, + "mrr_at_5": 0.33243, + "ndcg_at_1": 0.241, + "ndcg_at_10": 0.19873, + "ndcg_at_100": 0.27606, + "ndcg_at_1000": 0.32811, + "ndcg_at_3": 0.19498, + "ndcg_at_5": 0.16813, + "precision_at_1": 0.241, + "precision_at_10": 0.1008, + "precision_at_100": 0.02122, + "precision_at_1000": 0.00337, + "precision_at_3": 0.182, + "precision_at_5": 0.1462, + "recall_at_1": 0.04903, + "recall_at_10": 0.20438, + "recall_at_100": 0.43043, + "recall_at_1000": 0.6841, + "recall_at_3": 0.11068, + "recall_at_5": 0.14818 + } +} \ No newline at end of file diff --git a/evaluation/mteb/SICK-R.json b/evaluation/mteb/SICK-R.json new file mode 100644 index 0000000000000000000000000000000000000000..fc7fe8107a58cde6c3f132ed46b6e5c523d84b86 --- /dev/null +++ b/evaluation/mteb/SICK-R.json @@ -0,0 +1,17 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.7858086597995997, + "spearman": 0.6963214182814992 + }, + "euclidean": { + "pearson": 0.7276175489042691, + "spearman": 0.6784965161872971 + }, + "evaluation_time": 199.7, + "manhattan": { + "pearson": 0.7273812689782593, + "spearman": 0.6783610439531278 + } + } +} \ No newline at end of file diff --git a/evaluation/mteb/STS12.json b/evaluation/mteb/STS12.json new file mode 100644 index 0000000000000000000000000000000000000000..bbaf0d7e7a0c8008f25db80a01a307cac51097f7 --- /dev/null +++ b/evaluation/mteb/STS12.json @@ -0,0 +1,19 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.7513970861325007, + "spearman": 0.675020551515597 + }, + "euclidean": { + "pearson": 0.6633415412418275, + "spearman": 0.6682145056673268 + }, + "evaluation_time": 100.4, + "manhattan": { + "pearson": 0.6655489484006415, + "spearman": 0.6695147433279057 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/STS13.json b/evaluation/mteb/STS13.json new file mode 100644 index 0000000000000000000000000000000000000000..0d1e09bf56d8efd516c0f6a578950a80f35082be --- /dev/null +++ b/evaluation/mteb/STS13.json @@ -0,0 +1,19 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.7885850536483447, + "spearman": 0.7916333501772059 + }, + "euclidean": { + "pearson": 0.7274090561408476, + "spearman": 0.7357374448302961 + }, + "evaluation_time": 43.53, + "manhattan": { + "pearson": 0.7292980654233225, + "spearman": 0.7372777155112589 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/STS14.json b/evaluation/mteb/STS14.json new file mode 100644 index 0000000000000000000000000000000000000000..c35624ed2a7bc194369fc12d71f1a6b36499e38f --- /dev/null +++ b/evaluation/mteb/STS14.json @@ -0,0 +1,19 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.7951125593897028, + "spearman": 0.7446048326701329 + }, + "euclidean": { + "pearson": 0.7087726087052986, + "spearman": 0.677721470654411 + }, + "evaluation_time": 107.32, + "manhattan": { + "pearson": 0.7105892792135637, + "spearman": 0.6793472619779036 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/STS15.json b/evaluation/mteb/STS15.json new file mode 100644 index 0000000000000000000000000000000000000000..0753a414d86c4d70aa0ea9670037dce391c1bc83 --- /dev/null +++ b/evaluation/mteb/STS15.json @@ -0,0 +1,19 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.838299348880489, + "spearman": 0.8447194637929274 + }, + "euclidean": { + "pearson": 0.7868768462480418, + "spearman": 0.7980526323901918 + }, + "evaluation_time": 90.96, + "manhattan": { + "pearson": 0.786810718151946, + "spearman": 0.797820584821254 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/STS16.json b/evaluation/mteb/STS16.json new file mode 100644 index 0000000000000000000000000000000000000000..dc9d5d011b82928806aea8098a5763a94b4acf65 --- /dev/null +++ b/evaluation/mteb/STS16.json @@ -0,0 +1,19 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.7999206664843005, + "spearman": 0.8096089203722137 + }, + "euclidean": { + "pearson": 0.7131216213716366, + "spearman": 0.7145258140049406 + }, + "evaluation_time": 41.56, + "manhattan": { + "pearson": 0.7126140340402836, + "spearman": 0.7138968946669431 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/STS17.json b/evaluation/mteb/STS17.json new file mode 100644 index 0000000000000000000000000000000000000000..b5a5edce09ae1520a14e9da3f9f9966f84d4d4a5 --- /dev/null +++ b/evaluation/mteb/STS17.json @@ -0,0 +1,21 @@ +{ + "test": { + "en-en": { + "cos_sim": { + "pearson": 0.8735697089594868, + "spearman": 0.8778202647220289 + }, + "euclidean": { + "pearson": 0.8420969668786666, + "spearman": 0.8391876425459982 + }, + "manhattan": { + "pearson": 0.8424429755612541, + "spearman": 0.8398826315103398 + } + }, + "evaluation_time": 8.66 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/STS22.json b/evaluation/mteb/STS22.json new file mode 100644 index 0000000000000000000000000000000000000000..ad7721716310b78b30d2c151f207d7f9a73a3c40 --- /dev/null +++ b/evaluation/mteb/STS22.json @@ -0,0 +1,21 @@ +{ + "test": { + "en": { + "cos_sim": { + "pearson": 0.6906962775868384, + "spearman": 0.6934889515492327 + }, + "euclidean": { + "pearson": 0.6928108180412313, + "spearman": 0.696437114853659 + }, + "manhattan": { + "pearson": 0.6939974983734993, + "spearman": 0.696905728448208 + } + }, + "evaluation_time": 78.83 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/STSBenchmark.json b/evaluation/mteb/STSBenchmark.json new file mode 100644 index 0000000000000000000000000000000000000000..0c720fa9aa042cc54cdda0cbe5a5b6c843853c58 --- /dev/null +++ b/evaluation/mteb/STSBenchmark.json @@ -0,0 +1,19 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.8242553734213958, + "spearman": 0.8138977341532744 + }, + "euclidean": { + "pearson": 0.7647494587945523, + "spearman": 0.7592794860531089 + }, + "evaluation_time": 39.38, + "manhattan": { + "pearson": 0.764768777169467, + "spearman": 0.759252673228599 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/SciDocsRR.json b/evaluation/mteb/SciDocsRR.json new file mode 100644 index 0000000000000000000000000000000000000000..e999794d56cc9d8d2e97c8d8e779254c987ed6dc --- /dev/null +++ b/evaluation/mteb/SciDocsRR.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 2310.92, + "map": 0.8078825425914723, + "mrr": 0.9460017197762296 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/SciFact.json b/evaluation/mteb/SciFact.json new file mode 100644 index 0000000000000000000000000000000000000000..ee25d276cf31b4ae2e266fd9aeb0178e95a4b645 --- /dev/null +++ b/evaluation/mteb/SciFact.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 981.11, + "map_at_1": 0.60633, + "map_at_10": 0.70197, + "map_at_100": 0.70758, + "map_at_1000": 0.70765, + "map_at_3": 0.67082, + "map_at_5": 0.69209, + "mrr_at_1": 0.63333, + "mrr_at_10": 0.7117, + "mrr_at_100": 0.71626, + "mrr_at_1000": 0.71633, + "mrr_at_3": 0.68833, + "mrr_at_5": 0.706, + "ndcg_at_1": 0.63333, + "ndcg_at_10": 0.74697, + "ndcg_at_100": 0.76986, + "ndcg_at_1000": 0.77225, + "ndcg_at_3": 0.69527, + "ndcg_at_5": 0.72816, + "precision_at_1": 0.63333, + "precision_at_10": 0.099, + "precision_at_100": 0.01103, + "precision_at_1000": 0.00112, + "precision_at_3": 0.26889, + "precision_at_5": 0.182, + "recall_at_1": 0.60633, + "recall_at_10": 0.87367, + "recall_at_100": 0.97333, + "recall_at_1000": 0.99333, + "recall_at_3": 0.73656, + "recall_at_5": 0.82083 + } +} \ No newline at end of file diff --git a/evaluation/mteb/SprintDuplicateQuestions.json b/evaluation/mteb/SprintDuplicateQuestions.json new file mode 100644 index 0000000000000000000000000000000000000000..d26b5bf37505424a7c3d3273f9cfc426123547cf --- /dev/null +++ b/evaluation/mteb/SprintDuplicateQuestions.json @@ -0,0 +1,48 @@ +{ + "test": { + "cos_sim": { + "accuracy": 0.9976633663366337, + "accuracy_threshold": 0.7178364992141724, + "ap": 0.9384024096781063, + "f1": 0.8808080808080808, + "f1_threshold": 0.7178364992141724, + "precision": 0.889795918367347, + "recall": 0.872 + }, + "dot": { + "accuracy": 0.9946336633663366, + "accuracy_threshold": 1165.947265625, + "ap": 0.7578127156965245, + "f1": 0.7141403865717193, + "f1_threshold": 1093.7952880859375, + "precision": 0.7267080745341615, + "recall": 0.702 + }, + "euclidean": { + "accuracy": 0.9967524752475247, + "accuracy_threshold": 26.32122039794922, + "ap": 0.8861274955249769, + "f1": 0.8230852211434736, + "f1_threshold": 27.092132568359375, + "precision": 0.8934426229508197, + "recall": 0.763 + }, + "evaluation_time": 139.2, + "manhattan": { + "accuracy": 0.9967722772277228, + "accuracy_threshold": 1360.433837890625, + "ap": 0.8877516158012779, + "f1": 0.8236536430834213, + "f1_threshold": 1421.690673828125, + "precision": 0.87248322147651, + "recall": 0.78 + }, + "max": { + "accuracy": 0.9976633663366337, + "ap": 0.9384024096781063, + "f1": 0.8808080808080808 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/StackExchangeClustering.json b/evaluation/mteb/StackExchangeClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..682720e14d4c89ba1e2ec371119297554586fab5 --- /dev/null +++ b/evaluation/mteb/StackExchangeClustering.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 9693.46, + "v_measure": 0.5920812266121527, + "v_measure_std": 0.06285309319740275 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/StackExchangeClusteringP2P.json b/evaluation/mteb/StackExchangeClusteringP2P.json new file mode 100644 index 0000000000000000000000000000000000000000..83793bdf60b7e4fa9420fc255dd52d327878c9d1 --- /dev/null +++ b/evaluation/mteb/StackExchangeClusteringP2P.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 12015.91, + "v_measure": 0.33954248554638056, + "v_measure_std": 0.014889014323282816 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/StackOverflowDupQuestions.json b/evaluation/mteb/StackOverflowDupQuestions.json new file mode 100644 index 0000000000000000000000000000000000000000..192dc7267abc737ef7535ac7b7a4169a8b5581ab --- /dev/null +++ b/evaluation/mteb/StackOverflowDupQuestions.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 1269.58, + "map": 0.5152800990025549, + "mrr": 0.5236039491554197 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/SummEval.json b/evaluation/mteb/SummEval.json new file mode 100644 index 0000000000000000000000000000000000000000..0c760bf36a516d8933d93018cc64106d581cf52b --- /dev/null +++ b/evaluation/mteb/SummEval.json @@ -0,0 +1,15 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.2457438758817976, + "spearman": 0.24747448399760644 + }, + "dot": { + "pearson": 0.2658901758418499, + "spearman": 0.25653620812462785 + }, + "evaluation_time": 1684.65 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/SummEvaldropnan.json b/evaluation/mteb/SummEvaldropnan.json new file mode 100644 index 0000000000000000000000000000000000000000..0c760bf36a516d8933d93018cc64106d581cf52b --- /dev/null +++ b/evaluation/mteb/SummEvaldropnan.json @@ -0,0 +1,15 @@ +{ + "test": { + "cos_sim": { + "pearson": 0.2457438758817976, + "spearman": 0.24747448399760644 + }, + "dot": { + "pearson": 0.2658901758418499, + "spearman": 0.25653620812462785 + }, + "evaluation_time": 1684.65 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/TRECCOVID.json b/evaluation/mteb/TRECCOVID.json new file mode 100644 index 0000000000000000000000000000000000000000..f64e7e3adacc49c2db1ae951050c28fdd0f7f913 --- /dev/null +++ b/evaluation/mteb/TRECCOVID.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 3150.52, + "map_at_1": 0.00253, + "map_at_10": 0.0214, + "map_at_100": 0.12873, + "map_at_1000": 0.31002, + "map_at_3": 0.00711, + "map_at_5": 0.01125, + "mrr_at_1": 0.96, + "mrr_at_10": 0.98, + "mrr_at_100": 0.98, + "mrr_at_1000": 0.98, + "mrr_at_3": 0.98, + "mrr_at_5": 0.98, + "ndcg_at_1": 0.94, + "ndcg_at_10": 0.84881, + "ndcg_at_100": 0.64694, + "ndcg_at_1000": 0.5685, + "ndcg_at_3": 0.90061, + "ndcg_at_5": 0.87155, + "precision_at_1": 0.96, + "precision_at_10": 0.888, + "precision_at_100": 0.657, + "precision_at_1000": 0.2508, + "precision_at_3": 0.92667, + "precision_at_5": 0.9, + "recall_at_1": 0.00253, + "recall_at_10": 0.02292, + "recall_at_100": 0.1578, + "recall_at_1000": 0.53015, + "recall_at_3": 0.00727, + "recall_at_5": 0.01162 + } +} \ No newline at end of file diff --git a/evaluation/mteb/Touche2020.json b/evaluation/mteb/Touche2020.json new file mode 100644 index 0000000000000000000000000000000000000000..bdcaaab5a15fa29c03e78a3f753c9dae84f65823 --- /dev/null +++ b/evaluation/mteb/Touche2020.json @@ -0,0 +1,37 @@ +{ + "dataset_version": null, + "mteb_version": "0.0.2", + "test": { + "evaluation_time": 5933.21, + "map_at_1": 0.02116, + "map_at_10": 0.09625, + "map_at_100": 0.15641, + "map_at_1000": 0.17127, + "map_at_3": 0.04316, + "map_at_5": 0.06208, + "mrr_at_1": 0.32653, + "mrr_at_10": 0.48084, + "mrr_at_100": 0.48631, + "mrr_at_1000": 0.48649, + "mrr_at_3": 0.42857, + "mrr_at_5": 0.46224, + "ndcg_at_1": 0.29592, + "ndcg_at_10": 0.25431, + "ndcg_at_100": 0.36344, + "ndcg_at_1000": 0.47676, + "ndcg_at_3": 0.26144, + "ndcg_at_5": 0.26304, + "precision_at_1": 0.32653, + "precision_at_10": 0.24082, + "precision_at_100": 0.07714, + "precision_at_1000": 0.01531, + "precision_at_3": 0.26531, + "precision_at_5": 0.26939, + "recall_at_1": 0.02116, + "recall_at_10": 0.16794, + "recall_at_100": 0.47452, + "recall_at_1000": 0.82312, + "recall_at_3": 0.05306, + "recall_at_5": 0.09306 + } +} \ No newline at end of file diff --git a/evaluation/mteb/ToxicConversationsClassification.json b/evaluation/mteb/ToxicConversationsClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..baefea446c2773548d6afb4b9e3eb4891c28b99e --- /dev/null +++ b/evaluation/mteb/ToxicConversationsClassification.json @@ -0,0 +1,14 @@ +{ + "test": { + "accuracy": 0.67709, + "accuracy_stderr": 0.052929278287163525, + "ap": 0.13541535578501715, + "ap_stderr": 0.009664239599244414, + "evaluation_time": 29864.89, + "f1": 0.525696199194468, + "f1_stderr": 0.0301544898714517, + "main_score": 0.13541535578501715 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/TweetSentimentExtractionClassification.json b/evaluation/mteb/TweetSentimentExtractionClassification.json new file mode 100644 index 0000000000000000000000000000000000000000..c5b576aaef0893e91a140eec547b8d5d80391420 --- /dev/null +++ b/evaluation/mteb/TweetSentimentExtractionClassification.json @@ -0,0 +1,12 @@ +{ + "test": { + "accuracy": 0.5685059422750425, + "accuracy_stderr": 0.023161878177526286, + "evaluation_time": 734.09, + "f1": 0.5723337736491058, + "f1_stderr": 0.0226476279206486, + "main_score": 0.5685059422750425 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/TwentyNewsgroupsClustering.json b/evaluation/mteb/TwentyNewsgroupsClustering.json new file mode 100644 index 0000000000000000000000000000000000000000..02ac3f4cc291e6356e3bfd1d49b14e012b2c9d73 --- /dev/null +++ b/evaluation/mteb/TwentyNewsgroupsClustering.json @@ -0,0 +1,9 @@ +{ + "test": { + "evaluation_time": 721.44, + "v_measure": 0.3946372298609047, + "v_measure_std": 0.05169775783296825 + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/TwitterSemEval2015.json b/evaluation/mteb/TwitterSemEval2015.json new file mode 100644 index 0000000000000000000000000000000000000000..48306fc40d79f26d84c7b826203c68f35334d975 --- /dev/null +++ b/evaluation/mteb/TwitterSemEval2015.json @@ -0,0 +1,48 @@ +{ + "test": { + "cos_sim": { + "accuracy": 0.8409131549144663, + "accuracy_threshold": 0.740990400314331, + "ap": 0.6686677647503386, + "f1": 0.6294631710362049, + "f1_threshold": 0.6761115789413452, + "precision": 0.5973933649289099, + "recall": 0.6651715039577837 + }, + "dot": { + "accuracy": 0.8027656911247542, + "accuracy_threshold": 1326.768310546875, + "ap": 0.5429172039861209, + "f1": 0.5477150537634409, + "f1_threshold": 947.6712646484375, + "precision": 0.4758660957571039, + "recall": 0.6451187335092349 + }, + "euclidean": { + "accuracy": 0.8276211480002384, + "accuracy_threshold": 27.691078186035156, + "ap": 0.6243039769075329, + "f1": 0.5919159053935678, + "f1_threshold": 30.392478942871094, + "precision": 0.5629611997143538, + "recall": 0.6240105540897097 + }, + "evaluation_time": 236.37, + "manhattan": { + "accuracy": 0.827561542588067, + "accuracy_threshold": 1430.9112548828125, + "ap": 0.6241882051995578, + "f1": 0.5932101002778785, + "f1_threshold": 1566.4383544921875, + "precision": 0.5471361711611321, + "recall": 0.6477572559366754 + }, + "max": { + "accuracy": 0.8409131549144663, + "ap": 0.6686677647503386, + "f1": 0.6294631710362049 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file diff --git a/evaluation/mteb/TwitterURLCorpus.json b/evaluation/mteb/TwitterURLCorpus.json new file mode 100644 index 0000000000000000000000000000000000000000..6811ead49ae0fbde86aaaff5ef24595f012450e0 --- /dev/null +++ b/evaluation/mteb/TwitterURLCorpus.json @@ -0,0 +1,48 @@ +{ + "test": { + "cos_sim": { + "accuracy": 0.8879574649745798, + "accuracy_threshold": 0.669963002204895, + "ap": 0.8528960532524223, + "f1": 0.7798460043358001, + "f1_threshold": 0.6535664796829224, + "precision": 0.7578090948714223, + "recall": 0.8032029565753003 + }, + "dot": { + "accuracy": 0.855939767920208, + "accuracy_threshold": 1020.4056396484375, + "ap": 0.7614131706694056, + "f1": 0.7270246298696869, + "f1_threshold": 895.0596923828125, + "precision": 0.6527012127894156, + "recall": 0.8204496458269172 + }, + "euclidean": { + "accuracy": 0.8672332828812046, + "accuracy_threshold": 29.07307243347168, + "ap": 0.8084854809178995, + "f1": 0.7247657499809552, + "f1_threshold": 31.70741844177246, + "precision": 0.7171717171717171, + "recall": 0.732522328303049 + }, + "evaluation_time": 926.22, + "manhattan": { + "accuracy": 0.867563162184189, + "accuracy_threshold": 1487.37158203125, + "ap": 0.8087598895575626, + "f1": 0.7254617892068091, + "f1_threshold": 1660.7451171875, + "precision": 0.6849268225960881, + "recall": 0.7710963966738528 + }, + "max": { + "accuracy": 0.8879574649745798, + "ap": 0.8528960532524223, + "f1": 0.7798460043358001 + } + }, + "dataset_version": null, + "mteb_version": "0.0.2" +} \ No newline at end of file