diff --git "a/data.json" "b/data.json" new file mode 100644--- /dev/null +++ "b/data.json" @@ -0,0 +1,11300 @@ +{ + "title": "PL-MTEB: Polish Massive Text Embedding Benchmark", + "description": "", + "paper": "https://arxiv.org/abs/2405.10138", + "citation": "@misc{powiata2024plmteb,\ntitle={PL-MTEB: Polish Massive Text Embedding Benchmark},\nauthor={Rafał Poświata and Sławomir Dadas and Michał Perełkiewicz},\nyear={2024},\neprint={2405.10138},\narchivePrefix={arXiv},\nprimaryClass={cs.CL}\n}", + "options": { + "closeable": true, + "showHelp": false, + "showFooter": false + }, + "metrics": [ + { + "id": "main_score", + "description": "Main score" + } + ], + "taskGroups": [ + { + "id": "Classification", + "description": "" + }, + { + "id": "Clustering", + "description": "" + }, + { + "id": "PairClassification", + "description": "" + }, + { + "id": "Retrieval", + "description": "" + }, + { + "id": "STS", + "description": "" + } + ], + "tasks": [ + { + "id": "CBD", + "name": "CBD", + "groupId": "Classification" + }, + { + "id": "PolEmo2.0-IN", + "name": "PolEmo2.0-IN", + "groupId": "Classification" + }, + { + "id": "PolEmo2.0-OUT", + "name": "PolEmo2.0-OUT", + "groupId": "Classification" + }, + { + "id": "AllegroReviews", + "name": "AllegroReviews", + "groupId": "Classification" + }, + { + "id": "PAC", + "name": "PAC", + "groupId": "Classification" + }, + { + "id": "MassiveIntentClassification", + "name": "MassiveIntentClassification", + "groupId": "Classification" + }, + { + "id": "MassiveScenarioClassification", + "name": "MassiveScenarioClassification", + "groupId": "Classification" + }, + { + "id": "SciField", + "name": "SciField", + "groupId": "Classification" + }, + { + "id": "EightTagsClustering", + "name": "EightTagsClustering", + "groupId": "Clustering" + }, + { + "id": "PlscClusteringS2S", + "name": "PlscClusteringS2S", + "groupId": "Clustering" + }, + { + "id": "PlscClusteringP2P", + "name": "PlscClusteringP2P", + "groupId": "Clustering" + }, + { + "id": "WikinewsPlClusteringS2S", + "name": "WikinewsPlClusteringS2S", + "groupId": "Clustering" + }, + { + "id": "WikinewsPlClusteringP2P", + "name": "WikinewsPlClusteringP2P", + "groupId": "Clustering" + }, + { + "id": "SICK-E-PL", + "name": "SICK-E-PL", + "groupId": "PairClassification" + }, + { + "id": "PPC", + "name": "PPC", + "groupId": "PairClassification" + }, + { + "id": "CDSC-E", + "name": "CDSC-E", + "groupId": "PairClassification" + }, + { + "id": "PSC", + "name": "PSC", + "groupId": "PairClassification" + }, + { + "id": "ArguAna-PL", + "name": "ArguAna-PL", + "groupId": "Retrieval" + }, + { + "id": "DBPedia-PL", + "name": "DBPedia-PL", + "groupId": "Retrieval" + }, + { + "id": "FiQA-PL", + "name": "FiQA-PL", + "groupId": "Retrieval" + }, + { + "id": "HotpotQA-PL", + "name": "HotpotQA-PL", + "groupId": "Retrieval" + }, + { + "id": "MSMARCO-PL", + "name": "MSMARCO-PL", + "groupId": "Retrieval" + }, + { + "id": "NFCorpus-PL", + "name": "NFCorpus-PL", + "groupId": "Retrieval" + }, + { + "id": "NQ-PL", + "name": "NQ-PL", + "groupId": "Retrieval" + }, + { + "id": "Quora-PL", + "name": "Quora-PL", + "groupId": "Retrieval" + }, + { + "id": "SCIDOCS-PL", + "name": "SCIDOCS-PL", + "groupId": "Retrieval" + }, + { + "id": "SciFact-PL", + "name": "SciFact-PL", + "groupId": "Retrieval" + }, + { + "id": "TRECCOVID-PL", + "name": "TRECCOVID-PL", + "groupId": "Retrieval" + }, + { + "id": "SciDefRetrieval", + "name": "SciDefRetrieval", + "groupId": "Retrieval" + }, + { + "id": "SICK-R-PL", + "name": "SICK-R-PL", + "groupId": "STS" + }, + { + "id": "CDSC-R", + "name": "CDSC-R", + "groupId": "STS" + }, + { + "id": "STS22", + "name": "STS22", + "groupId": "STS" + } + ], + "models": [ + { + "id": "LaBSE", + "name": "LaBSE", + "url": "https://huggingface.co/LaBSE" + }, + { + "id": "distiluse-base-multilingual-cased-v2", + "name": "distiluse-base-multilingual-cased-v2", + "url": "https://huggingface.co/distiluse-base-multilingual-cased-v2" + }, + { + "id": "paraphrase-multilingual-MiniLM-L12-v2", + "name": "paraphrase-multilingual-MiniLM-L12-v2", + "url": "https://huggingface.co/paraphrase-multilingual-MiniLM-L12-v2" + }, + { + "id": "paraphrase-multilingual-mpnet-base-v2", + "name": "paraphrase-multilingual-mpnet-base-v2", + "url": "https://huggingface.co/paraphrase-multilingual-mpnet-base-v2" + }, + { + "id": "multilingual-e5-small", + "name": "intfloat/multilingual-e5-small", + "url": "https://huggingface.co/intfloat/multilingual-e5-small" + }, + { + "id": "multilingual-e5-base", + "name": "intfloat/multilingual-e5-base", + "url": "https://huggingface.co/intfloat/multilingual-e5-base" + }, + { + "id": "multilingual-e5-large", + "name": "intfloat/multilingual-e5-large", + "url": "https://huggingface.co/intfloat/multilingual-e5-large" + }, + { + "id": "bge-m3", + "name": "BAAI/bge-m3", + "url": "https://huggingface.co/BAAI/bge-m3" + }, + { + "id": "st-polish-paraphrase-from-mpnet", + "name": "sdadas/st-polish-paraphrase-from-mpnet", + "url": "https://huggingface.co/sdadas/st-polish-paraphrase-from-mpnet" + }, + { + "id": "st-polish-paraphrase-from-distilroberta", + "name": "sdadas/st-polish-paraphrase-from-distilroberta", + "url": "https://huggingface.co/sdadas/st-polish-paraphrase-from-distilroberta" + }, + { + "id": "silver-retriever-base-v1", + "name": "ipipan/silver-retriever-base-v1", + "url": "https://huggingface.co/ipipan/silver-retriever-base-v1" + }, + { + "id": "mmlw-e5-small", + "name": "sdadas/mmlw-e5-small", + "url": "https://huggingface.co/sdadas/mmlw-e5-small" + }, + { + "id": "mmlw-e5-base", + "name": "sdadas/mmlw-e5-base", + "url": "https://huggingface.co/sdadas/mmlw-e5-base" + }, + { + "id": "mmlw-e5-large", + "name": "sdadas/mmlw-e5-large", + "url": "https://huggingface.co/sdadas/mmlw-e5-large" + }, + { + "id": "mmlw-roberta-base", + "name": "sdadas/mmlw-roberta-base", + "url": "https://huggingface.co/sdadas/mmlw-roberta-base" + }, + { + "id": "mmlw-roberta-large", + "name": "sdadas/mmlw-roberta-large", + "url": "https://huggingface.co/sdadas/mmlw-roberta-large" + } + ], + "results": [ + { + "id": "bge-m3", + "tags": [], + "results": { + "CBD": { + "accuracy": 69.23, + "accuracy_stderr": 3.40413, + "ap": 22.08922, + "ap_stderr": 2.74023, + "evaluation_time": 1211.0, + "f1": 58.50981, + "f1_stderr": 3.22339, + "main_score": 69.23 + }, + "AllegroReviews": { + "accuracy": 45.80517, + "accuracy_stderr": 4.85244, + "evaluation_time": 1177.0, + "f1": 39.46514, + "f1_stderr": 2.44887, + "main_score": 45.80517 + }, + "PlscClusteringS2S": { + "evaluation_time": 2254.0, + "v_measure": 37.14482, + "v_measures": 36.88264, + "main_score": 37.14482 + }, + "ArguAna-PL": { + "evaluation_time": 5331.0, + "map_at_1": 25.107, + "map_at_10": 41.171, + "map_at_100": 42.181, + "map_at_1000": 42.194, + "map_at_20": 41.951, + "map_at_3": 36.166, + "map_at_5": 39.097, + "mrr_at_1": 25.676, + "mrr_at_10": 41.381, + "mrr_at_100": 42.391, + "mrr_at_1000": 42.404, + "mrr_at_20": 42.161, + "mrr_at_3": 36.463, + "mrr_at_5": 39.283, + "ndcg_at_1": 25.107, + "ndcg_at_10": 50.089, + "ndcg_at_100": 54.346, + "ndcg_at_1000": 54.678, + "ndcg_at_20": 52.847, + "ndcg_at_3": 39.791, + "ndcg_at_5": 45.083, + "precision_at_1": 25.107, + "precision_at_10": 7.859, + "precision_at_100": 0.971, + "precision_at_1000": 0.1, + "precision_at_20": 4.467, + "precision_at_3": 16.761, + "precision_at_5": 12.632, + "recall_at_1": 25.107, + "recall_at_10": 78.592, + "recall_at_100": 97.084, + "recall_at_1000": 99.644, + "recall_at_20": 89.331, + "recall_at_3": 50.284, + "recall_at_5": 63.158, + "main_score": 50.089 + }, + "PolEmo2.0-OUT": { + "accuracy": 49.39271, + "accuracy_stderr": 4.02524, + "evaluation_time": 1016.0, + "f1": 40.77141, + "f1_stderr": 2.75647, + "main_score": 49.39271 + }, + "PAC": { + "accuracy": 69.15146, + "accuracy_stderr": 4.1835, + "ap": 77.15019, + "ap_stderr": 2.60913, + "evaluation_time": 1201.0, + "f1": 66.39082, + "f1_stderr": 4.278, + "main_score": 69.15146 + }, + "SciField": { + "accuracy": 80.29903, + "accuracy_stderr": 2.65444, + "evaluation_time": 1602.0, + "f1": 79.96645, + "f1_stderr": 2.79662, + "main_score": 80.29903 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 83.26539, + "cos_sim.accuracy_threshold": 91.02463, + "cos_sim.ap": 77.39401, + "cos_sim.f1": 69.55041, + "cos_sim.f1_threshold": 86.31442, + "cos_sim.precision": 66.64491, + "cos_sim.recall": 72.7208, + "dot.accuracy": 83.24501, + "dot.accuracy_threshold": 91.01562, + "dot.ap": 77.35475, + "dot.f1": 69.5803, + "dot.f1_threshold": 86.71875, + "dot.precision": 67.81609, + "dot.recall": 71.43875, + "euclidean.accuracy": 83.26539, + "euclidean.accuracy_threshold": 42.37412, + "euclidean.ap": 77.39281, + "euclidean.f1": 69.55041, + "euclidean.f1_threshold": 52.31562, + "euclidean.precision": 66.64491, + "euclidean.recall": 72.7208, + "evaluation_time": 551.0, + "manhattan.accuracy": 83.18386, + "manhattan.accuracy_threshold": 1092.63882, + "manhattan.ap": 77.30867, + "manhattan.f1": 69.62504, + "manhattan.f1_threshold": 1313.00753, + "manhattan.precision": 67.332, + "manhattan.recall": 72.07977, + "max.accuracy": 83.26539, + "max.ap": 77.39401, + "max.f1": 69.62504, + "main_score": 77.39401 + }, + "HotpotQA-PL": { + "evaluation_time": 837810.0, + "map_at_1": 37.238, + "map_at_10": 55.464, + "map_at_100": 56.316, + "map_at_1000": 56.387, + "map_at_20": 55.976, + "map_at_3": 52.439, + "map_at_5": 54.318, + "mrr_at_1": 74.477, + "mrr_at_10": 80.313, + "mrr_at_100": 80.549, + "mrr_at_1000": 80.561, + "mrr_at_20": 80.464, + "mrr_at_3": 79.242, + "mrr_at_5": 79.884, + "ndcg_at_1": 74.477, + "ndcg_at_10": 64.163, + "ndcg_at_100": 67.205, + "ndcg_at_1000": 68.649, + "ndcg_at_20": 65.473, + "ndcg_at_3": 59.736, + "ndcg_at_5": 62.168, + "precision_at_1": 74.477, + "precision_at_10": 13.244, + "precision_at_100": 1.562, + "precision_at_1000": 0.175, + "precision_at_20": 7.044, + "precision_at_3": 37.614, + "precision_at_5": 24.494, + "recall_at_1": 37.238, + "recall_at_10": 66.219, + "recall_at_100": 78.109, + "recall_at_1000": 87.718, + "recall_at_20": 70.439, + "recall_at_3": 56.421, + "recall_at_5": 61.236, + "main_score": 64.163 + }, + "PPC": { + "cos_sim.accuracy": 82.2, + "cos_sim.accuracy_threshold": 86.57274, + "cos_sim.ap": 92.29288, + "cos_sim.f1": 85.24046, + "cos_sim.f1_threshold": 86.38702, + "cos_sim.precision": 85.38206, + "cos_sim.recall": 85.09934, + "dot.accuracy": 82.2, + "dot.accuracy_threshold": 86.42578, + "dot.ap": 92.25376, + "dot.f1": 85.2649, + "dot.f1_threshold": 86.32812, + "dot.precision": 85.2649, + "dot.recall": 85.2649, + "euclidean.accuracy": 82.2, + "euclidean.accuracy_threshold": 51.82122, + "euclidean.ap": 92.29325, + "euclidean.f1": 85.24046, + "euclidean.f1_threshold": 52.17993, + "euclidean.precision": 85.38206, + "euclidean.recall": 85.09934, + "evaluation_time": 229.0, + "manhattan.accuracy": 81.9, + "manhattan.accuracy_threshold": 1297.01855, + "manhattan.ap": 92.2084, + "manhattan.f1": 85.07832, + "manhattan.f1_threshold": 1319.8164, + "manhattan.precision": 84.72906, + "manhattan.recall": 85.43046, + "max.accuracy": 82.2, + "max.ap": 92.29325, + "max.f1": 85.2649, + "main_score": 92.29288 + }, + "EightTagsClustering": { + "evaluation_time": 5947.0, + "main_score": 27.58241, + "v_measure": 27.58241, + "v_measure_std": 3.77757, + "v_measures": 25.10631 + }, + "NFCorpus-PL": { + "evaluation_time": 2757.0, + "map_at_1": 5.02, + "map_at_10": 11.298, + "map_at_100": 13.553, + "map_at_1000": 14.702, + "map_at_20": 12.219, + "map_at_3": 8.232, + "map_at_5": 9.605, + "mrr_at_1": 43.653, + "mrr_at_10": 51.235, + "mrr_at_100": 51.89, + "mrr_at_1000": 51.942, + "mrr_at_20": 51.655, + "mrr_at_3": 49.226, + "mrr_at_5": 50.17, + "ndcg_at_1": 41.176, + "ndcg_at_10": 30.648, + "ndcg_at_100": 27.086, + "ndcg_at_1000": 35.423, + "ndcg_at_20": 28.252, + "ndcg_at_3": 36.08, + "ndcg_at_5": 33.8, + "precision_at_1": 43.344, + "precision_at_10": 22.539, + "precision_at_100": 6.616, + "precision_at_1000": 1.868, + "precision_at_20": 16.378, + "precision_at_3": 33.746, + "precision_at_5": 29.102, + "recall_at_1": 5.02, + "recall_at_10": 15.092, + "recall_at_100": 26.47, + "recall_at_1000": 56.796, + "recall_at_20": 17.91, + "recall_at_3": 9.235, + "recall_at_5": 11.427, + "main_score": 30.648 + }, + "SciFact-PL": { + "evaluation_time": 3737.0, + "map_at_1": 46.833, + "map_at_10": 56.269, + "map_at_100": 57.076, + "map_at_1000": 57.111, + "map_at_20": 56.851, + "map_at_3": 53.864, + "map_at_5": 54.953, + "mrr_at_1": 49.333, + "mrr_at_10": 57.574, + "mrr_at_100": 58.278, + "mrr_at_1000": 58.307, + "mrr_at_20": 58.082, + "mrr_at_3": 55.778, + "mrr_at_5": 56.661, + "ndcg_at_1": 49.333, + "ndcg_at_10": 60.827, + "ndcg_at_100": 64.355, + "ndcg_at_1000": 65.41, + "ndcg_at_20": 62.814, + "ndcg_at_3": 56.655, + "ndcg_at_5": 58.236, + "precision_at_1": 49.333, + "precision_at_10": 8.333, + "precision_at_100": 1.013, + "precision_at_1000": 0.111, + "precision_at_20": 4.6, + "precision_at_3": 22.556, + "precision_at_5": 14.667, + "recall_at_1": 46.833, + "recall_at_10": 73.422, + "recall_at_100": 89.1, + "recall_at_1000": 97.667, + "recall_at_20": 81.033, + "recall_at_3": 61.822, + "recall_at_5": 66.028, + "main_score": 60.827 + }, + "MassiveScenarioClassification": { + "accuracy": 72.02757, + "accuracy_stderr": 1.72272, + "f1": 71.32873, + "f1_stderr": 1.53579, + "main_score": 72.02757 + }, + "FiQA-PL": { + "evaluation_time": 21963.0, + "map_at_1": 16.159, + "map_at_10": 25.324, + "map_at_100": 26.919, + "map_at_1000": 27.093, + "map_at_20": 26.226, + "map_at_3": 21.848, + "map_at_5": 23.797, + "mrr_at_1": 31.79, + "mrr_at_10": 39.793, + "mrr_at_100": 40.655, + "mrr_at_1000": 40.718, + "mrr_at_20": 40.336, + "mrr_at_3": 37.423, + "mrr_at_5": 38.619, + "ndcg_at_1": 31.79, + "ndcg_at_10": 32.234, + "ndcg_at_100": 38.475, + "ndcg_at_1000": 41.985, + "ndcg_at_20": 34.742, + "ndcg_at_3": 28.799, + "ndcg_at_5": 29.954, + "precision_at_1": 31.79, + "precision_at_10": 8.951, + "precision_at_100": 1.517, + "precision_at_1000": 0.213, + "precision_at_20": 5.44, + "precision_at_3": 18.673, + "precision_at_5": 14.105, + "recall_at_1": 16.159, + "recall_at_10": 38.166, + "recall_at_100": 61.756, + "recall_at_1000": 83.371, + "recall_at_20": 46.093, + "recall_at_3": 25.881, + "recall_at_5": 31.088, + "main_score": 32.234 + }, + "SICK-R-PL": { + "cos_sim.pearson": 77.92638, + "cos_sim.spearman": 74.53951, + "euclidean.pearson": 75.68002, + "euclidean.spearman": 74.53961, + "evaluation_time": 694.0, + "manhattan.pearson": 75.67823, + "manhattan.spearman": 74.51476, + "main_score": 74.53951 + }, + "CDSC-R": { + "cos_sim.pearson": 92.07099, + "cos_sim.spearman": 91.85006, + "euclidean.pearson": 89.95132, + "euclidean.spearman": 91.8485, + "evaluation_time": 154.0, + "manhattan.pearson": 89.90229, + "manhattan.spearman": 91.77693, + "main_score": 91.85006 + }, + "SciDefRetrieval": { + "evaluation_time": 398.0, + "map_at_1": 69.514, + "map_at_10": 76.52, + "map_at_100": 76.845, + "map_at_1000": 76.865, + "map_at_20": 76.724, + "map_at_3": 75.209, + "map_at_5": 76.159, + "mrr_at_1": 69.514, + "mrr_at_10": 76.52, + "mrr_at_100": 76.845, + "mrr_at_1000": 76.865, + "mrr_at_20": 76.724, + "mrr_at_3": 75.209, + "mrr_at_5": 76.159, + "ndcg_at_1": 69.514, + "ndcg_at_10": 79.616, + "ndcg_at_100": 81.155, + "ndcg_at_1000": 81.653, + "ndcg_at_20": 80.36, + "ndcg_at_3": 77.074, + "ndcg_at_5": 78.779, + "precision_at_1": 69.514, + "precision_at_10": 8.91, + "precision_at_100": 0.962, + "precision_at_1000": 0.1, + "precision_at_20": 4.602, + "precision_at_3": 27.491, + "precision_at_5": 17.32, + "recall_at_1": 69.514, + "recall_at_10": 89.102, + "recall_at_100": 96.171, + "recall_at_1000": 100.0, + "recall_at_20": 92.047, + "recall_at_3": 82.474, + "recall_at_5": 86.598, + "main_score": 79.616 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 818.0, + "v_measure": 23.22095, + "v_measures": 26.04313, + "main_score": 23.22095 + }, + "PSC": { + "cos_sim.accuracy": 97.86642, + "cos_sim.accuracy_threshold": 61.1107, + "cos_sim.ap": 99.17135, + "cos_sim.f1": 96.47779, + "cos_sim.f1_threshold": 61.1107, + "cos_sim.precision": 96.92308, + "cos_sim.recall": 96.03659, + "dot.accuracy": 97.86642, + "dot.accuracy_threshold": 61.13281, + "dot.ap": 99.16962, + "dot.f1": 96.47779, + "dot.f1_threshold": 61.13281, + "dot.precision": 96.92308, + "dot.recall": 96.03659, + "euclidean.accuracy": 97.86642, + "euclidean.accuracy_threshold": 88.1966, + "euclidean.ap": 99.1717, + "euclidean.f1": 96.47779, + "euclidean.f1_threshold": 88.1966, + "euclidean.precision": 96.92308, + "euclidean.recall": 96.03659, + "evaluation_time": 424.0, + "manhattan.accuracy": 97.95918, + "manhattan.accuracy_threshold": 2284.08488, + "manhattan.ap": 99.16728, + "manhattan.f1": 96.68675, + "manhattan.f1_threshold": 2284.08488, + "manhattan.precision": 95.53571, + "manhattan.recall": 97.86585, + "max.accuracy": 97.95918, + "max.ap": 99.1717, + "max.f1": 96.68675, + "main_score": 99.17135 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 4069.0, + "v_measure": 42.03094, + "v_measures": 46.07095, + "main_score": 42.03094 + }, + "NQ-PL": { + "evaluation_time": 667440.0, + "map_at_1": 29.143, + "map_at_10": 42.41, + "map_at_100": 43.441, + "map_at_1000": 43.487, + "map_at_20": 43.072, + "map_at_3": 38.855, + "map_at_5": 40.907, + "mrr_at_1": 32.764, + "mrr_at_10": 44.784, + "mrr_at_100": 45.599, + "mrr_at_1000": 45.635, + "mrr_at_20": 45.314, + "mrr_at_3": 41.812, + "mrr_at_5": 43.568, + "ndcg_at_1": 32.764, + "ndcg_at_10": 48.917, + "ndcg_at_100": 53.423, + "ndcg_at_1000": 54.558, + "ndcg_at_20": 51.08, + "ndcg_at_3": 42.234, + "ndcg_at_5": 45.639, + "precision_at_1": 32.764, + "precision_at_10": 7.929, + "precision_at_100": 1.045, + "precision_at_1000": 0.115, + "precision_at_20": 4.47, + "precision_at_3": 19.032, + "precision_at_5": 13.413, + "recall_at_1": 29.143, + "recall_at_10": 66.3, + "recall_at_100": 86.131, + "recall_at_1000": 94.73, + "recall_at_20": 74.336, + "recall_at_3": 49.025, + "recall_at_5": 56.839, + "main_score": 48.917 + }, + "STS22": { + "cos_sim.pearson": 40.66208, + "cos_sim.spearman": 42.81962, + "euclidean.pearson": 27.80387, + "euclidean.spearman": 42.81962, + "manhattan.pearson": 27.8995, + "manhattan.spearman": 42.75492, + "main_score": 42.81962 + }, + "DBPedia-PL": { + "evaluation_time": 711215.0, + "map_at_1": 7.129, + "map_at_10": 14.988, + "map_at_100": 21.109, + "map_at_1000": 22.471, + "map_at_20": 17.347, + "map_at_3": 10.923, + "map_at_5": 12.75, + "mrr_at_1": 54.5, + "mrr_at_10": 64.466, + "mrr_at_100": 64.986, + "mrr_at_1000": 64.993, + "mrr_at_20": 64.798, + "mrr_at_3": 62.792, + "mrr_at_5": 63.679, + "ndcg_at_1": 44.5, + "ndcg_at_10": 33.483, + "ndcg_at_100": 37.958, + "ndcg_at_1000": 44.887, + "ndcg_at_20": 33.239, + "ndcg_at_3": 37.452, + "ndcg_at_5": 35.077, + "precision_at_1": 54.5, + "precision_at_10": 26.8, + "precision_at_100": 8.933, + "precision_at_1000": 1.81, + "precision_at_20": 20.587, + "precision_at_3": 40.667, + "precision_at_5": 33.85, + "recall_at_1": 7.129, + "recall_at_10": 20.053, + "recall_at_100": 43.828, + "recall_at_1000": 65.823, + "recall_at_20": 26.485, + "recall_at_3": 12.55, + "recall_at_5": 15.545, + "main_score": 33.483 + }, + "Quora-PL": { + "evaluation_time": 37467.0, + "map_at_1": 65.309, + "map_at_10": 79.031, + "map_at_100": 79.755, + "map_at_1000": 79.779, + "map_at_20": 79.49, + "map_at_3": 75.883, + "map_at_5": 77.825, + "mrr_at_1": 75.2, + "mrr_at_10": 82.358, + "mrr_at_100": 82.544, + "mrr_at_1000": 82.548, + "mrr_at_20": 82.484, + "mrr_at_3": 81.057, + "mrr_at_5": 81.899, + "ndcg_at_1": 75.32, + "ndcg_at_10": 83.418, + "ndcg_at_100": 85.15, + "ndcg_at_1000": 85.376, + "ndcg_at_20": 84.28, + "ndcg_at_3": 79.865, + "ndcg_at_5": 81.753, + "precision_at_1": 75.32, + "precision_at_10": 12.822, + "precision_at_100": 1.501, + "precision_at_1000": 0.156, + "precision_at_20": 6.863, + "precision_at_3": 35.073, + "precision_at_5": 23.236, + "recall_at_1": 65.309, + "recall_at_10": 92.214, + "recall_at_100": 98.543, + "recall_at_1000": 99.787, + "recall_at_20": 95.03, + "recall_at_3": 82.179, + "recall_at_5": 87.347, + "main_score": 83.418 + }, + "CDSC-E": { + "cos_sim.accuracy": 89.0, + "cos_sim.accuracy_threshold": 93.72418, + "cos_sim.ap": 76.99228, + "cos_sim.f1": 66.85879, + "cos_sim.f1_threshold": 90.46653, + "cos_sim.precision": 73.88535, + "cos_sim.recall": 61.05263, + "dot.accuracy": 89.0, + "dot.accuracy_threshold": 93.75, + "dot.ap": 76.90247, + "dot.f1": 66.86047, + "dot.f1_threshold": 90.57617, + "dot.precision": 74.67532, + "dot.recall": 60.52632, + "euclidean.accuracy": 89.0, + "euclidean.accuracy_threshold": 35.431, + "euclidean.ap": 76.99774, + "euclidean.f1": 66.85879, + "euclidean.f1_threshold": 43.66966, + "euclidean.precision": 73.88535, + "euclidean.recall": 61.05263, + "evaluation_time": 112.0, + "manhattan.accuracy": 89.0, + "manhattan.accuracy_threshold": 892.33813, + "manhattan.ap": 76.92851, + "manhattan.f1": 67.44186, + "manhattan.f1_threshold": 1103.21369, + "manhattan.precision": 75.32468, + "manhattan.recall": 61.05263, + "max.accuracy": 89.0, + "max.ap": 76.99774, + "max.f1": 67.44186, + "main_score": 76.99228 + }, + "TRECCOVID-PL": { + "evaluation_time": 89012.0, + "map_at_1": 0.173, + "map_at_10": 1.066, + "map_at_100": 5.563, + "map_at_1000": 14.777, + "map_at_20": 1.797, + "map_at_3": 0.423, + "map_at_5": 0.68, + "mrr_at_1": 56.0, + "mrr_at_10": 69.152, + "mrr_at_100": 69.649, + "mrr_at_1000": 69.649, + "mrr_at_20": 69.649, + "mrr_at_3": 66.333, + "mrr_at_5": 68.333, + "ndcg_at_1": 55.0, + "ndcg_at_10": 48.775, + "ndcg_at_100": 39.196, + "ndcg_at_1000": 38.479, + "ndcg_at_20": 46.719, + "ndcg_at_3": 53.184, + "ndcg_at_5": 52.522, + "precision_at_1": 58.0, + "precision_at_10": 51.2, + "precision_at_100": 40.66, + "precision_at_1000": 17.802, + "precision_at_20": 49.2, + "precision_at_3": 54.667, + "precision_at_5": 56.0, + "recall_at_1": 0.173, + "recall_at_10": 1.328, + "recall_at_100": 9.257, + "recall_at_1000": 37.243, + "recall_at_20": 2.455, + "recall_at_3": 0.464, + "recall_at_5": 0.786, + "main_score": 48.775 + }, + "SCIDOCS-PL": { + "evaluation_time": 14899.0, + "map_at_1": 3.228, + "map_at_10": 8.284, + "map_at_100": 9.811, + "map_at_1000": 10.051, + "map_at_20": 9.02, + "map_at_3": 6.083, + "map_at_5": 7.224, + "mrr_at_1": 15.9, + "mrr_at_10": 25.414, + "mrr_at_100": 26.529, + "mrr_at_1000": 26.609, + "mrr_at_20": 26.03, + "mrr_at_3": 22.583, + "mrr_at_5": 24.228, + "ndcg_at_1": 15.9, + "ndcg_at_10": 14.435, + "ndcg_at_100": 21.034, + "ndcg_at_1000": 25.913, + "ndcg_at_20": 16.578, + "ndcg_at_3": 13.875, + "ndcg_at_5": 12.135, + "precision_at_1": 15.9, + "precision_at_10": 7.5, + "precision_at_100": 1.711, + "precision_at_1000": 0.289, + "precision_at_20": 4.995, + "precision_at_3": 13.233, + "precision_at_5": 10.82, + "recall_at_1": 3.228, + "recall_at_10": 15.225, + "recall_at_100": 34.743, + "recall_at_1000": 58.678, + "recall_at_20": 20.245, + "recall_at_3": 8.043, + "recall_at_5": 10.972, + "main_score": 14.435 + }, + "PolEmo2.0-IN": { + "accuracy": 74.54294, + "accuracy_stderr": 4.97799, + "evaluation_time": 1268.0, + "f1": 71.54823, + "f1_stderr": 4.71791, + "main_score": 74.54294 + }, + "PlscClusteringP2P": { + "evaluation_time": 13897.0, + "v_measure": 40.78584, + "v_measures": 40.91014, + "main_score": 40.78584 + }, + "MSMARCO-PL": { + "evaluation_time": 1737692.0, + "map_at_1": 14.471, + "map_at_10": 23.842, + "map_at_100": 25.027, + "map_at_1000": 25.103, + "map_at_20": 24.528, + "map_at_3": 20.7, + "map_at_5": 22.561, + "mrr_at_1": 14.957, + "mrr_at_10": 24.334, + "mrr_at_100": 25.487, + "mrr_at_1000": 25.556, + "mrr_at_20": 25.006, + "mrr_at_3": 21.213, + "mrr_at_5": 23.06, + "ndcg_at_1": 14.957, + "ndcg_at_10": 29.276, + "ndcg_at_100": 35.205, + "ndcg_at_1000": 37.223, + "ndcg_at_20": 31.712, + "ndcg_at_3": 22.875, + "ndcg_at_5": 26.203, + "precision_at_1": 14.957, + "precision_at_10": 4.802, + "precision_at_100": 0.78, + "precision_at_1000": 0.095, + "precision_at_20": 2.908, + "precision_at_3": 9.914, + "precision_at_5": 7.625, + "recall_at_1": 14.471, + "recall_at_10": 46.057, + "recall_at_100": 73.914, + "recall_at_1000": 89.611, + "recall_at_20": 55.476, + "recall_at_3": 28.686, + "recall_at_5": 36.691, + "main_score": 29.276 + }, + "MassiveIntentClassification": { + "accuracy": 67.90854, + "accuracy_stderr": 1.47007, + "f1": 64.88948, + "f1_stderr": 1.17338, + "main_score": 67.90854 + } + } + }, + { + "id": "paraphrase-multilingual-mpnet-base-v2", + "tags": [], + "results": { + "CBD": { + "accuracy": 65.0, + "accuracy_stderr": 4.68828, + "ap": 20.4727, + "ap_stderr": 2.92693, + "evaluation_time": 632.0, + "f1": 55.19605, + "f1_stderr": 3.83382, + "main_score": 65.0 + }, + "AllegroReviews": { + "accuracy": 33.85686, + "accuracy_stderr": 4.11228, + "evaluation_time": 271.0, + "f1": 30.59573, + "f1_stderr": 2.93365, + "main_score": 33.85686 + }, + "PlscClusteringS2S": { + "evaluation_time": 1552.0, + "v_measure": 38.0382, + "v_measures": 38.25592, + "main_score": 38.0382 + }, + "ArguAna-PL": { + "evaluation_time": 1090.0, + "map_at_1": 20.697, + "map_at_10": 34.578, + "map_at_100": 35.858, + "map_at_1000": 35.881, + "map_at_3": 29.848, + "map_at_5": 32.732, + "mrr_at_1": 21.266, + "mrr_at_10": 34.808, + "mrr_at_100": 36.088, + "mrr_at_1000": 36.111, + "mrr_at_3": 30.109, + "mrr_at_5": 32.965, + "ndcg_at_1": 20.697, + "ndcg_at_10": 42.616, + "ndcg_at_100": 48.428, + "ndcg_at_1000": 48.99, + "ndcg_at_3": 32.916, + "ndcg_at_5": 38.123, + "precision_at_1": 20.697, + "precision_at_10": 6.842, + "precision_at_100": 0.947, + "precision_at_1000": 0.099, + "precision_at_3": 13.94, + "precision_at_5": 10.896, + "recall_at_1": 20.697, + "recall_at_10": 68.421, + "recall_at_100": 94.737, + "recall_at_1000": 99.075, + "recall_at_3": 41.821, + "recall_at_5": 54.481, + "main_score": 42.616 + }, + "PolEmo2.0-OUT": { + "accuracy": 19.97976, + "accuracy_stderr": 6.61237, + "evaluation_time": 189.0, + "f1": 20.3431, + "f1_stderr": 4.8922, + "main_score": 19.97976 + }, + "PAC": { + "accuracy": 63.75905, + "accuracy_stderr": 4.28969, + "ap": 74.74541, + "ap_stderr": 2.75726, + "evaluation_time": 305.0, + "f1": 61.25248, + "f1_stderr": 4.82675, + "main_score": 63.75905 + }, + "SciField": { + "accuracy": 82.0306, + "accuracy_stderr": 2.2652, + "evaluation_time": 667.0, + "f1": 81.59709, + "f1_stderr": 2.47284, + "main_score": 82.0306 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 83.55075, + "cos_sim.accuracy_threshold": 89.26644, + "cos_sim.ap": 77.22027, + "cos_sim.f1": 70.6533, + "cos_sim.f1_threshold": 86.91501, + "cos_sim.precision": 68.63667, + "cos_sim.recall": 72.79202, + "dot.accuracy": 76.92621, + "dot.accuracy_threshold": 600.15211, + "dot.ap": 60.07445, + "dot.f1": 58.71178, + "dot.f1_threshold": 496.64803, + "dot.precision": 50.33079, + "dot.recall": 70.4416, + "euclidean.accuracy": 83.20424, + "euclidean.accuracy_threshold": 103.01859, + "euclidean.ap": 76.62612, + "euclidean.f1": 70.12547, + "euclidean.f1_threshold": 131.98493, + "euclidean.precision": 66.92557, + "euclidean.recall": 73.64672, + "evaluation_time": 244.0, + "manhattan.accuracy": 83.16347, + "manhattan.accuracy_threshold": 2239.74266, + "manhattan.ap": 76.64447, + "manhattan.f1": 70.4022, + "manhattan.f1_threshold": 2800.61264, + "manhattan.precision": 68.03987, + "manhattan.recall": 72.93447, + "max.accuracy": 83.55075, + "max.ap": 77.22027, + "max.f1": 70.6533, + "main_score": 77.22027 + }, + "HotpotQA-PL": { + "evaluation_time": 353808.0, + "map_at_1": 16.759, + "map_at_10": 22.576, + "map_at_100": 23.388, + "map_at_1000": 23.492, + "map_at_3": 20.907, + "map_at_5": 21.842, + "mrr_at_1": 33.518, + "mrr_at_10": 40.624, + "mrr_at_100": 41.344, + "mrr_at_1000": 41.405, + "mrr_at_3": 38.74, + "mrr_at_5": 39.819, + "ndcg_at_1": 33.518, + "ndcg_at_10": 29.363, + "ndcg_at_100": 33.184, + "ndcg_at_1000": 35.785, + "ndcg_at_3": 26.033, + "ndcg_at_5": 27.665, + "precision_at_1": 33.518, + "precision_at_10": 6.384, + "precision_at_100": 0.942, + "precision_at_1000": 0.129, + "precision_at_3": 16.255, + "precision_at_5": 11.047, + "recall_at_1": 16.759, + "recall_at_10": 31.918, + "recall_at_100": 47.117, + "recall_at_1000": 64.537, + "recall_at_3": 24.382, + "recall_at_5": 27.616, + "main_score": 29.363 + }, + "PPC": { + "cos_sim.accuracy": 88.2, + "cos_sim.accuracy_threshold": 83.3201, + "cos_sim.ap": 93.67153, + "cos_sim.f1": 90.63729, + "cos_sim.f1_threshold": 82.55664, + "cos_sim.precision": 86.35682, + "cos_sim.recall": 95.36424, + "dot.accuracy": 79.7, + "dot.accuracy_threshold": 537.06031, + "dot.ap": 88.16456, + "dot.f1": 84.37259, + "dot.f1_threshold": 537.06031, + "dot.precision": 78.84892, + "dot.recall": 90.72848, + "euclidean.accuracy": 87.9, + "euclidean.accuracy_threshold": 158.67469, + "euclidean.ap": 93.63958, + "euclidean.f1": 90.43887, + "euclidean.f1_threshold": 166.70549, + "euclidean.precision": 85.8631, + "euclidean.recall": 95.5298, + "evaluation_time": 145.0, + "manhattan.accuracy": 87.8, + "manhattan.accuracy_threshold": 3265.40298, + "manhattan.ap": 93.65267, + "manhattan.f1": 90.32258, + "manhattan.f1_threshold": 3495.98083, + "manhattan.precision": 86.05697, + "manhattan.recall": 95.03311, + "max.accuracy": 88.2, + "max.ap": 93.67153, + "max.f1": 90.63729, + "main_score": 93.67153 + }, + "EightTagsClustering": { + "evaluation_time": 3458.0, + "main_score": 25.76417, + "v_measure": 25.76417, + "v_measure_std": 3.1384, + "v_measures": 25.51325 + }, + "NFCorpus-PL": { + "evaluation_time": 423.0, + "map_at_1": 2.534, + "map_at_10": 5.384, + "map_at_100": 7.098, + "map_at_1000": 8.248, + "map_at_3": 3.881, + "map_at_5": 4.464, + "mrr_at_1": 25.387, + "mrr_at_10": 32.067, + "mrr_at_100": 32.974, + "mrr_at_1000": 33.057, + "mrr_at_3": 30.134, + "mrr_at_5": 31.109, + "ndcg_at_1": 23.529, + "ndcg_at_10": 18.53, + "ndcg_at_100": 17.737, + "ndcg_at_1000": 27.417, + "ndcg_at_3": 20.974, + "ndcg_at_5": 19.666, + "precision_at_1": 25.077, + "precision_at_10": 14.644, + "precision_at_100": 5.39, + "precision_at_1000": 1.841, + "precision_at_3": 19.917, + "precision_at_5": 17.647, + "recall_at_1": 2.534, + "recall_at_10": 8.281, + "recall_at_100": 19.261, + "recall_at_1000": 52.49, + "recall_at_3": 4.532, + "recall_at_5": 5.551, + "main_score": 18.53 + }, + "SciFact-PL": { + "evaluation_time": 542.0, + "map_at_1": 26.778, + "map_at_10": 36.525, + "map_at_100": 37.444, + "map_at_1000": 37.511, + "map_at_3": 33.832, + "map_at_5": 35.659, + "mrr_at_1": 28.0, + "mrr_at_10": 37.302, + "mrr_at_100": 38.063, + "mrr_at_1000": 38.121, + "mrr_at_3": 34.778, + "mrr_at_5": 36.561, + "ndcg_at_1": 28.0, + "ndcg_at_10": 41.531, + "ndcg_at_100": 46.265, + "ndcg_at_1000": 48.171, + "ndcg_at_3": 36.45, + "ndcg_at_5": 39.626, + "precision_at_1": 28.0, + "precision_at_10": 6.333, + "precision_at_100": 0.907, + "precision_at_1000": 0.107, + "precision_at_3": 15.667, + "precision_at_5": 11.2, + "recall_at_1": 26.778, + "recall_at_10": 55.922, + "recall_at_100": 78.822, + "recall_at_1000": 94.0, + "recall_at_3": 42.372, + "recall_at_5": 50.522, + "main_score": 41.531 + }, + "MassiveScenarioClassification": { + "accuracy": 68.98453, + "accuracy_stderr": 1.86726, + "f1": 68.68067, + "f1_stderr": 1.27435, + "main_score": 68.98453 + }, + "FiQA-PL": { + "evaluation_time": 4685.0, + "map_at_1": 7.409, + "map_at_10": 11.059, + "map_at_100": 12.166, + "map_at_1000": 12.351, + "map_at_3": 9.666, + "map_at_5": 10.436, + "mrr_at_1": 14.198, + "mrr_at_10": 19.137, + "mrr_at_100": 20.124, + "mrr_at_1000": 20.225, + "mrr_at_3": 17.464, + "mrr_at_5": 18.436, + "ndcg_at_1": 14.198, + "ndcg_at_10": 14.683, + "ndcg_at_100": 20.231, + "ndcg_at_1000": 24.818, + "ndcg_at_3": 13.141, + "ndcg_at_5": 13.601, + "precision_at_1": 14.198, + "precision_at_10": 4.213, + "precision_at_100": 0.977, + "precision_at_1000": 0.177, + "precision_at_3": 8.745, + "precision_at_5": 6.543, + "recall_at_1": 7.409, + "recall_at_10": 17.564, + "recall_at_100": 39.575, + "recall_at_1000": 68.726, + "recall_at_3": 11.672, + "recall_at_5": 14.094, + "main_score": 14.683 + }, + "SICK-R-PL": { + "cos_sim.pearson": 79.85472, + "cos_sim.spearman": 73.12822, + "euclidean.pearson": 76.92803, + "euclidean.spearman": 73.16464, + "evaluation_time": 290.0, + "manhattan.pearson": 76.92901, + "manhattan.spearman": 73.10307, + "main_score": 73.12822 + }, + "CDSC-R": { + "cos_sim.pearson": 87.85363, + "cos_sim.spearman": 88.79684, + "euclidean.pearson": 89.60743, + "euclidean.spearman": 89.30708, + "evaluation_time": 108.0, + "manhattan.pearson": 89.42806, + "manhattan.spearman": 89.11899, + "main_score": 88.79684 + }, + "SciDefRetrieval": { + "evaluation_time": 157.0, + "map_at_1": 31.37, + "map_at_10": 40.488, + "map_at_100": 41.494, + "map_at_1000": 41.567, + "map_at_20": 41.097, + "map_at_3": 38.169, + "map_at_5": 39.318, + "mrr_at_1": 31.517, + "mrr_at_10": 40.549, + "mrr_at_100": 41.554, + "mrr_at_1000": 41.629, + "mrr_at_20": 41.159, + "mrr_at_3": 38.193, + "mrr_at_5": 39.379, + "ndcg_at_1": 31.37, + "ndcg_at_10": 45.245, + "ndcg_at_100": 50.395, + "ndcg_at_1000": 52.304, + "ndcg_at_20": 47.427, + "ndcg_at_3": 40.358, + "ndcg_at_5": 42.425, + "precision_at_1": 31.37, + "precision_at_10": 6.038, + "precision_at_100": 0.851, + "precision_at_1000": 0.1, + "precision_at_20": 3.446, + "precision_at_3": 15.562, + "precision_at_5": 10.339, + "recall_at_1": 31.37, + "recall_at_10": 60.383, + "recall_at_100": 85.125, + "recall_at_1000": 100.0, + "recall_at_20": 68.925, + "recall_at_3": 46.686, + "recall_at_5": 51.694, + "main_score": 45.245 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 567.0, + "v_measure": 26.36732, + "v_measures": 28.20673, + "main_score": 26.36732 + }, + "PSC": { + "cos_sim.accuracy": 96.66048, + "cos_sim.accuracy_threshold": 67.5139, + "cos_sim.ap": 98.25582, + "cos_sim.f1": 94.52888, + "cos_sim.f1_threshold": 67.5139, + "cos_sim.precision": 94.24242, + "cos_sim.recall": 94.81707, + "dot.accuracy": 87.94063, + "dot.accuracy_threshold": 288.95206, + "dot.ap": 88.07347, + "dot.f1": 81.42077, + "dot.f1_threshold": 269.52176, + "dot.precision": 73.76238, + "dot.recall": 90.85366, + "euclidean.accuracy": 95.26902, + "euclidean.accuracy_threshold": 180.63406, + "euclidean.ap": 97.28221, + "euclidean.f1": 92.1659, + "euclidean.f1_threshold": 181.12392, + "euclidean.precision": 92.87926, + "euclidean.recall": 91.46341, + "evaluation_time": 161.0, + "manhattan.accuracy": 95.36178, + "manhattan.accuracy_threshold": 3936.24344, + "manhattan.ap": 97.29532, + "manhattan.f1": 92.44713, + "manhattan.f1_threshold": 3936.24344, + "manhattan.precision": 91.61677, + "manhattan.recall": 93.29268, + "max.accuracy": 96.66048, + "max.ap": 98.25582, + "max.f1": 94.52888, + "main_score": 98.25582 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 941.0, + "v_measure": 35.64889, + "v_measures": 34.77451, + "main_score": 35.64889 + }, + "NQ-PL": { + "evaluation_time": 201402.0, + "map_at_1": 7.022, + "map_at_10": 12.283, + "map_at_100": 13.27, + "map_at_1000": 13.366, + "map_at_3": 10.47, + "map_at_5": 11.423, + "mrr_at_1": 7.995, + "mrr_at_10": 13.519, + "mrr_at_100": 14.439, + "mrr_at_1000": 14.522, + "mrr_at_3": 11.718, + "mrr_at_5": 12.667, + "ndcg_at_1": 7.995, + "ndcg_at_10": 15.637, + "ndcg_at_100": 20.692, + "ndcg_at_1000": 23.42, + "ndcg_at_3": 11.898, + "ndcg_at_5": 13.582, + "precision_at_1": 7.995, + "precision_at_10": 2.969, + "precision_at_100": 0.585, + "precision_at_1000": 0.085, + "precision_at_3": 5.784, + "precision_at_5": 4.444, + "recall_at_1": 7.022, + "recall_at_10": 24.788, + "recall_at_100": 48.346, + "recall_at_1000": 69.467, + "recall_at_3": 14.791, + "recall_at_5": 18.687, + "main_score": 15.637 + }, + "STS22": { + "cos_sim.pearson": 31.43453, + "cos_sim.spearman": 33.6411, + "euclidean.pearson": 17.70048, + "euclidean.spearman": 29.30685, + "manhattan.pearson": 17.48715, + "manhattan.spearman": 29.38504, + "main_score": 33.6411 + }, + "DBPedia-PL": { + "evaluation_time": 305919.0, + "map_at_1": 4.251, + "map_at_10": 8.446, + "map_at_100": 11.349, + "map_at_1000": 12.193, + "map_at_3": 6.442, + "map_at_5": 7.22, + "mrr_at_1": 36.0, + "mrr_at_10": 44.717, + "mrr_at_100": 45.491, + "mrr_at_1000": 45.522, + "mrr_at_3": 42.5, + "mrr_at_5": 43.65, + "ndcg_at_1": 27.5, + "ndcg_at_10": 20.185, + "ndcg_at_100": 22.868, + "ndcg_at_1000": 28.86, + "ndcg_at_3": 23.121, + "ndcg_at_5": 21.31, + "precision_at_1": 36.0, + "precision_at_10": 16.625, + "precision_at_100": 5.227, + "precision_at_1000": 1.299, + "precision_at_3": 27.083, + "precision_at_5": 21.65, + "recall_at_1": 4.251, + "recall_at_10": 12.392, + "recall_at_100": 28.574, + "recall_at_1000": 48.763, + "recall_at_3": 7.538, + "recall_at_5": 9.187, + "main_score": 20.185 + }, + "Quora-PL": { + "evaluation_time": 20321.0, + "map_at_1": 61.142, + "map_at_10": 74.575, + "map_at_100": 75.429, + "map_at_1000": 75.456, + "map_at_3": 71.393, + "map_at_5": 73.373, + "mrr_at_1": 70.55, + "mrr_at_10": 78.063, + "mrr_at_100": 78.35, + "mrr_at_1000": 78.357, + "mrr_at_3": 76.543, + "mrr_at_5": 77.497, + "ndcg_at_1": 70.57, + "ndcg_at_10": 79.183, + "ndcg_at_100": 81.443, + "ndcg_at_1000": 81.798, + "ndcg_at_3": 75.372, + "ndcg_at_5": 77.416, + "precision_at_1": 70.57, + "precision_at_10": 12.283, + "precision_at_100": 1.483, + "precision_at_1000": 0.155, + "precision_at_3": 33.18, + "precision_at_5": 22.18, + "recall_at_1": 61.142, + "recall_at_10": 88.614, + "recall_at_100": 97.15, + "recall_at_1000": 99.382, + "recall_at_3": 77.938, + "recall_at_5": 83.384, + "main_score": 79.183 + }, + "CDSC-E": { + "cos_sim.accuracy": 89.1, + "cos_sim.accuracy_threshold": 93.97279, + "cos_sim.ap": 75.76397, + "cos_sim.f1": 68.0, + "cos_sim.f1_threshold": 93.37923, + "cos_sim.precision": 74.375, + "cos_sim.recall": 62.63158, + "dot.accuracy": 84.8, + "dot.accuracy_threshold": 708.96988, + "dot.ap": 58.84174, + "dot.f1": 57.74648, + "dot.f1_threshold": 616.4609, + "dot.precision": 52.11864, + "dot.recall": 64.73684, + "euclidean.accuracy": 88.9, + "euclidean.accuracy_threshold": 67.18214, + "euclidean.ap": 75.33366, + "euclidean.f1": 66.85083, + "euclidean.f1_threshold": 101.147, + "euclidean.precision": 70.34884, + "euclidean.recall": 63.68421, + "evaluation_time": 88.0, + "manhattan.accuracy": 88.9, + "manhattan.accuracy_threshold": 1748.79913, + "manhattan.ap": 75.15904, + "manhattan.f1": 66.85083, + "manhattan.f1_threshold": 2144.70367, + "manhattan.precision": 70.34884, + "manhattan.recall": 63.68421, + "max.accuracy": 89.1, + "max.ap": 75.76397, + "max.f1": 68.0, + "main_score": 75.76397 + }, + "TRECCOVID-PL": { + "evaluation_time": 13337.0, + "map_at_1": 0.107, + "map_at_10": 0.718, + "map_at_100": 3.477, + "map_at_1000": 7.355, + "map_at_3": 0.304, + "map_at_5": 0.429, + "mrr_at_1": 40.0, + "mrr_at_10": 54.519, + "mrr_at_100": 55.198, + "mrr_at_1000": 55.198, + "mrr_at_3": 50.667, + "mrr_at_5": 53.567, + "ndcg_at_1": 39.0, + "ndcg_at_10": 35.383, + "ndcg_at_100": 24.86, + "ndcg_at_1000": 23.088, + "ndcg_at_3": 40.838, + "ndcg_at_5": 38.11, + "precision_at_1": 40.0, + "precision_at_10": 36.8, + "precision_at_100": 25.26, + "precision_at_1000": 10.03, + "precision_at_3": 44.667, + "precision_at_5": 40.8, + "recall_at_1": 0.107, + "recall_at_10": 0.951, + "recall_at_100": 6.094, + "recall_at_1000": 22.2, + "recall_at_3": 0.351, + "recall_at_5": 0.53, + "main_score": 35.383 + }, + "SCIDOCS-PL": { + "evaluation_time": 2304.0, + "map_at_1": 2.65, + "map_at_10": 6.277, + "map_at_100": 7.522, + "map_at_1000": 7.771, + "map_at_3": 4.613, + "map_at_5": 5.489, + "mrr_at_1": 13.0, + "mrr_at_10": 20.058, + "mrr_at_100": 21.229, + "mrr_at_1000": 21.331, + "mrr_at_3": 17.45, + "mrr_at_5": 19.06, + "ndcg_at_1": 13.0, + "ndcg_at_10": 11.178, + "ndcg_at_100": 16.99, + "ndcg_at_1000": 22.26, + "ndcg_at_3": 10.521, + "ndcg_at_5": 9.361, + "precision_at_1": 13.0, + "precision_at_10": 5.82, + "precision_at_100": 1.431, + "precision_at_1000": 0.27, + "precision_at_3": 9.867, + "precision_at_5": 8.3, + "recall_at_1": 2.65, + "recall_at_10": 11.823, + "recall_at_100": 29.058, + "recall_at_1000": 54.873, + "recall_at_3": 6.03, + "recall_at_5": 8.448, + "main_score": 11.178 + }, + "PolEmo2.0-IN": { + "accuracy": 62.78393, + "accuracy_stderr": 4.33013, + "evaluation_time": 227.0, + "f1": 62.85485, + "f1_stderr": 2.91347, + "main_score": 62.78393 + }, + "PlscClusteringP2P": { + "evaluation_time": 2133.0, + "v_measure": 41.15439, + "v_measures": 41.79101, + "main_score": 41.15439 + }, + "MSMARCO-PL": { + "evaluation_time": 728128.0, + "map_at_1": 5.528, + "map_at_10": 9.761, + "map_at_100": 10.536, + "map_at_1000": 10.622, + "map_at_3": 8.207, + "map_at_5": 9.036, + "mrr_at_1": 5.688, + "mrr_at_10": 10.005, + "mrr_at_100": 10.771, + "mrr_at_1000": 10.854, + "mrr_at_3": 8.405, + "mrr_at_5": 9.265, + "ndcg_at_1": 5.702, + "ndcg_at_10": 12.447, + "ndcg_at_100": 16.633, + "ndcg_at_1000": 19.249, + "ndcg_at_3": 9.163, + "ndcg_at_5": 10.667, + "precision_at_1": 5.702, + "precision_at_10": 2.172, + "precision_at_100": 0.435, + "precision_at_1000": 0.066, + "precision_at_3": 4.054, + "precision_at_5": 3.192, + "recall_at_1": 5.528, + "recall_at_10": 20.85, + "recall_at_100": 41.184, + "recall_at_1000": 62.267, + "recall_at_3": 11.738, + "recall_at_5": 15.353, + "main_score": 12.447 + }, + "MassiveIntentClassification": { + "accuracy": 64.29052, + "accuracy_stderr": 1.35028, + "f1": 61.69424, + "f1_stderr": 1.15491, + "main_score": 64.29052 + } + } + }, + { + "id": "st-polish-paraphrase-from-distilroberta", + "tags": [], + "results": { + "CBD": { + "accuracy": 70.27, + "accuracy_stderr": 6.65658, + "ap": 23.09734, + "ap_stderr": 4.30917, + "evaluation_time": 1206.0, + "f1": 59.12637, + "f1_stderr": 5.92886, + "main_score": 70.27 + }, + "AllegroReviews": { + "accuracy": 34.50298, + "accuracy_stderr": 4.16771, + "evaluation_time": 292.0, + "f1": 31.18559, + "f1_stderr": 2.97517, + "main_score": 34.50298 + }, + "PlscClusteringS2S": { + "evaluation_time": 1500.0, + "v_measure": 40.4555, + "v_measures": 40.59808, + "main_score": 40.4555 + }, + "ArguAna-PL": { + "evaluation_time": 1656.0, + "map_at_1": 24.609, + "map_at_10": 40.616, + "map_at_100": 41.719, + "map_at_1000": 41.728, + "map_at_3": 35.775, + "map_at_5": 38.578, + "mrr_at_1": 25.605, + "mrr_at_10": 40.994, + "mrr_at_100": 42.09, + "mrr_at_1000": 42.099, + "mrr_at_3": 36.131, + "mrr_at_5": 38.915, + "ndcg_at_1": 24.609, + "ndcg_at_10": 49.417, + "ndcg_at_100": 54.095, + "ndcg_at_1000": 54.312, + "ndcg_at_3": 39.428, + "ndcg_at_5": 44.502, + "precision_at_1": 24.609, + "precision_at_10": 7.752, + "precision_at_100": 0.979, + "precision_at_1000": 0.1, + "precision_at_3": 16.667, + "precision_at_5": 12.475, + "recall_at_1": 24.609, + "recall_at_10": 77.525, + "recall_at_100": 97.937, + "recall_at_1000": 99.573, + "recall_at_3": 50.0, + "recall_at_5": 62.376, + "main_score": 49.417 + }, + "PolEmo2.0-OUT": { + "accuracy": 38.583, + "accuracy_stderr": 5.76848, + "evaluation_time": 224.0, + "f1": 33.28002, + "f1_stderr": 4.2286, + "main_score": 38.583 + }, + "PAC": { + "accuracy": 64.60469, + "accuracy_stderr": 4.7236, + "ap": 75.25323, + "ap_stderr": 2.72422, + "evaluation_time": 283.0, + "f1": 62.05209, + "f1_stderr": 5.19516, + "main_score": 64.60469 + }, + "SciField": { + "accuracy": 82.06537, + "accuracy_stderr": 3.13837, + "evaluation_time": 363.0, + "f1": 81.76849, + "f1_stderr": 3.32448, + "main_score": 82.06537 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 84.16225, + "cos_sim.accuracy_threshold": 91.19149, + "cos_sim.ap": 79.63053, + "cos_sim.f1": 71.98908, + "cos_sim.f1_threshold": 86.88762, + "cos_sim.precision": 69.08972, + "cos_sim.recall": 75.14245, + "dot.accuracy": 79.59641, + "dot.accuracy_threshold": 15560.48584, + "dot.ap": 68.41789, + "dot.f1": 64.15425, + "dot.f1_threshold": 13179.78973, + "dot.precision": 54.38336, + "dot.recall": 78.20513, + "euclidean.accuracy": 83.83612, + "euclidean.accuracy_threshold": 561.96575, + "euclidean.ap": 78.75762, + "euclidean.f1": 71.44306, + "euclidean.f1_threshold": 666.10136, + "euclidean.precision": 67.97428, + "euclidean.recall": 75.2849, + "evaluation_time": 252.0, + "manhattan.accuracy": 83.81574, + "manhattan.accuracy_threshold": 12046.45233, + "manhattan.ap": 78.87254, + "manhattan.f1": 71.41919, + "manhattan.f1_threshold": 14970.69397, + "manhattan.precision": 66.28049, + "manhattan.recall": 77.42165, + "max.accuracy": 84.16225, + "max.ap": 79.63053, + "max.f1": 71.98908, + "main_score": 79.63053 + }, + "HotpotQA-PL": { + "evaluation_time": 413587.0, + "map_at_1": 13.349, + "map_at_10": 17.759, + "map_at_100": 18.431, + "map_at_1000": 18.528, + "map_at_3": 16.43, + "map_at_5": 17.111, + "mrr_at_1": 26.698, + "mrr_at_10": 32.77, + "mrr_at_100": 33.516, + "mrr_at_1000": 33.586, + "mrr_at_3": 31.02, + "mrr_at_5": 31.939, + "ndcg_at_1": 26.698, + "ndcg_at_10": 23.469, + "ndcg_at_100": 27.065, + "ndcg_at_1000": 29.634, + "ndcg_at_3": 20.628, + "ndcg_at_5": 21.891, + "precision_at_1": 26.698, + "precision_at_10": 5.153, + "precision_at_100": 0.808, + "precision_at_1000": 0.115, + "precision_at_3": 12.856, + "precision_at_5": 8.716, + "recall_at_1": 13.349, + "recall_at_10": 25.766, + "recall_at_100": 40.385, + "recall_at_1000": 57.603, + "recall_at_3": 19.284, + "recall_at_5": 21.789, + "main_score": 23.469 + }, + "PPC": { + "cos_sim.accuracy": 88.3, + "cos_sim.accuracy_threshold": 85.84399, + "cos_sim.ap": 93.29086, + "cos_sim.f1": 90.75369, + "cos_sim.f1_threshold": 82.58413, + "cos_sim.precision": 85.50512, + "cos_sim.recall": 96.68874, + "dot.accuracy": 75.2, + "dot.accuracy_threshold": 12881.42395, + "dot.ap": 86.63422, + "dot.f1": 80.66465, + "dot.f1_threshold": 11022.88208, + "dot.precision": 74.16667, + "dot.recall": 88.4106, + "euclidean.accuracy": 88.7, + "euclidean.accuracy_threshold": 643.12625, + "euclidean.ap": 93.27246, + "euclidean.f1": 90.93775, + "euclidean.f1_threshold": 710.09407, + "euclidean.precision": 86.76692, + "euclidean.recall": 95.5298, + "evaluation_time": 112.0, + "manhattan.accuracy": 88.8, + "manhattan.accuracy_threshold": 15290.75012, + "manhattan.ap": 93.30814, + "manhattan.f1": 91.19497, + "manhattan.f1_threshold": 15611.19843, + "manhattan.precision": 86.82635, + "manhattan.recall": 96.02649, + "max.accuracy": 88.8, + "max.ap": 93.30814, + "max.f1": 91.19497, + "main_score": 93.29086 + }, + "EightTagsClustering": { + "evaluation_time": 3421.0, + "main_score": 32.74821, + "v_measure": 32.74821, + "v_measure_std": 5.12489, + "v_measures": 26.56176 + }, + "NFCorpus-PL": { + "evaluation_time": 1932.0, + "map_at_1": 2.744, + "map_at_10": 6.633, + "map_at_100": 8.974, + "map_at_1000": 10.341, + "map_at_3": 4.815, + "map_at_5": 5.525, + "mrr_at_1": 29.721, + "mrr_at_10": 39.839, + "mrr_at_100": 40.69, + "mrr_at_1000": 40.75, + "mrr_at_3": 37.41, + "mrr_at_5": 38.958, + "ndcg_at_1": 28.173, + "ndcg_at_10": 22.486, + "ndcg_at_100": 22.195, + "ndcg_at_1000": 31.867, + "ndcg_at_3": 25.982, + "ndcg_at_5": 24.223, + "precision_at_1": 29.412, + "precision_at_10": 17.461, + "precision_at_100": 6.467, + "precision_at_1000": 1.967, + "precision_at_3": 24.561, + "precision_at_5": 21.672, + "recall_at_1": 2.744, + "recall_at_10": 10.334, + "recall_at_100": 24.303, + "recall_at_1000": 58.316, + "recall_at_3": 6.283, + "recall_at_5": 7.513, + "main_score": 22.486 + }, + "SciFact-PL": { + "evaluation_time": 2761.0, + "map_at_1": 35.567, + "map_at_10": 44.921, + "map_at_100": 45.808, + "map_at_1000": 45.867, + "map_at_3": 42.183, + "map_at_5": 43.875, + "mrr_at_1": 37.667, + "mrr_at_10": 46.168, + "mrr_at_100": 46.935, + "mrr_at_1000": 46.985, + "mrr_at_3": 43.889, + "mrr_at_5": 45.222, + "ndcg_at_1": 37.667, + "ndcg_at_10": 49.495, + "ndcg_at_100": 54.243, + "ndcg_at_1000": 55.689, + "ndcg_at_3": 44.57, + "ndcg_at_5": 47.083, + "precision_at_1": 37.667, + "precision_at_10": 7.067, + "precision_at_100": 0.97, + "precision_at_1000": 0.11, + "precision_at_3": 17.889, + "precision_at_5": 12.4, + "recall_at_1": 35.567, + "recall_at_10": 62.367, + "recall_at_100": 85.456, + "recall_at_1000": 96.667, + "recall_at_3": 49.133, + "recall_at_5": 55.133, + "main_score": 49.495 + }, + "MassiveScenarioClassification": { + "accuracy": 70.01009, + "accuracy_stderr": 1.73612, + "f1": 69.7344, + "f1_stderr": 1.19327, + "main_score": 70.01009 + }, + "FiQA-PL": { + "evaluation_time": 7803.0, + "map_at_1": 8.822, + "map_at_10": 14.431, + "map_at_100": 15.701, + "map_at_1000": 15.91, + "map_at_3": 12.229, + "map_at_5": 13.217, + "mrr_at_1": 17.747, + "mrr_at_10": 24.417, + "mrr_at_100": 25.342, + "mrr_at_1000": 25.441, + "mrr_at_3": 22.145, + "mrr_at_5": 23.125, + "ndcg_at_1": 17.747, + "ndcg_at_10": 19.578, + "ndcg_at_100": 25.274, + "ndcg_at_1000": 29.889, + "ndcg_at_3": 16.548, + "ndcg_at_5": 17.259, + "precision_at_1": 17.747, + "precision_at_10": 5.787, + "precision_at_100": 1.156, + "precision_at_1000": 0.193, + "precision_at_3": 11.06, + "precision_at_5": 8.302, + "recall_at_1": 8.822, + "recall_at_10": 24.918, + "recall_at_100": 46.954, + "recall_at_1000": 75.956, + "recall_at_3": 14.998, + "recall_at_5": 18.178, + "main_score": 19.578 + }, + "SICK-R-PL": { + "cos_sim.pearson": 82.71716, + "cos_sim.spearman": 76.37139, + "euclidean.pearson": 80.36072, + "euclidean.spearman": 75.67272, + "evaluation_time": 272.0, + "manhattan.pearson": 80.24531, + "manhattan.spearman": 75.61799, + "main_score": 76.37139 + }, + "CDSC-R": { + "cos_sim.pearson": 88.98237, + "cos_sim.spearman": 89.6163, + "euclidean.pearson": 90.38875, + "euclidean.spearman": 89.83228, + "evaluation_time": 91.0, + "manhattan.pearson": 90.28804, + "manhattan.spearman": 89.72354, + "main_score": 89.6163 + }, + "SciDefRetrieval": { + "evaluation_time": 175.0, + "map_at_1": 38.292, + "map_at_10": 48.374, + "map_at_100": 49.243, + "map_at_1000": 49.291, + "map_at_20": 48.91, + "map_at_3": 45.876, + "map_at_5": 47.239, + "mrr_at_1": 38.144, + "mrr_at_10": 48.298, + "mrr_at_100": 49.179, + "mrr_at_1000": 49.227, + "mrr_at_20": 48.84, + "mrr_at_3": 45.852, + "mrr_at_5": 47.177, + "ndcg_at_1": 38.292, + "ndcg_at_10": 53.544, + "ndcg_at_100": 57.893, + "ndcg_at_1000": 59.112, + "ndcg_at_20": 55.51, + "ndcg_at_3": 48.318, + "ndcg_at_5": 50.759, + "precision_at_1": 38.292, + "precision_at_10": 6.996, + "precision_at_100": 0.906, + "precision_at_1000": 0.1, + "precision_at_20": 3.888, + "precision_at_3": 18.459, + "precision_at_5": 12.253, + "recall_at_1": 38.292, + "recall_at_10": 69.956, + "recall_at_100": 90.574, + "recall_at_1000": 100.0, + "recall_at_20": 77.761, + "recall_at_3": 55.376, + "recall_at_5": 61.267, + "main_score": 53.544 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 599.0, + "v_measure": 32.47178, + "v_measures": 31.04682, + "main_score": 32.47178 + }, + "PSC": { + "cos_sim.accuracy": 97.03154, + "cos_sim.accuracy_threshold": 64.11148, + "cos_sim.ap": 99.10327, + "cos_sim.f1": 95.25223, + "cos_sim.f1_threshold": 61.44285, + "cos_sim.precision": 92.77457, + "cos_sim.recall": 97.86585, + "dot.accuracy": 95.54731, + "dot.accuracy_threshold": 4817.89627, + "dot.ap": 96.07153, + "dot.f1": 92.86754, + "dot.f1_threshold": 4491.56418, + "dot.precision": 88.85794, + "dot.recall": 97.2561, + "euclidean.accuracy": 95.17625, + "euclidean.accuracy_threshold": 765.60431, + "euclidean.ap": 97.24052, + "euclidean.f1": 92.2619, + "euclidean.f1_threshold": 767.84821, + "euclidean.precision": 90.11628, + "euclidean.recall": 94.5122, + "evaluation_time": 139.0, + "manhattan.accuracy": 94.99072, + "manhattan.accuracy_threshold": 16177.58484, + "manhattan.ap": 97.23251, + "manhattan.f1": 91.91617, + "manhattan.f1_threshold": 16495.63904, + "manhattan.precision": 90.29412, + "manhattan.recall": 93.59756, + "max.accuracy": 97.03154, + "max.ap": 99.10327, + "max.f1": 95.25223, + "main_score": 99.10327 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 960.0, + "v_measure": 41.28874, + "v_measures": 42.71887, + "main_score": 41.28874 + }, + "NQ-PL": { + "evaluation_time": 591192.0, + "map_at_1": 8.575, + "map_at_10": 15.48, + "map_at_100": 16.542, + "map_at_1000": 16.639, + "map_at_3": 13.085, + "map_at_5": 14.402, + "mrr_at_1": 9.705, + "mrr_at_10": 17.002, + "mrr_at_100": 17.938, + "mrr_at_1000": 18.021, + "mrr_at_3": 14.557, + "mrr_at_5": 15.92, + "ndcg_at_1": 9.705, + "ndcg_at_10": 19.828, + "ndcg_at_100": 25.176, + "ndcg_at_1000": 27.877, + "ndcg_at_3": 14.938, + "ndcg_at_5": 17.294, + "precision_at_1": 9.705, + "precision_at_10": 3.74, + "precision_at_100": 0.684, + "precision_at_1000": 0.094, + "precision_at_3": 7.242, + "precision_at_5": 5.672, + "recall_at_1": 8.575, + "recall_at_10": 31.759, + "recall_at_100": 56.629, + "recall_at_1000": 77.441, + "recall_at_3": 18.854, + "recall_at_5": 24.324, + "main_score": 19.828 + }, + "STS22": { + "cos_sim.pearson": 39.63038, + "cos_sim.spearman": 40.35977, + "euclidean.pearson": 30.27238, + "euclidean.spearman": 40.2476, + "manhattan.pearson": 30.31154, + "manhattan.spearman": 39.99383, + "main_score": 40.35977 + }, + "DBPedia-PL": { + "evaluation_time": 346035.0, + "map_at_1": 4.584, + "map_at_10": 8.758, + "map_at_100": 10.911, + "map_at_1000": 11.567, + "map_at_3": 6.868, + "map_at_5": 7.67, + "mrr_at_1": 36.0, + "mrr_at_10": 44.921, + "mrr_at_100": 45.621, + "mrr_at_1000": 45.672, + "mrr_at_3": 43.0, + "mrr_at_5": 44.0, + "ndcg_at_1": 26.5, + "ndcg_at_10": 19.817, + "ndcg_at_100": 21.491, + "ndcg_at_1000": 26.981, + "ndcg_at_3": 23.0, + "ndcg_at_5": 21.126, + "precision_at_1": 36.0, + "precision_at_10": 15.7, + "precision_at_100": 4.472, + "precision_at_1000": 1.11, + "precision_at_3": 26.667, + "precision_at_5": 21.4, + "recall_at_1": 4.584, + "recall_at_10": 12.594, + "recall_at_100": 25.266, + "recall_at_1000": 44.475, + "recall_at_3": 7.862, + "recall_at_5": 9.476, + "main_score": 19.817 + }, + "Quora-PL": { + "evaluation_time": 20465.0, + "map_at_1": 62.94, + "map_at_10": 76.635, + "map_at_100": 77.417, + "map_at_1000": 77.442, + "map_at_3": 73.438, + "map_at_5": 75.399, + "mrr_at_1": 72.59, + "mrr_at_10": 79.97, + "mrr_at_100": 80.208, + "mrr_at_1000": 80.214, + "mrr_at_3": 78.57, + "mrr_at_5": 79.463, + "ndcg_at_1": 72.66, + "ndcg_at_10": 81.166, + "ndcg_at_100": 83.147, + "ndcg_at_1000": 83.439, + "ndcg_at_3": 77.451, + "ndcg_at_5": 79.431, + "precision_at_1": 72.66, + "precision_at_10": 12.586, + "precision_at_100": 1.495, + "precision_at_1000": 0.155, + "precision_at_3": 34.103, + "precision_at_5": 22.704, + "recall_at_1": 62.94, + "recall_at_10": 90.413, + "recall_at_100": 97.865, + "recall_at_1000": 99.65, + "recall_at_3": 79.937, + "recall_at_5": 85.301, + "main_score": 81.166 + }, + "CDSC-E": { + "cos_sim.accuracy": 89.0, + "cos_sim.accuracy_threshold": 94.88986, + "cos_sim.ap": 75.99465, + "cos_sim.f1": 66.86747, + "cos_sim.f1_threshold": 94.80755, + "cos_sim.precision": 78.16901, + "cos_sim.recall": 58.42105, + "dot.accuracy": 85.7, + "dot.accuracy_threshold": 16685.1181, + "dot.ap": 58.71114, + "dot.f1": 56.86654, + "dot.f1_threshold": 12936.09314, + "dot.precision": 44.95413, + "dot.recall": 77.36842, + "euclidean.accuracy": 88.5, + "euclidean.accuracy_threshold": 293.41235, + "euclidean.ap": 75.15287, + "euclidean.f1": 66.31579, + "euclidean.f1_threshold": 501.10111, + "euclidean.precision": 66.31579, + "euclidean.recall": 66.31579, + "evaluation_time": 74.0, + "manhattan.accuracy": 88.7, + "manhattan.accuracy_threshold": 8184.71603, + "manhattan.ap": 75.16699, + "manhattan.f1": 66.12903, + "manhattan.f1_threshold": 10449.04175, + "manhattan.precision": 67.58242, + "manhattan.recall": 64.73684, + "max.accuracy": 89.0, + "max.ap": 75.99465, + "max.f1": 66.86747, + "main_score": 75.99465 + }, + "TRECCOVID-PL": { + "evaluation_time": 56997.0, + "map_at_1": 0.157, + "map_at_10": 0.811, + "map_at_100": 3.981, + "map_at_1000": 9.286, + "map_at_3": 0.352, + "map_at_5": 0.502, + "mrr_at_1": 56.0, + "mrr_at_10": 64.722, + "mrr_at_100": 65.051, + "mrr_at_1000": 65.081, + "mrr_at_3": 61.667, + "mrr_at_5": 63.967, + "ndcg_at_1": 53.0, + "ndcg_at_10": 38.974, + "ndcg_at_100": 28.302, + "ndcg_at_1000": 28.092, + "ndcg_at_3": 45.682, + "ndcg_at_5": 43.289, + "precision_at_1": 56.0, + "precision_at_10": 40.2, + "precision_at_100": 28.66, + "precision_at_1000": 12.806, + "precision_at_3": 48.0, + "precision_at_5": 45.6, + "recall_at_1": 0.157, + "recall_at_10": 1.017, + "recall_at_100": 6.664, + "recall_at_1000": 27.299, + "recall_at_3": 0.378, + "recall_at_5": 0.583, + "main_score": 38.974 + }, + "SCIDOCS-PL": { + "evaluation_time": 13341.0, + "map_at_1": 2.64, + "map_at_10": 6.799, + "map_at_100": 8.155, + "map_at_1000": 8.432, + "map_at_3": 4.756, + "map_at_5": 5.666, + "mrr_at_1": 13.1, + "mrr_at_10": 20.96, + "mrr_at_100": 22.181, + "mrr_at_1000": 22.272, + "mrr_at_3": 18.183, + "mrr_at_5": 19.663, + "ndcg_at_1": 13.1, + "ndcg_at_10": 12.15, + "ndcg_at_100": 18.51, + "ndcg_at_1000": 24.094, + "ndcg_at_3": 10.962, + "ndcg_at_5": 9.649, + "precision_at_1": 13.1, + "precision_at_10": 6.56, + "precision_at_100": 1.59, + "precision_at_1000": 0.294, + "precision_at_3": 10.333, + "precision_at_5": 8.58, + "recall_at_1": 2.64, + "recall_at_10": 13.288, + "recall_at_100": 32.335, + "recall_at_1000": 59.813, + "recall_at_3": 6.26, + "recall_at_5": 8.693, + "main_score": 12.15 + }, + "PolEmo2.0-IN": { + "accuracy": 67.06371, + "accuracy_stderr": 3.61748, + "evaluation_time": 287.0, + "f1": 66.51008, + "f1_stderr": 3.2817, + "main_score": 67.06371 + }, + "PlscClusteringP2P": { + "evaluation_time": 2762.0, + "v_measure": 43.43455, + "v_measures": 42.61913, + "main_score": 43.43455 + }, + "MSMARCO-PL": { + "evaluation_time": 1071601.0, + "map_at_1": 7.538, + "map_at_10": 13.04, + "map_at_100": 13.959, + "map_at_1000": 14.054, + "map_at_3": 10.92, + "map_at_5": 12.09, + "mrr_at_1": 7.765, + "mrr_at_10": 13.386, + "mrr_at_100": 14.297, + "mrr_at_1000": 14.388, + "mrr_at_3": 11.249, + "mrr_at_5": 12.416, + "ndcg_at_1": 7.765, + "ndcg_at_10": 16.515, + "ndcg_at_100": 21.567, + "ndcg_at_1000": 24.379, + "ndcg_at_3": 12.128, + "ndcg_at_5": 14.224, + "precision_at_1": 7.765, + "precision_at_10": 2.851, + "precision_at_100": 0.547, + "precision_at_1000": 0.079, + "precision_at_3": 5.287, + "precision_at_5": 4.226, + "recall_at_1": 7.538, + "recall_at_10": 27.305, + "recall_at_100": 52.04, + "recall_at_1000": 74.475, + "recall_at_3": 15.308, + "recall_at_5": 20.339, + "main_score": 16.515 + }, + "MassiveIntentClassification": { + "accuracy": 64.80834, + "accuracy_stderr": 1.20602, + "f1": 63.04589, + "f1_stderr": 1.08248, + "main_score": 64.80834 + } + } + }, + { + "id": "mmlw-roberta-large", + "tags": [], + "results": { + "CBD": { + "accuracy": 69.33, + "accuracy_stderr": 5.4529, + "ap": 22.97241, + "ap_stderr": 3.72318, + "evaluation_time": 661.0, + "f1": 58.91072, + "f1_stderr": 4.41993, + "main_score": 69.33 + }, + "AllegroReviews": { + "accuracy": 47.48509, + "accuracy_stderr": 2.84616, + "evaluation_time": 598.0, + "f1": 42.33328, + "f1_stderr": 2.25616, + "main_score": 47.48509 + }, + "PlscClusteringS2S": { + "evaluation_time": 2305.0, + "v_measure": 43.06503, + "v_measures": 42.45422, + "main_score": 43.06503 + }, + "ArguAna-PL": { + "evaluation_time": 5178.0, + "map_at_1": 38.834, + "map_at_10": 55.229, + "map_at_100": 55.792, + "map_at_1000": 55.794, + "map_at_3": 51.233, + "map_at_5": 53.772, + "mrr_at_1": 39.687, + "mrr_at_10": 55.596, + "mrr_at_100": 56.157, + "mrr_at_1000": 56.158, + "mrr_at_3": 51.66, + "mrr_at_5": 54.135, + "ndcg_at_1": 38.834, + "ndcg_at_10": 63.402, + "ndcg_at_100": 65.78, + "ndcg_at_1000": 65.816, + "ndcg_at_3": 55.349, + "ndcg_at_5": 59.892, + "precision_at_1": 38.834, + "precision_at_10": 8.905, + "precision_at_100": 0.994, + "precision_at_1000": 0.1, + "precision_at_3": 22.428, + "precision_at_5": 15.647, + "recall_at_1": 38.834, + "recall_at_10": 89.047, + "recall_at_100": 99.36, + "recall_at_1000": 99.644, + "recall_at_3": 67.283, + "recall_at_5": 78.236, + "main_score": 63.402 + }, + "PolEmo2.0-OUT": { + "accuracy": 53.7247, + "accuracy_stderr": 4.76307, + "evaluation_time": 571.0, + "f1": 43.06059, + "f1_stderr": 3.39674, + "main_score": 53.7247 + }, + "PAC": { + "accuracy": 64.69447, + "accuracy_stderr": 5.26935, + "ap": 75.58029, + "ap_stderr": 3.21097, + "evaluation_time": 495.0, + "f1": 62.36104, + "f1_stderr": 5.6098, + "main_score": 64.69447 + }, + "SciField": { + "accuracy": 85.00695, + "accuracy_stderr": 2.07326, + "evaluation_time": 798.0, + "f1": 84.7291, + "f1_stderr": 2.0912, + "main_score": 85.00695 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 87.42356, + "cos_sim.accuracy_threshold": 96.27669, + "cos_sim.ap": 84.46532, + "cos_sim.f1": 77.18911, + "cos_sim.f1_threshold": 96.18283, + "cos_sim.precision": 79.83257, + "cos_sim.recall": 74.7151, + "dot.accuracy": 81.10477, + "dot.accuracy_threshold": 32625.0, + "dot.ap": 71.12232, + "dot.f1": 68.13456, + "dot.f1_threshold": 31875.0, + "dot.precision": 59.69989, + "dot.recall": 79.34473, + "euclidean.accuracy": 87.21973, + "euclidean.accuracy_threshold": 489.53189, + "euclidean.ap": 84.33078, + "euclidean.f1": 76.81931, + "euclidean.f1_threshold": 535.3071, + "euclidean.precision": 76.57466, + "euclidean.recall": 77.06553, + "evaluation_time": 383.0, + "manhattan.accuracy": 87.21973, + "manhattan.accuracy_threshold": 12107.75356, + "manhattan.ap": 84.35651, + "manhattan.f1": 76.87004, + "manhattan.f1_threshold": 14189.12632, + "manhattan.precision": 74.4823, + "manhattan.recall": 79.41595, + "max.accuracy": 87.42356, + "max.ap": 84.46532, + "max.f1": 77.18911, + "main_score": 84.46532 + }, + "HotpotQA-PL": { + "evaluation_time": 1047625.0, + "map_at_1": 39.399, + "map_at_10": 62.604, + "map_at_100": 63.475, + "map_at_1000": 63.534, + "map_at_3": 58.871, + "map_at_5": 61.217, + "mrr_at_1": 78.758, + "mrr_at_10": 84.584, + "mrr_at_100": 84.753, + "mrr_at_1000": 84.759, + "mrr_at_3": 83.657, + "mrr_at_5": 84.283, + "ndcg_at_1": 78.798, + "ndcg_at_10": 71.04, + "ndcg_at_100": 74.048, + "ndcg_at_1000": 75.163, + "ndcg_at_3": 65.862, + "ndcg_at_5": 68.776, + "precision_at_1": 78.798, + "precision_at_10": 14.949, + "precision_at_100": 1.731, + "precision_at_1000": 0.188, + "precision_at_3": 42.237, + "precision_at_5": 27.635, + "recall_at_1": 39.399, + "recall_at_10": 74.747, + "recall_at_100": 86.529, + "recall_at_1000": 93.849, + "recall_at_3": 63.356, + "recall_at_5": 69.088, + "main_score": 71.04 + }, + "PPC": { + "cos_sim.accuracy": 88.4, + "cos_sim.accuracy_threshold": 95.78413, + "cos_sim.ap": 93.56463, + "cos_sim.f1": 90.73634, + "cos_sim.f1_threshold": 95.08161, + "cos_sim.precision": 86.94992, + "cos_sim.recall": 94.86755, + "dot.accuracy": 75.3, + "dot.accuracy_threshold": 31100.0, + "dot.ap": 83.06946, + "dot.f1": 81.50888, + "dot.f1_threshold": 30525.0, + "dot.precision": 73.6631, + "dot.recall": 91.22517, + "euclidean.accuracy": 88.8, + "euclidean.accuracy_threshold": 524.8874, + "euclidean.ap": 93.53974, + "euclidean.f1": 90.87948, + "euclidean.f1_threshold": 524.8874, + "euclidean.precision": 89.42308, + "euclidean.recall": 92.38411, + "evaluation_time": 179.0, + "manhattan.accuracy": 88.8, + "manhattan.accuracy_threshold": 13302.61191, + "manhattan.ap": 93.5421, + "manhattan.f1": 90.85072, + "manhattan.f1_threshold": 14049.2552, + "manhattan.precision": 88.16199, + "manhattan.recall": 93.70861, + "max.accuracy": 88.8, + "max.ap": 93.56463, + "max.f1": 90.87948, + "main_score": 93.56463 + }, + "EightTagsClustering": { + "evaluation_time": 5409.0, + "main_score": 32.0272, + "v_measure": 32.0272, + "v_measure_std": 4.21476, + "v_measures": 32.34234 + }, + "NFCorpus-PL": { + "evaluation_time": 3428.0, + "map_at_1": 5.604, + "map_at_10": 12.684, + "map_at_100": 16.274, + "map_at_1000": 17.669, + "map_at_3": 9.347, + "map_at_5": 10.752, + "mrr_at_1": 43.963, + "mrr_at_10": 52.94, + "mrr_at_100": 53.571, + "mrr_at_1000": 53.613, + "mrr_at_3": 51.032, + "mrr_at_5": 52.193, + "ndcg_at_1": 41.486, + "ndcg_at_10": 33.937, + "ndcg_at_100": 31.726, + "ndcg_at_1000": 40.331, + "ndcg_at_3": 39.217, + "ndcg_at_5": 36.521, + "precision_at_1": 43.034, + "precision_at_10": 25.325, + "precision_at_100": 8.022, + "precision_at_1000": 2.063, + "precision_at_3": 36.945, + "precision_at_5": 31.517, + "recall_at_1": 5.604, + "recall_at_10": 16.554, + "recall_at_100": 33.113, + "recall_at_1000": 62.832, + "recall_at_3": 10.397, + "recall_at_5": 12.63, + "main_score": 33.937 + }, + "SciFact-PL": { + "evaluation_time": 4890.0, + "map_at_1": 56.05, + "map_at_10": 65.933, + "map_at_100": 66.571, + "map_at_1000": 66.6, + "map_at_3": 63.489, + "map_at_5": 64.918, + "mrr_at_1": 59.0, + "mrr_at_10": 67.026, + "mrr_at_100": 67.559, + "mrr_at_1000": 67.586, + "mrr_at_3": 65.444, + "mrr_at_5": 66.278, + "ndcg_at_1": 59.0, + "ndcg_at_10": 70.233, + "ndcg_at_100": 72.789, + "ndcg_at_1000": 73.637, + "ndcg_at_3": 66.407, + "ndcg_at_5": 68.206, + "precision_at_1": 59.0, + "precision_at_10": 9.367, + "precision_at_100": 1.06, + "precision_at_1000": 0.113, + "precision_at_3": 26.222, + "precision_at_5": 17.067, + "recall_at_1": 56.05, + "recall_at_10": 82.089, + "recall_at_100": 93.167, + "recall_at_1000": 100.0, + "recall_at_3": 71.822, + "recall_at_5": 76.483, + "main_score": 70.233 + }, + "MassiveScenarioClassification": { + "accuracy": 77.84465, + "accuracy_stderr": 1.62199, + "f1": 77.73439, + "f1_stderr": 1.50185, + "main_score": 77.84465 + }, + "FiQA-PL": { + "evaluation_time": 32605.0, + "map_at_1": 19.873, + "map_at_10": 32.923, + "map_at_100": 34.819, + "map_at_1000": 34.99, + "map_at_3": 28.501, + "map_at_5": 31.088, + "mrr_at_1": 40.432, + "mrr_at_10": 49.243, + "mrr_at_100": 50.014, + "mrr_at_1000": 50.055, + "mrr_at_3": 47.145, + "mrr_at_5": 48.171, + "ndcg_at_1": 40.586, + "ndcg_at_10": 40.887, + "ndcg_at_100": 47.701, + "ndcg_at_1000": 50.624, + "ndcg_at_3": 37.143, + "ndcg_at_5": 38.329, + "precision_at_1": 40.586, + "precision_at_10": 11.497, + "precision_at_100": 1.838, + "precision_at_1000": 0.237, + "precision_at_3": 25.0, + "precision_at_5": 18.549, + "recall_at_1": 19.873, + "recall_at_10": 48.073, + "recall_at_100": 73.473, + "recall_at_1000": 90.94, + "recall_at_3": 33.645, + "recall_at_5": 39.711, + "main_score": 40.887 + }, + "SICK-R-PL": { + "cos_sim.pearson": 83.0563, + "cos_sim.spearman": 79.90633, + "euclidean.pearson": 81.56427, + "euclidean.spearman": 80.05439, + "evaluation_time": 440.0, + "manhattan.pearson": 81.52958, + "manhattan.spearman": 80.03875, + "main_score": 79.90633 + }, + "CDSC-R": { + "cos_sim.pearson": 92.10889, + "cos_sim.spearman": 92.54126, + "euclidean.pearson": 91.99022, + "euclidean.spearman": 92.55236, + "evaluation_time": 131.0, + "manhattan.pearson": 91.9217, + "manhattan.spearman": 92.47798, + "main_score": 92.54126 + }, + "SciDefRetrieval": { + "evaluation_time": 341.0, + "map_at_1": 67.305, + "map_at_10": 75.254, + "map_at_100": 75.651, + "map_at_1000": 75.661, + "map_at_20": 75.506, + "map_at_3": 73.883, + "map_at_5": 74.723, + "mrr_at_1": 67.599, + "mrr_at_10": 75.406, + "mrr_at_100": 75.813, + "mrr_at_1000": 75.823, + "mrr_at_20": 75.668, + "mrr_at_3": 74.055, + "mrr_at_5": 74.865, + "ndcg_at_1": 67.305, + "ndcg_at_10": 78.711, + "ndcg_at_100": 80.573, + "ndcg_at_1000": 80.837, + "ndcg_at_20": 79.613, + "ndcg_at_3": 75.91, + "ndcg_at_5": 77.424, + "precision_at_1": 67.305, + "precision_at_10": 8.94, + "precision_at_100": 0.979, + "precision_at_1000": 0.1, + "precision_at_20": 4.647, + "precision_at_3": 27.246, + "precision_at_5": 17.084, + "recall_at_1": 67.305, + "recall_at_10": 89.396, + "recall_at_100": 97.938, + "recall_at_1000": 100.0, + "recall_at_20": 92.931, + "recall_at_3": 81.738, + "recall_at_5": 85.42, + "main_score": 78.711 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 802.0, + "v_measure": 36.8707, + "v_measures": 34.4898, + "main_score": 36.8707 + }, + "PSC": { + "cos_sim.accuracy": 97.03154, + "cos_sim.accuracy_threshold": 90.37622, + "cos_sim.ap": 98.63208, + "cos_sim.f1": 95.06173, + "cos_sim.f1_threshold": 90.37622, + "cos_sim.precision": 96.25, + "cos_sim.recall": 93.90244, + "dot.accuracy": 86.82746, + "dot.accuracy_threshold": 28500.0, + "dot.ap": 83.7745, + "dot.f1": 79.30535, + "dot.f1_threshold": 28325.0, + "dot.precision": 75.48209, + "dot.recall": 83.53659, + "euclidean.accuracy": 97.03154, + "euclidean.accuracy_threshold": 804.77224, + "euclidean.ap": 98.80678, + "euclidean.f1": 95.20958, + "euclidean.f1_threshold": 809.6788, + "euclidean.precision": 93.52941, + "euclidean.recall": 96.95122, + "evaluation_time": 262.0, + "manhattan.accuracy": 97.21707, + "manhattan.accuracy_threshold": 20228.04848, + "manhattan.ap": 98.82279, + "manhattan.f1": 95.44073, + "manhattan.f1_threshold": 20228.04848, + "manhattan.precision": 95.15152, + "manhattan.recall": 95.73171, + "max.accuracy": 97.21707, + "max.ap": 98.82279, + "max.f1": 95.44073, + "main_score": 98.63208 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 2103.0, + "v_measure": 45.78817, + "v_measures": 48.24325, + "main_score": 45.78817 + }, + "NQ-PL": { + "evaluation_time": 875845.0, + "map_at_1": 26.642, + "map_at_10": 40.368, + "map_at_100": 41.487, + "map_at_1000": 41.528, + "map_at_3": 36.292, + "map_at_5": 38.548, + "mrr_at_1": 30.156, + "mrr_at_10": 42.853, + "mrr_at_100": 43.742, + "mrr_at_1000": 43.772, + "mrr_at_3": 39.47, + "mrr_at_5": 41.366, + "ndcg_at_1": 30.214, + "ndcg_at_10": 47.62, + "ndcg_at_100": 52.486, + "ndcg_at_1000": 53.482, + "ndcg_at_3": 39.864, + "ndcg_at_5": 43.645, + "precision_at_1": 30.214, + "precision_at_10": 8.03, + "precision_at_100": 1.074, + "precision_at_1000": 0.117, + "precision_at_3": 18.183, + "precision_at_5": 13.105, + "recall_at_1": 26.642, + "recall_at_10": 67.282, + "recall_at_100": 88.632, + "recall_at_1000": 96.109, + "recall_at_3": 47.048, + "recall_at_5": 55.791, + "main_score": 47.62 + }, + "STS22": { + "cos_sim.pearson": 35.93847, + "cos_sim.spearman": 39.31544, + "euclidean.pearson": 30.19744, + "euclidean.spearman": 39.96661, + "manhattan.pearson": 30.19561, + "manhattan.spearman": 39.82255, + "main_score": 39.31544 + }, + "DBPedia-PL": { + "evaluation_time": 914606.0, + "map_at_1": 8.683, + "map_at_10": 18.9, + "map_at_100": 26.933, + "map_at_1000": 28.558, + "map_at_3": 13.638, + "map_at_5": 15.9, + "mrr_at_1": 63.75, + "mrr_at_10": 73.566, + "mrr_at_100": 73.817, + "mrr_at_1000": 73.824, + "mrr_at_3": 71.875, + "mrr_at_5": 73.2, + "ndcg_at_1": 53.125, + "ndcg_at_10": 40.271, + "ndcg_at_100": 45.51, + "ndcg_at_1000": 52.968, + "ndcg_at_3": 45.122, + "ndcg_at_5": 42.306, + "precision_at_1": 63.75, + "precision_at_10": 31.55, + "precision_at_100": 10.44, + "precision_at_1000": 2.01, + "precision_at_3": 48.333, + "precision_at_5": 40.5, + "recall_at_1": 8.683, + "recall_at_10": 24.63, + "recall_at_100": 51.762, + "recall_at_1000": 75.65, + "recall_at_3": 15.136, + "recall_at_5": 18.678, + "main_score": 40.271 + }, + "Quora-PL": { + "evaluation_time": 36609.0, + "map_at_1": 67.021, + "map_at_10": 81.362, + "map_at_100": 82.067, + "map_at_1000": 82.084, + "map_at_3": 78.223, + "map_at_5": 80.219, + "mrr_at_1": 77.17, + "mrr_at_10": 84.222, + "mrr_at_100": 84.376, + "mrr_at_1000": 84.379, + "mrr_at_3": 83.003, + "mrr_at_5": 83.834, + "ndcg_at_1": 77.29, + "ndcg_at_10": 85.506, + "ndcg_at_100": 87.0, + "ndcg_at_1000": 87.143, + "ndcg_at_3": 82.17, + "ndcg_at_5": 84.057, + "precision_at_1": 77.29, + "precision_at_10": 13.15, + "precision_at_100": 1.522, + "precision_at_1000": 0.156, + "precision_at_3": 36.173, + "precision_at_5": 23.988, + "recall_at_1": 67.021, + "recall_at_10": 93.943, + "recall_at_100": 99.167, + "recall_at_1000": 99.929, + "recall_at_3": 84.558, + "recall_at_5": 89.697, + "main_score": 85.506 + }, + "CDSC-E": { + "cos_sim.accuracy": 89.8, + "cos_sim.accuracy_threshold": 98.22768, + "cos_sim.ap": 79.8704, + "cos_sim.f1": 68.53933, + "cos_sim.f1_threshold": 96.25511, + "cos_sim.precision": 73.49398, + "cos_sim.recall": 64.21053, + "dot.accuracy": 86.1, + "dot.accuracy_threshold": 31675.0, + "dot.ap": 63.68498, + "dot.f1": 63.61746, + "dot.f1_threshold": 30500.0, + "dot.precision": 52.57732, + "dot.recall": 80.52632, + "euclidean.accuracy": 89.8, + "euclidean.accuracy_threshold": 344.20012, + "euclidean.ap": 79.75271, + "euclidean.f1": 68.46361, + "euclidean.f1_threshold": 518.01078, + "euclidean.precision": 70.16575, + "euclidean.recall": 66.84211, + "evaluation_time": 97.0, + "manhattan.accuracy": 89.7, + "manhattan.accuracy_threshold": 8423.18966, + "manhattan.ap": 79.64633, + "manhattan.f1": 68.49315, + "manhattan.f1_threshold": 12775.22458, + "manhattan.precision": 71.42857, + "manhattan.recall": 65.78947, + "max.accuracy": 89.8, + "max.ap": 79.8704, + "max.f1": 68.53933, + "main_score": 79.8704 + }, + "TRECCOVID-PL": { + "evaluation_time": 126771.0, + "map_at_1": 0.21, + "map_at_10": 1.768, + "map_at_100": 9.448, + "map_at_1000": 21.728, + "map_at_3": 0.603, + "map_at_5": 0.961, + "mrr_at_1": 80.0, + "mrr_at_10": 88.667, + "mrr_at_100": 88.667, + "mrr_at_1000": 88.667, + "mrr_at_3": 87.667, + "mrr_at_5": 88.667, + "ndcg_at_1": 77.0, + "ndcg_at_10": 70.814, + "ndcg_at_100": 52.532, + "ndcg_at_1000": 45.636, + "ndcg_at_3": 76.542, + "ndcg_at_5": 73.24, + "precision_at_1": 80.0, + "precision_at_10": 75.0, + "precision_at_100": 53.88, + "precision_at_1000": 20.002, + "precision_at_3": 80.0, + "precision_at_5": 76.4, + "recall_at_1": 0.21, + "recall_at_10": 2.012, + "recall_at_100": 12.782, + "recall_at_1000": 42.05, + "recall_at_3": 0.644, + "recall_at_5": 1.04, + "main_score": 70.814 + }, + "SCIDOCS-PL": { + "evaluation_time": 18567.0, + "map_at_1": 4.523, + "map_at_10": 11.584, + "map_at_100": 13.705, + "map_at_1000": 14.039, + "map_at_3": 8.188, + "map_at_5": 9.922, + "mrr_at_1": 22.1, + "mrr_at_10": 32.947, + "mrr_at_100": 34.11, + "mrr_at_1000": 34.163, + "mrr_at_3": 29.633, + "mrr_at_5": 31.658, + "ndcg_at_1": 22.2, + "ndcg_at_10": 19.466, + "ndcg_at_100": 27.725, + "ndcg_at_1000": 33.539, + "ndcg_at_3": 18.26, + "ndcg_at_5": 16.265, + "precision_at_1": 22.2, + "precision_at_10": 10.11, + "precision_at_100": 2.204, + "precision_at_1000": 0.36, + "precision_at_3": 17.1, + "precision_at_5": 14.44, + "recall_at_1": 4.523, + "recall_at_10": 20.497, + "recall_at_100": 44.757, + "recall_at_1000": 73.147, + "recall_at_3": 10.413, + "recall_at_5": 14.638, + "main_score": 19.466 + }, + "PolEmo2.0-IN": { + "accuracy": 76.84211, + "accuracy_stderr": 4.04745, + "evaluation_time": 710.0, + "f1": 75.49714, + "f1_stderr": 2.6732, + "main_score": 76.84211 + }, + "PlscClusteringP2P": { + "evaluation_time": 6535.0, + "v_measure": 45.65366, + "v_measures": 45.56237, + "main_score": 45.65366 + }, + "MSMARCO-PL": { + "evaluation_time": 2129341.0, + "map_at_1": 19.598, + "map_at_10": 30.454, + "map_at_100": 31.601, + "map_at_1000": 31.66, + "map_at_3": 27.118, + "map_at_5": 28.943, + "mrr_at_1": 20.1, + "mrr_at_10": 30.978, + "mrr_at_100": 32.057, + "mrr_at_1000": 32.112, + "mrr_at_3": 27.679, + "mrr_at_5": 29.493, + "ndcg_at_1": 20.158, + "ndcg_at_10": 36.63, + "ndcg_at_100": 42.291, + "ndcg_at_1000": 43.828, + "ndcg_at_3": 29.745, + "ndcg_at_5": 33.024, + "precision_at_1": 20.158, + "precision_at_10": 5.812, + "precision_at_100": 0.868, + "precision_at_1000": 0.1, + "precision_at_3": 12.689, + "precision_at_5": 9.295, + "recall_at_1": 19.598, + "recall_at_10": 55.597, + "recall_at_100": 82.143, + "recall_at_1000": 94.015, + "recall_at_3": 36.72, + "recall_at_5": 44.606, + "main_score": 36.63 + }, + "MassiveIntentClassification": { + "accuracy": 74.8117, + "accuracy_stderr": 0.84444, + "f1": 72.02597, + "f1_stderr": 0.7131, + "main_score": 74.8117 + } + } + }, + { + "id": "multilingual-e5-small", + "tags": [], + "results": { + "CBD": { + "accuracy": 63.25, + "accuracy_stderr": 7.61121, + "ap": 18.38711, + "ap_stderr": 2.01701, + "evaluation_time": 760.0, + "f1": 52.81581, + "f1_stderr": 4.38167, + "main_score": 63.25 + }, + "AllegroReviews": { + "accuracy": 37.41551, + "accuracy_stderr": 3.8327, + "evaluation_time": 219.0, + "f1": 32.71479, + "f1_stderr": 2.60801, + "main_score": 37.41551 + }, + "PlscClusteringS2S": { + "evaluation_time": 1488.0, + "v_measure": 37.84382, + "v_measures": 37.8494, + "main_score": 37.84382 + }, + "ArguAna-PL": { + "evaluation_time": 1393.0, + "map_at_1": 18.492, + "map_at_10": 30.271, + "map_at_100": 31.574, + "map_at_1000": 31.619, + "map_at_3": 25.984, + "map_at_5": 28.274, + "mrr_at_1": 19.417, + "mrr_at_10": 30.599, + "mrr_at_100": 31.901, + "mrr_at_1000": 31.948, + "mrr_at_3": 26.316, + "mrr_at_5": 28.613, + "ndcg_at_1": 18.492, + "ndcg_at_10": 37.432, + "ndcg_at_100": 43.511, + "ndcg_at_1000": 44.661, + "ndcg_at_3": 28.442, + "ndcg_at_5": 32.556, + "precision_at_1": 18.492, + "precision_at_10": 6.067, + "precision_at_100": 0.885, + "precision_at_1000": 0.098, + "precision_at_3": 11.854, + "precision_at_5": 9.104, + "recall_at_1": 18.492, + "recall_at_10": 60.669, + "recall_at_100": 88.549, + "recall_at_1000": 97.511, + "recall_at_3": 35.562, + "recall_at_5": 45.519, + "main_score": 37.432 + }, + "PolEmo2.0-OUT": { + "accuracy": 39.12955, + "accuracy_stderr": 4.99639, + "evaluation_time": 208.0, + "f1": 33.03897, + "f1_stderr": 3.61457, + "main_score": 39.12955 + }, + "PAC": { + "accuracy": 70.54735, + "accuracy_stderr": 3.30036, + "ap": 78.40261, + "ap_stderr": 3.11459, + "evaluation_time": 262.0, + "f1": 68.04233, + "f1_stderr": 4.24747, + "main_score": 70.54735 + }, + "SciField": { + "accuracy": 79.90264, + "accuracy_stderr": 2.02315, + "evaluation_time": 713.0, + "f1": 79.38097, + "f1_stderr": 2.09945, + "main_score": 79.90264 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 78.96453, + "cos_sim.accuracy_threshold": 96.98362, + "cos_sim.ap": 66.33915, + "cos_sim.f1": 61.95264, + "cos_sim.f1_threshold": 94.55237, + "cos_sim.precision": 56.87909, + "cos_sim.recall": 68.01994, + "dot.accuracy": 78.96453, + "dot.accuracy_threshold": 96.98363, + "dot.ap": 66.33906, + "dot.f1": 61.95264, + "dot.f1_threshold": 94.55237, + "dot.precision": 56.87909, + "dot.recall": 68.01994, + "euclidean.accuracy": 78.96453, + "euclidean.accuracy_threshold": 24.56167, + "euclidean.ap": 66.33917, + "euclidean.f1": 61.95264, + "euclidean.f1_threshold": 33.00797, + "euclidean.precision": 56.87909, + "euclidean.recall": 68.01994, + "evaluation_time": 239.0, + "manhattan.accuracy": 78.94415, + "manhattan.accuracy_threshold": 390.29424, + "manhattan.ap": 66.21187, + "manhattan.f1": 61.81592, + "manhattan.f1_threshold": 530.33609, + "manhattan.precision": 54.85651, + "manhattan.recall": 70.79772, + "max.accuracy": 78.96453, + "max.ap": 66.33917, + "max.f1": 61.95264, + "main_score": 66.33915 + }, + "HotpotQA-PL": { + "evaluation_time": 235898.0, + "map_at_1": 35.226, + "map_at_10": 51.295, + "map_at_100": 52.109, + "map_at_1000": 52.182, + "map_at_3": 48.546, + "map_at_5": 50.226, + "mrr_at_1": 70.452, + "mrr_at_10": 76.858, + "mrr_at_100": 77.142, + "mrr_at_1000": 77.157, + "mrr_at_3": 75.746, + "mrr_at_5": 76.411, + "ndcg_at_1": 70.452, + "ndcg_at_10": 60.153, + "ndcg_at_100": 63.156, + "ndcg_at_1000": 64.651, + "ndcg_at_3": 56.017, + "ndcg_at_5": 58.246, + "precision_at_1": 70.452, + "precision_at_10": 12.4, + "precision_at_100": 1.475, + "precision_at_1000": 0.167, + "precision_at_3": 35.192, + "precision_at_5": 22.885, + "recall_at_1": 35.226, + "recall_at_10": 61.999, + "recall_at_100": 73.747, + "recall_at_1000": 83.673, + "recall_at_3": 52.789, + "recall_at_5": 57.211, + "main_score": 60.153 + }, + "PPC": { + "cos_sim.accuracy": 73.7, + "cos_sim.accuracy_threshold": 94.12979, + "cos_sim.ap": 86.71754, + "cos_sim.f1": 79.2423, + "cos_sim.f1_threshold": 94.12979, + "cos_sim.precision": 75.71644, + "cos_sim.recall": 83.11258, + "dot.accuracy": 73.7, + "dot.accuracy_threshold": 94.12979, + "dot.ap": 86.71754, + "dot.f1": 79.2423, + "dot.f1_threshold": 94.12979, + "dot.precision": 75.71644, + "dot.recall": 83.11258, + "euclidean.accuracy": 73.7, + "euclidean.accuracy_threshold": 34.26428, + "euclidean.ap": 86.71754, + "euclidean.f1": 79.2423, + "euclidean.f1_threshold": 34.26428, + "euclidean.precision": 75.71644, + "euclidean.recall": 83.11258, + "evaluation_time": 112.0, + "manhattan.accuracy": 73.8, + "manhattan.accuracy_threshold": 490.68441, + "manhattan.ap": 86.79319, + "manhattan.f1": 79.17329, + "manhattan.f1_threshold": 529.11692, + "manhattan.precision": 76.14679, + "manhattan.recall": 82.45033, + "max.accuracy": 73.8, + "max.ap": 86.79319, + "max.f1": 79.2423, + "main_score": 86.71754 + }, + "EightTagsClustering": { + "evaluation_time": 3438.0, + "main_score": 23.16028, + "v_measure": 23.16028, + "v_measure_std": 2.83645, + "v_measures": 21.37302 + }, + "NFCorpus-PL": { + "evaluation_time": 803.0, + "map_at_1": 4.407, + "map_at_10": 9.074, + "map_at_100": 10.979, + "map_at_1000": 11.998, + "map_at_3": 6.767, + "map_at_5": 7.769, + "mrr_at_1": 36.533, + "mrr_at_10": 44.367, + "mrr_at_100": 45.012, + "mrr_at_1000": 45.068, + "mrr_at_3": 41.692, + "mrr_at_5": 42.699, + "ndcg_at_1": 34.365, + "ndcg_at_10": 26.477, + "ndcg_at_100": 23.385, + "ndcg_at_1000": 31.781, + "ndcg_at_3": 30.883, + "ndcg_at_5": 28.598, + "precision_at_1": 35.604, + "precision_at_10": 19.412, + "precision_at_100": 5.656, + "precision_at_1000": 1.765, + "precision_at_3": 29.205, + "precision_at_5": 24.458, + "recall_at_1": 4.407, + "recall_at_10": 12.758, + "recall_at_100": 23.896, + "recall_at_1000": 53.454, + "recall_at_3": 7.721, + "recall_at_5": 9.566, + "main_score": 26.477 + }, + "SciFact-PL": { + "evaluation_time": 1068.0, + "map_at_1": 48.928, + "map_at_10": 58.24, + "map_at_100": 58.981, + "map_at_1000": 59.019, + "map_at_3": 55.922, + "map_at_5": 57.051, + "mrr_at_1": 51.667, + "mrr_at_10": 60.161, + "mrr_at_100": 60.702, + "mrr_at_1000": 60.736, + "mrr_at_3": 58.5, + "mrr_at_5": 59.117, + "ndcg_at_1": 51.667, + "ndcg_at_10": 62.757, + "ndcg_at_100": 66.033, + "ndcg_at_1000": 67.022, + "ndcg_at_3": 58.694, + "ndcg_at_5": 60.121, + "precision_at_1": 51.667, + "precision_at_10": 8.367, + "precision_at_100": 1.02, + "precision_at_1000": 0.11, + "precision_at_3": 23.0, + "precision_at_5": 14.867, + "recall_at_1": 48.928, + "recall_at_10": 74.722, + "recall_at_100": 89.6, + "recall_at_1000": 97.333, + "recall_at_3": 63.283, + "recall_at_5": 66.95, + "main_score": 62.757 + }, + "MassiveScenarioClassification": { + "accuracy": 64.25353, + "accuracy_stderr": 2.18465, + "f1": 63.25008, + "f1_stderr": 1.91229, + "main_score": 64.25353 + }, + "FiQA-PL": { + "evaluation_time": 5874.0, + "map_at_1": 10.6, + "map_at_10": 16.833, + "map_at_100": 18.174, + "map_at_1000": 18.337, + "map_at_3": 14.785, + "map_at_5": 15.78, + "mrr_at_1": 21.451, + "mrr_at_10": 28.219, + "mrr_at_100": 29.237, + "mrr_at_1000": 29.31, + "mrr_at_3": 26.543, + "mrr_at_5": 27.307, + "ndcg_at_1": 21.451, + "ndcg_at_10": 22.034, + "ndcg_at_100": 28.275, + "ndcg_at_1000": 31.812, + "ndcg_at_3": 19.899, + "ndcg_at_5": 20.239, + "precision_at_1": 21.451, + "precision_at_10": 6.157, + "precision_at_100": 1.233, + "precision_at_1000": 0.184, + "precision_at_3": 13.22, + "precision_at_5": 9.444, + "recall_at_1": 10.6, + "recall_at_10": 26.439, + "recall_at_100": 51.004, + "recall_at_1000": 73.071, + "recall_at_3": 18.355, + "recall_at_5": 21.258, + "main_score": 22.034 + }, + "SICK-R-PL": { + "cos_sim.pearson": 74.59483, + "cos_sim.spearman": 69.45617, + "euclidean.pearson": 71.35579, + "euclidean.spearman": 69.45618, + "evaluation_time": 263.0, + "manhattan.pearson": 71.30818, + "manhattan.spearman": 69.30666, + "main_score": 69.45617 + }, + "CDSC-R": { + "cos_sim.pearson": 90.27322, + "cos_sim.spearman": 90.27184, + "euclidean.pearson": 88.29695, + "euclidean.spearman": 90.27189, + "evaluation_time": 81.0, + "manhattan.pearson": 88.30777, + "manhattan.spearman": 90.26048, + "main_score": 90.27184 + }, + "SciDefRetrieval": { + "evaluation_time": 172.0, + "map_at_1": 68.336, + "map_at_10": 75.526, + "map_at_100": 75.913, + "map_at_1000": 75.928, + "map_at_20": 75.792, + "map_at_3": 74.104, + "map_at_5": 75.083, + "mrr_at_1": 68.336, + "mrr_at_10": 75.475, + "mrr_at_100": 75.875, + "mrr_at_1000": 75.891, + "mrr_at_20": 75.756, + "mrr_at_3": 74.08, + "mrr_at_5": 75.0, + "ndcg_at_1": 68.336, + "ndcg_at_10": 78.759, + "ndcg_at_100": 80.604, + "ndcg_at_1000": 80.949, + "ndcg_at_20": 79.752, + "ndcg_at_3": 75.883, + "ndcg_at_5": 77.645, + "precision_at_1": 68.336, + "precision_at_10": 8.881, + "precision_at_100": 0.973, + "precision_at_1000": 0.1, + "precision_at_20": 4.639, + "precision_at_3": 27.0, + "precision_at_5": 17.054, + "recall_at_1": 68.336, + "recall_at_10": 88.807, + "recall_at_100": 97.349, + "recall_at_1000": 100.0, + "recall_at_20": 92.784, + "recall_at_3": 81.001, + "recall_at_5": 85.272, + "main_score": 78.759 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 602.0, + "v_measure": 24.48954, + "v_measures": 19.71789, + "main_score": 24.48954 + }, + "PSC": { + "cos_sim.accuracy": 97.95918, + "cos_sim.accuracy_threshold": 88.13713, + "cos_sim.ap": 99.23504, + "cos_sim.f1": 96.64634, + "cos_sim.f1_threshold": 88.01633, + "cos_sim.precision": 96.64634, + "cos_sim.recall": 96.64634, + "dot.accuracy": 97.95918, + "dot.accuracy_threshold": 88.13713, + "dot.ap": 99.23504, + "dot.f1": 96.64634, + "dot.f1_threshold": 88.01633, + "dot.precision": 96.64634, + "dot.recall": 96.64634, + "euclidean.accuracy": 97.95918, + "euclidean.accuracy_threshold": 48.70908, + "euclidean.ap": 99.23504, + "euclidean.f1": 96.64634, + "euclidean.f1_threshold": 48.95598, + "euclidean.precision": 96.64634, + "euclidean.recall": 96.64634, + "evaluation_time": 115.0, + "manhattan.accuracy": 97.95918, + "manhattan.accuracy_threshold": 768.45217, + "manhattan.ap": 99.23042, + "manhattan.f1": 96.65653, + "manhattan.f1_threshold": 768.45217, + "manhattan.precision": 96.36364, + "manhattan.recall": 96.95122, + "max.accuracy": 97.95918, + "max.ap": 99.23504, + "max.f1": 96.65653, + "main_score": 99.23504 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 989.0, + "v_measure": 41.28106, + "v_measures": 41.16208, + "main_score": 41.28106 + }, + "NQ-PL": { + "evaluation_time": 176652.0, + "map_at_1": 23.054, + "map_at_10": 34.434, + "map_at_100": 35.47, + "map_at_1000": 35.525, + "map_at_3": 30.954, + "map_at_5": 32.987, + "mrr_at_1": 26.159, + "mrr_at_10": 36.563, + "mrr_at_100": 37.413, + "mrr_at_1000": 37.456, + "mrr_at_3": 33.589, + "mrr_at_5": 35.303, + "ndcg_at_1": 26.13, + "ndcg_at_10": 40.46, + "ndcg_at_100": 45.177, + "ndcg_at_1000": 46.594, + "ndcg_at_3": 33.848, + "ndcg_at_5": 37.225, + "precision_at_1": 26.13, + "precision_at_10": 6.77, + "precision_at_100": 0.943, + "precision_at_1000": 0.108, + "precision_at_3": 15.45, + "precision_at_5": 11.205, + "recall_at_1": 23.054, + "recall_at_10": 56.856, + "recall_at_100": 77.967, + "recall_at_1000": 88.75, + "recall_at_3": 39.627, + "recall_at_5": 47.424, + "main_score": 40.46 + }, + "STS22": { + "cos_sim.pearson": 27.95184, + "cos_sim.spearman": 35.79609, + "euclidean.pearson": 17.44199, + "euclidean.spearman": 35.89471, + "manhattan.pearson": 17.3926, + "manhattan.spearman": 36.12887, + "main_score": 35.79609 + }, + "DBPedia-PL": { + "evaluation_time": 172642.0, + "map_at_1": 6.498, + "map_at_10": 13.277, + "map_at_100": 17.685, + "map_at_1000": 18.698, + "map_at_3": 10.102, + "map_at_5": 11.496, + "mrr_at_1": 48.5, + "mrr_at_10": 58.362, + "mrr_at_100": 58.851, + "mrr_at_1000": 58.878, + "mrr_at_3": 55.917, + "mrr_at_5": 57.704, + "ndcg_at_1": 39.125, + "ndcg_at_10": 29.272, + "ndcg_at_100": 31.95, + "ndcg_at_1000": 38.449, + "ndcg_at_3": 33.383, + "ndcg_at_5": 31.538, + "precision_at_1": 48.5, + "precision_at_10": 22.9, + "precision_at_100": 6.835, + "precision_at_1000": 1.37, + "precision_at_3": 36.083, + "precision_at_5": 30.65, + "recall_at_1": 6.498, + "recall_at_10": 17.793, + "recall_at_100": 35.776, + "recall_at_1000": 57.372, + "recall_at_3": 11.469, + "recall_at_5": 14.047, + "main_score": 29.272 + }, + "Quora-PL": { + "evaluation_time": 18105.0, + "map_at_1": 60.863, + "map_at_10": 73.922, + "map_at_100": 74.74, + "map_at_1000": 74.774, + "map_at_3": 70.815, + "map_at_5": 72.697, + "mrr_at_1": 69.98, + "mrr_at_10": 77.738, + "mrr_at_100": 78.022, + "mrr_at_1000": 78.028, + "mrr_at_3": 76.208, + "mrr_at_5": 77.169, + "ndcg_at_1": 70.03, + "ndcg_at_10": 78.702, + "ndcg_at_100": 80.978, + "ndcg_at_1000": 81.378, + "ndcg_at_3": 74.83, + "ndcg_at_5": 76.855, + "precision_at_1": 70.03, + "precision_at_10": 12.071, + "precision_at_100": 1.458, + "precision_at_1000": 0.154, + "precision_at_3": 32.65, + "precision_at_5": 21.734, + "recall_at_1": 60.863, + "recall_at_10": 88.355, + "recall_at_100": 96.991, + "recall_at_1000": 99.37, + "recall_at_3": 77.584, + "recall_at_5": 82.974, + "main_score": 78.702 + }, + "CDSC-E": { + "cos_sim.accuracy": 88.3, + "cos_sim.accuracy_threshold": 98.24151, + "cos_sim.ap": 69.6958, + "cos_sim.f1": 62.2093, + "cos_sim.f1_threshold": 97.0529, + "cos_sim.precision": 69.48052, + "cos_sim.recall": 56.31579, + "dot.accuracy": 88.3, + "dot.accuracy_threshold": 98.24151, + "dot.ap": 69.69753, + "dot.f1": 62.2093, + "dot.f1_threshold": 97.0529, + "dot.precision": 69.48052, + "dot.recall": 56.31579, + "euclidean.accuracy": 88.3, + "euclidean.accuracy_threshold": 18.75336, + "euclidean.ap": 69.6958, + "euclidean.f1": 62.2093, + "euclidean.f1_threshold": 24.2778, + "euclidean.precision": 69.48052, + "euclidean.recall": 56.31579, + "evaluation_time": 69.0, + "manhattan.accuracy": 88.3, + "manhattan.accuracy_threshold": 293.40804, + "manhattan.ap": 69.84044, + "manhattan.f1": 62.39067, + "manhattan.f1_threshold": 377.45063, + "manhattan.precision": 69.93464, + "manhattan.recall": 56.31579, + "max.accuracy": 88.3, + "max.ap": 69.84044, + "max.f1": 62.39067, + "main_score": 69.6958 + }, + "TRECCOVID-PL": { + "evaluation_time": 24644.0, + "map_at_1": 0.22, + "map_at_10": 1.677, + "map_at_100": 9.026, + "map_at_1000": 20.775, + "map_at_3": 0.609, + "map_at_5": 0.912, + "mrr_at_1": 86.0, + "mrr_at_10": 91.733, + "mrr_at_100": 91.733, + "mrr_at_1000": 91.733, + "mrr_at_3": 91.333, + "mrr_at_5": 91.733, + "ndcg_at_1": 82.0, + "ndcg_at_10": 70.918, + "ndcg_at_100": 51.22, + "ndcg_at_1000": 43.863, + "ndcg_at_3": 75.212, + "ndcg_at_5": 72.294, + "precision_at_1": 86.0, + "precision_at_10": 75.2, + "precision_at_100": 52.08, + "precision_at_1000": 19.538, + "precision_at_3": 82.0, + "precision_at_5": 77.2, + "recall_at_1": 0.22, + "recall_at_10": 1.905, + "recall_at_100": 12.071, + "recall_at_1000": 40.882, + "recall_at_3": 0.643, + "recall_at_5": 0.983, + "main_score": 70.918 + }, + "SCIDOCS-PL": { + "evaluation_time": 3992.0, + "map_at_1": 2.655, + "map_at_10": 6.528, + "map_at_100": 7.611, + "map_at_1000": 7.825, + "map_at_3": 4.845, + "map_at_5": 5.703, + "mrr_at_1": 13.1, + "mrr_at_10": 20.829, + "mrr_at_100": 21.861, + "mrr_at_1000": 21.961, + "mrr_at_3": 18.267, + "mrr_at_5": 19.772, + "ndcg_at_1": 13.1, + "ndcg_at_10": 11.6, + "ndcg_at_100": 16.726, + "ndcg_at_1000": 21.464, + "ndcg_at_3": 11.115, + "ndcg_at_5": 9.725, + "precision_at_1": 13.1, + "precision_at_10": 6.02, + "precision_at_100": 1.351, + "precision_at_1000": 0.25, + "precision_at_3": 10.533, + "precision_at_5": 8.62, + "recall_at_1": 2.655, + "recall_at_10": 12.228, + "recall_at_100": 27.48, + "recall_at_1000": 50.778, + "recall_at_3": 6.408, + "recall_at_5": 8.753, + "main_score": 11.6 + }, + "PolEmo2.0-IN": { + "accuracy": 67.35457, + "accuracy_stderr": 5.71002, + "evaluation_time": 251.0, + "f1": 66.20995, + "f1_stderr": 4.39889, + "main_score": 67.35457 + }, + "PlscClusteringP2P": { + "evaluation_time": 2730.0, + "v_measure": 42.56376, + "v_measures": 42.87389, + "main_score": 42.56376 + }, + "MSMARCO-PL": { + "evaluation_time": 462230.0, + "map_at_1": 13.943, + "map_at_10": 22.159, + "map_at_100": 23.294, + "map_at_1000": 23.364, + "map_at_3": 19.493, + "map_at_5": 21.012, + "mrr_at_1": 14.355, + "mrr_at_10": 22.607, + "mrr_at_100": 23.711, + "mrr_at_1000": 23.778, + "mrr_at_3": 19.931, + "mrr_at_5": 21.46, + "ndcg_at_1": 14.355, + "ndcg_at_10": 26.945, + "ndcg_at_100": 32.68, + "ndcg_at_1000": 34.688, + "ndcg_at_3": 21.461, + "ndcg_at_5": 24.188, + "precision_at_1": 14.355, + "precision_at_10": 4.348, + "precision_at_100": 0.727, + "precision_at_1000": 0.09, + "precision_at_3": 9.241, + "precision_at_5": 6.911, + "recall_at_1": 13.943, + "recall_at_10": 41.702, + "recall_at_100": 68.777, + "recall_at_1000": 84.715, + "recall_at_3": 26.745, + "recall_at_5": 33.295, + "main_score": 26.945 + }, + "MassiveIntentClassification": { + "accuracy": 57.40417, + "accuracy_stderr": 1.66989, + "f1": 53.78744, + "f1_stderr": 1.1338, + "main_score": 57.40417 + } + } + }, + { + "id": "distiluse-base-multilingual-cased-v2", + "tags": [], + "results": { + "CBD": { + "accuracy": 60.0, + "accuracy_stderr": 7.70078, + "ap": 17.75094, + "ap_stderr": 2.74937, + "evaluation_time": 779.0, + "f1": 50.64816, + "f1_stderr": 5.34868, + "main_score": 60.0 + }, + "AllegroReviews": { + "accuracy": 28.03181, + "accuracy_stderr": 3.1747, + "evaluation_time": 282.0, + "f1": 25.7093, + "f1_stderr": 2.12201, + "main_score": 28.03181 + }, + "PlscClusteringS2S": { + "evaluation_time": 1375.0, + "v_measure": 29.84655, + "v_measures": 32.49936, + "main_score": 29.84655 + }, + "ArguAna-PL": { + "evaluation_time": 1498.0, + "map_at_1": 18.065, + "map_at_10": 29.667, + "map_at_100": 31.014, + "map_at_1000": 31.061, + "map_at_3": 25.545, + "map_at_5": 27.658, + "mrr_at_1": 18.208, + "mrr_at_10": 29.755, + "mrr_at_100": 31.096, + "mrr_at_1000": 31.142, + "mrr_at_3": 25.652, + "mrr_at_5": 27.715, + "ndcg_at_1": 18.065, + "ndcg_at_10": 36.702, + "ndcg_at_100": 43.188, + "ndcg_at_1000": 44.324, + "ndcg_at_3": 28.074, + "ndcg_at_5": 31.897, + "precision_at_1": 18.065, + "precision_at_10": 5.946, + "precision_at_100": 0.895, + "precision_at_1000": 0.098, + "precision_at_3": 11.807, + "precision_at_5": 8.947, + "recall_at_1": 18.065, + "recall_at_10": 59.459, + "recall_at_100": 89.545, + "recall_at_1000": 98.293, + "recall_at_3": 35.42, + "recall_at_5": 44.737, + "main_score": 36.702 + }, + "PolEmo2.0-OUT": { + "accuracy": 30.0, + "accuracy_stderr": 5.25677, + "evaluation_time": 201.0, + "f1": 23.64203, + "f1_stderr": 3.32255, + "main_score": 30.0 + }, + "PAC": { + "accuracy": 68.1726, + "accuracy_stderr": 3.95191, + "ap": 76.82381, + "ap_stderr": 2.30383, + "evaluation_time": 382.0, + "f1": 65.58689, + "f1_stderr": 3.99685, + "main_score": 68.1726 + }, + "SciField": { + "accuracy": 76.74548, + "accuracy_stderr": 1.78036, + "evaluation_time": 701.0, + "f1": 76.03207, + "f1_stderr": 2.12037, + "main_score": 76.74548 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 78.14921, + "cos_sim.accuracy_threshold": 88.69869, + "cos_sim.ap": 62.04702, + "cos_sim.f1": 60.23795, + "cos_sim.f1_threshold": 82.14853, + "cos_sim.precision": 53.74302, + "cos_sim.recall": 68.51852, + "dot.accuracy": 73.31839, + "dot.accuracy_threshold": 59.81869, + "dot.ap": 48.45565, + "dot.f1": 52.51153, + "dot.f1_threshold": 37.2081, + "dot.precision": 39.82333, + "dot.recall": 77.06553, + "euclidean.accuracy": 78.23074, + "euclidean.accuracy_threshold": 34.56655, + "euclidean.ap": 62.41283, + "euclidean.f1": 60.71108, + "euclidean.f1_threshold": 47.12331, + "euclidean.precision": 52.29027, + "euclidean.recall": 72.36467, + "evaluation_time": 211.0, + "manhattan.accuracy": 76.51855, + "manhattan.accuracy_threshold": 496.91877, + "manhattan.ap": 59.39869, + "manhattan.f1": 58.0518, + "manhattan.f1_threshold": 842.38262, + "manhattan.precision": 47.99814, + "manhattan.recall": 73.43305, + "max.accuracy": 78.23074, + "max.ap": 62.41283, + "max.f1": 60.71108, + "main_score": 62.04702 + }, + "HotpotQA-PL": { + "evaluation_time": 541639.0, + "map_at_1": 10.479, + "map_at_10": 15.51, + "map_at_100": 16.309, + "map_at_1000": 16.419, + "map_at_3": 13.907, + "map_at_5": 14.807, + "mrr_at_1": 20.959, + "mrr_at_10": 27.55, + "mrr_at_100": 28.398, + "mrr_at_1000": 28.476, + "mrr_at_3": 25.528, + "mrr_at_5": 26.704, + "ndcg_at_1": 20.959, + "ndcg_at_10": 20.833, + "ndcg_at_100": 24.89, + "ndcg_at_1000": 27.74, + "ndcg_at_3": 17.505, + "ndcg_at_5": 19.149, + "precision_at_1": 20.959, + "precision_at_10": 4.903, + "precision_at_100": 0.816, + "precision_at_1000": 0.12, + "precision_at_3": 11.33, + "precision_at_5": 8.1, + "recall_at_1": 10.479, + "recall_at_10": 24.517, + "recall_at_100": 40.79, + "recall_at_1000": 59.845, + "recall_at_3": 16.995, + "recall_at_5": 20.25, + "main_score": 20.833 + }, + "PPC": { + "cos_sim.accuracy": 75.3, + "cos_sim.accuracy_threshold": 84.51151, + "cos_sim.ap": 86.83363, + "cos_sim.f1": 79.38697, + "cos_sim.f1_threshold": 79.55562, + "cos_sim.precision": 73.89444, + "cos_sim.recall": 85.76159, + "dot.accuracy": 73.0, + "dot.accuracy_threshold": 54.35673, + "dot.ap": 83.27559, + "dot.f1": 78.84465, + "dot.f1_threshold": 53.57717, + "dot.precision": 74.5938, + "dot.recall": 83.60927, + "euclidean.accuracy": 73.8, + "euclidean.accuracy_threshold": 43.585, + "euclidean.ap": 85.98738, + "euclidean.f1": 78.42227, + "euclidean.f1_threshold": 52.15912, + "euclidean.precision": 73.58491, + "euclidean.recall": 83.9404, + "evaluation_time": 107.0, + "manhattan.accuracy": 71.0, + "manhattan.accuracy_threshold": 713.0887, + "manhattan.ap": 84.6881, + "manhattan.f1": 76.50696, + "manhattan.f1_threshold": 885.23226, + "manhattan.precision": 71.73913, + "manhattan.recall": 81.95364, + "max.accuracy": 75.3, + "max.ap": 86.83363, + "max.f1": 79.38697, + "main_score": 86.83363 + }, + "EightTagsClustering": { + "evaluation_time": 2578.0, + "main_score": 14.7746, + "v_measure": 14.7746, + "v_measure_std": 2.67502, + "v_measures": 15.74541 + }, + "NFCorpus-PL": { + "evaluation_time": 557.0, + "map_at_1": 1.501, + "map_at_10": 3.904, + "map_at_100": 5.564, + "map_at_1000": 6.602, + "map_at_3": 2.637, + "map_at_5": 3.196, + "mrr_at_1": 21.053, + "mrr_at_10": 30.109, + "mrr_at_100": 30.913, + "mrr_at_1000": 31.0, + "mrr_at_3": 27.451, + "mrr_at_5": 29.092, + "ndcg_at_1": 19.659, + "ndcg_at_10": 16.278, + "ndcg_at_100": 15.975, + "ndcg_at_1000": 25.09, + "ndcg_at_3": 18.416, + "ndcg_at_5": 17.719, + "precision_at_1": 21.053, + "precision_at_10": 13.189, + "precision_at_100": 4.997, + "precision_at_1000": 1.743, + "precision_at_3": 18.266, + "precision_at_5": 16.347, + "recall_at_1": 1.501, + "recall_at_10": 6.982, + "recall_at_100": 19.396, + "recall_at_1000": 50.922, + "recall_at_3": 3.401, + "recall_at_5": 4.757, + "main_score": 16.278 + }, + "SciFact-PL": { + "evaluation_time": 783.0, + "map_at_1": 22.444, + "map_at_10": 28.971, + "map_at_100": 29.998, + "map_at_1000": 30.096, + "map_at_3": 26.75, + "map_at_5": 28.014, + "mrr_at_1": 23.333, + "mrr_at_10": 30.212, + "mrr_at_100": 30.996, + "mrr_at_1000": 31.085, + "mrr_at_3": 28.0, + "mrr_at_5": 29.317, + "ndcg_at_1": 23.333, + "ndcg_at_10": 33.025, + "ndcg_at_100": 37.963, + "ndcg_at_1000": 40.758, + "ndcg_at_3": 28.57, + "ndcg_at_5": 30.756, + "precision_at_1": 23.333, + "precision_at_10": 5.0, + "precision_at_100": 0.78, + "precision_at_1000": 0.103, + "precision_at_3": 11.778, + "precision_at_5": 8.267, + "recall_at_1": 22.444, + "recall_at_10": 44.344, + "recall_at_100": 67.889, + "recall_at_1000": 90.0, + "recall_at_3": 32.194, + "recall_at_5": 37.583, + "main_score": 33.025 + }, + "MassiveScenarioClassification": { + "accuracy": 61.28783, + "accuracy_stderr": 1.50006, + "f1": 59.36496, + "f1_stderr": 1.5505, + "main_score": 61.28783 + }, + "FiQA-PL": { + "evaluation_time": 7543.0, + "map_at_1": 3.716, + "map_at_10": 5.821, + "map_at_100": 6.61, + "map_at_1000": 6.78, + "map_at_3": 5.124, + "map_at_5": 5.396, + "mrr_at_1": 6.636, + "mrr_at_10": 10.201, + "mrr_at_100": 11.148, + "mrr_at_1000": 11.261, + "mrr_at_3": 9.079, + "mrr_at_5": 9.511, + "ndcg_at_1": 6.636, + "ndcg_at_10": 8.015, + "ndcg_at_100": 12.319, + "ndcg_at_1000": 16.874, + "ndcg_at_3": 6.922, + "ndcg_at_5": 7.085, + "precision_at_1": 6.636, + "precision_at_10": 2.346, + "precision_at_100": 0.654, + "precision_at_1000": 0.145, + "precision_at_3": 4.527, + "precision_at_5": 3.333, + "recall_at_1": 3.716, + "recall_at_10": 10.238, + "recall_at_100": 27.388, + "recall_at_1000": 56.29, + "recall_at_3": 6.744, + "recall_at_5": 7.564, + "main_score": 8.015 + }, + "SICK-R-PL": { + "cos_sim.pearson": 71.5414, + "cos_sim.spearman": 65.52718, + "euclidean.pearson": 68.89634, + "euclidean.spearman": 65.68193, + "evaluation_time": 311.0, + "manhattan.pearson": 67.4223, + "manhattan.spearman": 63.63216, + "main_score": 65.52718 + }, + "CDSC-R": { + "cos_sim.pearson": 87.12515, + "cos_sim.spearman": 87.66769, + "euclidean.pearson": 85.00056, + "euclidean.spearman": 87.37247, + "evaluation_time": 126.0, + "manhattan.pearson": 84.69066, + "manhattan.spearman": 86.96851, + "main_score": 87.66769 + }, + "SciDefRetrieval": { + "evaluation_time": 178.0, + "map_at_1": 25.773, + "map_at_10": 36.221, + "map_at_100": 37.281, + "map_at_1000": 37.347, + "map_at_20": 36.837, + "map_at_3": 32.867, + "map_at_5": 34.995, + "mrr_at_1": 25.773, + "mrr_at_10": 36.22, + "mrr_at_100": 37.28, + "mrr_at_1000": 37.347, + "mrr_at_20": 36.836, + "mrr_at_3": 32.867, + "mrr_at_5": 34.995, + "ndcg_at_1": 25.773, + "ndcg_at_10": 41.835, + "ndcg_at_100": 47.195, + "ndcg_at_1000": 49.084, + "ndcg_at_20": 44.17, + "ndcg_at_3": 35.134, + "ndcg_at_5": 38.931, + "precision_at_1": 25.773, + "precision_at_10": 5.965, + "precision_at_100": 0.85, + "precision_at_1000": 0.1, + "precision_at_20": 3.454, + "precision_at_3": 13.893, + "precision_at_5": 10.162, + "recall_at_1": 25.773, + "recall_at_10": 59.647, + "recall_at_100": 84.978, + "recall_at_1000": 100.0, + "recall_at_20": 69.072, + "recall_at_3": 41.679, + "recall_at_5": 50.81, + "main_score": 41.835 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 485.0, + "v_measure": 13.95986, + "v_measures": 14.00178, + "main_score": 13.95986 + }, + "PSC": { + "cos_sim.accuracy": 94.06308, + "cos_sim.accuracy_threshold": 39.57714, + "cos_sim.ap": 96.34568, + "cos_sim.f1": 90.61584, + "cos_sim.f1_threshold": 39.57714, + "cos_sim.precision": 87.28814, + "cos_sim.recall": 94.20732, + "dot.accuracy": 93.2282, + "dot.accuracy_threshold": 25.80665, + "dot.ap": 94.89335, + "dot.f1": 89.58631, + "dot.f1_threshold": 25.3258, + "dot.precision": 84.18231, + "dot.recall": 95.73171, + "euclidean.accuracy": 93.04267, + "euclidean.accuracy_threshold": 86.25168, + "euclidean.ap": 95.00721, + "euclidean.f1": 88.51455, + "euclidean.f1_threshold": 87.127, + "euclidean.precision": 88.92308, + "euclidean.recall": 88.10976, + "evaluation_time": 185.0, + "manhattan.accuracy": 93.04267, + "manhattan.accuracy_threshold": 1552.18983, + "manhattan.ap": 94.76012, + "manhattan.f1": 88.40804, + "manhattan.f1_threshold": 1552.18983, + "manhattan.precision": 89.65517, + "manhattan.recall": 87.19512, + "max.accuracy": 94.06308, + "max.ap": 96.34568, + "max.f1": 90.61584, + "main_score": 96.34568 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 922.0, + "v_measure": 37.01115, + "v_measures": 41.73829, + "main_score": 37.01115 + }, + "NQ-PL": { + "evaluation_time": 319038.0, + "map_at_1": 2.013, + "map_at_10": 4.117, + "map_at_100": 4.808, + "map_at_1000": 4.926, + "map_at_3": 3.101, + "map_at_5": 3.519, + "mrr_at_1": 2.346, + "mrr_at_10": 4.705, + "mrr_at_100": 5.379, + "mrr_at_1000": 5.488, + "mrr_at_3": 3.607, + "mrr_at_5": 4.054, + "ndcg_at_1": 2.346, + "ndcg_at_10": 5.851, + "ndcg_at_100": 9.806, + "ndcg_at_1000": 13.475, + "ndcg_at_3": 3.612, + "ndcg_at_5": 4.38, + "precision_at_1": 2.346, + "precision_at_10": 1.263, + "precision_at_100": 0.359, + "precision_at_1000": 0.071, + "precision_at_3": 1.786, + "precision_at_5": 1.489, + "recall_at_1": 2.013, + "recall_at_10": 10.815, + "recall_at_100": 30.0, + "recall_at_1000": 59.038, + "recall_at_3": 4.606, + "recall_at_5": 6.431, + "main_score": 5.851 + }, + "STS22": { + "cos_sim.pearson": 26.73252, + "cos_sim.spearman": 34.57702, + "euclidean.pearson": 11.55572, + "euclidean.spearman": 33.83303, + "manhattan.pearson": 11.30958, + "manhattan.spearman": 33.45114, + "main_score": 34.57702 + }, + "DBPedia-PL": { + "evaluation_time": 494609.0, + "map_at_1": 2.222, + "map_at_10": 4.563, + "map_at_100": 6.718, + "map_at_1000": 7.524, + "map_at_3": 3.376, + "map_at_5": 3.847, + "mrr_at_1": 20.0, + "mrr_at_10": 28.52, + "mrr_at_100": 29.437, + "mrr_at_1000": 29.493, + "mrr_at_3": 25.417, + "mrr_at_5": 26.967, + "ndcg_at_1": 13.875, + "ndcg_at_10": 12.356, + "ndcg_at_100": 15.871, + "ndcg_at_1000": 22.354, + "ndcg_at_3": 12.939, + "ndcg_at_5": 12.413, + "precision_at_1": 20.0, + "precision_at_10": 11.3, + "precision_at_100": 4.43, + "precision_at_1000": 1.296, + "precision_at_3": 15.417, + "precision_at_5": 13.3, + "recall_at_1": 2.222, + "recall_at_10": 7.798, + "recall_at_100": 22.0, + "recall_at_1000": 43.974, + "recall_at_3": 4.147, + "recall_at_5": 5.343, + "main_score": 12.356 + }, + "Quora-PL": { + "evaluation_time": 27524.0, + "map_at_1": 54.755, + "map_at_10": 66.975, + "map_at_100": 67.879, + "map_at_1000": 67.925, + "map_at_3": 63.898, + "map_at_5": 65.717, + "mrr_at_1": 63.11, + "mrr_at_10": 71.035, + "mrr_at_100": 71.422, + "mrr_at_1000": 71.437, + "mrr_at_3": 69.215, + "mrr_at_5": 70.309, + "ndcg_at_1": 63.09, + "ndcg_at_10": 71.953, + "ndcg_at_100": 74.821, + "ndcg_at_1000": 75.514, + "ndcg_at_3": 67.721, + "ndcg_at_5": 69.828, + "precision_at_1": 63.09, + "precision_at_10": 11.165, + "precision_at_100": 1.399, + "precision_at_1000": 0.151, + "precision_at_3": 29.617, + "precision_at_5": 19.82, + "recall_at_1": 54.755, + "recall_at_10": 82.218, + "recall_at_100": 93.648, + "recall_at_1000": 98.125, + "recall_at_3": 70.367, + "recall_at_5": 76.007, + "main_score": 71.953 + }, + "CDSC-E": { + "cos_sim.accuracy": 88.0, + "cos_sim.accuracy_threshold": 93.67988, + "cos_sim.ap": 71.82922, + "cos_sim.f1": 65.12702, + "cos_sim.f1_threshold": 78.09259, + "cos_sim.precision": 58.02469, + "cos_sim.recall": 74.21053, + "dot.accuracy": 84.9, + "dot.accuracy_threshold": 59.12673, + "dot.ap": 57.85735, + "dot.f1": 58.19861, + "dot.f1_threshold": 49.0155, + "dot.precision": 51.85185, + "dot.recall": 66.31579, + "euclidean.accuracy": 88.2, + "euclidean.accuracy_threshold": 27.13625, + "euclidean.ap": 72.00106, + "euclidean.f1": 63.89497, + "euclidean.f1_threshold": 55.01508, + "euclidean.precision": 54.68165, + "euclidean.recall": 76.84211, + "evaluation_time": 79.0, + "manhattan.accuracy": 87.7, + "manhattan.accuracy_threshold": 447.90249, + "manhattan.ap": 70.9227, + "manhattan.f1": 63.82022, + "manhattan.f1_threshold": 935.80818, + "manhattan.precision": 55.68627, + "manhattan.recall": 74.73684, + "max.accuracy": 88.2, + "max.ap": 72.00106, + "max.f1": 65.12702, + "main_score": 71.82922 + }, + "TRECCOVID-PL": { + "evaluation_time": 19650.0, + "map_at_1": 0.054, + "map_at_10": 0.258, + "map_at_100": 0.9, + "map_at_1000": 2.011, + "map_at_3": 0.136, + "map_at_5": 0.178, + "mrr_at_1": 20.0, + "mrr_at_10": 34.828, + "mrr_at_100": 35.599, + "mrr_at_1000": 35.686, + "mrr_at_3": 32.0, + "mrr_at_5": 33.4, + "ndcg_at_1": 18.0, + "ndcg_at_10": 16.907, + "ndcg_at_100": 10.305, + "ndcg_at_1000": 10.621, + "ndcg_at_3": 20.492, + "ndcg_at_5": 18.721, + "precision_at_1": 20.0, + "precision_at_10": 17.6, + "precision_at_100": 10.3, + "precision_at_1000": 4.84, + "precision_at_3": 22.667, + "precision_at_5": 20.0, + "recall_at_1": 0.054, + "recall_at_10": 0.419, + "recall_at_100": 2.377, + "recall_at_1000": 10.489, + "recall_at_3": 0.183, + "recall_at_5": 0.257, + "main_score": 16.907 + }, + "SCIDOCS-PL": { + "evaluation_time": 3478.0, + "map_at_1": 1.255, + "map_at_10": 3.38, + "map_at_100": 4.382, + "map_at_1000": 4.591, + "map_at_3": 2.347, + "map_at_5": 2.884, + "mrr_at_1": 6.2, + "mrr_at_10": 11.485, + "mrr_at_100": 12.677, + "mrr_at_1000": 12.799, + "mrr_at_3": 9.083, + "mrr_at_5": 10.588, + "ndcg_at_1": 6.2, + "ndcg_at_10": 6.502, + "ndcg_at_100": 11.604, + "ndcg_at_1000": 16.545, + "ndcg_at_3": 5.48, + "ndcg_at_5": 5.216, + "precision_at_1": 6.2, + "precision_at_10": 3.63, + "precision_at_100": 1.108, + "precision_at_1000": 0.231, + "precision_at_3": 5.3, + "precision_at_5": 4.92, + "recall_at_1": 1.255, + "recall_at_10": 7.355, + "recall_at_100": 22.51, + "recall_at_1000": 46.848, + "recall_at_3": 3.23, + "recall_at_5": 4.98, + "main_score": 6.502 + }, + "PolEmo2.0-IN": { + "accuracy": 48.83657, + "accuracy_stderr": 2.88288, + "evaluation_time": 256.0, + "f1": 50.19095, + "f1_stderr": 3.37134, + "main_score": 48.83657 + }, + "PlscClusteringP2P": { + "evaluation_time": 2826.0, + "v_measure": 37.17276, + "v_measures": 37.04616, + "main_score": 37.17276 + }, + "MSMARCO-PL": { + "evaluation_time": 1167211.0, + "map_at_1": 1.934, + "map_at_10": 3.529, + "map_at_100": 3.962, + "map_at_1000": 4.036, + "map_at_3": 2.918, + "map_at_5": 3.248, + "mrr_at_1": 1.963, + "mrr_at_10": 3.596, + "mrr_at_100": 4.035, + "mrr_at_1000": 4.109, + "mrr_at_3": 2.968, + "mrr_at_5": 3.306, + "ndcg_at_1": 1.948, + "ndcg_at_10": 4.565, + "ndcg_at_100": 7.165, + "ndcg_at_1000": 9.613, + "ndcg_at_3": 3.274, + "ndcg_at_5": 3.868, + "precision_at_1": 1.948, + "precision_at_10": 0.807, + "precision_at_100": 0.219, + "precision_at_1000": 0.043, + "precision_at_3": 1.452, + "precision_at_5": 1.163, + "recall_at_1": 1.934, + "recall_at_10": 7.851, + "recall_at_100": 21.026, + "recall_at_1000": 41.104, + "recall_at_3": 4.26, + "recall_at_5": 5.683, + "main_score": 4.565 + }, + "MassiveIntentClassification": { + "accuracy": 53.1002, + "accuracy_stderr": 1.97225, + "f1": 48.49101, + "f1_stderr": 1.99358, + "main_score": 53.1002 + } + } + }, + { + "id": "multilingual-e5-base", + "tags": [], + "results": { + "CBD": { + "accuracy": 62.66, + "accuracy_stderr": 5.93164, + "ap": 18.88654, + "ap_stderr": 2.04852, + "evaluation_time": 793.0, + "f1": 53.12289, + "f1_stderr": 3.74864, + "main_score": 62.66 + }, + "AllegroReviews": { + "accuracy": 40.85487, + "accuracy_stderr": 3.68545, + "evaluation_time": 328.0, + "f1": 35.49399, + "f1_stderr": 1.90415, + "main_score": 40.85487 + }, + "PlscClusteringS2S": { + "evaluation_time": 1581.0, + "v_measure": 37.98251, + "v_measures": 37.04655, + "main_score": 37.98251 + }, + "ArguAna-PL": { + "evaluation_time": 2404.0, + "map_at_1": 19.915, + "map_at_10": 34.435, + "map_at_100": 35.732, + "map_at_1000": 35.752, + "map_at_3": 29.801, + "map_at_5": 32.336, + "mrr_at_1": 20.839, + "mrr_at_10": 34.783, + "mrr_at_100": 36.088, + "mrr_at_1000": 36.107, + "mrr_at_3": 30.145, + "mrr_at_5": 32.645, + "ndcg_at_1": 19.915, + "ndcg_at_10": 42.809, + "ndcg_at_100": 48.585, + "ndcg_at_1000": 49.061, + "ndcg_at_3": 33.169, + "ndcg_at_5": 37.753, + "precision_at_1": 19.915, + "precision_at_10": 6.97, + "precision_at_100": 0.956, + "precision_at_1000": 0.099, + "precision_at_3": 14.32, + "precision_at_5": 10.825, + "recall_at_1": 19.915, + "recall_at_10": 69.701, + "recall_at_100": 95.59, + "recall_at_1000": 99.218, + "recall_at_3": 42.959, + "recall_at_5": 54.125, + "main_score": 42.809 + }, + "PolEmo2.0-OUT": { + "accuracy": 43.90688, + "accuracy_stderr": 6.03527, + "evaluation_time": 287.0, + "f1": 36.24763, + "f1_stderr": 3.58576, + "main_score": 43.90688 + }, + "PAC": { + "accuracy": 70.86881, + "accuracy_stderr": 3.02738, + "ap": 77.82949, + "ap_stderr": 2.36788, + "evaluation_time": 313.0, + "f1": 67.88234, + "f1_stderr": 3.3815, + "main_score": 70.86881 + }, + "SciField": { + "accuracy": 81.40473, + "accuracy_stderr": 2.2149, + "evaluation_time": 789.0, + "f1": 80.95077, + "f1_stderr": 2.55786, + "main_score": 81.40473 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 80.43212, + "cos_sim.accuracy_threshold": 96.98166, + "cos_sim.ap": 68.76523, + "cos_sim.f1": 63.70861, + "cos_sim.f1_threshold": 95.26811, + "cos_sim.precision": 59.5297, + "cos_sim.recall": 68.51852, + "dot.accuracy": 80.43212, + "dot.accuracy_threshold": 96.98167, + "dot.ap": 68.76523, + "dot.f1": 63.70861, + "dot.f1_threshold": 95.26811, + "dot.precision": 59.5297, + "dot.recall": 68.51852, + "euclidean.accuracy": 80.43212, + "euclidean.accuracy_threshold": 24.56964, + "euclidean.ap": 68.76529, + "euclidean.f1": 63.70861, + "euclidean.f1_threshold": 30.76326, + "euclidean.precision": 59.5297, + "euclidean.recall": 68.51852, + "evaluation_time": 265.0, + "manhattan.accuracy": 80.35059, + "manhattan.accuracy_threshold": 557.26089, + "manhattan.ap": 68.6516, + "manhattan.f1": 63.68403, + "manhattan.f1_threshold": 648.31533, + "manhattan.precision": 62.52574, + "manhattan.recall": 64.88604, + "max.accuracy": 80.43212, + "max.ap": 68.76529, + "max.f1": 63.70861, + "main_score": 68.76523 + }, + "HotpotQA-PL": { + "evaluation_time": 432137.0, + "map_at_1": 36.82, + "map_at_10": 54.862, + "map_at_100": 55.717, + "map_at_1000": 55.785, + "map_at_3": 52.115, + "map_at_5": 53.806, + "mrr_at_1": 73.639, + "mrr_at_10": 79.728, + "mrr_at_100": 79.962, + "mrr_at_1000": 79.974, + "mrr_at_3": 78.656, + "mrr_at_5": 79.294, + "ndcg_at_1": 73.639, + "ndcg_at_10": 63.516, + "ndcg_at_100": 66.588, + "ndcg_at_1000": 67.997, + "ndcg_at_3": 59.414, + "ndcg_at_5": 61.621, + "precision_at_1": 73.639, + "precision_at_10": 13.084, + "precision_at_100": 1.549, + "precision_at_1000": 0.174, + "precision_at_3": 37.529, + "precision_at_5": 24.254, + "recall_at_1": 36.82, + "recall_at_10": 65.422, + "recall_at_100": 77.475, + "recall_at_1000": 86.867, + "recall_at_3": 56.293, + "recall_at_5": 60.635, + "main_score": 63.516 + }, + "PPC": { + "cos_sim.accuracy": 77.0, + "cos_sim.accuracy_threshold": 95.78815, + "cos_sim.ap": 88.01297, + "cos_sim.f1": 80.88235, + "cos_sim.f1_threshold": 95.44735, + "cos_sim.precision": 79.83871, + "cos_sim.recall": 81.95364, + "dot.accuracy": 77.0, + "dot.accuracy_threshold": 95.78815, + "dot.ap": 88.01297, + "dot.f1": 80.88235, + "dot.f1_threshold": 95.44735, + "dot.precision": 79.83871, + "dot.recall": 81.95364, + "euclidean.accuracy": 77.0, + "euclidean.accuracy_threshold": 29.02363, + "euclidean.ap": 88.01297, + "euclidean.f1": 80.88235, + "euclidean.f1_threshold": 30.175, + "euclidean.precision": 79.83871, + "euclidean.recall": 81.95364, + "evaluation_time": 131.0, + "manhattan.accuracy": 77.0, + "manhattan.accuracy_threshold": 638.55839, + "manhattan.ap": 87.98491, + "manhattan.f1": 80.71313, + "manhattan.f1_threshold": 671.57731, + "manhattan.precision": 79.04762, + "manhattan.recall": 82.45033, + "max.accuracy": 77.0, + "max.ap": 88.01297, + "max.f1": 80.88235, + "main_score": 88.01297 + }, + "EightTagsClustering": { + "evaluation_time": 3494.0, + "main_score": 24.71708, + "v_measure": 24.71708, + "v_measure_std": 2.79357, + "v_measures": 26.11333 + }, + "NFCorpus-PL": { + "evaluation_time": 1352.0, + "map_at_1": 4.253, + "map_at_10": 8.905, + "map_at_100": 11.089, + "map_at_1000": 12.136, + "map_at_3": 6.621, + "map_at_5": 7.752, + "mrr_at_1": 37.771, + "mrr_at_10": 45.079, + "mrr_at_100": 45.876, + "mrr_at_1000": 45.92, + "mrr_at_3": 42.776, + "mrr_at_5": 44.061, + "ndcg_at_1": 35.449, + "ndcg_at_10": 25.978, + "ndcg_at_100": 23.881, + "ndcg_at_1000": 32.089, + "ndcg_at_3": 30.568, + "ndcg_at_5": 29.29, + "precision_at_1": 37.152, + "precision_at_10": 19.164, + "precision_at_100": 6.065, + "precision_at_1000": 1.784, + "precision_at_3": 28.689, + "precision_at_5": 25.697, + "recall_at_1": 4.253, + "recall_at_10": 11.725, + "recall_at_100": 25.346, + "recall_at_1000": 54.083, + "recall_at_3": 7.271, + "recall_at_5": 9.111, + "main_score": 25.978 + }, + "SciFact-PL": { + "evaluation_time": 1851.0, + "map_at_1": 46.9, + "map_at_10": 57.299, + "map_at_100": 57.908, + "map_at_1000": 57.955, + "map_at_3": 54.711, + "map_at_5": 56.076, + "mrr_at_1": 49.0, + "mrr_at_10": 58.644, + "mrr_at_100": 59.134, + "mrr_at_1000": 59.176, + "mrr_at_3": 56.722, + "mrr_at_5": 57.806, + "ndcg_at_1": 49.0, + "ndcg_at_10": 62.111, + "ndcg_at_100": 64.93, + "ndcg_at_1000": 66.224, + "ndcg_at_3": 57.727, + "ndcg_at_5": 59.718, + "precision_at_1": 49.0, + "precision_at_10": 8.467, + "precision_at_100": 1.0, + "precision_at_1000": 0.111, + "precision_at_3": 22.778, + "precision_at_5": 15.067, + "recall_at_1": 46.9, + "recall_at_10": 75.167, + "recall_at_100": 88.033, + "recall_at_1000": 98.333, + "recall_at_3": 63.628, + "recall_at_5": 68.5, + "main_score": 62.111 + }, + "MassiveScenarioClassification": { + "accuracy": 66.10625, + "accuracy_stderr": 2.62014, + "f1": 64.8591, + "f1_stderr": 2.71898, + "main_score": 66.10625 + }, + "FiQA-PL": { + "evaluation_time": 11071.0, + "map_at_1": 12.896, + "map_at_10": 19.524, + "map_at_100": 20.932, + "map_at_1000": 21.126, + "map_at_3": 16.946, + "map_at_5": 18.198, + "mrr_at_1": 25.926, + "mrr_at_10": 32.843, + "mrr_at_100": 33.751, + "mrr_at_1000": 33.835, + "mrr_at_3": 30.736, + "mrr_at_5": 31.8, + "ndcg_at_1": 25.926, + "ndcg_at_10": 25.522, + "ndcg_at_100": 31.624, + "ndcg_at_1000": 35.735, + "ndcg_at_3": 22.779, + "ndcg_at_5": 23.355, + "precision_at_1": 25.926, + "precision_at_10": 7.145, + "precision_at_100": 1.319, + "precision_at_1000": 0.203, + "precision_at_3": 15.072, + "precision_at_5": 11.019, + "recall_at_1": 12.896, + "recall_at_10": 30.399, + "recall_at_100": 53.912, + "recall_at_1000": 79.627, + "recall_at_3": 20.119, + "recall_at_5": 23.89, + "main_score": 25.522 + }, + "SICK-R-PL": { + "cos_sim.pearson": 76.50023, + "cos_sim.spearman": 71.22785, + "euclidean.pearson": 73.2383, + "euclidean.spearman": 71.22787, + "evaluation_time": 307.0, + "manhattan.pearson": 73.3459, + "manhattan.spearman": 71.23465, + "main_score": 71.22785 + }, + "CDSC-R": { + "cos_sim.pearson": 89.98329, + "cos_sim.spearman": 90.08096, + "euclidean.pearson": 88.42047, + "euclidean.spearman": 90.08096, + "evaluation_time": 115.0, + "manhattan.pearson": 88.48742, + "manhattan.spearman": 90.10918, + "main_score": 90.08096 + }, + "SciDefRetrieval": { + "evaluation_time": 208.0, + "map_at_1": 69.514, + "map_at_10": 76.455, + "map_at_100": 76.784, + "map_at_1000": 76.797, + "map_at_20": 76.676, + "map_at_3": 74.988, + "map_at_5": 75.849, + "mrr_at_1": 69.809, + "mrr_at_10": 76.641, + "mrr_at_100": 76.968, + "mrr_at_1000": 76.98, + "mrr_at_20": 76.861, + "mrr_at_3": 75.16, + "mrr_at_5": 76.051, + "ndcg_at_1": 69.514, + "ndcg_at_10": 79.802, + "ndcg_at_100": 81.368, + "ndcg_at_1000": 81.691, + "ndcg_at_20": 80.616, + "ndcg_at_3": 76.795, + "ndcg_at_5": 78.329, + "precision_at_1": 69.514, + "precision_at_10": 9.028, + "precision_at_100": 0.975, + "precision_at_1000": 0.1, + "precision_at_20": 4.676, + "precision_at_3": 27.344, + "precision_at_5": 17.143, + "recall_at_1": 69.514, + "recall_at_10": 90.28, + "recall_at_100": 97.496, + "recall_at_1000": 100.0, + "recall_at_20": 93.52, + "recall_at_3": 82.032, + "recall_at_5": 85.714, + "main_score": 79.802 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 596.0, + "v_measure": 27.55881, + "v_measures": 26.7642, + "main_score": 27.55881 + }, + "PSC": { + "cos_sim.accuracy": 97.58813, + "cos_sim.accuracy_threshold": 87.9205, + "cos_sim.ap": 99.14178, + "cos_sim.f1": 96.08434, + "cos_sim.f1_threshold": 87.9205, + "cos_sim.precision": 94.94048, + "cos_sim.recall": 97.2561, + "dot.accuracy": 97.58813, + "dot.accuracy_threshold": 87.9205, + "dot.ap": 99.14178, + "dot.f1": 96.08434, + "dot.f1_threshold": 87.9205, + "dot.precision": 94.94048, + "dot.recall": 97.2561, + "euclidean.accuracy": 97.58813, + "euclidean.accuracy_threshold": 49.15176, + "euclidean.ap": 99.14178, + "euclidean.f1": 96.08434, + "euclidean.f1_threshold": 49.15176, + "euclidean.precision": 94.94048, + "euclidean.recall": 97.2561, + "evaluation_time": 181.0, + "manhattan.accuracy": 97.30983, + "manhattan.accuracy_threshold": 1085.48641, + "manhattan.ap": 99.10226, + "manhattan.f1": 95.61271, + "manhattan.f1_threshold": 1085.48641, + "manhattan.precision": 94.89489, + "manhattan.recall": 96.34146, + "max.accuracy": 97.58813, + "max.ap": 99.14178, + "max.f1": 96.08434, + "main_score": 99.14178 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 1228.0, + "v_measure": 38.1514, + "v_measures": 40.4328, + "main_score": 38.1514 + }, + "NQ-PL": { + "evaluation_time": 344272.0, + "map_at_1": 26.222, + "map_at_10": 38.483, + "map_at_100": 39.456, + "map_at_1000": 39.509, + "map_at_3": 35.005, + "map_at_5": 37.08, + "mrr_at_1": 29.519, + "mrr_at_10": 40.736, + "mrr_at_100": 41.528, + "mrr_at_1000": 41.569, + "mrr_at_3": 37.741, + "mrr_at_5": 39.53, + "ndcg_at_1": 29.519, + "ndcg_at_10": 44.803, + "ndcg_at_100": 49.257, + "ndcg_at_1000": 50.602, + "ndcg_at_3": 38.19, + "ndcg_at_5": 41.668, + "precision_at_1": 29.519, + "precision_at_10": 7.338, + "precision_at_100": 0.989, + "precision_at_1000": 0.112, + "precision_at_3": 17.314, + "precision_at_5": 12.37, + "recall_at_1": 26.222, + "recall_at_10": 61.868, + "recall_at_100": 81.771, + "recall_at_1000": 91.927, + "recall_at_3": 44.658, + "recall_at_5": 52.735, + "main_score": 44.803 + }, + "STS22": { + "cos_sim.pearson": 28.45839, + "cos_sim.spearman": 34.06761, + "euclidean.pearson": 17.96598, + "euclidean.spearman": 34.06761, + "manhattan.pearson": 18.1, + "manhattan.spearman": 33.96698, + "main_score": 34.06761 + }, + "DBPedia-PL": { + "evaluation_time": 366950.0, + "map_at_1": 7.149, + "map_at_10": 13.631, + "map_at_100": 18.72, + "map_at_1000": 19.919, + "map_at_3": 10.271, + "map_at_5": 11.706, + "mrr_at_1": 52.25, + "mrr_at_10": 60.581, + "mrr_at_100": 61.005, + "mrr_at_1000": 61.043, + "mrr_at_3": 58.375, + "mrr_at_5": 59.788, + "ndcg_at_1": 42.375, + "ndcg_at_10": 30.226, + "ndcg_at_100": 33.733, + "ndcg_at_1000": 40.535, + "ndcg_at_3": 34.48, + "ndcg_at_5": 32.053, + "precision_at_1": 52.25, + "precision_at_10": 23.4, + "precision_at_100": 7.413, + "precision_at_1000": 1.485, + "precision_at_3": 36.417, + "precision_at_5": 30.45, + "recall_at_1": 7.149, + "recall_at_10": 17.906, + "recall_at_100": 38.215, + "recall_at_1000": 60.893, + "recall_at_3": 11.264, + "recall_at_5": 13.934, + "main_score": 30.226 + }, + "Quora-PL": { + "evaluation_time": 20505.0, + "map_at_1": 62.794, + "map_at_10": 76.571, + "map_at_100": 77.359, + "map_at_1000": 77.386, + "map_at_3": 73.377, + "map_at_5": 75.401, + "mrr_at_1": 72.53, + "mrr_at_10": 80.205, + "mrr_at_100": 80.439, + "mrr_at_1000": 80.444, + "mrr_at_3": 78.712, + "mrr_at_5": 79.729, + "ndcg_at_1": 72.48, + "ndcg_at_10": 81.219, + "ndcg_at_100": 83.213, + "ndcg_at_1000": 83.497, + "ndcg_at_3": 77.426, + "ndcg_at_5": 79.57, + "precision_at_1": 72.48, + "precision_at_10": 12.504, + "precision_at_100": 1.484, + "precision_at_1000": 0.155, + "precision_at_3": 33.983, + "precision_at_5": 22.69, + "recall_at_1": 62.794, + "recall_at_10": 90.653, + "recall_at_100": 98.021, + "recall_at_1000": 99.643, + "recall_at_3": 80.103, + "recall_at_5": 85.804, + "main_score": 81.219 + }, + "CDSC-E": { + "cos_sim.accuracy": 88.6, + "cos_sim.accuracy_threshold": 98.29718, + "cos_sim.ap": 72.66746, + "cos_sim.f1": 64.95726, + "cos_sim.f1_threshold": 97.22315, + "cos_sim.precision": 70.80745, + "cos_sim.recall": 60.0, + "dot.accuracy": 88.6, + "dot.accuracy_threshold": 98.29718, + "dot.ap": 72.66746, + "dot.f1": 64.95726, + "dot.f1_threshold": 97.22315, + "dot.precision": 70.80745, + "dot.recall": 60.0, + "euclidean.accuracy": 88.6, + "euclidean.accuracy_threshold": 18.4543, + "euclidean.ap": 72.66746, + "euclidean.f1": 64.95726, + "euclidean.f1_threshold": 23.56625, + "euclidean.precision": 70.80745, + "euclidean.recall": 60.0, + "evaluation_time": 69.0, + "manhattan.accuracy": 88.5, + "manhattan.accuracy_threshold": 448.28091, + "manhattan.ap": 72.54677, + "manhattan.f1": 64.61538, + "manhattan.f1_threshold": 477.84376, + "manhattan.precision": 77.77778, + "manhattan.recall": 55.26316, + "max.accuracy": 88.6, + "max.ap": 72.66746, + "max.f1": 64.95726, + "main_score": 72.66746 + }, + "TRECCOVID-PL": { + "evaluation_time": 44875.0, + "map_at_1": 0.208, + "map_at_10": 1.636, + "map_at_100": 8.769, + "map_at_1000": 22.009, + "map_at_3": 0.594, + "map_at_5": 0.908, + "mrr_at_1": 74.0, + "mrr_at_10": 85.233, + "mrr_at_100": 85.233, + "mrr_at_1000": 85.233, + "mrr_at_3": 84.333, + "mrr_at_5": 85.233, + "ndcg_at_1": 69.0, + "ndcg_at_10": 66.056, + "ndcg_at_100": 50.036, + "ndcg_at_1000": 46.794, + "ndcg_at_3": 70.827, + "ndcg_at_5": 68.466, + "precision_at_1": 74.0, + "precision_at_10": 71.6, + "precision_at_100": 51.7, + "precision_at_1000": 20.932, + "precision_at_3": 78.0, + "precision_at_5": 74.8, + "recall_at_1": 0.208, + "recall_at_10": 1.893, + "recall_at_100": 12.425, + "recall_at_1000": 44.612, + "recall_at_3": 0.64, + "recall_at_5": 1.0, + "main_score": 66.056 + }, + "SCIDOCS-PL": { + "evaluation_time": 7332.0, + "map_at_1": 3.013, + "map_at_10": 7.053, + "map_at_100": 8.344, + "map_at_1000": 8.573, + "map_at_3": 5.303, + "map_at_5": 6.118, + "mrr_at_1": 14.9, + "mrr_at_10": 22.242, + "mrr_at_100": 23.35, + "mrr_at_1000": 23.443, + "mrr_at_3": 19.767, + "mrr_at_5": 21.037, + "ndcg_at_1": 14.9, + "ndcg_at_10": 12.355, + "ndcg_at_100": 18.344, + "ndcg_at_1000": 23.405, + "ndcg_at_3": 12.023, + "ndcg_at_5": 10.221, + "precision_at_1": 14.9, + "precision_at_10": 6.35, + "precision_at_100": 1.515, + "precision_at_1000": 0.274, + "precision_at_3": 11.267, + "precision_at_5": 8.86, + "recall_at_1": 3.013, + "recall_at_10": 12.858, + "recall_at_100": 30.702, + "recall_at_1000": 55.75, + "recall_at_3": 6.838, + "recall_at_5": 8.948, + "main_score": 12.355 + }, + "PolEmo2.0-IN": { + "accuracy": 67.65928, + "accuracy_stderr": 5.21455, + "evaluation_time": 367.0, + "f1": 66.65834, + "f1_stderr": 4.61212, + "main_score": 67.65928 + }, + "PlscClusteringP2P": { + "evaluation_time": 3701.0, + "v_measure": 42.55158, + "v_measures": 41.43822, + "main_score": 42.55158 + }, + "MSMARCO-PL": { + "evaluation_time": 898449.0, + "map_at_1": 15.491, + "map_at_10": 24.398, + "map_at_100": 25.549, + "map_at_1000": 25.62, + "map_at_3": 21.559, + "map_at_5": 23.105, + "mrr_at_1": 16.046, + "mrr_at_10": 24.944, + "mrr_at_100": 26.058, + "mrr_at_1000": 26.125, + "mrr_at_3": 22.147, + "mrr_at_5": 23.658, + "ndcg_at_1": 16.046, + "ndcg_at_10": 29.52, + "ndcg_at_100": 35.29, + "ndcg_at_1000": 37.206, + "ndcg_at_3": 23.678, + "ndcg_at_5": 26.429, + "precision_at_1": 16.046, + "precision_at_10": 4.732, + "precision_at_100": 0.766, + "precision_at_1000": 0.093, + "precision_at_3": 10.124, + "precision_at_5": 7.461, + "recall_at_1": 15.491, + "recall_at_10": 45.254, + "recall_at_100": 72.418, + "recall_at_1000": 87.373, + "recall_at_3": 29.26, + "recall_at_5": 35.864, + "main_score": 29.52 + }, + "MassiveIntentClassification": { + "accuracy": 61.03564, + "accuracy_stderr": 1.451, + "f1": 56.88428, + "f1_stderr": 1.23205, + "main_score": 61.03564 + } + } + }, + { + "id": "paraphrase-multilingual-MiniLM-L12-v2", + "tags": [], + "results": { + "CBD": { + "accuracy": 57.68, + "accuracy_stderr": 9.66197, + "ap": 17.31965, + "ap_stderr": 2.50665, + "evaluation_time": 1177.0, + "f1": 49.07073, + "f1_stderr": 5.66149, + "main_score": 57.68 + }, + "AllegroReviews": { + "accuracy": 30.88469, + "accuracy_stderr": 4.06658, + "evaluation_time": 186.0, + "f1": 28.05477, + "f1_stderr": 2.89384, + "main_score": 30.88469 + }, + "PlscClusteringS2S": { + "evaluation_time": 1550.0, + "v_measure": 36.90179, + "v_measures": 37.49968, + "main_score": 36.90179 + }, + "ArguAna-PL": { + "evaluation_time": 746.0, + "map_at_1": 17.568, + "map_at_10": 30.198, + "map_at_100": 31.615, + "map_at_1000": 31.646, + "map_at_3": 25.616, + "map_at_5": 28.269, + "mrr_at_1": 17.852, + "mrr_at_10": 30.337, + "mrr_at_100": 31.748, + "mrr_at_1000": 31.779, + "mrr_at_3": 25.842, + "mrr_at_5": 28.427, + "ndcg_at_1": 17.568, + "ndcg_at_10": 37.834, + "ndcg_at_100": 44.425, + "ndcg_at_1000": 45.238, + "ndcg_at_3": 28.396, + "ndcg_at_5": 33.204, + "precision_at_1": 17.568, + "precision_at_10": 6.245, + "precision_at_100": 0.927, + "precision_at_1000": 0.099, + "precision_at_3": 12.162, + "precision_at_5": 9.644, + "recall_at_1": 17.568, + "recall_at_10": 62.447, + "recall_at_100": 92.674, + "recall_at_1000": 99.075, + "recall_at_3": 36.486, + "recall_at_5": 48.222, + "main_score": 37.834 + }, + "PolEmo2.0-OUT": { + "accuracy": 28.70445, + "accuracy_stderr": 6.57555, + "evaluation_time": 151.0, + "f1": 26.11843, + "f1_stderr": 4.8635, + "main_score": 28.70445 + }, + "PAC": { + "accuracy": 65.7631, + "accuracy_stderr": 3.24492, + "ap": 75.12507, + "ap_stderr": 1.92656, + "evaluation_time": 245.0, + "f1": 62.62728, + "f1_stderr": 3.59131, + "main_score": 65.7631 + }, + "SciField": { + "accuracy": 79.29068, + "accuracy_stderr": 2.12523, + "evaluation_time": 667.0, + "f1": 78.85997, + "f1_stderr": 2.3016, + "main_score": 79.29068 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 81.69588, + "cos_sim.accuracy_threshold": 88.44593, + "cos_sim.ap": 71.94251, + "cos_sim.f1": 66.48295, + "cos_sim.f1_threshold": 84.35401, + "cos_sim.precision": 64.37625, + "cos_sim.recall": 68.73219, + "dot.accuracy": 75.92744, + "dot.accuracy_threshold": 1881.69899, + "dot.ap": 56.94456, + "dot.f1": 58.07033, + "dot.f1_threshold": 1541.41512, + "dot.precision": 50.23401, + "dot.recall": 68.80342, + "euclidean.accuracy": 81.32898, + "euclidean.accuracy_threshold": 207.56972, + "euclidean.ap": 71.80318, + "euclidean.f1": 66.51116, + "euclidean.f1_threshold": 261.35826, + "euclidean.precision": 62.49217, + "euclidean.recall": 71.08262, + "evaluation_time": 230.0, + "manhattan.accuracy": 81.36975, + "manhattan.accuracy_threshold": 3402.93121, + "manhattan.ap": 71.8088, + "manhattan.f1": 66.23465, + "manhattan.f1_threshold": 4004.08096, + "manhattan.precision": 63.54712, + "manhattan.recall": 69.15954, + "max.accuracy": 81.69588, + "max.ap": 71.94251, + "max.f1": 66.51116, + "main_score": 71.94251 + }, + "HotpotQA-PL": { + "evaluation_time": 210729.0, + "map_at_1": 12.512, + "map_at_10": 17.207, + "map_at_100": 17.919, + "map_at_1000": 18.013, + "map_at_3": 15.774, + "map_at_5": 16.584, + "mrr_at_1": 25.024, + "mrr_at_10": 31.159, + "mrr_at_100": 31.936, + "mrr_at_1000": 32.006, + "mrr_at_3": 29.338, + "mrr_at_5": 30.378, + "ndcg_at_1": 25.024, + "ndcg_at_10": 22.757, + "ndcg_at_100": 26.417, + "ndcg_at_1000": 28.942, + "ndcg_at_3": 19.776, + "ndcg_at_5": 21.247, + "precision_at_1": 25.024, + "precision_at_10": 5.097, + "precision_at_100": 0.803, + "precision_at_1000": 0.114, + "precision_at_3": 12.501, + "precision_at_5": 8.667, + "recall_at_1": 12.512, + "recall_at_10": 25.483, + "recall_at_100": 40.169, + "recall_at_1000": 57.144, + "recall_at_3": 18.751, + "recall_at_5": 21.668, + "main_score": 22.757 + }, + "PPC": { + "cos_sim.accuracy": 84.5, + "cos_sim.accuracy_threshold": 81.81483, + "cos_sim.ap": 91.79697, + "cos_sim.f1": 87.80109, + "cos_sim.f1_threshold": 78.60287, + "cos_sim.precision": 82.72328, + "cos_sim.recall": 93.54305, + "dot.accuracy": 70.8, + "dot.accuracy_threshold": 1274.0963, + "dot.ap": 79.39532, + "dot.f1": 78.94003, + "dot.f1_threshold": 1027.43511, + "dot.precision": 68.19277, + "dot.recall": 93.70861, + "euclidean.accuracy": 86.0, + "euclidean.accuracy_threshold": 257.25, + "euclidean.ap": 92.36259, + "euclidean.f1": 88.87183, + "euclidean.f1_threshold": 294.80319, + "euclidean.precision": 82.83262, + "euclidean.recall": 95.86093, + "evaluation_time": 122.0, + "manhattan.accuracy": 85.9, + "manhattan.accuracy_threshold": 3914.46648, + "manhattan.ap": 92.31762, + "manhattan.f1": 88.73457, + "manhattan.f1_threshold": 4547.84622, + "manhattan.precision": 83.09249, + "manhattan.recall": 95.19868, + "max.accuracy": 86.0, + "max.ap": 92.36259, + "max.f1": 88.87183, + "main_score": 91.79697 + }, + "EightTagsClustering": { + "evaluation_time": 3593.0, + "main_score": 24.17887, + "v_measure": 24.17887, + "v_measure_std": 3.30654, + "v_measures": 27.46885 + }, + "NFCorpus-PL": { + "evaluation_time": 359.0, + "map_at_1": 1.575, + "map_at_10": 4.339, + "map_at_100": 6.206, + "map_at_1000": 7.276, + "map_at_3": 2.925, + "map_at_5": 3.531, + "mrr_at_1": 22.91, + "mrr_at_10": 30.034, + "mrr_at_100": 30.949, + "mrr_at_1000": 31.044, + "mrr_at_3": 27.915, + "mrr_at_5": 29.045, + "ndcg_at_1": 20.898, + "ndcg_at_10": 17.158, + "ndcg_at_100": 16.722, + "ndcg_at_1000": 25.944, + "ndcg_at_3": 19.258, + "ndcg_at_5": 18.473, + "precision_at_1": 22.91, + "precision_at_10": 14.303, + "precision_at_100": 5.44, + "precision_at_1000": 1.767, + "precision_at_3": 18.885, + "precision_at_5": 17.276, + "recall_at_1": 1.575, + "recall_at_10": 7.483, + "recall_at_100": 19.062, + "recall_at_1000": 52.622, + "recall_at_3": 3.737, + "recall_at_5": 5.041, + "main_score": 17.158 + }, + "SciFact-PL": { + "evaluation_time": 437.0, + "map_at_1": 30.111, + "map_at_10": 36.59, + "map_at_100": 37.624, + "map_at_1000": 37.698, + "map_at_3": 34.472, + "map_at_5": 35.418, + "mrr_at_1": 32.333, + "mrr_at_10": 38.093, + "mrr_at_100": 39.006, + "mrr_at_1000": 39.068, + "mrr_at_3": 36.444, + "mrr_at_5": 37.111, + "ndcg_at_1": 32.333, + "ndcg_at_10": 40.235, + "ndcg_at_100": 45.374, + "ndcg_at_1000": 47.609, + "ndcg_at_3": 36.287, + "ndcg_at_5": 37.679, + "precision_at_1": 32.333, + "precision_at_10": 5.667, + "precision_at_100": 0.847, + "precision_at_1000": 0.105, + "precision_at_3": 14.333, + "precision_at_5": 9.533, + "recall_at_1": 30.111, + "recall_at_10": 50.139, + "recall_at_100": 74.422, + "recall_at_1000": 92.333, + "recall_at_3": 39.306, + "recall_at_5": 42.694, + "main_score": 40.235 + }, + "MassiveScenarioClassification": { + "accuracy": 65.04371, + "accuracy_stderr": 1.92686, + "f1": 64.57997, + "f1_stderr": 1.94605, + "main_score": 65.04371 + }, + "FiQA-PL": { + "evaluation_time": 2717.0, + "map_at_1": 5.1, + "map_at_10": 9.091, + "map_at_100": 9.975, + "map_at_1000": 10.162, + "map_at_3": 7.697, + "map_at_5": 8.262, + "mrr_at_1": 9.877, + "mrr_at_10": 14.951, + "mrr_at_100": 15.939, + "mrr_at_1000": 16.046, + "mrr_at_3": 13.297, + "mrr_at_5": 14.053, + "ndcg_at_1": 9.877, + "ndcg_at_10": 12.492, + "ndcg_at_100": 17.086, + "ndcg_at_1000": 21.667, + "ndcg_at_3": 10.427, + "ndcg_at_5": 10.883, + "precision_at_1": 9.877, + "precision_at_10": 3.611, + "precision_at_100": 0.84, + "precision_at_1000": 0.164, + "precision_at_3": 6.739, + "precision_at_5": 5.031, + "recall_at_1": 5.1, + "recall_at_10": 17.045, + "recall_at_100": 34.625, + "recall_at_1000": 63.284, + "recall_at_3": 10.295, + "recall_at_5": 12.283, + "main_score": 12.492 + }, + "SICK-R-PL": { + "cos_sim.pearson": 75.76394, + "cos_sim.spearman": 68.76769, + "euclidean.pearson": 73.89374, + "euclidean.spearman": 69.403, + "evaluation_time": 260.0, + "manhattan.pearson": 73.85105, + "manhattan.spearman": 69.37073, + "main_score": 68.76769 + }, + "CDSC-R": { + "cos_sim.pearson": 88.86837, + "cos_sim.spearman": 88.97695, + "euclidean.pearson": 89.36682, + "euclidean.spearman": 89.48135, + "evaluation_time": 93.0, + "manhattan.pearson": 89.37394, + "manhattan.spearman": 89.4965, + "main_score": 88.97695 + }, + "SciDefRetrieval": { + "evaluation_time": 151.0, + "map_at_1": 23.27, + "map_at_10": 31.066, + "map_at_100": 32.239, + "map_at_1000": 32.332, + "map_at_20": 31.681, + "map_at_3": 28.596, + "map_at_5": 30.304, + "mrr_at_1": 23.27, + "mrr_at_10": 31.049, + "mrr_at_100": 32.219, + "mrr_at_1000": 32.312, + "mrr_at_20": 31.662, + "mrr_at_3": 28.547, + "mrr_at_5": 30.263, + "ndcg_at_1": 23.27, + "ndcg_at_10": 35.203, + "ndcg_at_100": 41.647, + "ndcg_at_1000": 44.171, + "ndcg_at_20": 37.463, + "ndcg_at_3": 30.295, + "ndcg_at_5": 33.35, + "precision_at_1": 23.27, + "precision_at_10": 4.831, + "precision_at_100": 0.801, + "precision_at_1000": 0.1, + "precision_at_20": 2.865, + "precision_at_3": 11.733, + "precision_at_5": 8.513, + "recall_at_1": 23.27, + "recall_at_10": 48.306, + "recall_at_100": 80.118, + "recall_at_1000": 100.0, + "recall_at_20": 57.29, + "recall_at_3": 35.199, + "recall_at_5": 42.563, + "main_score": 35.203 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 565.0, + "v_measure": 24.28831, + "v_measures": 21.80536, + "main_score": 24.28831 + }, + "PSC": { + "cos_sim.accuracy": 94.99072, + "cos_sim.accuracy_threshold": 62.02089, + "cos_sim.ap": 97.14385, + "cos_sim.f1": 91.79331, + "cos_sim.f1_threshold": 62.02089, + "cos_sim.precision": 91.51515, + "cos_sim.recall": 92.07317, + "dot.accuracy": 93.13544, + "dot.accuracy_threshold": 492.37061, + "dot.ap": 92.35062, + "dot.f1": 89.45869, + "dot.f1_threshold": 492.37061, + "dot.precision": 83.95722, + "dot.recall": 95.73171, + "euclidean.accuracy": 94.06308, + "euclidean.accuracy_threshold": 267.66911, + "euclidean.ap": 96.27172, + "euclidean.f1": 90.44776, + "euclidean.f1_threshold": 271.3799, + "euclidean.precision": 88.59649, + "euclidean.recall": 92.37805, + "evaluation_time": 111.0, + "manhattan.accuracy": 94.15584, + "manhattan.accuracy_threshold": 4137.04376, + "manhattan.ap": 96.15378, + "manhattan.f1": 90.35222, + "manhattan.f1_threshold": 4137.04376, + "manhattan.precision": 90.76923, + "manhattan.recall": 89.93902, + "max.accuracy": 94.99072, + "max.ap": 97.14385, + "max.f1": 91.79331, + "main_score": 97.14385 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 791.0, + "v_measure": 39.21223, + "v_measures": 38.60525, + "main_score": 39.21223 + }, + "NQ-PL": { + "evaluation_time": 111559.0, + "map_at_1": 5.359, + "map_at_10": 9.675, + "map_at_100": 10.575, + "map_at_1000": 10.674, + "map_at_3": 8.047, + "map_at_5": 8.892, + "mrr_at_1": 6.257, + "mrr_at_10": 10.797, + "mrr_at_100": 11.667, + "mrr_at_1000": 11.752, + "mrr_at_3": 9.154, + "mrr_at_5": 10.02, + "ndcg_at_1": 6.257, + "ndcg_at_10": 12.563, + "ndcg_at_100": 17.235, + "ndcg_at_1000": 20.007, + "ndcg_at_3": 9.162, + "ndcg_at_5": 10.683, + "precision_at_1": 6.257, + "precision_at_10": 2.433, + "precision_at_100": 0.509, + "precision_at_1000": 0.078, + "precision_at_3": 4.452, + "precision_at_5": 3.523, + "recall_at_1": 5.359, + "recall_at_10": 20.507, + "recall_at_100": 42.306, + "recall_at_1000": 63.748, + "recall_at_3": 11.298, + "recall_at_5": 14.871, + "main_score": 12.563 + }, + "STS22": { + "cos_sim.pearson": 31.95767, + "cos_sim.spearman": 33.72707, + "euclidean.pearson": 17.32696, + "euclidean.spearman": 32.14937, + "manhattan.pearson": 18.80229, + "manhattan.spearman": 32.72083, + "main_score": 33.72707 + }, + "DBPedia-PL": { + "evaluation_time": 143088.0, + "map_at_1": 3.54, + "map_at_10": 7.116, + "map_at_100": 9.562, + "map_at_1000": 10.272, + "map_at_3": 5.183, + "map_at_5": 6.109, + "mrr_at_1": 31.25, + "mrr_at_10": 40.546, + "mrr_at_100": 41.163, + "mrr_at_1000": 41.22, + "mrr_at_3": 38.25, + "mrr_at_5": 39.5, + "ndcg_at_1": 23.75, + "ndcg_at_10": 18.005, + "ndcg_at_100": 19.892, + "ndcg_at_1000": 25.654, + "ndcg_at_3": 20.073, + "ndcg_at_5": 19.052, + "precision_at_1": 31.25, + "precision_at_10": 15.375, + "precision_at_100": 4.662, + "precision_at_1000": 1.114, + "precision_at_3": 23.417, + "precision_at_5": 19.65, + "recall_at_1": 3.54, + "recall_at_10": 11.014, + "recall_at_100": 24.831, + "recall_at_1000": 44.286, + "recall_at_3": 6.121, + "recall_at_5": 8.2, + "main_score": 18.005 + }, + "Quora-PL": { + "evaluation_time": 18099.0, + "map_at_1": 59.641, + "map_at_10": 72.575, + "map_at_100": 73.452, + "map_at_1000": 73.484, + "map_at_3": 69.468, + "map_at_5": 71.363, + "mrr_at_1": 68.78, + "mrr_at_10": 76.139, + "mrr_at_100": 76.464, + "mrr_at_1000": 76.474, + "mrr_at_3": 74.603, + "mrr_at_5": 75.576, + "ndcg_at_1": 68.8, + "ndcg_at_10": 77.179, + "ndcg_at_100": 79.624, + "ndcg_at_1000": 80.063, + "ndcg_at_3": 73.353, + "ndcg_at_5": 75.372, + "precision_at_1": 68.8, + "precision_at_10": 11.97, + "precision_at_100": 1.46, + "precision_at_1000": 0.154, + "precision_at_3": 32.29, + "precision_at_5": 21.538, + "recall_at_1": 59.641, + "recall_at_10": 86.566, + "recall_at_100": 95.973, + "recall_at_1000": 98.802, + "recall_at_3": 75.749, + "recall_at_5": 81.196, + "main_score": 77.179 + }, + "CDSC-E": { + "cos_sim.accuracy": 88.7, + "cos_sim.accuracy_threshold": 92.71072, + "cos_sim.ap": 72.22191, + "cos_sim.f1": 66.47399, + "cos_sim.f1_threshold": 91.19754, + "cos_sim.precision": 73.71795, + "cos_sim.recall": 60.52632, + "dot.accuracy": 83.2, + "dot.accuracy_threshold": 2101.31016, + "dot.ap": 52.79368, + "dot.f1": 56.2724, + "dot.f1_threshold": 1350.2243, + "dot.precision": 42.66304, + "dot.recall": 82.63158, + "euclidean.accuracy": 88.4, + "euclidean.accuracy_threshold": 159.45673, + "euclidean.ap": 72.33336, + "euclidean.f1": 65.51724, + "euclidean.f1_threshold": 179.43041, + "euclidean.precision": 72.1519, + "euclidean.recall": 60.0, + "evaluation_time": 72.0, + "manhattan.accuracy": 88.3, + "manhattan.accuracy_threshold": 2438.27648, + "manhattan.ap": 72.27077, + "manhattan.f1": 65.35211, + "manhattan.f1_threshold": 2837.10308, + "manhattan.precision": 70.30303, + "manhattan.recall": 61.05263, + "max.accuracy": 88.7, + "max.ap": 72.33336, + "max.f1": 66.47399, + "main_score": 72.22191 + }, + "TRECCOVID-PL": { + "evaluation_time": 10590.0, + "map_at_1": 0.148, + "map_at_10": 0.679, + "map_at_100": 3.479, + "map_at_1000": 7.293, + "map_at_3": 0.291, + "map_at_5": 0.407, + "mrr_at_1": 50.0, + "mrr_at_10": 59.269, + "mrr_at_100": 59.969, + "mrr_at_1000": 59.997, + "mrr_at_3": 55.333, + "mrr_at_5": 57.233, + "ndcg_at_1": 49.0, + "ndcg_at_10": 34.384, + "ndcg_at_100": 24.562, + "ndcg_at_1000": 22.856, + "ndcg_at_3": 39.581, + "ndcg_at_5": 36.283, + "precision_at_1": 50.0, + "precision_at_10": 35.6, + "precision_at_100": 24.94, + "precision_at_1000": 10.144, + "precision_at_3": 39.333, + "precision_at_5": 36.4, + "recall_at_1": 0.148, + "recall_at_10": 0.897, + "recall_at_100": 5.94, + "recall_at_1000": 21.586, + "recall_at_3": 0.303, + "recall_at_5": 0.467, + "main_score": 34.384 + }, + "SCIDOCS-PL": { + "evaluation_time": 1561.0, + "map_at_1": 2.53, + "map_at_10": 5.653, + "map_at_100": 6.88, + "map_at_1000": 7.111, + "map_at_3": 4.145, + "map_at_5": 4.9, + "mrr_at_1": 12.5, + "mrr_at_10": 18.929, + "mrr_at_100": 20.014, + "mrr_at_1000": 20.118, + "mrr_at_3": 16.483, + "mrr_at_5": 17.723, + "ndcg_at_1": 12.5, + "ndcg_at_10": 10.257, + "ndcg_at_100": 15.933, + "ndcg_at_1000": 21.04, + "ndcg_at_3": 9.604, + "ndcg_at_5": 8.437, + "precision_at_1": 12.5, + "precision_at_10": 5.33, + "precision_at_100": 1.36, + "precision_at_1000": 0.26, + "precision_at_3": 8.867, + "precision_at_5": 7.36, + "recall_at_1": 2.53, + "recall_at_10": 10.818, + "recall_at_100": 27.618, + "recall_at_1000": 52.888, + "recall_at_3": 5.405, + "recall_at_5": 7.47, + "main_score": 10.257 + }, + "PolEmo2.0-IN": { + "accuracy": 57.75623, + "accuracy_stderr": 4.16112, + "evaluation_time": 167.0, + "f1": 58.77209, + "f1_stderr": 3.07355, + "main_score": 57.75623 + }, + "PlscClusteringP2P": { + "evaluation_time": 2134.0, + "v_measure": 40.69791, + "v_measures": 40.68628, + "main_score": 40.69791 + }, + "MSMARCO-PL": { + "evaluation_time": 380837.0, + "map_at_1": 4.651, + "map_at_10": 8.213, + "map_at_100": 8.908, + "map_at_1000": 8.99, + "map_at_3": 6.981, + "map_at_5": 7.605, + "mrr_at_1": 4.785, + "mrr_at_10": 8.409, + "mrr_at_100": 9.105, + "mrr_at_1000": 9.185, + "mrr_at_3": 7.142, + "mrr_at_5": 7.784, + "ndcg_at_1": 4.799, + "ndcg_at_10": 10.393, + "ndcg_at_100": 14.197, + "ndcg_at_1000": 16.732, + "ndcg_at_3": 7.787, + "ndcg_at_5": 8.93, + "precision_at_1": 4.799, + "precision_at_10": 1.781, + "precision_at_100": 0.376, + "precision_at_1000": 0.06, + "precision_at_3": 3.424, + "precision_at_5": 2.633, + "recall_at_1": 4.651, + "recall_at_10": 17.18, + "recall_at_100": 35.737, + "recall_at_1000": 56.233, + "recall_at_3": 9.955, + "recall_at_5": 12.741, + "main_score": 10.393 + }, + "MassiveIntentClassification": { + "accuracy": 59.43174, + "accuracy_stderr": 1.12613, + "f1": 57.19053, + "f1_stderr": 1.06581, + "main_score": 59.43174 + } + } + }, + { + "id": "mmlw-e5-large", + "tags": [], + "results": { + "CBD": { + "accuracy": 66.15, + "accuracy_stderr": 7.81706, + "ap": 21.69514, + "ap_stderr": 2.73966, + "evaluation_time": 6044.0, + "f1": 56.48143, + "f1_stderr": 5.13266, + "main_score": 66.15 + }, + "AllegroReviews": { + "accuracy": 37.6839, + "accuracy_stderr": 3.17048, + "evaluation_time": 681.0, + "f1": 34.19193, + "f1_stderr": 2.39509, + "main_score": 37.6839 + }, + "PlscClusteringS2S": { + "evaluation_time": 2268.0, + "v_measure": 39.60627, + "v_measures": 39.9118, + "main_score": 39.60627 + }, + "ArguAna-PL": { + "evaluation_time": 7143.0, + "map_at_1": 38.407, + "map_at_10": 55.147, + "map_at_100": 55.757, + "map_at_1000": 55.761, + "map_at_3": 51.268, + "map_at_5": 53.697, + "mrr_at_1": 40.043, + "mrr_at_10": 55.841, + "mrr_at_100": 56.459, + "mrr_at_1000": 56.463, + "mrr_at_3": 52.074, + "mrr_at_5": 54.365, + "ndcg_at_1": 38.407, + "ndcg_at_10": 63.248, + "ndcg_at_100": 65.717, + "ndcg_at_1000": 65.79, + "ndcg_at_3": 55.404, + "ndcg_at_5": 59.76, + "precision_at_1": 38.407, + "precision_at_10": 8.862, + "precision_at_100": 0.991, + "precision_at_1000": 0.1, + "precision_at_3": 22.451, + "precision_at_5": 15.576, + "recall_at_1": 38.407, + "recall_at_10": 88.62, + "recall_at_100": 99.075, + "recall_at_1000": 99.573, + "recall_at_3": 67.354, + "recall_at_5": 77.881, + "main_score": 63.248 + }, + "PolEmo2.0-OUT": { + "accuracy": 43.13765, + "accuracy_stderr": 4.86711, + "evaluation_time": 658.0, + "f1": 36.15557, + "f1_stderr": 2.82459, + "main_score": 43.13765 + }, + "PAC": { + "accuracy": 63.76774, + "accuracy_stderr": 4.30467, + "ap": 74.87897, + "ap_stderr": 2.7666, + "evaluation_time": 598.0, + "f1": 61.42051, + "f1_stderr": 4.55659, + "main_score": 63.76774 + }, + "SciField": { + "accuracy": 84.28373, + "accuracy_stderr": 2.49961, + "evaluation_time": 825.0, + "f1": 84.0008, + "f1_stderr": 2.55545, + "main_score": 84.28373 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 82.65389, + "cos_sim.accuracy_threshold": 97.20789, + "cos_sim.ap": 76.13749, + "cos_sim.f1": 66.64355, + "cos_sim.f1_threshold": 95.49442, + "cos_sim.precision": 64.93243, + "cos_sim.recall": 68.44729, + "dot.accuracy": 76.0905, + "dot.accuracy_threshold": 27750.0, + "dot.ap": 58.05897, + "dot.f1": 61.18108, + "dot.f1_threshold": 26625.0, + "dot.precision": 50.39189, + "dot.recall": 77.849, + "euclidean.accuracy": 82.61313, + "euclidean.accuracy_threshold": 410.12101, + "euclidean.ap": 76.1029, + "euclidean.f1": 66.50783, + "euclidean.f1_threshold": 516.58405, + "euclidean.precision": 63.6897, + "euclidean.recall": 69.58689, + "evaluation_time": 413.0, + "manhattan.accuracy": 82.67428, + "manhattan.accuracy_threshold": 10692.56287, + "manhattan.ap": 76.12712, + "manhattan.f1": 66.597, + "manhattan.f1_threshold": 12923.06403, + "manhattan.precision": 65.16701, + "manhattan.recall": 68.09117, + "max.accuracy": 82.67428, + "max.ap": 76.13749, + "max.f1": 66.64355, + "main_score": 76.13749 + }, + "HotpotQA-PL": { + "evaluation_time": 1178870.0, + "map_at_1": 39.095, + "map_at_10": 62.529, + "map_at_100": 63.425, + "map_at_1000": 63.483, + "map_at_3": 58.887, + "map_at_5": 61.186, + "mrr_at_1": 78.123, + "mrr_at_10": 84.231, + "mrr_at_100": 84.408, + "mrr_at_1000": 84.414, + "mrr_at_3": 83.286, + "mrr_at_5": 83.94, + "ndcg_at_1": 78.19, + "ndcg_at_10": 70.938, + "ndcg_at_100": 73.992, + "ndcg_at_1000": 75.1, + "ndcg_at_3": 65.863, + "ndcg_at_5": 68.755, + "precision_at_1": 78.19, + "precision_at_10": 14.949, + "precision_at_100": 1.733, + "precision_at_1000": 0.188, + "precision_at_3": 42.381, + "precision_at_5": 27.711, + "recall_at_1": 39.095, + "recall_at_10": 74.747, + "recall_at_100": 86.631, + "recall_at_1000": 93.923, + "recall_at_3": 63.572, + "recall_at_5": 69.278, + "main_score": 70.938 + }, + "PPC": { + "cos_sim.accuracy": 82.1, + "cos_sim.accuracy_threshold": 96.76954, + "cos_sim.ap": 91.09417, + "cos_sim.f1": 84.78438, + "cos_sim.f1_threshold": 95.94262, + "cos_sim.precision": 83.36, + "cos_sim.recall": 86.25828, + "dot.accuracy": 75.9, + "dot.accuracy_threshold": 28825.0, + "dot.ap": 86.82681, + "dot.f1": 80.53797, + "dot.f1_threshold": 28325.0, + "dot.precision": 77.12121, + "dot.recall": 84.27152, + "euclidean.accuracy": 81.6, + "euclidean.accuracy_threshold": 427.59366, + "euclidean.ap": 90.81249, + "euclidean.f1": 84.35374, + "euclidean.f1_threshold": 454.72871, + "euclidean.precision": 86.71329, + "euclidean.recall": 82.11921, + "evaluation_time": 182.0, + "manhattan.accuracy": 81.6, + "manhattan.accuracy_threshold": 10774.40897, + "manhattan.ap": 90.81273, + "manhattan.f1": 84.33531, + "manhattan.f1_threshold": 11717.16617, + "manhattan.precision": 86.30849, + "manhattan.recall": 82.45033, + "max.accuracy": 82.1, + "max.ap": 91.09417, + "max.f1": 84.78438, + "main_score": 91.09417 + }, + "EightTagsClustering": { + "evaluation_time": 5641.0, + "main_score": 31.83948, + "v_measure": 31.83948, + "v_measure_std": 3.84667, + "v_measures": 34.87326 + }, + "NFCorpus-PL": { + "evaluation_time": 3811.0, + "map_at_1": 5.772, + "map_at_10": 12.708, + "map_at_100": 16.194, + "map_at_1000": 17.63, + "map_at_3": 9.34, + "map_at_5": 10.741, + "mrr_at_1": 43.344, + "mrr_at_10": 53.429, + "mrr_at_100": 53.887, + "mrr_at_1000": 53.925, + "mrr_at_3": 51.342, + "mrr_at_5": 52.456, + "ndcg_at_1": 41.641, + "ndcg_at_10": 34.028, + "ndcg_at_100": 31.613, + "ndcg_at_1000": 40.428, + "ndcg_at_3": 38.991, + "ndcg_at_5": 36.704, + "precision_at_1": 43.034, + "precision_at_10": 25.325, + "precision_at_100": 7.889, + "precision_at_1000": 2.069, + "precision_at_3": 36.739, + "precision_at_5": 32.074, + "recall_at_1": 5.772, + "recall_at_10": 16.827, + "recall_at_100": 32.346, + "recall_at_1000": 62.739, + "recall_at_3": 10.56, + "recall_at_5": 12.655, + "main_score": 34.028 + }, + "SciFact-PL": { + "evaluation_time": 5426.0, + "map_at_1": 57.261, + "map_at_10": 66.924, + "map_at_100": 67.443, + "map_at_1000": 67.478, + "map_at_3": 64.859, + "map_at_5": 65.719, + "mrr_at_1": 60.333, + "mrr_at_10": 67.954, + "mrr_at_100": 68.42, + "mrr_at_1000": 68.45, + "mrr_at_3": 66.444, + "mrr_at_5": 67.128, + "ndcg_at_1": 60.333, + "ndcg_at_10": 71.209, + "ndcg_at_100": 73.37, + "ndcg_at_1000": 74.287, + "ndcg_at_3": 67.668, + "ndcg_at_5": 68.644, + "precision_at_1": 60.333, + "precision_at_10": 9.467, + "precision_at_100": 1.053, + "precision_at_1000": 0.113, + "precision_at_3": 26.778, + "precision_at_5": 16.933, + "recall_at_1": 57.261, + "recall_at_10": 83.256, + "recall_at_100": 92.767, + "recall_at_1000": 100.0, + "recall_at_3": 72.933, + "recall_at_5": 75.744, + "main_score": 71.209 + }, + "MassiveScenarioClassification": { + "accuracy": 75.269, + "accuracy_stderr": 1.74818, + "f1": 75.55876, + "f1_stderr": 1.22447, + "main_score": 75.269 + }, + "FiQA-PL": { + "evaluation_time": 32844.0, + "map_at_1": 19.088, + "map_at_10": 31.818, + "map_at_100": 33.689, + "map_at_1000": 33.86, + "map_at_3": 27.399, + "map_at_5": 29.945, + "mrr_at_1": 38.117, + "mrr_at_10": 47.668, + "mrr_at_100": 48.428, + "mrr_at_1000": 48.475, + "mrr_at_3": 45.242, + "mrr_at_5": 46.716, + "ndcg_at_1": 38.272, + "ndcg_at_10": 39.903, + "ndcg_at_100": 46.661, + "ndcg_at_1000": 49.625, + "ndcg_at_3": 35.921, + "ndcg_at_5": 37.558, + "precision_at_1": 38.272, + "precision_at_10": 11.358, + "precision_at_100": 1.819, + "precision_at_1000": 0.235, + "precision_at_3": 24.434, + "precision_at_5": 18.395, + "recall_at_1": 19.088, + "recall_at_10": 47.356, + "recall_at_100": 72.451, + "recall_at_1000": 90.257, + "recall_at_3": 32.931, + "recall_at_5": 39.878, + "main_score": 39.903 + }, + "SICK-R-PL": { + "cos_sim.pearson": 81.23898, + "cos_sim.spearman": 76.0417, + "euclidean.pearson": 78.96475, + "euclidean.spearman": 76.07209, + "evaluation_time": 582.0, + "manhattan.pearson": 78.92666, + "manhattan.spearman": 76.04968, + "main_score": 76.0417 + }, + "CDSC-R": { + "cos_sim.pearson": 93.73949, + "cos_sim.spearman": 93.50397, + "euclidean.pearson": 92.47499, + "euclidean.spearman": 93.39316, + "evaluation_time": 173.0, + "manhattan.pearson": 92.4725, + "manhattan.spearman": 93.3674, + "main_score": 93.50397 + }, + "SciDefRetrieval": { + "evaluation_time": 341.0, + "map_at_1": 67.452, + "map_at_10": 75.481, + "map_at_100": 75.815, + "map_at_1000": 75.829, + "map_at_20": 75.691, + "map_at_3": 74.325, + "map_at_5": 74.833, + "mrr_at_1": 67.747, + "mrr_at_10": 75.65, + "mrr_at_100": 75.982, + "mrr_at_1000": 75.996, + "mrr_at_20": 75.859, + "mrr_at_3": 74.472, + "mrr_at_5": 75.032, + "ndcg_at_1": 67.452, + "ndcg_at_10": 78.989, + "ndcg_at_100": 80.626, + "ndcg_at_1000": 80.958, + "ndcg_at_20": 79.766, + "ndcg_at_3": 76.534, + "ndcg_at_5": 77.447, + "precision_at_1": 67.452, + "precision_at_10": 8.984, + "precision_at_100": 0.975, + "precision_at_1000": 0.1, + "precision_at_20": 4.647, + "precision_at_3": 27.639, + "precision_at_5": 17.025, + "recall_at_1": 67.452, + "recall_at_10": 89.838, + "recall_at_100": 97.496, + "recall_at_1000": 100.0, + "recall_at_20": 92.931, + "recall_at_3": 82.916, + "recall_at_5": 85.125, + "main_score": 78.989 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 783.0, + "v_measure": 26.13264, + "v_measures": 22.28249, + "main_score": 26.13264 + }, + "PSC": { + "cos_sim.accuracy": 98.05195, + "cos_sim.accuracy_threshold": 86.45528, + "cos_sim.ap": 99.5271, + "cos_sim.f1": 96.83258, + "cos_sim.f1_threshold": 86.3822, + "cos_sim.precision": 95.8209, + "cos_sim.recall": 97.86585, + "dot.accuracy": 92.30056, + "dot.accuracy_threshold": 25150.0, + "dot.ap": 94.12759, + "dot.f1": 87.00906, + "dot.f1_threshold": 24575.0, + "dot.precision": 86.22754, + "dot.recall": 87.80488, + "euclidean.accuracy": 98.05195, + "euclidean.accuracy_threshold": 859.70559, + "euclidean.ap": 99.49403, + "euclidean.f1": 96.81335, + "euclidean.f1_threshold": 871.08104, + "euclidean.precision": 96.37462, + "euclidean.recall": 97.2561, + "evaluation_time": 363.0, + "manhattan.accuracy": 98.05195, + "manhattan.accuracy_threshold": 22060.04714, + "manhattan.ap": 99.50121, + "manhattan.f1": 96.81335, + "manhattan.f1_threshold": 22209.01625, + "manhattan.precision": 96.37462, + "manhattan.recall": 97.2561, + "max.accuracy": 98.05195, + "max.ap": 99.5271, + "max.f1": 96.83258, + "main_score": 99.5271 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 2448.0, + "v_measure": 47.27901, + "v_measures": 51.40563, + "main_score": 47.27901 + }, + "NQ-PL": { + "evaluation_time": 995618.0, + "map_at_1": 26.101, + "map_at_10": 39.912, + "map_at_100": 41.037, + "map_at_1000": 41.077, + "map_at_3": 35.691, + "map_at_5": 38.155, + "mrr_at_1": 29.403, + "mrr_at_10": 42.377, + "mrr_at_100": 43.249, + "mrr_at_1000": 43.277, + "mrr_at_3": 38.794, + "mrr_at_5": 40.933, + "ndcg_at_1": 29.519, + "ndcg_at_10": 47.33, + "ndcg_at_100": 52.171, + "ndcg_at_1000": 53.125, + "ndcg_at_3": 39.316, + "ndcg_at_5": 43.457, + "precision_at_1": 29.519, + "precision_at_10": 8.03, + "precision_at_100": 1.075, + "precision_at_1000": 0.117, + "precision_at_3": 18.009, + "precision_at_5": 13.221, + "recall_at_1": 26.101, + "recall_at_10": 67.504, + "recall_at_100": 88.647, + "recall_at_1000": 95.771, + "recall_at_3": 46.669, + "recall_at_5": 56.24, + "main_score": 47.33 + }, + "STS22": { + "cos_sim.pearson": 39.13987, + "cos_sim.spearman": 40.40195, + "euclidean.pearson": 29.14566, + "euclidean.spearman": 39.9739, + "manhattan.pearson": 29.17299, + "manhattan.spearman": 39.85682, + "main_score": 40.40195 + }, + "DBPedia-PL": { + "evaluation_time": 1084037.0, + "map_at_1": 8.434, + "map_at_10": 18.424, + "map_at_100": 26.428, + "map_at_1000": 28.002, + "map_at_3": 13.502, + "map_at_5": 15.577, + "mrr_at_1": 63.0, + "mrr_at_10": 72.714, + "mrr_at_100": 73.021, + "mrr_at_1000": 73.028, + "mrr_at_3": 70.75, + "mrr_at_5": 72.3, + "ndcg_at_1": 52.75, + "ndcg_at_10": 39.84, + "ndcg_at_100": 44.989, + "ndcg_at_1000": 52.533, + "ndcg_at_3": 45.198, + "ndcg_at_5": 42.015, + "precision_at_1": 63.0, + "precision_at_10": 31.05, + "precision_at_100": 10.26, + "precision_at_1000": 1.988, + "precision_at_3": 48.25, + "precision_at_5": 40.45, + "recall_at_1": 8.434, + "recall_at_10": 24.004, + "recall_at_100": 51.428, + "recall_at_1000": 75.712, + "recall_at_3": 15.015, + "recall_at_5": 18.283, + "main_score": 39.84 + }, + "Quora-PL": { + "evaluation_time": 39944.0, + "map_at_1": 67.448, + "map_at_10": 81.566, + "map_at_100": 82.284, + "map_at_1000": 82.301, + "map_at_3": 78.425, + "map_at_5": 80.434, + "mrr_at_1": 77.61, + "mrr_at_10": 84.467, + "mrr_at_100": 84.632, + "mrr_at_1000": 84.634, + "mrr_at_3": 83.288, + "mrr_at_5": 84.095, + "ndcg_at_1": 77.66, + "ndcg_at_10": 85.632, + "ndcg_at_100": 87.166, + "ndcg_at_1000": 87.306, + "ndcg_at_3": 82.323, + "ndcg_at_5": 84.22, + "precision_at_1": 77.66, + "precision_at_10": 13.136, + "precision_at_100": 1.522, + "precision_at_1000": 0.156, + "precision_at_3": 36.153, + "precision_at_5": 23.982, + "recall_at_1": 67.448, + "recall_at_10": 93.832, + "recall_at_100": 99.212, + "recall_at_1000": 99.94, + "recall_at_3": 84.539, + "recall_at_5": 89.71, + "main_score": 85.632 + }, + "CDSC-E": { + "cos_sim.accuracy": 89.4, + "cos_sim.accuracy_threshold": 99.13042, + "cos_sim.ap": 76.83229, + "cos_sim.f1": 65.36585, + "cos_sim.f1_threshold": 94.77479, + "cos_sim.precision": 60.90909, + "cos_sim.recall": 70.52632, + "dot.accuracy": 84.1, + "dot.accuracy_threshold": 27550.0, + "dot.ap": 57.26072, + "dot.f1": 62.75395, + "dot.f1_threshold": 27025.0, + "dot.precision": 54.94071, + "dot.recall": 73.15789, + "euclidean.accuracy": 89.4, + "euclidean.accuracy_threshold": 229.88521, + "euclidean.ap": 76.59095, + "euclidean.f1": 65.21739, + "euclidean.f1_threshold": 558.34741, + "euclidean.precision": 60.26786, + "euclidean.recall": 71.05263, + "evaluation_time": 102.0, + "manhattan.accuracy": 89.4, + "manhattan.accuracy_threshold": 5868.15304, + "manhattan.ap": 76.58826, + "manhattan.f1": 64.72019, + "manhattan.f1_threshold": 14141.74601, + "manhattan.precision": 60.181, + "manhattan.recall": 70.0, + "max.accuracy": 89.4, + "max.ap": 76.83229, + "max.f1": 65.36585, + "main_score": 76.83229 + }, + "TRECCOVID-PL": { + "evaluation_time": 133080.0, + "map_at_1": 0.22, + "map_at_10": 1.693, + "map_at_100": 9.281, + "map_at_1000": 21.463, + "map_at_3": 0.609, + "map_at_5": 0.957, + "mrr_at_1": 80.0, + "mrr_at_10": 88.733, + "mrr_at_100": 88.733, + "mrr_at_1000": 88.733, + "mrr_at_3": 88.333, + "mrr_at_5": 88.733, + "ndcg_at_1": 79.0, + "ndcg_at_10": 71.177, + "ndcg_at_100": 52.479, + "ndcg_at_1000": 45.333, + "ndcg_at_3": 77.48, + "ndcg_at_5": 76.137, + "precision_at_1": 82.0, + "precision_at_10": 74.0, + "precision_at_100": 53.68, + "precision_at_1000": 19.954, + "precision_at_3": 80.667, + "precision_at_5": 80.8, + "recall_at_1": 0.22, + "recall_at_10": 1.934, + "recall_at_100": 12.728, + "recall_at_1000": 41.869, + "recall_at_3": 0.637, + "recall_at_5": 1.042, + "main_score": 71.177 + }, + "SCIDOCS-PL": { + "evaluation_time": 21815.0, + "map_at_1": 4.393, + "map_at_10": 11.472, + "map_at_100": 13.585, + "map_at_1000": 13.918, + "map_at_3": 8.212, + "map_at_5": 9.864, + "mrr_at_1": 21.7, + "mrr_at_10": 32.268, + "mrr_at_100": 33.495, + "mrr_at_1000": 33.548, + "mrr_at_3": 29.15, + "mrr_at_5": 30.91, + "ndcg_at_1": 21.6, + "ndcg_at_10": 19.126, + "ndcg_at_100": 27.496, + "ndcg_at_1000": 33.274, + "ndcg_at_3": 18.196, + "ndcg_at_5": 15.945, + "precision_at_1": 21.6, + "precision_at_10": 9.94, + "precision_at_100": 2.2, + "precision_at_1000": 0.359, + "precision_at_3": 17.2, + "precision_at_5": 14.12, + "recall_at_1": 4.393, + "recall_at_10": 20.167, + "recall_at_100": 44.678, + "recall_at_1000": 72.868, + "recall_at_3": 10.473, + "recall_at_5": 14.313, + "main_score": 19.126 + }, + "PolEmo2.0-IN": { + "accuracy": 69.45983, + "accuracy_stderr": 3.315, + "evaluation_time": 828.0, + "f1": 68.60918, + "f1_stderr": 1.62032, + "main_score": 69.45983 + }, + "PlscClusteringP2P": { + "evaluation_time": 8105.0, + "v_measure": 43.59005, + "v_measures": 43.699, + "main_score": 43.59005 + }, + "MSMARCO-PL": { + "evaluation_time": 2489265.0, + "map_at_1": 19.44, + "map_at_10": 30.264, + "map_at_100": 31.438, + "map_at_1000": 31.495, + "map_at_3": 26.735, + "map_at_5": 28.716, + "mrr_at_1": 19.914, + "mrr_at_10": 30.754, + "mrr_at_100": 31.877, + "mrr_at_1000": 31.929, + "mrr_at_3": 27.299, + "mrr_at_5": 29.254, + "ndcg_at_1": 20.014, + "ndcg_at_10": 36.472, + "ndcg_at_100": 42.231, + "ndcg_at_1000": 43.744, + "ndcg_at_3": 29.268, + "ndcg_at_5": 32.79, + "precision_at_1": 20.014, + "precision_at_10": 5.814, + "precision_at_100": 0.871, + "precision_at_1000": 0.1, + "precision_at_3": 12.426, + "precision_at_5": 9.238, + "recall_at_1": 19.44, + "recall_at_10": 55.535, + "recall_at_100": 82.444, + "recall_at_1000": 94.217, + "recall_at_3": 35.963, + "recall_at_5": 44.367, + "main_score": 36.472 + }, + "MassiveIntentClassification": { + "accuracy": 72.01412, + "accuracy_stderr": 1.25453, + "f1": 70.04544, + "f1_stderr": 1.17356, + "main_score": 72.01412 + } + } + }, + { + "id": "silver-retriever-base-v1", + "tags": [], + "results": { + "CBD": { + "accuracy": 68.51, + "accuracy_stderr": 6.45654, + "ap": 20.65444, + "ap_stderr": 2.80113, + "evaluation_time": 868.0, + "f1": 57.11806, + "f1_stderr": 4.43025, + "main_score": 68.51 + }, + "AllegroReviews": { + "accuracy": 33.3499, + "accuracy_stderr": 4.36748, + "evaluation_time": 324.0, + "f1": 30.60295, + "f1_stderr": 2.90865, + "main_score": 33.3499 + }, + "PlscClusteringS2S": { + "evaluation_time": 1483.0, + "v_measure": 37.75644, + "v_measures": 37.14334, + "main_score": 37.75644 + }, + "ArguAna-PL": { + "evaluation_time": 2257.0, + "map_at_1": 21.266, + "map_at_10": 35.751, + "map_at_100": 36.967, + "map_at_1000": 36.988, + "map_at_3": 31.235, + "map_at_5": 33.536, + "mrr_at_1": 22.76, + "mrr_at_10": 36.378, + "mrr_at_100": 37.566, + "mrr_at_1000": 37.587, + "mrr_at_3": 31.686, + "mrr_at_5": 34.182, + "ndcg_at_1": 21.266, + "ndcg_at_10": 44.125, + "ndcg_at_100": 49.518, + "ndcg_at_1000": 50.047, + "ndcg_at_3": 34.636, + "ndcg_at_5": 38.804, + "precision_at_1": 21.266, + "precision_at_10": 7.105, + "precision_at_100": 0.952, + "precision_at_1000": 0.099, + "precision_at_3": 14.841, + "precision_at_5": 10.939, + "recall_at_1": 21.266, + "recall_at_10": 71.053, + "recall_at_100": 95.164, + "recall_at_1000": 99.289, + "recall_at_3": 44.523, + "recall_at_5": 54.694, + "main_score": 44.125 + }, + "PolEmo2.0-OUT": { + "accuracy": 44.69636, + "accuracy_stderr": 5.36861, + "evaluation_time": 271.0, + "f1": 36.27101, + "f1_stderr": 4.22645, + "main_score": 44.69636 + }, + "PAC": { + "accuracy": 66.26412, + "accuracy_stderr": 3.79704, + "ap": 76.2948, + "ap_stderr": 2.15503, + "evaluation_time": 291.0, + "f1": 64.04838, + "f1_stderr": 3.60923, + "main_score": 66.26412 + }, + "SciField": { + "accuracy": 80.74409, + "accuracy_stderr": 1.96137, + "evaluation_time": 459.0, + "f1": 80.48703, + "f1_stderr": 2.09382, + "main_score": 80.74409 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 76.39625, + "cos_sim.accuracy_threshold": 97.92456, + "cos_sim.ap": 58.18506, + "cos_sim.f1": 59.22035, + "cos_sim.f1_threshold": 94.50015, + "cos_sim.precision": 48.39584, + "cos_sim.recall": 76.28205, + "dot.accuracy": 72.80881, + "dot.accuracy_threshold": 43375.0, + "dot.ap": 47.12278, + "dot.f1": 51.93548, + "dot.f1_threshold": 38750.0, + "dot.precision": 38.38556, + "dot.recall": 80.27066, + "euclidean.accuracy": 76.41663, + "euclidean.accuracy_threshold": 424.18567, + "euclidean.ap": 58.4496, + "euclidean.f1": 59.52712, + "euclidean.f1_threshold": 677.02533, + "euclidean.precision": 48.83615, + "euclidean.recall": 76.21083, + "evaluation_time": 241.0, + "manhattan.accuracy": 76.62046, + "manhattan.accuracy_threshold": 9258.85792, + "manhattan.ap": 58.53553, + "manhattan.f1": 59.5721, + "manhattan.f1_threshold": 14739.08023, + "manhattan.precision": 48.83827, + "manhattan.recall": 76.35328, + "max.accuracy": 76.62046, + "max.ap": 58.53553, + "max.f1": 59.5721, + "main_score": 58.18506 + }, + "HotpotQA-PL": { + "evaluation_time": 437712.0, + "map_at_1": 26.779, + "map_at_10": 36.155, + "map_at_100": 37.031, + "map_at_1000": 37.116, + "map_at_3": 33.848, + "map_at_5": 35.159, + "mrr_at_1": 53.464, + "mrr_at_10": 61.513, + "mrr_at_100": 62.035, + "mrr_at_1000": 62.062, + "mrr_at_3": 59.759, + "mrr_at_5": 60.812, + "ndcg_at_1": 53.558, + "ndcg_at_10": 45.135, + "ndcg_at_100": 48.774, + "ndcg_at_1000": 50.672, + "ndcg_at_3": 41.078, + "ndcg_at_5": 43.12, + "precision_at_1": 53.558, + "precision_at_10": 9.468, + "precision_at_100": 1.234, + "precision_at_1000": 0.149, + "precision_at_3": 25.46, + "precision_at_5": 16.891, + "recall_at_1": 26.779, + "recall_at_10": 47.34, + "recall_at_100": 61.722, + "recall_at_1000": 74.423, + "recall_at_3": 38.19, + "recall_at_5": 42.228, + "main_score": 45.135 + }, + "PPC": { + "cos_sim.accuracy": 73.1, + "cos_sim.accuracy_threshold": 95.32224, + "cos_sim.ap": 85.32697, + "cos_sim.f1": 79.45205, + "cos_sim.f1_threshold": 94.8858, + "cos_sim.precision": 73.52113, + "cos_sim.recall": 86.42384, + "dot.accuracy": 60.9, + "dot.accuracy_threshold": 33800.0, + "dot.ap": 67.68535, + "dot.f1": 75.26612, + "dot.f1_threshold": 29150.0, + "dot.precision": 60.52367, + "dot.recall": 99.50331, + "euclidean.accuracy": 72.8, + "euclidean.accuracy_threshold": 547.60285, + "euclidean.ap": 85.7607, + "euclidean.f1": 79.25373, + "euclidean.f1_threshold": 657.26322, + "euclidean.precision": 72.14674, + "euclidean.recall": 87.91391, + "evaluation_time": 115.0, + "manhattan.accuracy": 73.1, + "manhattan.accuracy_threshold": 13125.13008, + "manhattan.ap": 85.98454, + "manhattan.f1": 79.26549, + "manhattan.f1_threshold": 13607.04643, + "manhattan.precision": 73.68421, + "manhattan.recall": 85.76159, + "max.accuracy": 73.1, + "max.ap": 85.98454, + "max.f1": 79.45205, + "main_score": 85.32697 + }, + "EightTagsClustering": { + "evaluation_time": 3425.0, + "main_score": 30.91116, + "v_measure": 30.91116, + "v_measure_std": 3.06392, + "v_measures": 29.41416 + }, + "NFCorpus-PL": { + "evaluation_time": 1306.0, + "map_at_1": 4.794, + "map_at_10": 10.049, + "map_at_100": 12.343, + "map_at_1000": 13.483, + "map_at_3": 7.307, + "map_at_5": 8.692, + "mrr_at_1": 40.248, + "mrr_at_10": 48.188, + "mrr_at_100": 48.895, + "mrr_at_1000": 48.945, + "mrr_at_3": 46.233, + "mrr_at_5": 47.255, + "ndcg_at_1": 38.545, + "ndcg_at_10": 28.554, + "ndcg_at_100": 25.661, + "ndcg_at_1000": 34.339, + "ndcg_at_3": 33.927, + "ndcg_at_5": 31.733, + "precision_at_1": 39.628, + "precision_at_10": 21.053, + "precision_at_100": 6.368, + "precision_at_1000": 1.871, + "precision_at_3": 31.889, + "precision_at_5": 27.492, + "recall_at_1": 4.794, + "recall_at_10": 13.168, + "recall_at_100": 26.029, + "recall_at_1000": 56.858, + "recall_at_3": 8.173, + "recall_at_5": 10.447, + "main_score": 28.554 + }, + "SciFact-PL": { + "evaluation_time": 1786.0, + "map_at_1": 41.444, + "map_at_10": 49.473, + "map_at_100": 50.385, + "map_at_1000": 50.427, + "map_at_3": 46.319, + "map_at_5": 48.088, + "mrr_at_1": 42.667, + "mrr_at_10": 51.266, + "mrr_at_100": 52.008, + "mrr_at_1000": 52.036, + "mrr_at_3": 49.056, + "mrr_at_5": 50.306, + "ndcg_at_1": 43.667, + "ndcg_at_10": 54.445, + "ndcg_at_100": 58.614, + "ndcg_at_1000": 59.691, + "ndcg_at_3": 48.647, + "ndcg_at_5": 51.583, + "precision_at_1": 43.667, + "precision_at_10": 7.6, + "precision_at_100": 0.987, + "precision_at_1000": 0.108, + "precision_at_3": 18.778, + "precision_at_5": 13.0, + "recall_at_1": 41.444, + "recall_at_10": 67.817, + "recall_at_100": 86.978, + "recall_at_1000": 95.433, + "recall_at_3": 52.606, + "recall_at_5": 59.711, + "main_score": 54.445 + }, + "MassiveScenarioClassification": { + "accuracy": 69.9731, + "accuracy_stderr": 0.96814, + "f1": 70.31009, + "f1_stderr": 0.81183, + "main_score": 69.9731 + }, + "FiQA-PL": { + "evaluation_time": 12327.0, + "map_at_1": 11.811, + "map_at_10": 18.909, + "map_at_100": 20.313, + "map_at_1000": 20.506, + "map_at_3": 16.172, + "map_at_5": 17.575, + "mrr_at_1": 23.457, + "mrr_at_10": 31.238, + "mrr_at_100": 32.232, + "mrr_at_1000": 32.296, + "mrr_at_3": 28.781, + "mrr_at_5": 30.216, + "ndcg_at_1": 23.302, + "ndcg_at_10": 24.947, + "ndcg_at_100": 31.022, + "ndcg_at_1000": 34.975, + "ndcg_at_3": 21.719, + "ndcg_at_5": 22.777, + "precision_at_1": 23.302, + "precision_at_10": 7.13, + "precision_at_100": 1.327, + "precision_at_1000": 0.2, + "precision_at_3": 14.352, + "precision_at_5": 11.049, + "recall_at_1": 11.811, + "recall_at_10": 30.758, + "recall_at_100": 53.716, + "recall_at_1000": 78.391, + "recall_at_3": 19.761, + "recall_at_5": 24.232, + "main_score": 24.947 + }, + "SICK-R-PL": { + "cos_sim.pearson": 75.23573, + "cos_sim.spearman": 67.26209, + "euclidean.pearson": 72.02335, + "euclidean.spearman": 67.43808, + "evaluation_time": 294.0, + "manhattan.pearson": 72.27728, + "manhattan.spearman": 67.54734, + "main_score": 67.26209 + }, + "CDSC-R": { + "cos_sim.pearson": 89.12746, + "cos_sim.spearman": 89.08665, + "euclidean.pearson": 88.39148, + "euclidean.spearman": 89.53805, + "evaluation_time": 90.0, + "manhattan.pearson": 88.58597, + "manhattan.spearman": 89.6847, + "main_score": 89.08665 + }, + "SciDefRetrieval": { + "evaluation_time": 205.0, + "map_at_1": 67.599, + "map_at_10": 75.537, + "map_at_100": 75.827, + "map_at_1000": 75.839, + "map_at_20": 75.691, + "map_at_3": 73.981, + "map_at_5": 75.005, + "mrr_at_1": 67.01, + "mrr_at_10": 75.211, + "mrr_at_100": 75.5, + "mrr_at_1000": 75.512, + "mrr_at_20": 75.357, + "mrr_at_3": 73.711, + "mrr_at_5": 74.713, + "ndcg_at_1": 67.599, + "ndcg_at_10": 79.162, + "ndcg_at_100": 80.659, + "ndcg_at_1000": 80.964, + "ndcg_at_20": 79.696, + "ndcg_at_3": 76.015, + "ndcg_at_5": 77.847, + "precision_at_1": 67.599, + "precision_at_10": 9.043, + "precision_at_100": 0.976, + "precision_at_1000": 0.1, + "precision_at_20": 4.624, + "precision_at_3": 27.295, + "precision_at_5": 17.261, + "recall_at_1": 67.599, + "recall_at_10": 90.427, + "recall_at_100": 97.644, + "recall_at_1000": 100.0, + "recall_at_20": 92.489, + "recall_at_3": 81.885, + "recall_at_5": 86.303, + "main_score": 79.162 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 586.0, + "v_measure": 33.07239, + "v_measures": 29.74406, + "main_score": 33.07239 + }, + "PSC": { + "cos_sim.accuracy": 97.30983, + "cos_sim.accuracy_threshold": 93.59357, + "cos_sim.ap": 98.45728, + "cos_sim.f1": 95.53159, + "cos_sim.f1_threshold": 93.59357, + "cos_sim.precision": 96.57321, + "cos_sim.recall": 94.5122, + "dot.accuracy": 93.78479, + "dot.accuracy_threshold": 44200.0, + "dot.ap": 95.89736, + "dot.f1": 89.85075, + "dot.f1_threshold": 44100.0, + "dot.precision": 88.0117, + "dot.recall": 91.76829, + "euclidean.accuracy": 97.4026, + "euclidean.accuracy_threshold": 806.01024, + "euclidean.ap": 98.5456, + "euclidean.f1": 95.73171, + "euclidean.f1_threshold": 806.01024, + "euclidean.precision": 95.73171, + "euclidean.recall": 95.73171, + "evaluation_time": 153.0, + "manhattan.accuracy": 97.49536, + "manhattan.accuracy_threshold": 17561.17266, + "manhattan.ap": 98.59568, + "manhattan.f1": 95.90288, + "manhattan.f1_threshold": 17561.17266, + "manhattan.precision": 95.46828, + "manhattan.recall": 96.34146, + "max.accuracy": 97.49536, + "max.ap": 98.59568, + "max.f1": 95.90288, + "main_score": 98.45728 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 1243.0, + "v_measure": 44.47839, + "v_measures": 46.30842, + "main_score": 44.47839 + }, + "NQ-PL": { + "evaluation_time": 335929.0, + "map_at_1": 19.527, + "map_at_10": 31.253, + "map_at_100": 32.477, + "map_at_1000": 32.539, + "map_at_3": 27.458, + "map_at_5": 29.559, + "mrr_at_1": 21.698, + "mrr_at_10": 33.292, + "mrr_at_100": 34.292, + "mrr_at_1000": 34.34, + "mrr_at_3": 29.881, + "mrr_at_5": 31.718, + "ndcg_at_1": 21.958, + "ndcg_at_10": 37.901, + "ndcg_at_100": 43.52, + "ndcg_at_1000": 45.091, + "ndcg_at_3": 30.479, + "ndcg_at_5": 34.043, + "precision_at_1": 21.958, + "precision_at_10": 6.634, + "precision_at_100": 0.983, + "precision_at_1000": 0.113, + "precision_at_3": 14.156, + "precision_at_5": 10.527, + "recall_at_1": 19.527, + "recall_at_10": 56.269, + "recall_at_100": 81.475, + "recall_at_1000": 93.325, + "recall_at_3": 36.653, + "recall_at_5": 44.921, + "main_score": 37.901 + }, + "STS22": { + "cos_sim.pearson": 32.24908, + "cos_sim.spearman": 38.69291, + "euclidean.pearson": 27.00967, + "euclidean.spearman": 38.87341, + "manhattan.pearson": 28.30658, + "manhattan.spearman": 39.51809, + "main_score": 38.69291 + }, + "DBPedia-PL": { + "evaluation_time": 366991.0, + "map_at_1": 5.363, + "map_at_10": 11.024, + "map_at_100": 15.516, + "map_at_1000": 16.632, + "map_at_3": 8.17, + "map_at_5": 9.45, + "mrr_at_1": 44.0, + "mrr_at_10": 53.998, + "mrr_at_100": 54.75, + "mrr_at_1000": 54.784, + "mrr_at_3": 52.042, + "mrr_at_5": 53.354, + "ndcg_at_1": 34.25, + "ndcg_at_10": 26.324, + "ndcg_at_100": 30.542, + "ndcg_at_1000": 37.309, + "ndcg_at_3": 29.353, + "ndcg_at_5": 27.871, + "precision_at_1": 44.0, + "precision_at_10": 21.175, + "precision_at_100": 7.183, + "precision_at_1000": 1.554, + "precision_at_3": 32.5, + "precision_at_5": 27.6, + "recall_at_1": 5.363, + "recall_at_10": 15.702, + "recall_at_100": 36.926, + "recall_at_1000": 58.855, + "recall_at_3": 9.69, + "recall_at_5": 12.119, + "main_score": 26.324 + }, + "Quora-PL": { + "evaluation_time": 20392.0, + "map_at_1": 59.379, + "map_at_10": 72.959, + "map_at_100": 73.802, + "map_at_1000": 73.835, + "map_at_3": 69.719, + "map_at_5": 71.699, + "mrr_at_1": 68.2, + "mrr_at_10": 76.617, + "mrr_at_100": 76.9, + "mrr_at_1000": 76.907, + "mrr_at_3": 74.938, + "mrr_at_5": 75.989, + "ndcg_at_1": 68.27, + "ndcg_at_10": 77.98, + "ndcg_at_100": 80.299, + "ndcg_at_1000": 80.686, + "ndcg_at_3": 73.867, + "ndcg_at_5": 76.044, + "precision_at_1": 68.27, + "precision_at_10": 12.113, + "precision_at_100": 1.467, + "precision_at_1000": 0.155, + "precision_at_3": 32.507, + "precision_at_5": 21.734, + "recall_at_1": 59.379, + "recall_at_10": 88.435, + "recall_at_100": 97.231, + "recall_at_1000": 99.515, + "recall_at_3": 76.977, + "recall_at_5": 82.759, + "main_score": 77.98 + }, + "CDSC-E": { + "cos_sim.accuracy": 87.7, + "cos_sim.accuracy_threshold": 99.03202, + "cos_sim.ap": 67.35367, + "cos_sim.f1": 61.65414, + "cos_sim.f1_threshold": 96.80819, + "cos_sim.precision": 58.85167, + "cos_sim.recall": 64.73684, + "dot.accuracy": 84.2, + "dot.accuracy_threshold": 42300.0, + "dot.ap": 51.52727, + "dot.f1": 55.0, + "dot.f1_threshold": 41625.0, + "dot.precision": 58.23529, + "dot.recall": 52.10526, + "euclidean.accuracy": 87.7, + "euclidean.accuracy_threshold": 266.70816, + "euclidean.ap": 67.52987, + "euclidean.f1": 61.77215, + "euclidean.f1_threshold": 511.13926, + "euclidean.precision": 59.5122, + "euclidean.recall": 64.21053, + "evaluation_time": 75.0, + "manhattan.accuracy": 87.7, + "manhattan.accuracy_threshold": 5786.02293, + "manhattan.ap": 67.56194, + "manhattan.f1": 61.5, + "manhattan.f1_threshold": 11298.96997, + "manhattan.precision": 58.57143, + "manhattan.recall": 64.73684, + "max.accuracy": 87.7, + "max.ap": 67.56194, + "max.f1": 61.77215, + "main_score": 67.35367 + }, + "TRECCOVID-PL": { + "evaluation_time": 44986.0, + "map_at_1": 0.172, + "map_at_10": 0.992, + "map_at_100": 5.417, + "map_at_1000": 13.347, + "map_at_3": 0.387, + "map_at_5": 0.584, + "mrr_at_1": 64.0, + "mrr_at_10": 72.338, + "mrr_at_100": 72.73, + "mrr_at_1000": 72.73, + "mrr_at_3": 70.0, + "mrr_at_5": 70.9, + "ndcg_at_1": 57.0, + "ndcg_at_10": 46.977, + "ndcg_at_100": 36.889, + "ndcg_at_1000": 34.89, + "ndcg_at_3": 50.346, + "ndcg_at_5": 48.703, + "precision_at_1": 64.0, + "precision_at_10": 49.8, + "precision_at_100": 38.24, + "precision_at_1000": 16.018, + "precision_at_3": 54.0, + "precision_at_5": 52.0, + "recall_at_1": 0.172, + "recall_at_10": 1.26, + "recall_at_100": 8.888, + "recall_at_1000": 33.491, + "recall_at_3": 0.411, + "recall_at_5": 0.681, + "main_score": 46.977 + }, + "SCIDOCS-PL": { + "evaluation_time": 7119.0, + "map_at_1": 2.618, + "map_at_10": 6.071, + "map_at_100": 7.207, + "map_at_1000": 7.418, + "map_at_3": 4.589, + "map_at_5": 5.228, + "mrr_at_1": 13.1, + "mrr_at_10": 20.13, + "mrr_at_100": 21.243, + "mrr_at_1000": 21.334, + "mrr_at_3": 17.5, + "mrr_at_5": 18.98, + "ndcg_at_1": 12.9, + "ndcg_at_10": 10.899, + "ndcg_at_100": 16.3, + "ndcg_at_1000": 20.905, + "ndcg_at_3": 10.484, + "ndcg_at_5": 8.904, + "precision_at_1": 12.9, + "precision_at_10": 5.63, + "precision_at_100": 1.349, + "precision_at_1000": 0.246, + "precision_at_3": 9.8, + "precision_at_5": 7.66, + "recall_at_1": 2.618, + "recall_at_10": 11.418, + "recall_at_100": 27.38, + "recall_at_1000": 50.062, + "recall_at_3": 5.968, + "recall_at_5": 7.778, + "main_score": 10.899 + }, + "PolEmo2.0-IN": { + "accuracy": 63.51801, + "accuracy_stderr": 4.86668, + "evaluation_time": 339.0, + "f1": 64.85842, + "f1_stderr": 3.42049, + "main_score": 63.51801 + }, + "PlscClusteringP2P": { + "evaluation_time": 3441.0, + "v_measure": 41.01517, + "v_measures": 39.57196, + "main_score": 41.01517 + }, + "MSMARCO-PL": { + "evaluation_time": 881437.0, + "map_at_1": 11.603, + "map_at_10": 20.233, + "map_at_100": 21.486, + "map_at_1000": 21.566, + "map_at_3": 17.141, + "map_at_5": 18.858, + "mrr_at_1": 11.734, + "mrr_at_10": 20.403, + "mrr_at_100": 21.629, + "mrr_at_1000": 21.704, + "mrr_at_3": 17.359, + "mrr_at_5": 19.036, + "ndcg_at_1": 11.877, + "ndcg_at_10": 25.466, + "ndcg_at_100": 31.842, + "ndcg_at_1000": 34.019, + "ndcg_at_3": 19.078, + "ndcg_at_5": 22.167, + "precision_at_1": 11.877, + "precision_at_10": 4.358, + "precision_at_100": 0.76, + "precision_at_1000": 0.095, + "precision_at_3": 8.386, + "precision_at_5": 6.596, + "recall_at_1": 11.603, + "recall_at_10": 41.79, + "recall_at_100": 72.015, + "recall_at_1000": 89.088, + "recall_at_3": 24.299, + "recall_at_5": 31.747, + "main_score": 25.466 + }, + "MassiveIntentClassification": { + "accuracy": 66.62744, + "accuracy_stderr": 1.49168, + "f1": 64.59835, + "f1_stderr": 0.9449, + "main_score": 66.62744 + } + } + }, + { + "id": "multilingual-e5-large", + "tags": [], + "results": { + "CBD": { + "accuracy": 69.9, + "accuracy_stderr": 4.83487, + "ap": 23.39642, + "ap_stderr": 2.5517, + "evaluation_time": 894.0, + "f1": 59.52955, + "f1_stderr": 3.44557, + "main_score": 69.9 + }, + "AllegroReviews": { + "accuracy": 41.14314, + "accuracy_stderr": 4.59444, + "evaluation_time": 650.0, + "f1": 36.18334, + "f1_stderr": 2.54235, + "main_score": 41.14314 + }, + "PlscClusteringS2S": { + "evaluation_time": 2328.0, + "v_measure": 40.34907, + "v_measures": 40.2096, + "main_score": 40.34907 + }, + "ArguAna-PL": { + "evaluation_time": 6812.0, + "map_at_1": 29.09, + "map_at_10": 44.392, + "map_at_100": 45.318, + "map_at_1000": 45.327, + "map_at_3": 39.331, + "map_at_5": 42.34, + "mrr_at_1": 30.085, + "mrr_at_10": 44.778, + "mrr_at_100": 45.696, + "mrr_at_1000": 45.705, + "mrr_at_3": 39.734, + "mrr_at_5": 42.658, + "ndcg_at_1": 29.09, + "ndcg_at_10": 53.024, + "ndcg_at_100": 56.964, + "ndcg_at_1000": 57.186, + "ndcg_at_3": 42.657, + "ndcg_at_5": 48.108, + "precision_at_1": 29.09, + "precision_at_10": 8.065, + "precision_at_100": 0.979, + "precision_at_1000": 0.1, + "precision_at_3": 17.425, + "precision_at_5": 13.115, + "recall_at_1": 29.09, + "recall_at_10": 80.654, + "recall_at_100": 97.866, + "recall_at_1000": 99.573, + "recall_at_3": 52.276, + "recall_at_5": 65.576, + "main_score": 53.024 + }, + "PolEmo2.0-OUT": { + "accuracy": 53.38057, + "accuracy_stderr": 5.13153, + "evaluation_time": 620.0, + "f1": 42.07784, + "f1_stderr": 2.88301, + "main_score": 53.38057 + }, + "PAC": { + "accuracy": 70.3678, + "accuracy_stderr": 2.75119, + "ap": 78.33261, + "ap_stderr": 2.33824, + "evaluation_time": 632.0, + "f1": 67.92306, + "f1_stderr": 3.20448, + "main_score": 70.3678 + }, + "SciField": { + "accuracy": 83.77608, + "accuracy_stderr": 2.19513, + "evaluation_time": 1142.0, + "f1": 83.4146, + "f1_stderr": 2.35763, + "main_score": 83.77608 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 82.85773, + "cos_sim.accuracy_threshold": 96.8908, + "cos_sim.ap": 75.95509, + "cos_sim.f1": 69.34211, + "cos_sim.f1_threshold": 94.82454, + "cos_sim.precision": 64.42543, + "cos_sim.recall": 75.07123, + "dot.accuracy": 82.85773, + "dot.accuracy_threshold": 96.8908, + "dot.ap": 75.95507, + "dot.f1": 69.34211, + "dot.f1_threshold": 94.82454, + "dot.precision": 64.42543, + "dot.recall": 75.07123, + "euclidean.accuracy": 82.85773, + "euclidean.accuracy_threshold": 24.93671, + "euclidean.ap": 75.95512, + "euclidean.f1": 69.34211, + "euclidean.f1_threshold": 32.17285, + "euclidean.precision": 64.42543, + "euclidean.recall": 75.07123, + "evaluation_time": 415.0, + "manhattan.accuracy": 82.75581, + "manhattan.accuracy_threshold": 675.66361, + "manhattan.ap": 75.83155, + "manhattan.f1": 69.34404, + "manhattan.f1_threshold": 805.40867, + "manhattan.precision": 65.40404, + "manhattan.recall": 73.78917, + "max.accuracy": 82.85773, + "max.ap": 75.95512, + "max.f1": 69.34404, + "main_score": 75.95509 + }, + "HotpotQA-PL": { + "evaluation_time": 1195254.0, + "map_at_1": 38.015, + "map_at_10": 59.034, + "map_at_100": 59.895, + "map_at_1000": 59.962, + "map_at_3": 55.942, + "map_at_5": 57.926, + "mrr_at_1": 76.03, + "mrr_at_10": 81.927, + "mrr_at_100": 82.137, + "mrr_at_1000": 82.146, + "mrr_at_3": 80.878, + "mrr_at_5": 81.558, + "ndcg_at_1": 76.03, + "ndcg_at_10": 67.408, + "ndcg_at_100": 70.447, + "ndcg_at_1000": 71.719, + "ndcg_at_3": 62.937, + "ndcg_at_5": 65.502, + "precision_at_1": 76.03, + "precision_at_10": 14.036, + "precision_at_100": 1.642, + "precision_at_1000": 0.181, + "precision_at_3": 40.221, + "precision_at_5": 26.161, + "recall_at_1": 38.015, + "recall_at_10": 70.182, + "recall_at_100": 82.086, + "recall_at_1000": 90.452, + "recall_at_3": 60.331, + "recall_at_5": 65.402, + "main_score": 67.408 + }, + "PPC": { + "cos_sim.accuracy": 83.5, + "cos_sim.accuracy_threshold": 96.21956, + "cos_sim.ap": 92.18431, + "cos_sim.f1": 86.25894, + "cos_sim.f1_threshold": 95.11582, + "cos_sim.precision": 82.90076, + "cos_sim.recall": 89.90066, + "dot.accuracy": 83.5, + "dot.accuracy_threshold": 96.21956, + "dot.ap": 92.18431, + "dot.f1": 86.25894, + "dot.f1_threshold": 95.11583, + "dot.precision": 82.90076, + "dot.recall": 89.90066, + "euclidean.accuracy": 83.5, + "euclidean.accuracy_threshold": 27.49704, + "euclidean.ap": 92.18431, + "euclidean.f1": 86.25894, + "euclidean.f1_threshold": 31.25437, + "euclidean.precision": 82.90076, + "euclidean.recall": 89.90066, + "evaluation_time": 217.0, + "manhattan.accuracy": 83.3, + "manhattan.accuracy_threshold": 697.40076, + "manhattan.ap": 92.13207, + "manhattan.f1": 86.07796, + "manhattan.f1_threshold": 794.97585, + "manhattan.precision": 82.84839, + "manhattan.recall": 89.56954, + "max.accuracy": 83.5, + "max.ap": 92.18431, + "max.f1": 86.25894, + "main_score": 92.18431 + }, + "EightTagsClustering": { + "evaluation_time": 5820.0, + "main_score": 33.67761, + "v_measure": 33.67761, + "v_measure_std": 4.24349, + "v_measures": 34.67861 + }, + "NFCorpus-PL": { + "evaluation_time": 3766.0, + "map_at_1": 4.748, + "map_at_10": 10.625, + "map_at_100": 13.183, + "map_at_1000": 14.358, + "map_at_3": 7.359, + "map_at_5": 9.085, + "mrr_at_1": 40.557, + "mrr_at_10": 49.176, + "mrr_at_100": 49.898, + "mrr_at_1000": 49.935, + "mrr_at_3": 46.904, + "mrr_at_5": 48.545, + "ndcg_at_1": 38.545, + "ndcg_at_10": 30.235, + "ndcg_at_100": 27.413, + "ndcg_at_1000": 35.811, + "ndcg_at_3": 34.061, + "ndcg_at_5": 33.123, + "precision_at_1": 39.628, + "precision_at_10": 22.508, + "precision_at_100": 6.789, + "precision_at_1000": 1.893, + "precision_at_3": 31.785, + "precision_at_5": 28.731, + "recall_at_1": 4.748, + "recall_at_10": 15.124, + "recall_at_100": 27.989, + "recall_at_1000": 57.155, + "recall_at_3": 8.333, + "recall_at_5": 12.004, + "main_score": 30.235 + }, + "SciFact-PL": { + "evaluation_time": 5377.0, + "map_at_1": 51.361, + "map_at_10": 61.057, + "map_at_100": 61.711, + "map_at_1000": 61.744, + "map_at_3": 58.611, + "map_at_5": 60.046, + "mrr_at_1": 54.0, + "mrr_at_10": 62.486, + "mrr_at_100": 63.012, + "mrr_at_1000": 63.041, + "mrr_at_3": 60.667, + "mrr_at_5": 61.783, + "ndcg_at_1": 54.0, + "ndcg_at_10": 65.663, + "ndcg_at_100": 68.497, + "ndcg_at_1000": 69.434, + "ndcg_at_3": 61.564, + "ndcg_at_5": 63.662, + "precision_at_1": 54.0, + "precision_at_10": 8.8, + "precision_at_100": 1.033, + "precision_at_1000": 0.111, + "precision_at_3": 24.556, + "precision_at_5": 16.133, + "recall_at_1": 51.361, + "recall_at_10": 78.211, + "recall_at_100": 90.767, + "recall_at_1000": 98.333, + "recall_at_3": 67.144, + "recall_at_5": 72.489, + "main_score": 65.663 + }, + "MassiveScenarioClassification": { + "accuracy": 69.81843, + "accuracy_stderr": 1.87933, + "f1": 69.16734, + "f1_stderr": 1.68286, + "main_score": 69.81843 + }, + "FiQA-PL": { + "evaluation_time": 33132.0, + "map_at_1": 16.775, + "map_at_10": 25.954, + "map_at_100": 27.587, + "map_at_1000": 27.777, + "map_at_3": 22.607, + "map_at_5": 24.476, + "mrr_at_1": 33.333, + "mrr_at_10": 41.046, + "mrr_at_100": 41.984, + "mrr_at_1000": 42.036, + "mrr_at_3": 38.76, + "mrr_at_5": 40.033, + "ndcg_at_1": 33.333, + "ndcg_at_10": 33.002, + "ndcg_at_100": 39.473, + "ndcg_at_1000": 43.066, + "ndcg_at_3": 29.532, + "ndcg_at_5": 30.724, + "precision_at_1": 33.333, + "precision_at_10": 9.12, + "precision_at_100": 1.568, + "precision_at_1000": 0.22, + "precision_at_3": 19.342, + "precision_at_5": 14.506, + "recall_at_1": 16.775, + "recall_at_10": 39.019, + "recall_at_100": 63.399, + "recall_at_1000": 85.371, + "recall_at_3": 26.633, + "recall_at_5": 31.857, + "main_score": 33.002 + }, + "SICK-R-PL": { + "cos_sim.pearson": 79.74907, + "cos_sim.spearman": 75.07907, + "euclidean.pearson": 76.69004, + "euclidean.spearman": 75.07908, + "evaluation_time": 563.0, + "manhattan.pearson": 76.58071, + "manhattan.spearman": 74.97525, + "main_score": 75.07907 + }, + "CDSC-R": { + "cos_sim.pearson": 91.32088, + "cos_sim.spearman": 90.99846, + "euclidean.pearson": 89.73364, + "euclidean.spearman": 90.99846, + "evaluation_time": 212.0, + "manhattan.pearson": 89.72744, + "manhattan.spearman": 90.95961, + "main_score": 90.99846 + }, + "SciDefRetrieval": { + "evaluation_time": 347.0, + "map_at_1": 74.227, + "map_at_10": 80.767, + "map_at_100": 81.044, + "map_at_1000": 81.048, + "map_at_20": 80.945, + "map_at_3": 79.504, + "map_at_5": 80.351, + "mrr_at_1": 74.08, + "mrr_at_10": 80.733, + "mrr_at_100": 81.009, + "mrr_at_1000": 81.013, + "mrr_at_20": 80.911, + "mrr_at_3": 79.504, + "mrr_at_5": 80.314, + "ndcg_at_1": 74.227, + "ndcg_at_10": 83.78, + "ndcg_at_100": 85.072, + "ndcg_at_1000": 85.199, + "ndcg_at_20": 84.419, + "ndcg_at_3": 81.284, + "ndcg_at_5": 82.805, + "precision_at_1": 74.227, + "precision_at_10": 9.308, + "precision_at_100": 0.99, + "precision_at_1000": 0.1, + "precision_at_20": 4.779, + "precision_at_3": 28.817, + "precision_at_5": 18.027, + "recall_at_1": 74.227, + "recall_at_10": 93.078, + "recall_at_100": 98.969, + "recall_at_1000": 100.0, + "recall_at_20": 95.582, + "recall_at_3": 86.451, + "recall_at_5": 90.133, + "main_score": 83.78 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 805.0, + "v_measure": 21.92161, + "v_measures": 25.89326, + "main_score": 21.92161 + }, + "PSC": { + "cos_sim.accuracy": 98.23748, + "cos_sim.accuracy_threshold": 87.46811, + "cos_sim.ap": 99.39446, + "cos_sim.f1": 97.09924, + "cos_sim.f1_threshold": 87.46811, + "cos_sim.precision": 97.24771, + "cos_sim.recall": 96.95122, + "dot.accuracy": 98.23748, + "dot.accuracy_threshold": 87.46811, + "dot.ap": 99.39446, + "dot.f1": 97.09924, + "dot.f1_threshold": 87.46811, + "dot.precision": 97.24771, + "dot.recall": 96.95122, + "euclidean.accuracy": 98.23748, + "euclidean.accuracy_threshold": 50.06361, + "euclidean.ap": 99.39446, + "euclidean.f1": 97.09924, + "euclidean.f1_threshold": 50.06361, + "euclidean.precision": 97.24771, + "euclidean.recall": 96.95122, + "evaluation_time": 374.0, + "manhattan.accuracy": 98.42301, + "manhattan.accuracy_threshold": 1281.50187, + "manhattan.ap": 99.38151, + "manhattan.f1": 97.41248, + "manhattan.f1_threshold": 1281.50187, + "manhattan.precision": 97.26444, + "manhattan.recall": 97.56098, + "max.accuracy": 98.42301, + "max.ap": 99.39446, + "max.f1": 97.41248, + "main_score": 99.39446 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 2420.0, + "v_measure": 36.94195, + "v_measures": 34.19828, + "main_score": 36.94195 + }, + "NQ-PL": { + "evaluation_time": 1002147.0, + "map_at_1": 32.537, + "map_at_10": 46.16, + "map_at_100": 47.109, + "map_at_1000": 47.149, + "map_at_3": 42.496, + "map_at_5": 44.753, + "mrr_at_1": 36.645, + "mrr_at_10": 48.745, + "mrr_at_100": 49.48, + "mrr_at_1000": 49.51, + "mrr_at_3": 45.795, + "mrr_at_5": 47.665, + "ndcg_at_1": 36.616, + "ndcg_at_10": 52.792, + "ndcg_at_100": 56.901, + "ndcg_at_1000": 57.886, + "ndcg_at_3": 46.035, + "ndcg_at_5": 49.759, + "precision_at_1": 36.616, + "precision_at_10": 8.407, + "precision_at_100": 1.074, + "precision_at_1000": 0.117, + "precision_at_3": 20.616, + "precision_at_5": 14.537, + "recall_at_1": 32.537, + "recall_at_10": 70.317, + "recall_at_100": 88.328, + "recall_at_1000": 95.802, + "recall_at_3": 52.993, + "recall_at_5": 61.556, + "main_score": 52.792 + }, + "STS22": { + "cos_sim.pearson": 26.77083, + "cos_sim.spearman": 34.65549, + "euclidean.pearson": 17.14155, + "euclidean.spearman": 34.55484, + "manhattan.pearson": 16.98851, + "manhattan.spearman": 34.89495, + "main_score": 34.65549 + }, + "DBPedia-PL": { + "evaluation_time": 1085639.0, + "map_at_1": 7.541, + "map_at_10": 16.348, + "map_at_100": 22.415, + "map_at_1000": 23.827, + "map_at_3": 11.907, + "map_at_5": 13.608, + "mrr_at_1": 57.75, + "mrr_at_10": 67.213, + "mrr_at_100": 67.58, + "mrr_at_1000": 67.594, + "mrr_at_3": 65.375, + "mrr_at_5": 66.663, + "ndcg_at_1": 47.625, + "ndcg_at_10": 35.824, + "ndcg_at_100": 39.389, + "ndcg_at_1000": 46.319, + "ndcg_at_3": 40.604, + "ndcg_at_5": 37.44, + "precision_at_1": 57.75, + "precision_at_10": 28.45, + "precision_at_100": 8.828, + "precision_at_1000": 1.707, + "precision_at_3": 43.833, + "precision_at_5": 36.1, + "recall_at_1": 7.541, + "recall_at_10": 21.612, + "recall_at_100": 44.176, + "recall_at_1000": 66.508, + "recall_at_3": 13.193, + "recall_at_5": 15.964, + "main_score": 35.824 + }, + "Quora-PL": { + "evaluation_time": 40865.0, + "map_at_1": 65.294, + "map_at_10": 79.295, + "map_at_100": 80.058, + "map_at_1000": 80.079, + "map_at_3": 76.207, + "map_at_5": 78.11, + "mrr_at_1": 75.17, + "mrr_at_10": 82.501, + "mrr_at_100": 82.705, + "mrr_at_1000": 82.708, + "mrr_at_3": 81.215, + "mrr_at_5": 82.057, + "ndcg_at_1": 75.21, + "ndcg_at_10": 83.654, + "ndcg_at_100": 85.422, + "ndcg_at_1000": 85.615, + "ndcg_at_3": 80.212, + "ndcg_at_5": 82.056, + "precision_at_1": 75.21, + "precision_at_10": 12.856, + "precision_at_100": 1.51, + "precision_at_1000": 0.156, + "precision_at_3": 35.277, + "precision_at_5": 23.348, + "recall_at_1": 65.294, + "recall_at_10": 92.461, + "recall_at_100": 98.771, + "recall_at_1000": 99.844, + "recall_at_3": 82.746, + "recall_at_5": 87.769, + "main_score": 83.654 + }, + "CDSC-E": { + "cos_sim.accuracy": 88.9, + "cos_sim.accuracy_threshold": 98.24677, + "cos_sim.ap": 74.47426, + "cos_sim.f1": 66.12022, + "cos_sim.f1_threshold": 96.41443, + "cos_sim.precision": 68.75, + "cos_sim.recall": 63.68421, + "dot.accuracy": 88.9, + "dot.accuracy_threshold": 98.24677, + "dot.ap": 74.47426, + "dot.f1": 66.12022, + "dot.f1_threshold": 96.41443, + "dot.precision": 68.75, + "dot.recall": 63.68421, + "euclidean.accuracy": 88.9, + "euclidean.accuracy_threshold": 18.72555, + "euclidean.ap": 74.47426, + "euclidean.f1": 66.12022, + "euclidean.f1_threshold": 26.77897, + "euclidean.precision": 68.75, + "euclidean.recall": 63.68421, + "evaluation_time": 120.0, + "manhattan.accuracy": 88.8, + "manhattan.accuracy_threshold": 470.96138, + "manhattan.ap": 74.42067, + "manhattan.f1": 66.27566, + "manhattan.f1_threshold": 620.85767, + "manhattan.precision": 74.83444, + "manhattan.recall": 59.47368, + "max.accuracy": 88.9, + "max.ap": 74.47426, + "max.f1": 66.27566, + "main_score": 74.47426 + }, + "TRECCOVID-PL": { + "evaluation_time": 133843.0, + "map_at_1": 0.212, + "map_at_10": 1.576, + "map_at_100": 9.322, + "map_at_1000": 23.549, + "map_at_3": 0.568, + "map_at_5": 0.891, + "mrr_at_1": 84.0, + "mrr_at_10": 91.25, + "mrr_at_100": 91.25, + "mrr_at_1000": 91.25, + "mrr_at_3": 91.0, + "mrr_at_5": 91.0, + "ndcg_at_1": 79.0, + "ndcg_at_10": 70.033, + "ndcg_at_100": 52.309, + "ndcg_at_1000": 48.782, + "ndcg_at_3": 75.715, + "ndcg_at_5": 74.467, + "precision_at_1": 84.0, + "precision_at_10": 71.8, + "precision_at_100": 53.2, + "precision_at_1000": 21.59, + "precision_at_3": 79.333, + "precision_at_5": 77.6, + "recall_at_1": 0.212, + "recall_at_10": 1.838, + "recall_at_100": 12.992, + "recall_at_1000": 46.159, + "recall_at_3": 0.602, + "recall_at_5": 0.972, + "main_score": 70.033 + }, + "SCIDOCS-PL": { + "evaluation_time": 21772.0, + "map_at_1": 3.385, + "map_at_10": 7.92, + "map_at_100": 9.355, + "map_at_1000": 9.61, + "map_at_3": 5.955, + "map_at_5": 6.929, + "mrr_at_1": 16.8, + "mrr_at_10": 25.151, + "mrr_at_100": 26.28, + "mrr_at_1000": 26.366, + "mrr_at_3": 22.567, + "mrr_at_5": 23.887, + "ndcg_at_1": 16.8, + "ndcg_at_10": 13.812, + "ndcg_at_100": 20.132, + "ndcg_at_1000": 25.401, + "ndcg_at_3": 13.613, + "ndcg_at_5": 11.589, + "precision_at_1": 16.8, + "precision_at_10": 7.01, + "precision_at_100": 1.625, + "precision_at_1000": 0.29, + "precision_at_3": 12.7, + "precision_at_5": 10.04, + "recall_at_1": 3.385, + "recall_at_10": 14.192, + "recall_at_100": 32.933, + "recall_at_1000": 58.842, + "recall_at_3": 7.678, + "recall_at_5": 10.133, + "main_score": 13.812 + }, + "PolEmo2.0-IN": { + "accuracy": 77.06371, + "accuracy_stderr": 6.86824, + "evaluation_time": 808.0, + "f1": 74.35627, + "f1_stderr": 5.57012, + "main_score": 77.06371 + }, + "PlscClusteringP2P": { + "evaluation_time": 8186.0, + "v_measure": 43.19819, + "v_measures": 42.56836, + "main_score": 43.19819 + }, + "MSMARCO-PL": { + "evaluation_time": 2520874.0, + "map_at_1": 17.907, + "map_at_10": 27.688, + "map_at_100": 28.836, + "map_at_1000": 28.898, + "map_at_3": 24.431, + "map_at_5": 26.289, + "mrr_at_1": 18.496, + "mrr_at_10": 28.259, + "mrr_at_100": 29.364, + "mrr_at_1000": 29.42, + "mrr_at_3": 25.036, + "mrr_at_5": 26.872, + "ndcg_at_1": 18.496, + "ndcg_at_10": 33.382, + "ndcg_at_100": 39.138, + "ndcg_at_1000": 40.817, + "ndcg_at_3": 26.742, + "ndcg_at_5": 30.049, + "precision_at_1": 18.496, + "precision_at_10": 5.312, + "precision_at_100": 0.825, + "precision_at_1000": 0.097, + "precision_at_3": 11.366, + "precision_at_5": 8.487, + "recall_at_1": 17.907, + "recall_at_10": 50.903, + "recall_at_100": 77.993, + "recall_at_1000": 91.129, + "recall_at_3": 32.868, + "recall_at_5": 40.786, + "main_score": 33.382 + }, + "MassiveIntentClassification": { + "accuracy": 65.07397, + "accuracy_stderr": 1.40679, + "f1": 62.05609, + "f1_stderr": 1.50626, + "main_score": 65.07397 + } + } + }, + { + "id": "LaBSE", + "tags": [], + "results": { + "CBD": { + "accuracy": 65.71, + "accuracy_stderr": 6.68662, + "ap": 20.56838, + "ap_stderr": 2.94411, + "evaluation_time": 1221.0, + "f1": 55.63771, + "f1_stderr": 4.80618, + "main_score": 65.71 + }, + "AllegroReviews": { + "accuracy": 34.89066, + "accuracy_stderr": 3.54161, + "evaluation_time": 292.0, + "f1": 31.46965, + "f1_stderr": 2.42739, + "main_score": 34.89066 + }, + "PlscClusteringS2S": { + "evaluation_time": 1651.0, + "v_measure": 33.09256, + "v_measures": 32.1688, + "main_score": 33.09256 + }, + "ArguAna-PL": { + "evaluation_time": 1709.0, + "map_at_1": 19.488, + "map_at_10": 31.569, + "map_at_100": 32.675, + "map_at_1000": 32.722, + "map_at_3": 27.857, + "map_at_5": 29.884, + "mrr_at_1": 20.199, + "mrr_at_10": 31.86, + "mrr_at_100": 32.968, + "mrr_at_1000": 33.016, + "mrr_at_3": 28.129, + "mrr_at_5": 30.146, + "ndcg_at_1": 19.488, + "ndcg_at_10": 38.52, + "ndcg_at_100": 44.086, + "ndcg_at_1000": 45.365, + "ndcg_at_3": 30.754, + "ndcg_at_5": 34.457, + "precision_at_1": 19.488, + "precision_at_10": 6.081, + "precision_at_100": 0.872, + "precision_at_1000": 0.097, + "precision_at_3": 13.063, + "precision_at_5": 9.659, + "recall_at_1": 19.488, + "recall_at_10": 60.811, + "recall_at_100": 87.198, + "recall_at_1000": 97.297, + "recall_at_3": 39.189, + "recall_at_5": 48.293, + "main_score": 38.52 + }, + "PolEmo2.0-OUT": { + "accuracy": 44.7166, + "accuracy_stderr": 3.54967, + "evaluation_time": 213.0, + "f1": 35.26179, + "f1_stderr": 3.68212, + "main_score": 44.7166 + }, + "PAC": { + "accuracy": 68.10599, + "accuracy_stderr": 3.43436, + "ap": 77.01976, + "ap_stderr": 2.96345, + "evaluation_time": 319.0, + "f1": 65.50427, + "f1_stderr": 4.39801, + "main_score": 68.10599 + }, + "SciField": { + "accuracy": 76.50904, + "accuracy_stderr": 1.61772, + "evaluation_time": 1205.0, + "f1": 75.8612, + "f1_stderr": 1.6718, + "main_score": 76.50904 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 78.00652, + "cos_sim.accuracy_threshold": 93.56789, + "cos_sim.ap": 63.76689, + "cos_sim.f1": 59.93031, + "cos_sim.f1_threshold": 82.75378, + "cos_sim.precision": 50.58824, + "cos_sim.recall": 73.50427, + "dot.accuracy": 77.94537, + "dot.accuracy_threshold": 93.55469, + "dot.ap": 63.68794, + "dot.f1": 59.91367, + "dot.f1_threshold": 82.42188, + "dot.precision": 50.26557, + "dot.recall": 74.1453, + "euclidean.accuracy": 78.00652, + "euclidean.accuracy_threshold": 35.86321, + "euclidean.ap": 63.76623, + "euclidean.f1": 59.93031, + "euclidean.f1_threshold": 58.73902, + "euclidean.precision": 50.58824, + "euclidean.recall": 73.50427, + "evaluation_time": 270.0, + "manhattan.accuracy": 77.43579, + "manhattan.accuracy_threshold": 818.84973, + "manhattan.ap": 62.70874, + "manhattan.f1": 59.34315, + "manhattan.f1_threshold": 1278.5696, + "manhattan.precision": 49.24812, + "manhattan.recall": 74.64387, + "max.accuracy": 78.00652, + "max.ap": 63.76689, + "max.f1": 59.93031, + "main_score": 63.76689 + }, + "HotpotQA-PL": { + "evaluation_time": 997966.0, + "map_at_1": 10.831, + "map_at_10": 14.937, + "map_at_100": 15.508, + "map_at_1000": 15.583, + "map_at_3": 13.75, + "map_at_5": 14.41, + "mrr_at_1": 21.621, + "mrr_at_10": 27.077, + "mrr_at_100": 27.747, + "mrr_at_1000": 27.815, + "mrr_at_3": 25.544, + "mrr_at_5": 26.385, + "ndcg_at_1": 21.661, + "ndcg_at_10": 19.719, + "ndcg_at_100": 22.802, + "ndcg_at_1000": 24.966, + "ndcg_at_3": 17.235, + "ndcg_at_5": 18.413, + "precision_at_1": 21.661, + "precision_at_10": 4.393, + "precision_at_100": 0.689, + "precision_at_1000": 0.098, + "precision_at_3": 10.894, + "precision_at_5": 7.471, + "recall_at_1": 10.831, + "recall_at_10": 21.965, + "recall_at_100": 34.456, + "recall_at_1000": 49.055, + "recall_at_3": 16.34, + "recall_at_5": 18.677, + "main_score": 19.719 + }, + "PPC": { + "cos_sim.accuracy": 74.0, + "cos_sim.accuracy_threshold": 89.80378, + "cos_sim.ap": 86.96614, + "cos_sim.f1": 78.71546, + "cos_sim.f1_threshold": 81.9341, + "cos_sim.precision": 71.70068, + "cos_sim.recall": 87.25166, + "dot.accuracy": 74.0, + "dot.accuracy_threshold": 89.84375, + "dot.ap": 86.92172, + "dot.f1": 78.76969, + "dot.f1_threshold": 82.27539, + "dot.precision": 72.01646, + "dot.recall": 86.92053, + "euclidean.accuracy": 74.0, + "euclidean.accuracy_threshold": 45.16735, + "euclidean.ap": 86.96798, + "euclidean.f1": 78.71546, + "euclidean.f1_threshold": 60.11117, + "euclidean.precision": 71.70068, + "euclidean.recall": 87.25166, + "evaluation_time": 129.0, + "manhattan.accuracy": 73.4, + "manhattan.accuracy_threshold": 957.03881, + "manhattan.ap": 86.74988, + "manhattan.f1": 78.51852, + "manhattan.f1_threshold": 1281.49325, + "manhattan.precision": 71.04558, + "manhattan.recall": 87.74834, + "max.accuracy": 74.0, + "max.ap": 86.96798, + "max.f1": 78.76969, + "main_score": 86.96614 + }, + "EightTagsClustering": { + "evaluation_time": 3582.0, + "main_score": 12.89462, + "v_measure": 12.89462, + "v_measure_std": 3.83402, + "v_measures": 12.39652 + }, + "NFCorpus-PL": { + "evaluation_time": 694.0, + "map_at_1": 2.028, + "map_at_10": 4.775, + "map_at_100": 6.304, + "map_at_1000": 7.355, + "map_at_3": 3.246, + "map_at_5": 3.952, + "mrr_at_1": 24.458, + "mrr_at_10": 32.016, + "mrr_at_100": 32.927, + "mrr_at_1000": 33.007, + "mrr_at_3": 29.463, + "mrr_at_5": 31.058, + "ndcg_at_1": 22.601, + "ndcg_at_10": 17.446, + "ndcg_at_100": 16.684, + "ndcg_at_1000": 26.05, + "ndcg_at_3": 20.332, + "ndcg_at_5": 19.346, + "precision_at_1": 24.458, + "precision_at_10": 14.025, + "precision_at_100": 5.034, + "precision_at_1000": 1.784, + "precision_at_3": 20.124, + "precision_at_5": 18.019, + "recall_at_1": 2.028, + "recall_at_10": 7.629, + "recall_at_100": 19.683, + "recall_at_1000": 51.362, + "recall_at_3": 3.778, + "recall_at_5": 5.244, + "main_score": 17.446 + }, + "SciFact-PL": { + "evaluation_time": 935.0, + "map_at_1": 28.111, + "map_at_10": 35.212, + "map_at_100": 36.034, + "map_at_1000": 36.112, + "map_at_3": 32.204, + "map_at_5": 33.82, + "mrr_at_1": 29.333, + "mrr_at_10": 36.478, + "mrr_at_100": 37.234, + "mrr_at_1000": 37.293, + "mrr_at_3": 33.667, + "mrr_at_5": 35.183, + "ndcg_at_1": 29.333, + "ndcg_at_10": 39.792, + "ndcg_at_100": 43.944, + "ndcg_at_1000": 46.277, + "ndcg_at_3": 33.945, + "ndcg_at_5": 36.679, + "precision_at_1": 29.333, + "precision_at_10": 5.9, + "precision_at_100": 0.82, + "precision_at_1000": 0.104, + "precision_at_3": 13.444, + "precision_at_5": 9.6, + "recall_at_1": 28.111, + "recall_at_10": 53.0, + "recall_at_100": 72.533, + "recall_at_1000": 91.367, + "recall_at_3": 37.389, + "recall_at_5": 43.75, + "main_score": 39.792 + }, + "MassiveScenarioClassification": { + "accuracy": 64.57969, + "accuracy_stderr": 1.69525, + "f1": 63.13196, + "f1_stderr": 1.39325, + "main_score": 64.57969 + }, + "FiQA-PL": { + "evaluation_time": 50713.0, + "map_at_1": 2.746, + "map_at_10": 5.266, + "map_at_100": 5.993, + "map_at_1000": 6.159, + "map_at_3": 4.273, + "map_at_5": 4.79, + "mrr_at_1": 6.019, + "mrr_at_10": 9.359, + "mrr_at_100": 10.323, + "mrr_at_1000": 10.446, + "mrr_at_3": 7.973, + "mrr_at_5": 8.683, + "ndcg_at_1": 6.019, + "ndcg_at_10": 7.633, + "ndcg_at_100": 11.869, + "ndcg_at_1000": 16.391, + "ndcg_at_3": 5.921, + "ndcg_at_5": 6.564, + "precision_at_1": 6.019, + "precision_at_10": 2.315, + "precision_at_100": 0.656, + "precision_at_1000": 0.142, + "precision_at_3": 3.961, + "precision_at_5": 3.241, + "recall_at_1": 2.746, + "recall_at_10": 10.818, + "recall_at_100": 27.827, + "recall_at_1000": 56.691, + "recall_at_3": 5.772, + "recall_at_5": 7.768, + "main_score": 7.633 + }, + "SICK-R-PL": { + "cos_sim.pearson": 71.64236, + "cos_sim.spearman": 65.89767, + "euclidean.pearson": 69.03315, + "euclidean.spearman": 65.89856, + "evaluation_time": 297.0, + "manhattan.pearson": 68.85781, + "manhattan.spearman": 65.22857, + "main_score": 65.89767 + }, + "CDSC-R": { + "cos_sim.pearson": 84.62394, + "cos_sim.spearman": 85.5259, + "euclidean.pearson": 82.60605, + "euclidean.spearman": 85.52344, + "evaluation_time": 105.0, + "manhattan.pearson": 82.6218, + "manhattan.spearman": 85.0346, + "main_score": 85.5259 + }, + "SciDefRetrieval": { + "evaluation_time": 275.0, + "map_at_1": 19.735, + "map_at_10": 28.869, + "map_at_100": 30.124, + "map_at_1000": 30.202, + "map_at_20": 29.589, + "map_at_3": 26.068, + "map_at_5": 27.644, + "mrr_at_1": 19.735, + "mrr_at_10": 28.825, + "mrr_at_100": 30.079, + "mrr_at_1000": 30.156, + "mrr_at_20": 29.544, + "mrr_at_3": 25.92, + "mrr_at_5": 27.599, + "ndcg_at_1": 19.735, + "ndcg_at_10": 33.978, + "ndcg_at_100": 40.612, + "ndcg_at_1000": 42.801, + "ndcg_at_20": 36.59, + "ndcg_at_3": 28.147, + "ndcg_at_5": 30.993, + "precision_at_1": 19.735, + "precision_at_10": 5.037, + "precision_at_100": 0.826, + "precision_at_1000": 0.1, + "precision_at_20": 3.034, + "precision_at_3": 11.389, + "precision_at_5": 8.218, + "recall_at_1": 19.735, + "recall_at_10": 50.368, + "recall_at_100": 82.622, + "recall_at_1000": 100.0, + "recall_at_20": 60.677, + "recall_at_3": 34.168, + "recall_at_5": 41.09, + "main_score": 33.978 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 708.0, + "v_measure": 23.27852, + "v_measures": 25.54269, + "main_score": 23.27852 + }, + "PSC": { + "cos_sim.accuracy": 95.17625, + "cos_sim.accuracy_threshold": 57.65963, + "cos_sim.ap": 97.41791, + "cos_sim.f1": 92.02454, + "cos_sim.f1_threshold": 57.65963, + "cos_sim.precision": 92.59259, + "cos_sim.recall": 91.46341, + "dot.accuracy": 95.17625, + "dot.accuracy_threshold": 57.61719, + "dot.ap": 97.41209, + "dot.f1": 92.02454, + "dot.f1_threshold": 57.61719, + "dot.precision": 92.59259, + "dot.recall": 91.46341, + "euclidean.accuracy": 95.17625, + "euclidean.accuracy_threshold": 92.02054, + "euclidean.ap": 97.41927, + "euclidean.f1": 92.02454, + "euclidean.f1_threshold": 92.02054, + "euclidean.precision": 92.59259, + "euclidean.recall": 91.46341, + "evaluation_time": 159.0, + "manhattan.accuracy": 94.89796, + "manhattan.accuracy_threshold": 2037.60121, + "manhattan.ap": 97.16291, + "manhattan.f1": 91.65402, + "manhattan.f1_threshold": 2037.60121, + "manhattan.precision": 91.23867, + "manhattan.recall": 92.07317, + "max.accuracy": 95.17625, + "max.ap": 97.41927, + "max.f1": 92.02454, + "main_score": 97.41791 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 1029.0, + "v_measure": 41.46015, + "v_measures": 46.92252, + "main_score": 41.46015 + }, + "NQ-PL": { + "evaluation_time": 280988.0, + "map_at_1": 4.14, + "map_at_10": 7.393, + "map_at_100": 8.122, + "map_at_1000": 8.217, + "map_at_3": 6.238, + "map_at_5": 6.752, + "mrr_at_1": 4.693, + "mrr_at_10": 8.259, + "mrr_at_100": 9.011, + "mrr_at_1000": 9.098, + "mrr_at_3": 7.054, + "mrr_at_5": 7.61, + "ndcg_at_1": 4.751, + "ndcg_at_10": 9.652, + "ndcg_at_100": 13.752, + "ndcg_at_1000": 16.526, + "ndcg_at_3": 7.145, + "ndcg_at_5": 8.087, + "precision_at_1": 4.751, + "precision_at_10": 1.854, + "precision_at_100": 0.423, + "precision_at_1000": 0.069, + "precision_at_3": 3.438, + "precision_at_5": 2.596, + "recall_at_1": 4.14, + "recall_at_10": 15.901, + "recall_at_100": 35.518, + "recall_at_1000": 57.076, + "recall_at_3": 8.951, + "recall_at_5": 11.184, + "main_score": 9.652 + }, + "STS22": { + "cos_sim.pearson": 34.16581, + "cos_sim.spearman": 39.28322, + "euclidean.pearson": 22.32148, + "euclidean.spearman": 39.31719, + "manhattan.pearson": 22.96328, + "manhattan.spearman": 39.00039, + "main_score": 39.28322 + }, + "DBPedia-PL": { + "evaluation_time": 345122.0, + "map_at_1": 2.994, + "map_at_10": 6.13, + "map_at_100": 8.427, + "map_at_1000": 8.993, + "map_at_3": 4.555, + "map_at_5": 5.283, + "mrr_at_1": 26.75, + "mrr_at_10": 36.538, + "mrr_at_100": 37.388, + "mrr_at_1000": 37.428, + "mrr_at_3": 34.083, + "mrr_at_5": 35.483, + "ndcg_at_1": 20.625, + "ndcg_at_10": 16.102, + "ndcg_at_100": 18.434, + "ndcg_at_1000": 23.482, + "ndcg_at_3": 17.79, + "ndcg_at_5": 17.011, + "precision_at_1": 27.0, + "precision_at_10": 13.525, + "precision_at_100": 4.312, + "precision_at_1000": 0.949, + "precision_at_3": 20.417, + "precision_at_5": 17.55, + "recall_at_1": 2.994, + "recall_at_10": 9.909, + "recall_at_100": 22.941, + "recall_at_1000": 40.691, + "recall_at_3": 5.827, + "recall_at_5": 7.486, + "main_score": 16.102 + }, + "Quora-PL": { + "evaluation_time": 19816.0, + "map_at_1": 57.697, + "map_at_10": 69.989, + "map_at_100": 70.834, + "map_at_1000": 70.875, + "map_at_3": 66.966, + "map_at_5": 68.771, + "mrr_at_1": 66.53, + "mrr_at_10": 74.377, + "mrr_at_100": 74.718, + "mrr_at_1000": 74.728, + "mrr_at_3": 72.655, + "mrr_at_5": 73.745, + "ndcg_at_1": 66.48, + "ndcg_at_10": 74.965, + "ndcg_at_100": 77.633, + "ndcg_at_1000": 78.204, + "ndcg_at_3": 70.937, + "ndcg_at_5": 73.003, + "precision_at_1": 66.48, + "precision_at_10": 11.502, + "precision_at_100": 1.418, + "precision_at_1000": 0.153, + "precision_at_3": 30.903, + "precision_at_5": 20.608, + "recall_at_1": 57.697, + "recall_at_10": 84.912, + "recall_at_100": 95.545, + "recall_at_1000": 99.067, + "recall_at_3": 73.585, + "recall_at_5": 79.12, + "main_score": 74.965 + }, + "CDSC-E": { + "cos_sim.accuracy": 87.0, + "cos_sim.accuracy_threshold": 96.98292, + "cos_sim.ap": 68.91216, + "cos_sim.f1": 60.82192, + "cos_sim.f1_threshold": 90.13995, + "cos_sim.precision": 63.42857, + "cos_sim.recall": 58.42105, + "dot.accuracy": 87.0, + "dot.accuracy_threshold": 96.97266, + "dot.ap": 68.85506, + "dot.f1": 60.98901, + "dot.f1_threshold": 90.23438, + "dot.precision": 63.7931, + "dot.recall": 58.42105, + "euclidean.accuracy": 87.0, + "euclidean.accuracy_threshold": 24.56084, + "euclidean.ap": 68.91864, + "euclidean.f1": 60.82192, + "euclidean.f1_threshold": 44.41664, + "euclidean.precision": 63.42857, + "euclidean.recall": 58.42105, + "evaluation_time": 87.0, + "manhattan.accuracy": 87.0, + "manhattan.accuracy_threshold": 588.39581, + "manhattan.ap": 68.56945, + "manhattan.f1": 60.52632, + "manhattan.f1_threshold": 988.46947, + "manhattan.precision": 60.52632, + "manhattan.recall": 60.52632, + "max.accuracy": 87.0, + "max.ap": 68.91864, + "max.f1": 60.98901, + "main_score": 68.91216 + }, + "TRECCOVID-PL": { + "evaluation_time": 21982.0, + "map_at_1": 0.082, + "map_at_10": 0.295, + "map_at_100": 0.743, + "map_at_1000": 1.809, + "map_at_3": 0.165, + "map_at_5": 0.222, + "mrr_at_1": 26.0, + "mrr_at_10": 40.4, + "mrr_at_100": 41.565, + "mrr_at_1000": 41.614, + "mrr_at_3": 37.667, + "mrr_at_5": 39.567, + "ndcg_at_1": 28.0, + "ndcg_at_10": 18.452, + "ndcg_at_100": 10.024, + "ndcg_at_1000": 11.532, + "ndcg_at_3": 24.754, + "ndcg_at_5": 22.693, + "precision_at_1": 28.0, + "precision_at_10": 17.8, + "precision_at_100": 9.52, + "precision_at_1000": 5.612, + "precision_at_3": 25.333, + "precision_at_5": 23.2, + "recall_at_1": 0.082, + "recall_at_10": 0.46, + "recall_at_100": 2.057, + "recall_at_1000": 11.467, + "recall_at_3": 0.205, + "recall_at_5": 0.304, + "main_score": 18.452 + }, + "SCIDOCS-PL": { + "evaluation_time": 4220.0, + "map_at_1": 1.428, + "map_at_10": 3.841, + "map_at_100": 4.798, + "map_at_1000": 5.02, + "map_at_3": 2.692, + "map_at_5": 3.239, + "mrr_at_1": 7.0, + "mrr_at_10": 13.373, + "mrr_at_100": 14.516, + "mrr_at_1000": 14.634, + "mrr_at_3": 11.0, + "mrr_at_5": 12.295, + "ndcg_at_1": 7.0, + "ndcg_at_10": 7.48, + "ndcg_at_100": 12.466, + "ndcg_at_1000": 17.586, + "ndcg_at_3": 6.466, + "ndcg_at_5": 5.913, + "precision_at_1": 7.0, + "precision_at_10": 4.15, + "precision_at_100": 1.15, + "precision_at_1000": 0.239, + "precision_at_3": 6.233, + "precision_at_5": 5.46, + "recall_at_1": 1.428, + "recall_at_10": 8.413, + "recall_at_100": 23.357, + "recall_at_1000": 48.493, + "recall_at_3": 3.793, + "recall_at_5": 5.533, + "main_score": 7.48 + }, + "PolEmo2.0-IN": { + "accuracy": 64.00277, + "accuracy_stderr": 3.72751, + "evaluation_time": 261.0, + "f1": 64.26616, + "f1_stderr": 2.68267, + "main_score": 64.00277 + }, + "PlscClusteringP2P": { + "evaluation_time": 2809.0, + "v_measure": 40.88456, + "v_measures": 41.33784, + "main_score": 40.88456 + }, + "MSMARCO-PL": { + "evaluation_time": 885208.0, + "map_at_1": 3.266, + "map_at_10": 5.66, + "map_at_100": 6.235, + "map_at_1000": 6.309, + "map_at_3": 4.688, + "map_at_5": 5.211, + "mrr_at_1": 3.309, + "mrr_at_10": 5.772, + "mrr_at_100": 6.353, + "mrr_at_1000": 6.428, + "mrr_at_3": 4.804, + "mrr_at_5": 5.322, + "ndcg_at_1": 3.324, + "ndcg_at_10": 7.22, + "ndcg_at_100": 10.425, + "ndcg_at_1000": 12.801, + "ndcg_at_3": 5.197, + "ndcg_at_5": 6.138, + "precision_at_1": 3.324, + "precision_at_10": 1.254, + "precision_at_100": 0.293, + "precision_at_1000": 0.05, + "precision_at_3": 2.249, + "precision_at_5": 1.814, + "recall_at_1": 3.266, + "recall_at_10": 12.136, + "recall_at_100": 27.9, + "recall_at_1000": 47.224, + "recall_at_3": 6.564, + "recall_at_5": 8.837, + "main_score": 7.22 + }, + "MassiveIntentClassification": { + "accuracy": 59.71083, + "accuracy_stderr": 1.71867, + "f1": 55.92281, + "f1_stderr": 1.56096, + "main_score": 59.71083 + } + } + }, + { + "id": "mmlw-e5-base", + "tags": [], + "results": { + "CBD": { + "accuracy": 65.54, + "accuracy_stderr": 7.45858, + "ap": 19.75396, + "ap_stderr": 2.63567, + "evaluation_time": 779.0, + "f1": 55.00481, + "f1_stderr": 4.91255, + "main_score": 65.54 + }, + "AllegroReviews": { + "accuracy": 36.38171, + "accuracy_stderr": 3.40129, + "evaluation_time": 352.0, + "f1": 32.77742, + "f1_stderr": 2.55354, + "main_score": 36.38171 + }, + "PlscClusteringS2S": { + "evaluation_time": 1479.0, + "v_measure": 39.19567, + "v_measures": 40.08926, + "main_score": 39.19567 + }, + "ArguAna-PL": { + "evaluation_time": 2467.0, + "map_at_1": 32.646, + "map_at_10": 49.488, + "map_at_100": 50.191, + "map_at_1000": 50.194, + "map_at_3": 44.749, + "map_at_5": 47.572, + "mrr_at_1": 34.211, + "mrr_at_10": 50.112, + "mrr_at_100": 50.836, + "mrr_at_1000": 50.839, + "mrr_at_3": 45.614, + "mrr_at_5": 48.242, + "ndcg_at_1": 32.646, + "ndcg_at_10": 58.396, + "ndcg_at_100": 61.285, + "ndcg_at_1000": 61.359, + "ndcg_at_3": 48.759, + "ndcg_at_5": 53.807, + "precision_at_1": 32.646, + "precision_at_10": 8.663, + "precision_at_100": 0.99, + "precision_at_1000": 0.1, + "precision_at_3": 20.128, + "precision_at_5": 14.509, + "recall_at_1": 32.646, + "recall_at_10": 86.629, + "recall_at_100": 99.004, + "recall_at_1000": 99.573, + "recall_at_3": 60.384, + "recall_at_5": 72.546, + "main_score": 58.396 + }, + "PolEmo2.0-OUT": { + "accuracy": 35.80972, + "accuracy_stderr": 5.76511, + "evaluation_time": 320.0, + "f1": 31.13082, + "f1_stderr": 3.91763, + "main_score": 35.80972 + }, + "PAC": { + "accuracy": 64.50333, + "accuracy_stderr": 4.08567, + "ap": 75.81507, + "ap_stderr": 2.99806, + "evaluation_time": 378.0, + "f1": 62.24618, + "f1_stderr": 4.97046, + "main_score": 64.50333 + }, + "SciField": { + "accuracy": 84.62448, + "accuracy_stderr": 2.39012, + "evaluation_time": 453.0, + "f1": 84.42644, + "f1_stderr": 2.44467, + "main_score": 84.62448 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 83.12271, + "cos_sim.accuracy_threshold": 97.33819, + "cos_sim.ap": 76.45283, + "cos_sim.f1": 68.69628, + "cos_sim.f1_threshold": 96.45675, + "cos_sim.precision": 69.09222, + "cos_sim.recall": 68.30484, + "dot.accuracy": 79.20913, + "dot.accuracy_threshold": 25600.0, + "dot.ap": 65.03147, + "dot.f1": 62.81216, + "dot.f1_threshold": 24550.0, + "dot.precision": 50.74561, + "dot.recall": 82.40741, + "euclidean.accuracy": 83.16347, + "euclidean.accuracy_threshold": 373.54845, + "euclidean.ap": 76.49406, + "euclidean.f1": 68.66738, + "euclidean.f1_threshold": 432.27229, + "euclidean.precision": 68.88889, + "euclidean.recall": 68.44729, + "evaluation_time": 275.0, + "manhattan.accuracy": 83.16347, + "manhattan.accuracy_threshold": 8243.33498, + "manhattan.ap": 76.50806, + "manhattan.f1": 68.73884, + "manhattan.f1_threshold": 9520.88101, + "manhattan.precision": 68.96057, + "manhattan.recall": 68.51852, + "max.accuracy": 83.16347, + "max.ap": 76.50806, + "max.f1": 68.73884, + "main_score": 76.45283 + }, + "HotpotQA-PL": { + "evaluation_time": 427263.0, + "map_at_1": 37.373, + "map_at_10": 57.509, + "map_at_100": 58.451, + "map_at_1000": 58.524, + "map_at_3": 54.064, + "map_at_5": 56.258, + "mrr_at_1": 74.895, + "mrr_at_10": 81.233, + "mrr_at_100": 81.461, + "mrr_at_1000": 81.47, + "mrr_at_3": 80.124, + "mrr_at_5": 80.862, + "ndcg_at_1": 74.747, + "ndcg_at_10": 66.249, + "ndcg_at_100": 69.513, + "ndcg_at_1000": 70.896, + "ndcg_at_3": 61.312, + "ndcg_at_5": 64.132, + "precision_at_1": 74.747, + "precision_at_10": 13.873, + "precision_at_100": 1.641, + "precision_at_1000": 0.182, + "precision_at_3": 38.987, + "precision_at_5": 25.621, + "recall_at_1": 37.373, + "recall_at_10": 69.365, + "recall_at_100": 82.039, + "recall_at_1000": 91.148, + "recall_at_3": 58.481, + "recall_at_5": 64.051, + "main_score": 66.249 + }, + "PPC": { + "cos_sim.accuracy": 85.4, + "cos_sim.accuracy_threshold": 96.77746, + "cos_sim.ap": 91.75882, + "cos_sim.f1": 87.79264, + "cos_sim.f1_threshold": 96.77746, + "cos_sim.precision": 88.68243, + "cos_sim.recall": 86.92053, + "dot.accuracy": 71.0, + "dot.accuracy_threshold": 25037.5, + "dot.ap": 82.97829, + "dot.f1": 78.77055, + "dot.f1_threshold": 24612.5, + "dot.precision": 69.30818, + "dot.recall": 91.22517, + "euclidean.accuracy": 85.2, + "euclidean.accuracy_threshold": 409.34122, + "euclidean.ap": 91.85246, + "euclidean.f1": 87.64608, + "euclidean.f1_threshold": 414.27158, + "euclidean.precision": 88.38384, + "euclidean.recall": 86.92053, + "evaluation_time": 139.0, + "manhattan.accuracy": 85.4, + "manhattan.accuracy_threshold": 9072.605, + "manhattan.ap": 91.85497, + "manhattan.f1": 87.77219, + "manhattan.f1_threshold": 9072.605, + "manhattan.precision": 88.81356, + "manhattan.recall": 86.75497, + "max.accuracy": 85.4, + "max.ap": 91.85497, + "max.f1": 87.79264, + "main_score": 91.75882 + }, + "EightTagsClustering": { + "evaluation_time": 3391.0, + "main_score": 32.44397, + "v_measure": 32.44397, + "v_measure_std": 3.39337, + "v_measures": 32.90074 + }, + "NFCorpus-PL": { + "evaluation_time": 1345.0, + "map_at_1": 5.348, + "map_at_10": 12.277, + "map_at_100": 15.804, + "map_at_1000": 17.277, + "map_at_3": 8.783, + "map_at_5": 10.314, + "mrr_at_1": 43.963, + "mrr_at_10": 52.46, + "mrr_at_100": 53.233, + "mrr_at_1000": 53.265, + "mrr_at_3": 50.464, + "mrr_at_5": 51.548, + "ndcg_at_1": 40.712, + "ndcg_at_10": 33.709, + "ndcg_at_100": 31.398, + "ndcg_at_1000": 40.042, + "ndcg_at_3": 37.85, + "ndcg_at_5": 36.261, + "precision_at_1": 43.344, + "precision_at_10": 25.851, + "precision_at_100": 8.279, + "precision_at_1000": 2.085, + "precision_at_3": 36.326, + "precision_at_5": 32.074, + "recall_at_1": 5.348, + "recall_at_10": 16.441, + "recall_at_100": 32.975, + "recall_at_1000": 64.357, + "recall_at_3": 9.842, + "recall_at_5": 12.463, + "main_score": 33.709 + }, + "SciFact-PL": { + "evaluation_time": 1835.0, + "map_at_1": 52.994, + "map_at_10": 63.612, + "map_at_100": 64.294, + "map_at_1000": 64.325, + "map_at_3": 61.341, + "map_at_5": 62.366, + "mrr_at_1": 56.667, + "mrr_at_10": 65.333, + "mrr_at_100": 65.894, + "mrr_at_1000": 65.919, + "mrr_at_3": 63.667, + "mrr_at_5": 64.367, + "ndcg_at_1": 56.333, + "ndcg_at_10": 68.292, + "ndcg_at_100": 71.136, + "ndcg_at_1000": 71.901, + "ndcg_at_3": 64.387, + "ndcg_at_5": 65.546, + "precision_at_1": 56.333, + "precision_at_10": 9.133, + "precision_at_100": 1.063, + "precision_at_1000": 0.113, + "precision_at_3": 25.556, + "precision_at_5": 16.267, + "recall_at_1": 52.994, + "recall_at_10": 81.178, + "recall_at_100": 93.767, + "recall_at_1000": 99.667, + "recall_at_3": 69.906, + "recall_at_5": 73.183, + "main_score": 68.292 + }, + "MassiveScenarioClassification": { + "accuracy": 74.76126, + "accuracy_stderr": 1.72218, + "f1": 74.72274, + "f1_stderr": 1.11217, + "main_score": 74.76126 + }, + "FiQA-PL": { + "evaluation_time": 11001.0, + "map_at_1": 16.489, + "map_at_10": 26.916, + "map_at_100": 28.582, + "map_at_1000": 28.774, + "map_at_3": 23.048, + "map_at_5": 24.977, + "mrr_at_1": 33.642, + "mrr_at_10": 41.987, + "mrr_at_100": 42.882, + "mrr_at_1000": 42.93, + "mrr_at_3": 39.48, + "mrr_at_5": 40.923, + "ndcg_at_1": 33.488, + "ndcg_at_10": 34.528, + "ndcg_at_100": 41.085, + "ndcg_at_1000": 44.474, + "ndcg_at_3": 30.469, + "ndcg_at_5": 31.618, + "precision_at_1": 33.488, + "precision_at_10": 9.784, + "precision_at_100": 1.639, + "precision_at_1000": 0.227, + "precision_at_3": 20.525, + "precision_at_5": 15.093, + "recall_at_1": 16.489, + "recall_at_10": 42.37, + "recall_at_100": 67.183, + "recall_at_1000": 87.211, + "recall_at_3": 27.69, + "recall_at_5": 33.408, + "main_score": 34.528 + }, + "SICK-R-PL": { + "cos_sim.pearson": 82.60225, + "cos_sim.spearman": 76.76667, + "euclidean.pearson": 80.16303, + "euclidean.spearman": 76.76132, + "evaluation_time": 308.0, + "manhattan.pearson": 80.11881, + "manhattan.spearman": 76.74246, + "main_score": 76.76667 + }, + "CDSC-R": { + "cos_sim.pearson": 93.18499, + "cos_sim.spearman": 93.26225, + "euclidean.pearson": 92.25462, + "euclidean.spearman": 93.18624, + "evaluation_time": 123.0, + "manhattan.pearson": 92.16291, + "manhattan.spearman": 93.14403, + "main_score": 93.26225 + }, + "SciDefRetrieval": { + "evaluation_time": 200.0, + "map_at_1": 59.057, + "map_at_10": 68.031, + "map_at_100": 68.52, + "map_at_1000": 68.545, + "map_at_20": 68.323, + "map_at_3": 66.397, + "map_at_5": 67.464, + "mrr_at_1": 59.205, + "mrr_at_10": 68.13, + "mrr_at_100": 68.62, + "mrr_at_1000": 68.645, + "mrr_at_20": 68.431, + "mrr_at_3": 66.568, + "mrr_at_5": 67.563, + "ndcg_at_1": 59.057, + "ndcg_at_10": 72.089, + "ndcg_at_100": 74.44, + "ndcg_at_1000": 75.023, + "ndcg_at_20": 73.139, + "ndcg_at_3": 68.751, + "ndcg_at_5": 70.684, + "precision_at_1": 59.057, + "precision_at_10": 8.468, + "precision_at_100": 0.956, + "precision_at_1000": 0.1, + "precision_at_20": 4.44, + "precision_at_3": 25.184, + "precision_at_5": 16.053, + "recall_at_1": 59.057, + "recall_at_10": 84.683, + "recall_at_100": 95.582, + "recall_at_1000": 100.0, + "recall_at_20": 88.807, + "recall_at_3": 75.552, + "recall_at_5": 80.265, + "main_score": 72.089 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 685.0, + "v_measure": 30.50422, + "v_measures": 27.96573, + "main_score": 30.50422 + }, + "PSC": { + "cos_sim.accuracy": 97.58813, + "cos_sim.accuracy_threshold": 89.0505, + "cos_sim.ap": 99.14945, + "cos_sim.f1": 96.06061, + "cos_sim.f1_threshold": 89.0505, + "cos_sim.precision": 95.48193, + "cos_sim.recall": 96.64634, + "dot.accuracy": 95.17625, + "dot.accuracy_threshold": 22800.0, + "dot.ap": 97.05593, + "dot.f1": 92.14502, + "dot.f1_threshold": 22800.0, + "dot.precision": 91.31737, + "dot.recall": 92.9878, + "euclidean.accuracy": 97.68089, + "euclidean.accuracy_threshold": 743.16293, + "euclidean.ap": 99.18538, + "euclidean.f1": 96.20637, + "euclidean.f1_threshold": 743.16293, + "euclidean.precision": 95.77039, + "euclidean.recall": 96.64634, + "evaluation_time": 187.0, + "manhattan.accuracy": 97.58813, + "manhattan.accuracy_threshold": 16203.00963, + "manhattan.ap": 99.17871, + "manhattan.f1": 96.02446, + "manhattan.f1_threshold": 16203.00963, + "manhattan.precision": 96.31902, + "manhattan.recall": 95.73171, + "max.accuracy": 97.68089, + "max.ap": 99.18538, + "max.f1": 96.20637, + "main_score": 99.14945 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 1262.0, + "v_measure": 47.54783, + "v_measures": 46.76784, + "main_score": 47.54783 + }, + "NQ-PL": { + "evaluation_time": 339563.0, + "map_at_1": 24.674, + "map_at_10": 37.672, + "map_at_100": 38.767, + "map_at_1000": 38.82, + "map_at_3": 33.823, + "map_at_5": 36.063, + "mrr_at_1": 27.839, + "mrr_at_10": 40.129, + "mrr_at_100": 41.008, + "mrr_at_1000": 41.048, + "mrr_at_3": 36.718, + "mrr_at_5": 38.841, + "ndcg_at_1": 27.839, + "ndcg_at_10": 44.604, + "ndcg_at_100": 49.51, + "ndcg_at_1000": 50.841, + "ndcg_at_3": 37.223, + "ndcg_at_5": 41.073, + "precision_at_1": 27.839, + "precision_at_10": 7.5, + "precision_at_100": 1.03, + "precision_at_1000": 0.116, + "precision_at_3": 17.005, + "precision_at_5": 12.399, + "recall_at_1": 24.674, + "recall_at_10": 63.323, + "recall_at_100": 85.088, + "recall_at_1000": 95.143, + "recall_at_3": 44.158, + "recall_at_5": 53.054, + "main_score": 44.604 + }, + "STS22": { + "cos_sim.pearson": 38.27448, + "cos_sim.spearman": 40.35665, + "euclidean.pearson": 29.1485, + "euclidean.spearman": 40.45532, + "manhattan.pearson": 29.12418, + "manhattan.spearman": 40.28014, + "main_score": 40.35665 + }, + "DBPedia-PL": { + "evaluation_time": 360117.0, + "map_at_1": 8.268, + "map_at_10": 17.391, + "map_at_100": 24.266, + "map_at_1000": 25.844, + "map_at_3": 12.636, + "map_at_5": 14.701, + "mrr_at_1": 62.75, + "mrr_at_10": 70.252, + "mrr_at_100": 70.601, + "mrr_at_1000": 70.613, + "mrr_at_3": 68.083, + "mrr_at_5": 69.371, + "ndcg_at_1": 51.875, + "ndcg_at_10": 37.185, + "ndcg_at_100": 41.949, + "ndcg_at_1000": 49.523, + "ndcg_at_3": 41.556, + "ndcg_at_5": 39.278, + "precision_at_1": 63.25, + "precision_at_10": 29.225, + "precision_at_100": 9.745, + "precision_at_1000": 2.046, + "precision_at_3": 43.833, + "precision_at_5": 37.9, + "recall_at_1": 8.268, + "recall_at_10": 22.542, + "recall_at_100": 48.154, + "recall_at_1000": 72.621, + "recall_at_3": 13.818, + "recall_at_5": 17.137, + "main_score": 37.185 + }, + "Quora-PL": { + "evaluation_time": 20293.0, + "map_at_1": 66.149, + "map_at_10": 80.133, + "map_at_100": 80.845, + "map_at_1000": 80.866, + "map_at_3": 76.983, + "map_at_5": 78.938, + "mrr_at_1": 76.09, + "mrr_at_10": 83.251, + "mrr_at_100": 83.422, + "mrr_at_1000": 83.425, + "mrr_at_3": 82.022, + "mrr_at_5": 82.831, + "ndcg_at_1": 76.15, + "ndcg_at_10": 84.438, + "ndcg_at_100": 86.048, + "ndcg_at_1000": 86.226, + "ndcg_at_3": 80.98, + "ndcg_at_5": 82.856, + "precision_at_1": 76.15, + "precision_at_10": 12.985, + "precision_at_100": 1.513, + "precision_at_1000": 0.156, + "precision_at_3": 35.563, + "precision_at_5": 23.586, + "recall_at_1": 66.149, + "recall_at_10": 93.195, + "recall_at_100": 98.924, + "recall_at_1000": 99.885, + "recall_at_3": 83.439, + "recall_at_5": 88.575, + "main_score": 84.438 + }, + "CDSC-E": { + "cos_sim.accuracy": 89.5, + "cos_sim.accuracy_threshold": 98.63745, + "cos_sim.ap": 77.26879, + "cos_sim.f1": 65.13158, + "cos_sim.f1_threshold": 98.30475, + "cos_sim.precision": 86.84211, + "cos_sim.recall": 52.10526, + "dot.accuracy": 88.0, + "dot.accuracy_threshold": 25750.0, + "dot.ap": 69.17236, + "dot.f1": 65.71429, + "dot.f1_threshold": 25750.0, + "dot.precision": 71.875, + "dot.recall": 60.52632, + "euclidean.accuracy": 89.5, + "euclidean.accuracy_threshold": 268.24049, + "euclidean.ap": 77.19054, + "euclidean.f1": 64.91803, + "euclidean.f1_threshold": 300.44596, + "euclidean.precision": 86.08696, + "euclidean.recall": 52.10526, + "evaluation_time": 87.0, + "manhattan.accuracy": 89.5, + "manhattan.accuracy_threshold": 5897.58702, + "manhattan.ap": 77.19532, + "manhattan.f1": 64.72492, + "manhattan.f1_threshold": 6787.03678, + "manhattan.precision": 84.03361, + "manhattan.recall": 52.63158, + "max.accuracy": 89.5, + "max.ap": 77.26879, + "max.f1": 65.71429, + "main_score": 77.26879 + }, + "TRECCOVID-PL": { + "evaluation_time": 44242.0, + "map_at_1": 0.231, + "map_at_10": 1.822, + "map_at_100": 10.134, + "map_at_1000": 24.859, + "map_at_3": 0.615, + "map_at_5": 0.994, + "mrr_at_1": 84.0, + "mrr_at_10": 90.4, + "mrr_at_100": 90.4, + "mrr_at_1000": 90.4, + "mrr_at_3": 89.0, + "mrr_at_5": 90.4, + "ndcg_at_1": 81.0, + "ndcg_at_10": 73.333, + "ndcg_at_100": 55.351, + "ndcg_at_1000": 49.875, + "ndcg_at_3": 76.866, + "ndcg_at_5": 75.472, + "precision_at_1": 86.0, + "precision_at_10": 78.2, + "precision_at_100": 57.18, + "precision_at_1000": 22.332, + "precision_at_3": 82.0, + "precision_at_5": 81.2, + "recall_at_1": 0.231, + "recall_at_10": 2.056, + "recall_at_100": 13.468, + "recall_at_1000": 47.039, + "recall_at_3": 0.648, + "recall_at_5": 1.088, + "main_score": 73.333 + }, + "SCIDOCS-PL": { + "evaluation_time": 7238.0, + "map_at_1": 3.688, + "map_at_10": 10.23, + "map_at_100": 12.077, + "map_at_1000": 12.382, + "map_at_3": 7.149, + "map_at_5": 8.689, + "mrr_at_1": 18.2, + "mrr_at_10": 28.817, + "mrr_at_100": 29.982, + "mrr_at_1000": 30.058, + "mrr_at_3": 25.983, + "mrr_at_5": 27.418, + "ndcg_at_1": 18.2, + "ndcg_at_10": 17.353, + "ndcg_at_100": 24.859, + "ndcg_at_1000": 30.535, + "ndcg_at_3": 16.17, + "ndcg_at_5": 14.235, + "precision_at_1": 18.2, + "precision_at_10": 9.19, + "precision_at_100": 2.01, + "precision_at_1000": 0.338, + "precision_at_3": 15.5, + "precision_at_5": 12.78, + "recall_at_1": 3.688, + "recall_at_10": 18.632, + "recall_at_100": 40.822, + "recall_at_1000": 68.552, + "recall_at_3": 9.423, + "recall_at_5": 12.943, + "main_score": 17.353 + }, + "PolEmo2.0-IN": { + "accuracy": 68.69806, + "accuracy_stderr": 3.56416, + "evaluation_time": 364.0, + "f1": 68.062, + "f1_stderr": 1.99933, + "main_score": 68.69806 + }, + "PlscClusteringP2P": { + "evaluation_time": 3681.0, + "v_measure": 44.48256, + "v_measures": 44.96783, + "main_score": 44.48256 + }, + "MSMARCO-PL": { + "evaluation_time": 885919.0, + "map_at_1": 16.754, + "map_at_10": 26.764, + "map_at_100": 27.929, + "map_at_1000": 27.995, + "map_at_3": 23.527, + "map_at_5": 25.343, + "mrr_at_1": 17.192, + "mrr_at_10": 27.141, + "mrr_at_100": 28.269, + "mrr_at_1000": 28.328, + "mrr_at_3": 23.906, + "mrr_at_5": 25.76, + "ndcg_at_1": 17.178, + "ndcg_at_10": 32.539, + "ndcg_at_100": 38.383, + "ndcg_at_1000": 40.132, + "ndcg_at_3": 25.884, + "ndcg_at_5": 29.15, + "precision_at_1": 17.178, + "precision_at_10": 5.268, + "precision_at_100": 0.823, + "precision_at_1000": 0.097, + "precision_at_3": 11.122, + "precision_at_5": 8.338, + "recall_at_1": 16.754, + "recall_at_10": 50.388, + "recall_at_100": 77.87, + "recall_at_1000": 91.55, + "recall_at_3": 32.186, + "recall_at_5": 40.048, + "main_score": 32.539 + }, + "MassiveIntentClassification": { + "accuracy": 70.92804, + "accuracy_stderr": 1.13622, + "f1": 68.70996, + "f1_stderr": 0.86896, + "main_score": 70.92804 + } + } + }, + { + "id": "mmlw-roberta-base", + "tags": [], + "results": { + "CBD": { + "accuracy": 68.11, + "accuracy_stderr": 4.97784, + "ap": 20.91663, + "ap_stderr": 3.95431, + "evaluation_time": 903.0, + "f1": 56.85805, + "f1_stderr": 5.18233, + "main_score": 68.11 + }, + "AllegroReviews": { + "accuracy": 40.25845, + "accuracy_stderr": 3.81456, + "evaluation_time": 332.0, + "f1": 35.59685, + "f1_stderr": 2.74157, + "main_score": 40.25845 + }, + "PlscClusteringS2S": { + "evaluation_time": 1611.0, + "v_measure": 42.03511, + "v_measures": 42.70007, + "main_score": 42.03511 + }, + "ArguAna-PL": { + "evaluation_time": 2264.0, + "map_at_1": 33.073, + "map_at_10": 50.223, + "map_at_100": 50.942, + "map_at_1000": 50.945, + "map_at_3": 45.721, + "map_at_5": 48.413, + "mrr_at_1": 34.424, + "mrr_at_10": 50.689, + "mrr_at_100": 51.438, + "mrr_at_1000": 51.441, + "mrr_at_3": 46.219, + "mrr_at_5": 48.921, + "ndcg_at_1": 33.073, + "ndcg_at_10": 59.021, + "ndcg_at_100": 61.902, + "ndcg_at_1000": 61.984, + "ndcg_at_3": 49.818, + "ndcg_at_5": 54.645, + "precision_at_1": 33.073, + "precision_at_10": 8.684, + "precision_at_100": 0.99, + "precision_at_1000": 0.1, + "precision_at_3": 20.555, + "precision_at_5": 14.666, + "recall_at_1": 33.073, + "recall_at_10": 86.842, + "recall_at_100": 99.004, + "recall_at_1000": 99.644, + "recall_at_3": 61.664, + "recall_at_5": 73.329, + "main_score": 59.021 + }, + "PolEmo2.0-OUT": { + "accuracy": 46.59919, + "accuracy_stderr": 2.2719, + "evaluation_time": 309.0, + "f1": 37.21639, + "f1_stderr": 1.8938, + "main_score": 46.59919 + }, + "PAC": { + "accuracy": 65.87315, + "accuracy_stderr": 4.80667, + "ap": 76.41967, + "ap_stderr": 3.24737, + "evaluation_time": 271.0, + "f1": 63.64417, + "f1_stderr": 5.23628, + "main_score": 65.87315 + }, + "SciField": { + "accuracy": 85.18081, + "accuracy_stderr": 2.03018, + "evaluation_time": 444.0, + "f1": 84.88733, + "f1_stderr": 2.06759, + "main_score": 85.18081 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 86.32287, + "cos_sim.accuracy_threshold": 96.91415, + "cos_sim.ap": 82.04303, + "cos_sim.f1": 74.95572, + "cos_sim.f1_threshold": 96.37605, + "cos_sim.precision": 74.55955, + "cos_sim.recall": 75.35613, + "dot.accuracy": 83.91765, + "dot.accuracy_threshold": 25450.0, + "dot.ap": 76.86087, + "dot.f1": 72.31937, + "dot.f1_threshold": 25100.0, + "dot.precision": 67.36325, + "dot.recall": 78.06268, + "euclidean.accuracy": 86.32287, + "euclidean.accuracy_threshold": 399.40623, + "euclidean.ap": 81.9649, + "euclidean.f1": 74.93797, + "euclidean.f1_threshold": 436.12951, + "euclidean.precision": 74.59421, + "euclidean.recall": 75.2849, + "evaluation_time": 246.0, + "manhattan.accuracy": 86.30249, + "manhattan.accuracy_threshold": 8722.44555, + "manhattan.ap": 81.92854, + "manhattan.f1": 74.80969, + "manhattan.f1_threshold": 9952.1641, + "manhattan.precision": 72.74563, + "manhattan.recall": 76.9943, + "max.accuracy": 86.32287, + "max.ap": 82.04303, + "max.f1": 74.95572, + "main_score": 82.04303 + }, + "HotpotQA-PL": { + "evaluation_time": 418901.0, + "map_at_1": 37.711, + "map_at_10": 57.868, + "map_at_100": 58.77, + "map_at_1000": 58.837, + "map_at_3": 54.401, + "map_at_5": 56.564, + "mrr_at_1": 75.449, + "mrr_at_10": 81.575, + "mrr_at_100": 81.783, + "mrr_at_1000": 81.792, + "mrr_at_3": 80.504, + "mrr_at_5": 81.172, + "ndcg_at_1": 75.422, + "ndcg_at_10": 66.635, + "ndcg_at_100": 69.85, + "ndcg_at_1000": 71.179, + "ndcg_at_3": 61.648, + "ndcg_at_5": 64.412, + "precision_at_1": 75.422, + "precision_at_10": 13.962, + "precision_at_100": 1.649, + "precision_at_1000": 0.183, + "precision_at_3": 39.172, + "precision_at_5": 25.691, + "recall_at_1": 37.711, + "recall_at_10": 69.811, + "recall_at_100": 82.471, + "recall_at_1000": 91.29, + "recall_at_3": 58.758, + "recall_at_5": 64.227, + "main_score": 66.635 + }, + "PPC": { + "cos_sim.accuracy": 87.7, + "cos_sim.accuracy_threshold": 96.20914, + "cos_sim.ap": 92.81319, + "cos_sim.f1": 90.04049, + "cos_sim.f1_threshold": 96.20914, + "cos_sim.precision": 88.1141, + "cos_sim.recall": 92.05298, + "dot.accuracy": 75.0, + "dot.accuracy_threshold": 24175.0, + "dot.ap": 83.6309, + "dot.f1": 80.76923, + "dot.f1_threshold": 24125.0, + "dot.precision": 75.43103, + "dot.recall": 86.92053, + "euclidean.accuracy": 87.7, + "euclidean.accuracy_threshold": 434.29561, + "euclidean.ap": 92.94772, + "euclidean.f1": 90.10459, + "euclidean.f1_threshold": 450.118, + "euclidean.precision": 87.63693, + "euclidean.recall": 92.71523, + "evaluation_time": 122.0, + "manhattan.accuracy": 87.8, + "manhattan.accuracy_threshold": 9622.84966, + "manhattan.ap": 92.95331, + "manhattan.f1": 90.0813, + "manhattan.f1_threshold": 9622.84966, + "manhattan.precision": 88.4984, + "manhattan.recall": 91.72185, + "max.accuracy": 87.8, + "max.ap": 92.95331, + "max.f1": 90.10459, + "main_score": 92.81319 + }, + "EightTagsClustering": { + "evaluation_time": 3518.0, + "main_score": 30.09348, + "v_measure": 30.09348, + "v_measure_std": 5.3466, + "v_measures": 27.70773 + }, + "NFCorpus-PL": { + "evaluation_time": 1307.0, + "map_at_1": 5.877, + "map_at_10": 12.817, + "map_at_100": 16.247, + "map_at_1000": 17.683, + "map_at_3": 9.334, + "map_at_5": 10.887, + "mrr_at_1": 45.201, + "mrr_at_10": 52.7, + "mrr_at_100": 53.426, + "mrr_at_1000": 53.461, + "mrr_at_3": 50.464, + "mrr_at_5": 51.827, + "ndcg_at_1": 41.95, + "ndcg_at_10": 34.145, + "ndcg_at_100": 31.556, + "ndcg_at_1000": 40.265, + "ndcg_at_3": 38.07, + "ndcg_at_5": 36.571, + "precision_at_1": 44.272, + "precision_at_10": 25.697, + "precision_at_100": 8.077, + "precision_at_1000": 2.084, + "precision_at_3": 36.017, + "precision_at_5": 31.703, + "recall_at_1": 5.877, + "recall_at_10": 16.986, + "recall_at_100": 32.719, + "recall_at_1000": 63.763, + "recall_at_3": 10.292, + "recall_at_5": 12.886, + "main_score": 34.145 + }, + "SciFact-PL": { + "evaluation_time": 1771.0, + "map_at_1": 49.428, + "map_at_10": 60.423, + "map_at_100": 61.037, + "map_at_1000": 61.066, + "map_at_3": 56.989, + "map_at_5": 59.042, + "mrr_at_1": 52.667, + "mrr_at_10": 61.746, + "mrr_at_100": 62.273, + "mrr_at_1000": 62.301, + "mrr_at_3": 59.278, + "mrr_at_5": 60.611, + "ndcg_at_1": 52.333, + "ndcg_at_10": 65.75, + "ndcg_at_100": 68.566, + "ndcg_at_1000": 69.314, + "ndcg_at_3": 59.768, + "ndcg_at_5": 62.808, + "precision_at_1": 52.333, + "precision_at_10": 9.167, + "precision_at_100": 1.063, + "precision_at_1000": 0.113, + "precision_at_3": 23.778, + "precision_at_5": 16.2, + "recall_at_1": 49.428, + "recall_at_10": 81.078, + "recall_at_100": 93.933, + "recall_at_1000": 99.667, + "recall_at_3": 65.061, + "recall_at_5": 72.667, + "main_score": 65.75 + }, + "MassiveScenarioClassification": { + "accuracy": 75.52791, + "accuracy_stderr": 1.68901, + "f1": 75.40539, + "f1_stderr": 1.22388, + "main_score": 75.52791 + }, + "FiQA-PL": { + "evaluation_time": 12037.0, + "map_at_1": 16.95, + "map_at_10": 27.422, + "map_at_100": 29.146, + "map_at_1000": 29.328, + "map_at_3": 23.736, + "map_at_5": 25.671, + "mrr_at_1": 33.796, + "mrr_at_10": 42.689, + "mrr_at_100": 43.522, + "mrr_at_1000": 43.563, + "mrr_at_3": 40.226, + "mrr_at_5": 41.685, + "ndcg_at_1": 33.642, + "ndcg_at_10": 35.008, + "ndcg_at_100": 41.839, + "ndcg_at_1000": 45.035, + "ndcg_at_3": 31.359, + "ndcg_at_5": 32.377, + "precision_at_1": 33.642, + "precision_at_10": 9.938, + "precision_at_100": 1.685, + "precision_at_1000": 0.227, + "precision_at_3": 21.142, + "precision_at_5": 15.586, + "recall_at_1": 16.95, + "recall_at_10": 42.286, + "recall_at_100": 68.519, + "recall_at_1000": 87.471, + "recall_at_3": 28.834, + "recall_at_5": 34.274, + "main_score": 35.008 + }, + "SICK-R-PL": { + "cos_sim.pearson": 83.07566, + "cos_sim.spearman": 79.20198, + "euclidean.pearson": 81.27875, + "euclidean.spearman": 79.2156, + "evaluation_time": 321.0, + "manhattan.pearson": 81.21559, + "manhattan.spearman": 79.15378, + "main_score": 79.20198 + }, + "CDSC-R": { + "cos_sim.pearson": 91.79559, + "cos_sim.spearman": 92.54382, + "euclidean.pearson": 92.02981, + "euclidean.spearman": 92.55482, + "evaluation_time": 89.0, + "manhattan.pearson": 91.95296, + "manhattan.spearman": 92.5028, + "main_score": 92.54382 + }, + "SciDefRetrieval": { + "evaluation_time": 214.0, + "map_at_1": 59.352, + "map_at_10": 68.128, + "map_at_100": 68.611, + "map_at_1000": 68.633, + "map_at_20": 68.406, + "map_at_3": 66.421, + "map_at_5": 67.423, + "mrr_at_1": 59.352, + "mrr_at_10": 68.134, + "mrr_at_100": 68.634, + "mrr_at_1000": 68.654, + "mrr_at_20": 68.435, + "mrr_at_3": 66.421, + "mrr_at_5": 67.496, + "ndcg_at_1": 59.352, + "ndcg_at_10": 72.284, + "ndcg_at_100": 74.568, + "ndcg_at_1000": 75.12, + "ndcg_at_20": 73.292, + "ndcg_at_3": 68.728, + "ndcg_at_5": 70.572, + "precision_at_1": 59.352, + "precision_at_10": 8.527, + "precision_at_100": 0.957, + "precision_at_1000": 0.1, + "precision_at_20": 4.462, + "precision_at_3": 25.135, + "precision_at_5": 15.994, + "recall_at_1": 59.352, + "recall_at_10": 85.272, + "recall_at_100": 95.729, + "recall_at_1000": 100.0, + "recall_at_20": 89.249, + "recall_at_3": 75.405, + "recall_at_5": 79.971, + "main_score": 72.284 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 572.0, + "v_measure": 37.2418, + "v_measures": 35.72023, + "main_score": 37.2418 + }, + "PSC": { + "cos_sim.accuracy": 96.19666, + "cos_sim.accuracy_threshold": 90.25397, + "cos_sim.ap": 98.61238, + "cos_sim.f1": 93.77845, + "cos_sim.f1_threshold": 90.1125, + "cos_sim.precision": 93.35347, + "cos_sim.recall": 94.20732, + "dot.accuracy": 94.89796, + "dot.accuracy_threshold": 22250.0, + "dot.ap": 97.02853, + "dot.f1": 91.85185, + "dot.f1_threshold": 22250.0, + "dot.precision": 89.33718, + "dot.recall": 94.5122, + "euclidean.accuracy": 96.38219, + "euclidean.accuracy_threshold": 699.52952, + "euclidean.ap": 98.58146, + "euclidean.f1": 94.0458, + "euclidean.f1_threshold": 699.52952, + "euclidean.precision": 94.1896, + "euclidean.recall": 93.90244, + "evaluation_time": 137.0, + "manhattan.accuracy": 96.47495, + "manhattan.accuracy_threshold": 15450.35173, + "manhattan.ap": 98.61082, + "manhattan.f1": 94.1896, + "manhattan.f1_threshold": 15450.35173, + "manhattan.precision": 94.47853, + "manhattan.recall": 93.90244, + "max.accuracy": 96.47495, + "max.ap": 98.61238, + "max.f1": 94.1896, + "main_score": 98.61238 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 1156.0, + "v_measure": 47.00401, + "v_measures": 44.96166, + "main_score": 47.00401 + }, + "NQ-PL": { + "evaluation_time": 330145.0, + "map_at_1": 25.476, + "map_at_10": 38.67, + "map_at_100": 39.784, + "map_at_1000": 39.831, + "map_at_3": 34.829, + "map_at_5": 37.025, + "mrr_at_1": 28.621, + "mrr_at_10": 41.13, + "mrr_at_100": 42.028, + "mrr_at_1000": 42.06, + "mrr_at_3": 37.877, + "mrr_at_5": 39.764, + "ndcg_at_1": 28.563, + "ndcg_at_10": 45.654, + "ndcg_at_100": 50.695, + "ndcg_at_1000": 51.874, + "ndcg_at_3": 38.359, + "ndcg_at_5": 42.045, + "precision_at_1": 28.563, + "precision_at_10": 7.645, + "precision_at_100": 1.052, + "precision_at_1000": 0.117, + "precision_at_3": 17.458, + "precision_at_5": 12.613, + "recall_at_1": 25.476, + "recall_at_10": 64.484, + "recall_at_100": 86.962, + "recall_at_1000": 95.872, + "recall_at_3": 45.527, + "recall_at_5": 54.029, + "main_score": 45.654 + }, + "STS22": { + "cos_sim.pearson": 36.39658, + "cos_sim.spearman": 40.36403, + "euclidean.pearson": 29.14342, + "euclidean.spearman": 40.43198, + "manhattan.pearson": 29.18632, + "manhattan.spearman": 40.50563, + "main_score": 40.36403 + }, + "DBPedia-PL": { + "evaluation_time": 347252.0, + "map_at_1": 7.83, + "map_at_10": 16.616, + "map_at_100": 23.629, + "map_at_1000": 25.236, + "map_at_3": 12.485, + "map_at_5": 14.077, + "mrr_at_1": 61.75, + "mrr_at_10": 69.852, + "mrr_at_100": 70.279, + "mrr_at_1000": 70.294, + "mrr_at_3": 68.375, + "mrr_at_5": 69.187, + "ndcg_at_1": 49.75, + "ndcg_at_10": 36.217, + "ndcg_at_100": 41.235, + "ndcg_at_1000": 48.952, + "ndcg_at_3": 41.669, + "ndcg_at_5": 38.285, + "precision_at_1": 61.5, + "precision_at_10": 28.5, + "precision_at_100": 9.572, + "precision_at_1000": 2.025, + "precision_at_3": 44.083, + "precision_at_5": 36.3, + "recall_at_1": 7.83, + "recall_at_10": 21.463, + "recall_at_100": 47.095, + "recall_at_1000": 71.883, + "recall_at_3": 13.891, + "recall_at_5": 16.327, + "main_score": 36.217 + }, + "Quora-PL": { + "evaluation_time": 19980.0, + "map_at_1": 66.134, + "map_at_10": 80.19, + "map_at_100": 80.937, + "map_at_1000": 80.956, + "map_at_3": 77.074, + "map_at_5": 79.054, + "mrr_at_1": 75.88, + "mrr_at_10": 83.226, + "mrr_at_100": 83.403, + "mrr_at_1000": 83.406, + "mrr_at_3": 82.032, + "mrr_at_5": 82.843, + "ndcg_at_1": 75.94, + "ndcg_at_10": 84.437, + "ndcg_at_100": 86.13, + "ndcg_at_1000": 86.293, + "ndcg_at_3": 81.078, + "ndcg_at_5": 83.0, + "precision_at_1": 75.94, + "precision_at_10": 12.954, + "precision_at_100": 1.514, + "precision_at_1000": 0.156, + "precision_at_3": 35.61, + "precision_at_5": 23.652, + "recall_at_1": 66.134, + "recall_at_10": 92.991, + "recall_at_100": 99.003, + "recall_at_1000": 99.86, + "recall_at_3": 83.643, + "recall_at_5": 88.811, + "main_score": 84.437 + }, + "CDSC-E": { + "cos_sim.accuracy": 89.2, + "cos_sim.accuracy_threshold": 96.97493, + "cos_sim.ap": 79.10412, + "cos_sim.f1": 70.0, + "cos_sim.f1_threshold": 96.74774, + "cos_sim.precision": 74.11765, + "cos_sim.recall": 66.31579, + "dot.accuracy": 88.2, + "dot.accuracy_threshold": 25650.0, + "dot.ap": 72.57184, + "dot.f1": 67.16418, + "dot.f1_threshold": 25200.0, + "dot.precision": 63.67925, + "dot.recall": 71.05263, + "euclidean.accuracy": 89.3, + "euclidean.accuracy_threshold": 406.8513, + "euclidean.ap": 79.01346, + "euclidean.f1": 70.19499, + "euclidean.f1_threshold": 407.42497, + "euclidean.precision": 74.55621, + "euclidean.recall": 66.31579, + "evaluation_time": 71.0, + "manhattan.accuracy": 89.3, + "manhattan.accuracy_threshold": 8984.2972, + "manhattan.ap": 79.01671, + "manhattan.f1": 70.02801, + "manhattan.f1_threshold": 8984.2972, + "manhattan.precision": 74.8503, + "manhattan.recall": 65.78947, + "max.accuracy": 89.3, + "max.ap": 79.10412, + "max.f1": 70.19499, + "main_score": 79.10412 + }, + "TRECCOVID-PL": { + "evaluation_time": 46149.0, + "map_at_1": 0.221, + "map_at_10": 1.788, + "map_at_100": 9.937, + "map_at_1000": 24.763, + "map_at_3": 0.579, + "map_at_5": 0.947, + "mrr_at_1": 78.0, + "mrr_at_10": 88.067, + "mrr_at_100": 88.067, + "mrr_at_1000": 88.067, + "mrr_at_3": 87.667, + "mrr_at_5": 88.067, + "ndcg_at_1": 76.0, + "ndcg_at_10": 71.332, + "ndcg_at_100": 54.805, + "ndcg_at_1000": 49.505, + "ndcg_at_3": 73.693, + "ndcg_at_5": 73.733, + "precision_at_1": 82.0, + "precision_at_10": 76.8, + "precision_at_100": 56.68, + "precision_at_1000": 22.236, + "precision_at_3": 78.667, + "precision_at_5": 79.2, + "recall_at_1": 0.221, + "recall_at_10": 2.033, + "recall_at_100": 13.432, + "recall_at_1000": 46.913, + "recall_at_3": 0.625, + "recall_at_5": 1.052, + "main_score": 71.332 + }, + "SCIDOCS-PL": { + "evaluation_time": 6589.0, + "map_at_1": 4.183, + "map_at_10": 10.626, + "map_at_100": 12.485, + "map_at_1000": 12.793, + "map_at_3": 7.531, + "map_at_5": 9.037, + "mrr_at_1": 20.5, + "mrr_at_10": 30.175, + "mrr_at_100": 31.356, + "mrr_at_1000": 31.421, + "mrr_at_3": 26.9, + "mrr_at_5": 28.69, + "ndcg_at_1": 20.6, + "ndcg_at_10": 17.84, + "ndcg_at_100": 25.518, + "ndcg_at_1000": 31.138, + "ndcg_at_3": 16.677, + "ndcg_at_5": 14.641, + "precision_at_1": 20.6, + "precision_at_10": 9.3, + "precision_at_100": 2.048, + "precision_at_1000": 0.34, + "precision_at_3": 15.533, + "precision_at_5": 12.84, + "recall_at_1": 4.183, + "recall_at_10": 18.862, + "recall_at_100": 41.592, + "recall_at_1000": 69.037, + "recall_at_3": 9.443, + "recall_at_5": 13.028, + "main_score": 17.84 + }, + "PolEmo2.0-IN": { + "accuracy": 71.7313, + "accuracy_stderr": 4.1047, + "evaluation_time": 346.0, + "f1": 71.17732, + "f1_stderr": 2.59645, + "main_score": 71.7313 + }, + "PlscClusteringP2P": { + "evaluation_time": 3269.0, + "v_measure": 45.39421, + "v_measures": 45.78117, + "main_score": 45.39421 + }, + "MSMARCO-PL": { + "evaluation_time": 840143.0, + "map_at_1": 17.033, + "map_at_10": 27.242, + "map_at_100": 28.452, + "map_at_1000": 28.515, + "map_at_3": 24.046, + "map_at_5": 25.841, + "mrr_at_1": 17.493, + "mrr_at_10": 27.67, + "mrr_at_100": 28.824, + "mrr_at_1000": 28.881, + "mrr_at_3": 24.53, + "mrr_at_5": 26.27, + "ndcg_at_1": 17.479, + "ndcg_at_10": 33.048, + "ndcg_at_100": 39.071, + "ndcg_at_1000": 40.74, + "ndcg_at_3": 26.493, + "ndcg_at_5": 29.701, + "precision_at_1": 17.479, + "precision_at_10": 5.324, + "precision_at_100": 0.838, + "precision_at_1000": 0.098, + "precision_at_3": 11.409, + "precision_at_5": 8.47, + "recall_at_1": 17.033, + "recall_at_10": 50.929, + "recall_at_100": 79.262, + "recall_at_1000": 92.239, + "recall_at_3": 33.06, + "recall_at_5": 40.747, + "main_score": 33.048 + }, + "MassiveIntentClassification": { + "accuracy": 72.31002, + "accuracy_stderr": 1.07686, + "f1": 69.61604, + "f1_stderr": 1.08109, + "main_score": 72.31002 + } + } + }, + { + "id": "mmlw-e5-small", + "tags": [], + "results": { + "CBD": { + "accuracy": 64.24, + "accuracy_stderr": 6.07556, + "ap": 17.9671, + "ap_stderr": 2.06695, + "evaluation_time": 579.0, + "f1": 52.97375, + "f1_stderr": 2.88582, + "main_score": 64.24 + }, + "AllegroReviews": { + "accuracy": 33.03181, + "accuracy_stderr": 3.77028, + "evaluation_time": 236.0, + "f1": 29.8003, + "f1_stderr": 2.98342, + "main_score": 33.03181 + }, + "PlscClusteringS2S": { + "evaluation_time": 1486.0, + "v_measure": 38.15418, + "v_measures": 38.21914, + "main_score": 38.15418 + }, + "ArguAna-PL": { + "evaluation_time": 1452.0, + "map_at_1": 28.805, + "map_at_10": 45.327, + "map_at_100": 46.17, + "map_at_1000": 46.177, + "map_at_3": 40.529, + "map_at_5": 43.335, + "mrr_at_1": 30.299, + "mrr_at_10": 45.763, + "mrr_at_100": 46.641, + "mrr_at_1000": 46.648, + "mrr_at_3": 41.074, + "mrr_at_5": 43.837, + "ndcg_at_1": 28.805, + "ndcg_at_10": 54.308, + "ndcg_at_100": 57.879, + "ndcg_at_1000": 58.048, + "ndcg_at_3": 44.502, + "ndcg_at_5": 49.519, + "precision_at_1": 28.805, + "precision_at_10": 8.286, + "precision_at_100": 0.984, + "precision_at_1000": 0.1, + "precision_at_3": 18.682, + "precision_at_5": 13.627, + "recall_at_1": 28.805, + "recall_at_10": 82.859, + "recall_at_100": 98.364, + "recall_at_1000": 99.644, + "recall_at_3": 56.046, + "recall_at_5": 68.137, + "main_score": 54.308 + }, + "PolEmo2.0-OUT": { + "accuracy": 40.32389, + "accuracy_stderr": 2.76492, + "evaluation_time": 228.0, + "f1": 32.33912, + "f1_stderr": 1.13559, + "main_score": 40.32389 + }, + "PAC": { + "accuracy": 63.11323, + "accuracy_stderr": 5.08451, + "ap": 74.53486, + "ap_stderr": 2.53312, + "evaluation_time": 246.0, + "f1": 60.63101, + "f1_stderr": 5.17886, + "main_score": 63.11323 + }, + "SciField": { + "accuracy": 82.56606, + "accuracy_stderr": 2.34407, + "evaluation_time": 353.0, + "f1": 82.24443, + "f1_stderr": 2.465, + "main_score": 82.56606 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 81.7978, + "cos_sim.accuracy_threshold": 97.41061, + "cos_sim.ap": 73.21295, + "cos_sim.f1": 66.57391, + "cos_sim.f1_threshold": 96.49787, + "cos_sim.precision": 65.05778, + "cos_sim.recall": 68.16239, + "dot.accuracy": 78.67917, + "dot.accuracy_threshold": 7950.0, + "dot.ap": 63.61944, + "dot.f1": 62.45014, + "dot.f1_threshold": 7756.25, + "dot.precision": 52.04179, + "dot.recall": 78.06268, + "euclidean.accuracy": 81.77742, + "euclidean.accuracy_threshold": 214.96674, + "euclidean.ap": 73.25054, + "euclidean.f1": 66.55064, + "euclidean.f1_threshold": 238.26016, + "euclidean.precision": 65.07828, + "euclidean.recall": 68.09117, + "evaluation_time": 233.0, + "manhattan.accuracy": 81.63473, + "manhattan.accuracy_threshold": 3192.76212, + "manhattan.ap": 72.99782, + "manhattan.f1": 66.36236, + "manhattan.f1_threshold": 3673.4705, + "manhattan.precision": 65.44321, + "manhattan.recall": 67.30769, + "max.accuracy": 81.7978, + "max.ap": 73.25054, + "max.f1": 66.57391, + "main_score": 73.21295 + }, + "HotpotQA-PL": { + "evaluation_time": 234530.0, + "map_at_1": 32.627, + "map_at_10": 48.041, + "map_at_100": 49.008, + "map_at_1000": 49.093, + "map_at_3": 44.774, + "map_at_5": 46.791, + "mrr_at_1": 65.28, + "mrr_at_10": 72.535, + "mrr_at_100": 72.892, + "mrr_at_1000": 72.909, + "mrr_at_3": 71.083, + "mrr_at_5": 71.985, + "ndcg_at_1": 65.253, + "ndcg_at_10": 57.137, + "ndcg_at_100": 60.783, + "ndcg_at_1000": 62.507, + "ndcg_at_3": 52.17, + "ndcg_at_5": 54.896, + "precision_at_1": 65.253, + "precision_at_10": 12.088, + "precision_at_100": 1.496, + "precision_at_1000": 0.172, + "precision_at_3": 32.96, + "precision_at_5": 21.931, + "recall_at_1": 32.627, + "recall_at_10": 60.439, + "recall_at_100": 74.808, + "recall_at_1000": 86.219, + "recall_at_3": 49.44, + "recall_at_5": 54.828, + "main_score": 57.137 + }, + "PPC": { + "cos_sim.accuracy": 80.1, + "cos_sim.accuracy_threshold": 97.2478, + "cos_sim.ap": 89.69526, + "cos_sim.f1": 83.37469, + "cos_sim.f1_threshold": 96.80371, + "cos_sim.precision": 83.30579, + "cos_sim.recall": 83.44371, + "dot.accuracy": 77.8, + "dot.accuracy_threshold": 7900.0, + "dot.ap": 87.72366, + "dot.f1": 82.83753, + "dot.f1_threshold": 7862.5, + "dot.precision": 76.80339, + "dot.recall": 89.90066, + "euclidean.accuracy": 80.1, + "euclidean.accuracy_threshold": 214.82262, + "euclidean.ap": 89.61317, + "euclidean.f1": 83.40215, + "euclidean.f1_threshold": 229.53855, + "euclidean.precision": 83.19605, + "euclidean.recall": 83.60927, + "evaluation_time": 115.0, + "manhattan.accuracy": 79.6, + "manhattan.accuracy_threshold": 3242.77079, + "manhattan.ap": 89.48364, + "manhattan.f1": 82.96296, + "manhattan.f1_threshold": 3583.01562, + "manhattan.precision": 82.48773, + "manhattan.recall": 83.44371, + "max.accuracy": 80.1, + "max.ap": 89.69526, + "max.f1": 83.40215, + "main_score": 89.69526 + }, + "EightTagsClustering": { + "evaluation_time": 3278.0, + "main_score": 31.61777, + "v_measure": 31.61777, + "v_measure_std": 1.4684, + "v_measures": 31.11917 + }, + "NFCorpus-PL": { + "evaluation_time": 819.0, + "map_at_1": 4.18, + "map_at_10": 9.524, + "map_at_100": 12.272, + "map_at_1000": 13.616, + "map_at_3": 6.717, + "map_at_5": 8.172, + "mrr_at_1": 37.152, + "mrr_at_10": 45.068, + "mrr_at_100": 46.026, + "mrr_at_1000": 46.085, + "mrr_at_3": 43.344, + "mrr_at_5": 44.412, + "ndcg_at_1": 34.52, + "ndcg_at_10": 27.604, + "ndcg_at_100": 26.013, + "ndcg_at_1000": 35.272, + "ndcg_at_3": 31.538, + "ndcg_at_5": 30.166, + "precision_at_1": 36.223, + "precision_at_10": 21.053, + "precision_at_100": 7.08, + "precision_at_1000": 1.993, + "precision_at_3": 30.031, + "precision_at_5": 26.997, + "recall_at_1": 4.18, + "recall_at_10": 12.901, + "recall_at_100": 27.438, + "recall_at_1000": 60.768, + "recall_at_3": 7.492, + "recall_at_5": 10.05, + "main_score": 27.604 + }, + "SciFact-PL": { + "evaluation_time": 1056.0, + "map_at_1": 43.528, + "map_at_10": 53.29, + "map_at_100": 54.342, + "map_at_1000": 54.377, + "map_at_3": 50.652, + "map_at_5": 52.248, + "mrr_at_1": 46.667, + "mrr_at_10": 55.286, + "mrr_at_100": 56.094, + "mrr_at_1000": 56.125, + "mrr_at_3": 53.222, + "mrr_at_5": 54.339, + "ndcg_at_1": 46.0, + "ndcg_at_10": 58.142, + "ndcg_at_100": 62.426, + "ndcg_at_1000": 63.396, + "ndcg_at_3": 53.53, + "ndcg_at_5": 55.842, + "precision_at_1": 46.0, + "precision_at_10": 7.967, + "precision_at_100": 1.023, + "precision_at_1000": 0.111, + "precision_at_3": 21.444, + "precision_at_5": 14.333, + "recall_at_1": 43.528, + "recall_at_10": 71.511, + "recall_at_100": 89.933, + "recall_at_1000": 97.667, + "recall_at_3": 59.067, + "recall_at_5": 64.789, + "main_score": 58.142 + }, + "MassiveScenarioClassification": { + "accuracy": 72.12508, + "accuracy_stderr": 1.54498, + "f1": 71.55887, + "f1_stderr": 1.15389, + "main_score": 72.12508 + }, + "FiQA-PL": { + "evaluation_time": 5818.0, + "map_at_1": 14.643, + "map_at_10": 23.166, + "map_at_100": 24.725, + "map_at_1000": 24.92, + "map_at_3": 20.166, + "map_at_5": 22.003, + "mrr_at_1": 29.63, + "mrr_at_10": 37.632, + "mrr_at_100": 38.512, + "mrr_at_1000": 38.578, + "mrr_at_3": 35.391, + "mrr_at_5": 36.857, + "ndcg_at_1": 29.167, + "ndcg_at_10": 29.749, + "ndcg_at_100": 35.983, + "ndcg_at_1000": 39.817, + "ndcg_at_3": 26.739, + "ndcg_at_5": 27.993, + "precision_at_1": 29.167, + "precision_at_10": 8.333, + "precision_at_100": 1.448, + "precision_at_1000": 0.213, + "precision_at_3": 17.747, + "precision_at_5": 13.58, + "recall_at_1": 14.643, + "recall_at_10": 35.247, + "recall_at_100": 59.151, + "recall_at_1000": 82.565, + "recall_at_3": 24.006, + "recall_at_5": 29.383, + "main_score": 29.749 + }, + "SICK-R-PL": { + "cos_sim.pearson": 79.62333, + "cos_sim.spearman": 73.70598, + "euclidean.pearson": 77.36033, + "euclidean.spearman": 73.7442, + "evaluation_time": 264.0, + "manhattan.pearson": 77.12735, + "manhattan.spearman": 73.53015, + "main_score": 73.70598 + }, + "CDSC-R": { + "cos_sim.pearson": 91.64169, + "cos_sim.spearman": 91.97552, + "euclidean.pearson": 90.87481, + "euclidean.spearman": 91.92363, + "evaluation_time": 84.0, + "manhattan.pearson": 90.84155, + "manhattan.spearman": 91.85927, + "main_score": 91.97552 + }, + "SciDefRetrieval": { + "evaluation_time": 162.0, + "map_at_1": 44.919, + "map_at_10": 53.925, + "map_at_100": 54.588, + "map_at_1000": 54.624, + "map_at_20": 54.313, + "map_at_3": 51.497, + "map_at_5": 52.786, + "mrr_at_1": 44.919, + "mrr_at_10": 53.948, + "mrr_at_100": 54.587, + "mrr_at_1000": 54.623, + "mrr_at_20": 54.313, + "mrr_at_3": 51.522, + "mrr_at_5": 52.744, + "ndcg_at_1": 44.919, + "ndcg_at_10": 58.77, + "ndcg_at_100": 62.297, + "ndcg_at_1000": 63.36, + "ndcg_at_20": 60.21, + "ndcg_at_3": 53.664, + "ndcg_at_5": 56.009, + "precision_at_1": 44.919, + "precision_at_10": 7.423, + "precision_at_100": 0.915, + "precision_at_1000": 0.1, + "precision_at_20": 3.999, + "precision_at_3": 19.98, + "precision_at_5": 13.137, + "recall_at_1": 44.919, + "recall_at_10": 74.227, + "recall_at_100": 91.458, + "recall_at_1000": 100.0, + "recall_at_20": 79.971, + "recall_at_3": 59.941, + "recall_at_5": 65.685, + "main_score": 58.77 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 623.0, + "v_measure": 33.43728, + "v_measures": 31.10732, + "main_score": 33.43728 + }, + "PSC": { + "cos_sim.accuracy": 96.93878, + "cos_sim.accuracy_threshold": 89.95102, + "cos_sim.ap": 98.86489, + "cos_sim.f1": 95.11111, + "cos_sim.f1_threshold": 89.95102, + "cos_sim.precision": 92.5072, + "cos_sim.recall": 97.86585, + "dot.accuracy": 95.73284, + "dot.accuracy_threshold": 7425.0, + "dot.ap": 97.82617, + "dot.f1": 93.21534, + "dot.f1_threshold": 7406.25, + "dot.precision": 90.28571, + "dot.recall": 96.34146, + "euclidean.accuracy": 96.93878, + "euclidean.accuracy_threshold": 394.05864, + "euclidean.ap": 98.84838, + "euclidean.f1": 95.11111, + "euclidean.f1_threshold": 408.2659, + "euclidean.precision": 92.5072, + "euclidean.recall": 97.86585, + "evaluation_time": 129.0, + "manhattan.accuracy": 96.84601, + "manhattan.accuracy_threshold": 6374.69693, + "manhattan.ap": 98.78659, + "manhattan.f1": 94.94048, + "manhattan.f1_threshold": 6374.69693, + "manhattan.precision": 92.73256, + "manhattan.recall": 97.2561, + "max.accuracy": 96.93878, + "max.ap": 98.86489, + "max.f1": 95.11111, + "main_score": 98.86489 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 944.0, + "v_measure": 47.82497, + "v_measures": 47.85904, + "main_score": 47.82497 + }, + "NQ-PL": { + "evaluation_time": 176090.0, + "map_at_1": 17.965, + "map_at_10": 28.04, + "map_at_100": 29.217, + "map_at_1000": 29.285, + "map_at_3": 24.818, + "map_at_5": 26.617, + "mrr_at_1": 20.22, + "mrr_at_10": 30.148, + "mrr_at_100": 31.138, + "mrr_at_1000": 31.19, + "mrr_at_3": 27.202, + "mrr_at_5": 28.885, + "ndcg_at_1": 20.365, + "ndcg_at_10": 33.832, + "ndcg_at_100": 39.33, + "ndcg_at_1000": 41.1, + "ndcg_at_3": 27.46, + "ndcg_at_5": 30.584, + "precision_at_1": 20.365, + "precision_at_10": 5.849, + "precision_at_100": 0.896, + "precision_at_1000": 0.107, + "precision_at_3": 12.64, + "precision_at_5": 9.334, + "recall_at_1": 17.965, + "recall_at_10": 49.503, + "recall_at_100": 74.351, + "recall_at_1000": 87.766, + "recall_at_3": 32.665, + "recall_at_5": 39.974, + "main_score": 33.832 + }, + "STS22": { + "cos_sim.pearson": 37.69694, + "cos_sim.spearman": 40.62714, + "euclidean.pearson": 30.21273, + "euclidean.spearman": 40.66643, + "manhattan.pearson": 29.90224, + "manhattan.spearman": 40.35576, + "main_score": 40.62714 + }, + "DBPedia-PL": { + "evaluation_time": 171428.0, + "map_at_1": 6.148, + "map_at_10": 12.871, + "map_at_100": 18.04, + "map_at_1000": 19.286, + "map_at_3": 9.156, + "map_at_5": 10.858, + "mrr_at_1": 53.25, + "mrr_at_10": 61.017, + "mrr_at_100": 61.484, + "mrr_at_1000": 61.508, + "mrr_at_3": 58.75, + "mrr_at_5": 60.375, + "ndcg_at_1": 41.0, + "ndcg_at_10": 30.281, + "ndcg_at_100": 33.956, + "ndcg_at_1000": 40.77, + "ndcg_at_3": 34.127, + "ndcg_at_5": 32.274, + "precision_at_1": 52.5, + "precision_at_10": 24.525, + "precision_at_100": 8.125, + "precision_at_1000": 1.728, + "precision_at_3": 37.083, + "precision_at_5": 32.15, + "recall_at_1": 6.148, + "recall_at_10": 17.866, + "recall_at_100": 39.213, + "recall_at_1000": 61.604, + "recall_at_3": 10.084, + "recall_at_5": 13.334, + "main_score": 30.281 + }, + "Quora-PL": { + "evaluation_time": 17477.0, + "map_at_1": 62.757, + "map_at_10": 76.56, + "map_at_100": 77.328, + "map_at_1000": 77.355, + "map_at_3": 73.288, + "map_at_5": 75.255, + "mrr_at_1": 72.28, + "mrr_at_10": 79.879, + "mrr_at_100": 80.121, + "mrr_at_1000": 80.127, + "mrr_at_3": 78.407, + "mrr_at_5": 79.357, + "ndcg_at_1": 72.33, + "ndcg_at_10": 81.151, + "ndcg_at_100": 83.107, + "ndcg_at_1000": 83.397, + "ndcg_at_3": 77.3, + "ndcg_at_5": 79.307, + "precision_at_1": 72.33, + "precision_at_10": 12.587, + "precision_at_100": 1.488, + "precision_at_1000": 0.155, + "precision_at_3": 33.943, + "precision_at_5": 22.61, + "recall_at_1": 62.757, + "recall_at_10": 90.616, + "recall_at_100": 97.905, + "recall_at_1000": 99.618, + "recall_at_3": 79.928, + "recall_at_5": 85.305, + "main_score": 81.151 + }, + "CDSC-E": { + "cos_sim.accuracy": 88.8, + "cos_sim.accuracy_threshold": 99.16647, + "cos_sim.ap": 76.68029, + "cos_sim.f1": 66.82353, + "cos_sim.f1_threshold": 96.01849, + "cos_sim.precision": 60.42553, + "cos_sim.recall": 74.73684, + "dot.accuracy": 88.1, + "dot.accuracy_threshold": 7962.5, + "dot.ap": 72.0491, + "dot.f1": 66.66667, + "dot.f1_threshold": 7887.5, + "dot.precision": 69.31818, + "dot.recall": 64.21053, + "euclidean.accuracy": 88.8, + "euclidean.accuracy_threshold": 106.70563, + "euclidean.ap": 76.63592, + "euclidean.f1": 67.13287, + "euclidean.f1_threshold": 257.44403, + "euclidean.precision": 60.25105, + "euclidean.recall": 75.78947, + "evaluation_time": 64.0, + "manhattan.accuracy": 88.9, + "manhattan.accuracy_threshold": 1673.76829, + "manhattan.ap": 76.54553, + "manhattan.f1": 66.66667, + "manhattan.f1_threshold": 3950.1478, + "manhattan.precision": 60.51502, + "manhattan.recall": 74.21053, + "max.accuracy": 88.9, + "max.ap": 76.68029, + "max.f1": 67.13287, + "main_score": 76.68029 + }, + "TRECCOVID-PL": { + "evaluation_time": 24450.0, + "map_at_1": 0.227, + "map_at_10": 1.338, + "map_at_100": 6.965, + "map_at_1000": 17.135, + "map_at_3": 0.53, + "map_at_5": 0.799, + "mrr_at_1": 84.0, + "mrr_at_10": 88.083, + "mrr_at_100": 88.432, + "mrr_at_1000": 88.432, + "mrr_at_3": 87.333, + "mrr_at_5": 87.833, + "ndcg_at_1": 76.0, + "ndcg_at_10": 58.199, + "ndcg_at_100": 43.23, + "ndcg_at_1000": 39.751, + "ndcg_at_3": 63.743, + "ndcg_at_5": 60.43, + "precision_at_1": 84.0, + "precision_at_10": 62.0, + "precision_at_100": 44.52, + "precision_at_1000": 17.746, + "precision_at_3": 67.333, + "precision_at_5": 63.2, + "recall_at_1": 0.227, + "recall_at_10": 1.627, + "recall_at_100": 10.601, + "recall_at_1000": 37.532, + "recall_at_3": 0.547, + "recall_at_5": 0.864, + "main_score": 58.199 + }, + "SCIDOCS-PL": { + "evaluation_time": 3985.0, + "map_at_1": 3.313, + "map_at_10": 8.56, + "map_at_100": 10.178, + "map_at_1000": 10.46, + "map_at_3": 6.094, + "map_at_5": 7.323, + "mrr_at_1": 16.3, + "mrr_at_10": 25.579, + "mrr_at_100": 26.717, + "mrr_at_1000": 26.799, + "mrr_at_3": 22.583, + "mrr_at_5": 24.298, + "ndcg_at_1": 16.3, + "ndcg_at_10": 14.789, + "ndcg_at_100": 21.731, + "ndcg_at_1000": 27.262, + "ndcg_at_3": 13.74, + "ndcg_at_5": 12.199, + "precision_at_1": 16.3, + "precision_at_10": 7.78, + "precision_at_100": 1.79, + "precision_at_1000": 0.313, + "precision_at_3": 12.933, + "precision_at_5": 10.86, + "recall_at_1": 3.313, + "recall_at_10": 15.772, + "recall_at_100": 36.392, + "recall_at_1000": 63.525, + "recall_at_3": 7.863, + "recall_at_5": 11.003, + "main_score": 14.789 + }, + "PolEmo2.0-IN": { + "accuracy": 63.96122, + "accuracy_stderr": 3.48624, + "evaluation_time": 254.0, + "f1": 64.398, + "f1_stderr": 2.18266, + "main_score": 63.96122 + }, + "PlscClusteringP2P": { + "evaluation_time": 2801.0, + "v_measure": 43.72413, + "v_measures": 44.29117, + "main_score": 43.72413 + }, + "MSMARCO-PL": { + "evaluation_time": 462839.0, + "map_at_1": 13.151, + "map_at_10": 21.179, + "map_at_100": 22.227, + "map_at_1000": 22.308, + "map_at_3": 18.473, + "map_at_5": 19.943, + "mrr_at_1": 13.467, + "mrr_at_10": 21.471, + "mrr_at_100": 22.509, + "mrr_at_1000": 22.585, + "mrr_at_3": 18.789, + "mrr_at_5": 20.262, + "ndcg_at_1": 13.539, + "ndcg_at_10": 25.943, + "ndcg_at_100": 31.387, + "ndcg_at_1000": 33.641, + "ndcg_at_3": 20.368, + "ndcg_at_5": 23.004, + "precision_at_1": 13.539, + "precision_at_10": 4.249, + "precision_at_100": 0.704, + "precision_at_1000": 0.09, + "precision_at_3": 8.782, + "precision_at_5": 6.605, + "recall_at_1": 13.151, + "recall_at_10": 40.698, + "recall_at_100": 66.71, + "recall_at_1000": 84.491, + "recall_at_3": 25.452, + "recall_at_5": 31.791, + "main_score": 25.943 + }, + "MassiveIntentClassification": { + "accuracy": 67.35373, + "accuracy_stderr": 1.53496, + "f1": 64.67852, + "f1_stderr": 1.12809, + "main_score": 67.35373 + } + } + }, + { + "id": "st-polish-paraphrase-from-mpnet", + "tags": [], + "results": { + "CBD": { + "accuracy": 67.48, + "accuracy_stderr": 5.85044, + "ap": 22.19817, + "ap_stderr": 4.0242, + "evaluation_time": 917.0, + "f1": 57.32903, + "f1_stderr": 5.06502, + "main_score": 67.48 + }, + "AllegroReviews": { + "accuracy": 34.55268, + "accuracy_stderr": 4.99046, + "evaluation_time": 299.0, + "f1": 31.17398, + "f1_stderr": 3.03482, + "main_score": 34.55268 + }, + "PlscClusteringS2S": { + "evaluation_time": 1542.0, + "v_measure": 40.83383, + "v_measures": 40.62464, + "main_score": 40.83383 + }, + "ArguAna-PL": { + "evaluation_time": 2184.0, + "map_at_1": 25.889, + "map_at_10": 42.596, + "map_at_100": 43.485, + "map_at_1000": 43.491, + "map_at_3": 37.447, + "map_at_5": 40.352, + "mrr_at_1": 26.956, + "mrr_at_10": 43.006, + "mrr_at_100": 43.881, + "mrr_at_1000": 43.887, + "mrr_at_3": 37.85, + "mrr_at_5": 40.791, + "ndcg_at_1": 25.889, + "ndcg_at_10": 51.869, + "ndcg_at_100": 55.735, + "ndcg_at_1000": 55.867, + "ndcg_at_3": 41.235, + "ndcg_at_5": 46.49, + "precision_at_1": 25.889, + "precision_at_10": 8.151, + "precision_at_100": 0.986, + "precision_at_1000": 0.1, + "precision_at_3": 17.402, + "precision_at_5": 13.001, + "recall_at_1": 25.889, + "recall_at_10": 81.508, + "recall_at_100": 98.578, + "recall_at_1000": 99.573, + "recall_at_3": 52.205, + "recall_at_5": 65.007, + "main_score": 51.869 + }, + "PolEmo2.0-OUT": { + "accuracy": 30.9919, + "accuracy_stderr": 6.58281, + "evaluation_time": 265.0, + "f1": 28.26114, + "f1_stderr": 3.61152, + "main_score": 30.9919 + }, + "PAC": { + "accuracy": 63.24935, + "accuracy_stderr": 4.19367, + "ap": 75.10295, + "ap_stderr": 2.86277, + "evaluation_time": 279.0, + "f1": 60.99255, + "f1_stderr": 4.90928, + "main_score": 63.24935 + }, + "SciField": { + "accuracy": 83.26147, + "accuracy_stderr": 2.77974, + "evaluation_time": 400.0, + "f1": 82.92412, + "f1_stderr": 2.89845, + "main_score": 83.26147 + }, + "SICK-E-PL": { + "cos_sim.accuracy": 84.56991, + "cos_sim.accuracy_threshold": 91.92629, + "cos_sim.ap": 80.56123, + "cos_sim.f1": 72.99864, + "cos_sim.f1_threshold": 87.73259, + "cos_sim.precision": 69.68912, + "cos_sim.recall": 76.63818, + "dot.accuracy": 80.33021, + "dot.accuracy_threshold": 738.64756, + "dot.ap": 70.07288, + "dot.f1": 66.46078, + "dot.f1_threshold": 648.66838, + "dot.precision": 58.66957, + "dot.recall": 76.63818, + "euclidean.accuracy": 84.48838, + "euclidean.accuracy_threshold": 126.73597, + "euclidean.ap": 80.29664, + "euclidean.f1": 73.10619, + "euclidean.f1_threshold": 142.93938, + "euclidean.precision": 68.25201, + "euclidean.recall": 78.7037, + "evaluation_time": 241.0, + "manhattan.accuracy": 84.50876, + "manhattan.accuracy_threshold": 2566.46252, + "manhattan.ap": 80.31061, + "manhattan.f1": 73.14815, + "manhattan.f1_threshold": 3077.00329, + "manhattan.precision": 68.2716, + "manhattan.recall": 78.77493, + "max.accuracy": 84.56991, + "max.ap": 80.56123, + "max.f1": 73.14815, + "main_score": 80.56123 + }, + "HotpotQA-PL": { + "evaluation_time": 422573.0, + "map_at_1": 18.251, + "map_at_10": 24.894, + "map_at_100": 25.756, + "map_at_1000": 25.858, + "map_at_3": 22.986, + "map_at_5": 24.01, + "mrr_at_1": 36.502, + "mrr_at_10": 43.904, + "mrr_at_100": 44.646, + "mrr_at_1000": 44.702, + "mrr_at_3": 41.956, + "mrr_at_5": 43.072, + "ndcg_at_1": 36.502, + "ndcg_at_10": 32.108, + "ndcg_at_100": 36.143, + "ndcg_at_1000": 38.615, + "ndcg_at_3": 28.436, + "ndcg_at_5": 30.181, + "precision_at_1": 36.502, + "precision_at_10": 6.995, + "precision_at_100": 1.021, + "precision_at_1000": 0.135, + "precision_at_3": 17.785, + "precision_at_5": 12.051, + "recall_at_1": 18.251, + "recall_at_10": 34.976, + "recall_at_100": 51.053, + "recall_at_1000": 67.583, + "recall_at_3": 26.678, + "recall_at_5": 30.128, + "main_score": 32.108 + }, + "PPC": { + "cos_sim.accuracy": 89.0, + "cos_sim.accuracy_threshold": 84.42103, + "cos_sim.ap": 93.48698, + "cos_sim.f1": 91.3522, + "cos_sim.f1_threshold": 84.42103, + "cos_sim.precision": 86.97605, + "cos_sim.recall": 96.19205, + "dot.accuracy": 80.5, + "dot.accuracy_threshold": 550.62819, + "dot.ap": 88.97348, + "dot.f1": 84.91704, + "dot.f1_threshold": 516.48026, + "dot.precision": 77.97784, + "dot.recall": 93.21192, + "euclidean.accuracy": 89.5, + "euclidean.accuracy_threshold": 137.51318, + "euclidean.ap": 93.63657, + "euclidean.f1": 91.69279, + "euclidean.f1_threshold": 151.75154, + "euclidean.precision": 87.05357, + "euclidean.recall": 96.8543, + "evaluation_time": 128.0, + "manhattan.accuracy": 89.6, + "manhattan.accuracy_threshold": 2966.82014, + "manhattan.ap": 93.67179, + "manhattan.f1": 91.74455, + "manhattan.f1_threshold": 3342.08717, + "manhattan.precision": 86.61765, + "manhattan.recall": 97.51656, + "max.accuracy": 89.6, + "max.ap": 93.67179, + "max.f1": 91.74455, + "main_score": 93.48698 + }, + "EightTagsClustering": { + "evaluation_time": 3400.0, + "main_score": 30.76528, + "v_measure": 30.76528, + "v_measure_std": 1.9524, + "v_measures": 29.00909 + }, + "NFCorpus-PL": { + "evaluation_time": 1276.0, + "map_at_1": 3.315, + "map_at_10": 7.398, + "map_at_100": 9.865, + "map_at_1000": 11.274, + "map_at_3": 5.343, + "map_at_5": 6.277, + "mrr_at_1": 32.817, + "mrr_at_10": 41.496, + "mrr_at_100": 42.182, + "mrr_at_1000": 42.24, + "mrr_at_3": 39.319, + "mrr_at_5": 40.372, + "ndcg_at_1": 31.579, + "ndcg_at_10": 24.049, + "ndcg_at_100": 23.452, + "ndcg_at_1000": 33.252, + "ndcg_at_3": 28.155, + "ndcg_at_5": 26.127, + "precision_at_1": 32.817, + "precision_at_10": 18.669, + "precision_at_100": 6.632, + "precision_at_1000": 2.016, + "precision_at_3": 27.038, + "precision_at_5": 23.282, + "recall_at_1": 3.315, + "recall_at_10": 10.817, + "recall_at_100": 26.288, + "recall_at_1000": 60.18, + "recall_at_3": 6.25, + "recall_at_5": 7.884, + "main_score": 24.049 + }, + "SciFact-PL": { + "evaluation_time": 1735.0, + "map_at_1": 36.028, + "map_at_10": 47.207, + "map_at_100": 48.081, + "map_at_1000": 48.138, + "map_at_3": 44.019, + "map_at_5": 45.969, + "mrr_at_1": 38.667, + "mrr_at_10": 48.298, + "mrr_at_100": 49.019, + "mrr_at_1000": 49.069, + "mrr_at_3": 45.389, + "mrr_at_5": 47.156, + "ndcg_at_1": 38.667, + "ndcg_at_10": 52.512, + "ndcg_at_100": 56.667, + "ndcg_at_1000": 57.995, + "ndcg_at_3": 46.643, + "ndcg_at_5": 49.709, + "precision_at_1": 38.667, + "precision_at_10": 7.6, + "precision_at_100": 0.99, + "precision_at_1000": 0.11, + "precision_at_3": 19.222, + "precision_at_5": 13.267, + "recall_at_1": 36.028, + "recall_at_10": 67.956, + "recall_at_100": 87.2, + "recall_at_1000": 97.333, + "recall_at_3": 52.306, + "recall_at_5": 59.639, + "main_score": 52.512 + }, + "MassiveScenarioClassification": { + "accuracy": 71.846, + "accuracy_stderr": 2.06534, + "f1": 71.87823, + "f1_stderr": 1.77975, + "main_score": 71.846 + }, + "FiQA-PL": { + "evaluation_time": 12109.0, + "map_at_1": 9.255, + "map_at_10": 16.43, + "map_at_100": 17.68, + "map_at_1000": 17.903, + "map_at_3": 13.783, + "map_at_5": 15.173, + "mrr_at_1": 19.444, + "mrr_at_10": 27.48, + "mrr_at_100": 28.328, + "mrr_at_1000": 28.42, + "mrr_at_3": 25.051, + "mrr_at_5": 26.463, + "ndcg_at_1": 19.444, + "ndcg_at_10": 22.273, + "ndcg_at_100": 27.962, + "ndcg_at_1000": 32.486, + "ndcg_at_3": 19.024, + "ndcg_at_5": 20.154, + "precision_at_1": 19.444, + "precision_at_10": 6.651, + "precision_at_100": 1.256, + "precision_at_1000": 0.202, + "precision_at_3": 13.066, + "precision_at_5": 9.969, + "recall_at_1": 9.255, + "recall_at_10": 28.572, + "recall_at_100": 50.762, + "recall_at_1000": 78.754, + "recall_at_3": 17.579, + "recall_at_5": 22.193, + "main_score": 22.273 + }, + "SICK-R-PL": { + "cos_sim.pearson": 81.34797, + "cos_sim.spearman": 76.17885, + "euclidean.pearson": 80.20222, + "euclidean.spearman": 76.55388, + "evaluation_time": 266.0, + "manhattan.pearson": 80.07979, + "manhattan.spearman": 76.4413, + "main_score": 76.17885 + }, + "CDSC-R": { + "cos_sim.pearson": 87.66389, + "cos_sim.spearman": 88.55497, + "euclidean.pearson": 90.20712, + "euclidean.spearman": 89.45135, + "evaluation_time": 82.0, + "manhattan.pearson": 89.99572, + "manhattan.spearman": 89.21846, + "main_score": 88.55497 + }, + "SciDefRetrieval": { + "evaluation_time": 199.0, + "map_at_1": 44.33, + "map_at_10": 52.643, + "map_at_100": 53.518, + "map_at_1000": 53.564, + "map_at_20": 53.203, + "map_at_3": 50.442, + "map_at_5": 51.723, + "mrr_at_1": 44.183, + "mrr_at_10": 52.556, + "mrr_at_100": 53.431, + "mrr_at_1000": 53.477, + "mrr_at_20": 53.117, + "mrr_at_3": 50.368, + "mrr_at_5": 51.635, + "ndcg_at_1": 44.33, + "ndcg_at_10": 57.158, + "ndcg_at_100": 61.345, + "ndcg_at_1000": 62.512, + "ndcg_at_20": 59.179, + "ndcg_at_3": 52.533, + "ndcg_at_5": 54.84, + "precision_at_1": 44.33, + "precision_at_10": 7.158, + "precision_at_100": 0.91, + "precision_at_1000": 0.1, + "precision_at_20": 3.976, + "precision_at_3": 19.539, + "precision_at_5": 12.842, + "recall_at_1": 44.33, + "recall_at_10": 71.576, + "recall_at_100": 91.016, + "recall_at_1000": 100.0, + "recall_at_20": 79.529, + "recall_at_3": 58.616, + "recall_at_5": 64.212, + "main_score": 57.158 + }, + "WikinewsPlClusteringS2S": { + "evaluation_time": 588.0, + "v_measure": 31.04312, + "v_measures": 39.46924, + "main_score": 31.04312 + }, + "PSC": { + "cos_sim.accuracy": 97.49536, + "cos_sim.accuracy_threshold": 69.92124, + "cos_sim.ap": 99.05286, + "cos_sim.f1": 95.9276, + "cos_sim.f1_threshold": 69.34247, + "cos_sim.precision": 94.92537, + "cos_sim.recall": 96.95122, + "dot.accuracy": 94.06308, + "dot.accuracy_threshold": 345.30458, + "dot.ap": 95.42203, + "dot.f1": 90.85714, + "dot.f1_threshold": 343.82024, + "dot.precision": 85.48387, + "dot.recall": 96.95122, + "euclidean.accuracy": 97.1243, + "euclidean.accuracy_threshold": 190.67175, + "euclidean.ap": 98.90886, + "euclidean.f1": 95.40741, + "euclidean.f1_threshold": 190.67175, + "euclidean.precision": 92.79539, + "euclidean.recall": 98.17073, + "evaluation_time": 160.0, + "manhattan.accuracy": 97.21707, + "manhattan.accuracy_threshold": 4069.13147, + "manhattan.ap": 98.93285, + "manhattan.f1": 95.54896, + "manhattan.f1_threshold": 4069.13147, + "manhattan.precision": 93.06358, + "manhattan.recall": 98.17073, + "max.accuracy": 97.49536, + "max.ap": 99.05286, + "max.f1": 95.9276, + "main_score": 99.05286 + }, + "WikinewsPlClusteringP2P": { + "evaluation_time": 1121.0, + "v_measure": 42.74802, + "v_measures": 50.35517, + "main_score": 42.74802 + }, + "NQ-PL": { + "evaluation_time": 332572.0, + "map_at_1": 10.899, + "map_at_10": 18.744, + "map_at_100": 19.834, + "map_at_1000": 19.926, + "map_at_3": 15.95, + "map_at_5": 17.53, + "mrr_at_1": 12.486, + "mrr_at_10": 20.448, + "mrr_at_100": 21.394, + "mrr_at_1000": 21.469, + "mrr_at_3": 17.734, + "mrr_at_5": 19.315, + "ndcg_at_1": 12.486, + "ndcg_at_10": 23.536, + "ndcg_at_100": 28.982, + "ndcg_at_1000": 31.559, + "ndcg_at_3": 17.951, + "ndcg_at_5": 20.724, + "precision_at_1": 12.486, + "precision_at_10": 4.374, + "precision_at_100": 0.752, + "precision_at_1000": 0.1, + "precision_at_3": 8.594, + "precision_at_5": 6.715, + "recall_at_1": 10.899, + "recall_at_10": 36.655, + "recall_at_100": 61.916, + "recall_at_1000": 81.921, + "recall_at_3": 21.958, + "recall_at_5": 28.394, + "main_score": 23.536 + }, + "STS22": { + "cos_sim.pearson": 34.05374, + "cos_sim.spearman": 37.33945, + "euclidean.pearson": 27.34542, + "euclidean.spearman": 36.88659, + "manhattan.pearson": 27.28778, + "manhattan.spearman": 36.95946, + "main_score": 37.33945 + }, + "DBPedia-PL": { + "evaluation_time": 348301.0, + "map_at_1": 5.453, + "map_at_10": 10.535, + "map_at_100": 13.95, + "map_at_1000": 14.88, + "map_at_3": 7.907, + "map_at_5": 9.087, + "mrr_at_1": 45.75, + "mrr_at_10": 54.137, + "mrr_at_100": 54.677, + "mrr_at_1000": 54.704, + "mrr_at_3": 51.958, + "mrr_at_5": 53.183, + "ndcg_at_1": 35.5, + "ndcg_at_10": 24.595, + "ndcg_at_100": 27.027, + "ndcg_at_1000": 33.514, + "ndcg_at_3": 29.052, + "ndcg_at_5": 26.553, + "precision_at_1": 45.75, + "precision_at_10": 19.675, + "precision_at_100": 5.995, + "precision_at_1000": 1.387, + "precision_at_3": 32.917, + "precision_at_5": 26.4, + "recall_at_1": 5.453, + "recall_at_10": 14.801, + "recall_at_100": 32.376, + "recall_at_1000": 53.714, + "recall_at_3": 8.772, + "recall_at_5": 10.883, + "main_score": 24.595 + }, + "Quora-PL": { + "evaluation_time": 20346.0, + "map_at_1": 63.521, + "map_at_10": 77.105, + "map_at_100": 77.929, + "map_at_1000": 77.953, + "map_at_3": 73.973, + "map_at_5": 75.945, + "mrr_at_1": 73.24, + "mrr_at_10": 80.419, + "mrr_at_100": 80.682, + "mrr_at_1000": 80.688, + "mrr_at_3": 79.052, + "mrr_at_5": 79.963, + "ndcg_at_1": 73.24, + "ndcg_at_10": 81.49, + "ndcg_at_100": 83.573, + "ndcg_at_1000": 83.838, + "ndcg_at_3": 77.887, + "ndcg_at_5": 79.896, + "precision_at_1": 73.24, + "precision_at_10": 12.556, + "precision_at_100": 1.5, + "precision_at_1000": 0.155, + "precision_at_3": 34.197, + "precision_at_5": 22.776, + "recall_at_1": 63.521, + "recall_at_10": 90.369, + "recall_at_100": 98.111, + "recall_at_1000": 99.687, + "recall_at_3": 80.315, + "recall_at_5": 85.69, + "main_score": 81.49 + }, + "CDSC-E": { + "cos_sim.accuracy": 89.1, + "cos_sim.accuracy_threshold": 97.04278, + "cos_sim.ap": 75.06349, + "cos_sim.f1": 65.94005, + "cos_sim.f1_threshold": 93.59006, + "cos_sim.precision": 68.36158, + "cos_sim.recall": 63.68421, + "dot.accuracy": 85.4, + "dot.accuracy_threshold": 799.4493, + "dot.ap": 62.10207, + "dot.f1": 58.39793, + "dot.f1_threshold": 742.58733, + "dot.precision": 57.36041, + "dot.recall": 59.47368, + "euclidean.accuracy": 88.8, + "euclidean.accuracy_threshold": 56.52208, + "euclidean.ap": 74.95533, + "euclidean.f1": 66.1157, + "euclidean.f1_threshold": 99.70531, + "euclidean.precision": 69.36416, + "euclidean.recall": 63.15789, + "evaluation_time": 73.0, + "manhattan.accuracy": 88.8, + "manhattan.accuracy_threshold": 1399.1251, + "manhattan.ap": 74.79209, + "manhattan.f1": 65.5914, + "manhattan.f1_threshold": 2183.0616, + "manhattan.precision": 67.03297, + "manhattan.recall": 64.21053, + "max.accuracy": 89.1, + "max.ap": 75.06349, + "max.f1": 66.1157, + "main_score": 75.06349 + }, + "TRECCOVID-PL": { + "evaluation_time": 44988.0, + "map_at_1": 0.118, + "map_at_10": 0.676, + "map_at_100": 3.143, + "map_at_1000": 7.164, + "map_at_3": 0.311, + "map_at_5": 0.427, + "mrr_at_1": 44.0, + "mrr_at_10": 57.626, + "mrr_at_100": 58.012, + "mrr_at_1000": 58.012, + "mrr_at_3": 53.667, + "mrr_at_5": 55.367, + "ndcg_at_1": 43.0, + "ndcg_at_10": 35.227, + "ndcg_at_100": 23.541, + "ndcg_at_1000": 24.191, + "ndcg_at_3": 41.123, + "ndcg_at_5": 37.806, + "precision_at_1": 44.0, + "precision_at_10": 36.4, + "precision_at_100": 23.52, + "precision_at_1000": 10.944, + "precision_at_3": 42.667, + "precision_at_5": 39.2, + "recall_at_1": 0.118, + "recall_at_10": 0.886, + "recall_at_100": 5.624, + "recall_at_1000": 23.926, + "recall_at_3": 0.35, + "recall_at_5": 0.5, + "main_score": 35.227 + }, + "SCIDOCS-PL": { + "evaluation_time": 6699.0, + "map_at_1": 3.05, + "map_at_10": 7.55, + "map_at_100": 9.062, + "map_at_1000": 9.328, + "map_at_3": 5.428, + "map_at_5": 6.541, + "mrr_at_1": 15.0, + "mrr_at_10": 23.11, + "mrr_at_100": 24.27, + "mrr_at_1000": 24.363, + "mrr_at_3": 20.15, + "mrr_at_5": 21.855, + "ndcg_at_1": 15.0, + "ndcg_at_10": 13.227, + "ndcg_at_100": 19.917, + "ndcg_at_1000": 25.4, + "ndcg_at_3": 12.282, + "ndcg_at_5": 10.991, + "precision_at_1": 15.0, + "precision_at_10": 6.96, + "precision_at_100": 1.676, + "precision_at_1000": 0.3, + "precision_at_3": 11.567, + "precision_at_5": 9.86, + "recall_at_1": 3.05, + "recall_at_10": 14.09, + "recall_at_100": 34.017, + "recall_at_1000": 61.027, + "recall_at_3": 7.02, + "recall_at_5": 9.97, + "main_score": 13.227 + }, + "PolEmo2.0-IN": { + "accuracy": 68.36565, + "accuracy_stderr": 3.88711, + "evaluation_time": 327.0, + "f1": 67.75952, + "f1_stderr": 2.93215, + "main_score": 68.36565 + }, + "PlscClusteringP2P": { + "evaluation_time": 3301.0, + "v_measure": 43.50095, + "v_measures": 43.49004, + "main_score": 43.50095 + }, + "MSMARCO-PL": { + "evaluation_time": 853558.0, + "map_at_1": 8.52, + "map_at_10": 14.271, + "map_at_100": 15.257, + "map_at_1000": 15.354, + "map_at_3": 12.096, + "map_at_5": 13.257, + "mrr_at_1": 8.797, + "mrr_at_10": 14.628, + "mrr_at_100": 15.606, + "mrr_at_1000": 15.697, + "mrr_at_3": 12.438, + "mrr_at_5": 13.603, + "ndcg_at_1": 8.797, + "ndcg_at_10": 17.913, + "ndcg_at_100": 23.198, + "ndcg_at_1000": 25.967, + "ndcg_at_3": 13.377, + "ndcg_at_5": 15.471, + "precision_at_1": 8.797, + "precision_at_10": 3.049, + "precision_at_100": 0.577, + "precision_at_1000": 0.082, + "precision_at_3": 5.783, + "precision_at_5": 4.539, + "recall_at_1": 8.52, + "recall_at_10": 29.266, + "recall_at_100": 54.862, + "recall_at_1000": 76.816, + "recall_at_3": 16.76, + "recall_at_5": 21.815, + "main_score": 17.913 + }, + "MassiveIntentClassification": { + "accuracy": 65.92804, + "accuracy_stderr": 1.25525, + "f1": 63.48919, + "f1_stderr": 1.01094, + "main_score": 65.92804 + } + } + } + ] +} \ No newline at end of file