diff --git "a/README.md" "b/README.md" --- "a/README.md" +++ "b/README.md" @@ -6,8 +6,23 @@ tags: - sentence-similarity - mteb model-index: -- name: SGPT-5.8B-weightedmean-msmarco-specb-bitfit +- name: MUG-B-1.6 results: + - task: + type: Classification + dataset: + type: mteb/amazon_counterfactual + name: MTEB AmazonCounterfactualClassification (en-ext) + config: en-ext + split: test + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + metrics: + - type: accuracy + value: 74.04047976011994 + - type: ap + value: 23.622442298323236 + - type: f1 + value: 61.681362134359354 - task: type: Classification dataset: @@ -15,14 +30,44 @@ model-index: name: MTEB AmazonCounterfactualClassification (en) config: en split: test - revision: 2d8a100785abf0ae21420d2a55b0c56e3e1ea996 + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + metrics: + - type: accuracy + value: 72.38805970149255 + - type: ap + value: 35.14527522183942 + - type: f1 + value: 66.40004634079556 + - task: + type: Classification + dataset: + type: mteb/amazon_counterfactual + name: MTEB AmazonCounterfactualClassification (de) + config: de + split: test + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + metrics: + - type: accuracy + value: 54.3254817987152 + - type: ap + value: 71.95259605308317 + - type: f1 + value: 52.50731386267296 + - task: + type: Classification + dataset: + type: mteb/amazon_counterfactual + name: MTEB AmazonCounterfactualClassification (ja) + config: ja + split: test + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 metrics: - type: accuracy - value: 69.22388059701493 + value: 56.33832976445397 - type: ap - value: 32.04724673950256 + value: 12.671021199223937 - type: f1 - value: 63.25719825770428 + value: 46.127586182990605 - task: type: Classification dataset: @@ -30,14 +75,14 @@ model-index: name: MTEB AmazonPolarityClassification config: default split: test - revision: 80714f8dcf8cefc218ef4f8c5a966dd83f75a0e1 + revision: e2d317d38cd51312af73b3d32a06d1a08b442046 metrics: - type: accuracy - value: 71.26109999999998 + value: 93.70805000000001 - type: ap - value: 66.16336378255403 + value: 90.58639913354553 - type: f1 - value: 70.89719145825303 + value: 93.69822635061847 - task: type: Classification dataset: @@ -45,81 +90,156 @@ model-index: name: MTEB AmazonReviewsClassification (en) config: en split: test - revision: c379a6705fec24a2493fa68e011692605f44e119 + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + metrics: + - type: accuracy + value: 50.85000000000001 + - type: f1 + value: 49.80013009020246 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (de) + config: de + split: test + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + metrics: + - type: accuracy + value: 27.203999999999994 + - type: f1 + value: 26.60134413072989 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (es) + config: es + split: test + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + metrics: + - type: accuracy + value: 34.878 + - type: f1 + value: 33.072592092252314 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (fr) + config: fr + split: test + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + metrics: + - type: accuracy + value: 31.557999999999993 + - type: f1 + value: 30.866094552542624 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (ja) + config: ja + split: test + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + metrics: + - type: accuracy + value: 22.706 + - type: f1 + value: 22.23195837325246 + - task: + type: Classification + dataset: + type: mteb/amazon_reviews_multi + name: MTEB AmazonReviewsClassification (zh) + config: zh + split: test + revision: 1399c76144fd37290681b995c656ef9b2e06e26d metrics: - type: accuracy - value: 39.19199999999999 + value: 22.349999999999998 - type: f1 - value: 38.580766731113826 + value: 21.80183891680617 - task: type: Retrieval dataset: - type: arguana + type: mteb/arguana name: MTEB ArguAna config: default split: test - revision: 5b3e3697907184a9b77a3c99ee9ea1a9cbb1e4e3 + revision: c22ab2a51041ffd869aaddef7af8d8215647e41a metrics: - type: map_at_1 - value: 27.311999999999998 + value: 41.892 - type: map_at_10 - value: 42.620000000000005 + value: 57.989999999999995 - type: map_at_100 - value: 43.707 + value: 58.45 - type: map_at_1000 - value: 43.714999999999996 + value: 58.453 + - type: map_at_20 + value: 58.392999999999994 - type: map_at_3 - value: 37.624 + value: 53.746 - type: map_at_5 - value: 40.498 + value: 56.566 - type: mrr_at_1 - value: 27.667 + value: 43.314 - type: mrr_at_10 - value: 42.737 + value: 58.535000000000004 - type: mrr_at_100 - value: 43.823 + value: 58.975 - type: mrr_at_1000 - value: 43.830999999999996 + value: 58.977999999999994 + - type: mrr_at_20 + value: 58.916999999999994 - type: mrr_at_3 - value: 37.743 + value: 54.303000000000004 - type: mrr_at_5 - value: 40.616 + value: 57.055 - type: ndcg_at_1 - value: 27.311999999999998 + value: 41.892 - type: ndcg_at_10 - value: 51.37500000000001 + value: 66.176 - type: ndcg_at_100 - value: 55.778000000000006 + value: 67.958 - type: ndcg_at_1000 - value: 55.96600000000001 + value: 68.00699999999999 + - type: ndcg_at_20 + value: 67.565 - type: ndcg_at_3 - value: 41.087 + value: 57.691 - type: ndcg_at_5 - value: 46.269 + value: 62.766 - type: precision_at_1 - value: 27.311999999999998 + value: 41.892 - type: precision_at_10 - value: 7.945 + value: 9.189 - type: precision_at_100 - value: 0.9820000000000001 + value: 0.993 - type: precision_at_1000 value: 0.1 + - type: precision_at_20 + value: 4.861 - type: precision_at_3 - value: 17.046 + value: 23.044 - type: precision_at_5 - value: 12.745000000000001 + value: 16.287 - type: recall_at_1 - value: 27.311999999999998 + value: 41.892 - type: recall_at_10 - value: 79.445 + value: 91.892 - type: recall_at_100 - value: 98.151 + value: 99.289 - type: recall_at_1000 - value: 99.57300000000001 + value: 99.644 + - type: recall_at_20 + value: 97.226 - type: recall_at_3 - value: 51.13799999999999 + value: 69.132 - type: recall_at_5 - value: 63.727000000000004 + value: 81.437 - task: type: Clustering dataset: @@ -127,10 +247,10 @@ model-index: name: MTEB ArxivClusteringP2P config: default split: test - revision: 0bbdb47bcbe3a90093699aefeed338a0f28a7ee8 + revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d metrics: - type: v_measure - value: 45.59037428592033 + value: 49.03486273664411 - task: type: Clustering dataset: @@ -138,10 +258,10 @@ model-index: name: MTEB ArxivClusteringS2S config: default split: test - revision: b73bd54100e5abfa6e3a23dcafb46fe4d2438dc3 + revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 metrics: - type: v_measure - value: 38.86371701986363 + value: 43.04797567338598 - task: type: Reranking dataset: @@ -149,12 +269,12 @@ model-index: name: MTEB AskUbuntuDupQuestions config: default split: test - revision: 4d853f94cd57d85ec13805aeeac3ae3e5eb4c49c + revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 metrics: - type: map - value: 61.625568691427766 + value: 64.29499572176032 - type: mrr - value: 75.83256386580486 + value: 77.28861627753592 - task: type: STS dataset: @@ -162,20 +282,20 @@ model-index: name: MTEB BIOSSES config: default split: test - revision: 9ee918f184421b6bd48b78f6c714d86546106103 + revision: d3fb88f8f02e40887cd149695127462bbcf29b4a metrics: - type: cos_sim_pearson - value: 89.96074355094802 + value: 89.53248242133246 - type: cos_sim_spearman - value: 86.2501580394454 + value: 88.38032705871927 - type: euclidean_pearson - value: 82.18427440380462 + value: 87.77994445569084 - type: euclidean_spearman - value: 80.14760935017947 + value: 88.38032705871927 - type: manhattan_pearson - value: 82.24621578156392 + value: 87.52369210088627 - type: manhattan_spearman - value: 80.00363016590163 + value: 88.27972235673434 - task: type: Classification dataset: @@ -183,12 +303,12 @@ model-index: name: MTEB Banking77Classification config: default split: test - revision: 44fa15921b4c889113cc5df03dd4901b49161ab7 + revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 metrics: - type: accuracy - value: 84.49350649350649 + value: 85.4090909090909 - type: f1 - value: 84.4249343233736 + value: 84.87743757972068 - task: type: Clustering dataset: @@ -196,10 +316,10 @@ model-index: name: MTEB BiorxivClusteringP2P config: default split: test - revision: 11d0121201d1f1f280e8cc8f3d98fb9c4d9f9c55 + revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 metrics: - type: v_measure - value: 36.551459722989385 + value: 39.73840151083438 - task: type: Clustering dataset: @@ -207,1045 +327,1195 @@ model-index: name: MTEB BiorxivClusteringS2S config: default split: test - revision: c0fab014e1bcb8d3a5e31b2088972a1e01547dc1 + revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 metrics: - type: v_measure - value: 33.69901851846774 + value: 36.565075977998966 - task: type: Retrieval dataset: - type: BeIR/cqadupstack + type: mteb/cqadupstack-android name: MTEB CQADupstackAndroidRetrieval config: default split: test - revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db + revision: f46a197baaae43b4f621051089b82a364682dfeb metrics: - type: map_at_1 - value: 30.499 + value: 33.082 - type: map_at_10 - value: 41.208 + value: 44.787 - type: map_at_100 - value: 42.638 + value: 46.322 - type: map_at_1000 - value: 42.754 + value: 46.446 + - type: map_at_20 + value: 45.572 - type: map_at_3 - value: 37.506 + value: 40.913 - type: map_at_5 - value: 39.422000000000004 + value: 42.922 - type: mrr_at_1 - value: 37.339 + value: 40.629 - type: mrr_at_10 - value: 47.051 + value: 51.119 - type: mrr_at_100 - value: 47.745 + value: 51.783 - type: mrr_at_1000 - value: 47.786 + value: 51.82 + - type: mrr_at_20 + value: 51.49700000000001 - type: mrr_at_3 - value: 44.086999999999996 + value: 48.355 - type: mrr_at_5 - value: 45.711 + value: 49.979 - type: ndcg_at_1 - value: 37.339 + value: 40.629 - type: ndcg_at_10 - value: 47.666 + value: 51.647 - type: ndcg_at_100 - value: 52.994 + value: 56.923 - type: ndcg_at_1000 - value: 54.928999999999995 + value: 58.682 + - type: ndcg_at_20 + value: 53.457 - type: ndcg_at_3 - value: 41.982 + value: 46.065 - type: ndcg_at_5 - value: 44.42 + value: 48.352000000000004 - type: precision_at_1 - value: 37.339 + value: 40.629 - type: precision_at_10 - value: 9.127 + value: 10.072000000000001 - type: precision_at_100 - value: 1.4749999999999999 + value: 1.5939999999999999 - type: precision_at_1000 - value: 0.194 + value: 0.20600000000000002 + - type: precision_at_20 + value: 5.908 - type: precision_at_3 - value: 20.076 + value: 22.222 - type: precision_at_5 - value: 14.449000000000002 + value: 15.937000000000001 - type: recall_at_1 - value: 30.499 + value: 33.082 - type: recall_at_10 - value: 60.328 + value: 64.55300000000001 - type: recall_at_100 - value: 82.57900000000001 + value: 86.86399999999999 - type: recall_at_1000 - value: 95.074 + value: 97.667 + - type: recall_at_20 + value: 70.988 - type: recall_at_3 - value: 44.17 + value: 48.067 - type: recall_at_5 - value: 50.94 + value: 54.763 - task: type: Retrieval dataset: - type: BeIR/cqadupstack + type: mteb/cqadupstack-english name: MTEB CQADupstackEnglishRetrieval config: default split: test - revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db + revision: ad9991cb51e31e31e430383c75ffb2885547b5f0 metrics: - type: map_at_1 - value: 30.613 + value: 32.272 - type: map_at_10 - value: 40.781 + value: 42.620000000000005 - type: map_at_100 - value: 42.018 + value: 43.936 - type: map_at_1000 - value: 42.132999999999996 + value: 44.066 + - type: map_at_20 + value: 43.349 - type: map_at_3 - value: 37.816 + value: 39.458 - type: map_at_5 - value: 39.389 + value: 41.351 - type: mrr_at_1 - value: 38.408 + value: 40.127 - type: mrr_at_10 - value: 46.631 + value: 48.437000000000005 - type: mrr_at_100 - value: 47.332 + value: 49.096000000000004 - type: mrr_at_1000 - value: 47.368 + value: 49.14 + - type: mrr_at_20 + value: 48.847 - type: mrr_at_3 - value: 44.384 + value: 46.21 - type: mrr_at_5 - value: 45.661 + value: 47.561 - type: ndcg_at_1 - value: 38.408 + value: 40.127 - type: ndcg_at_10 - value: 46.379999999999995 + value: 48.209999999999994 - type: ndcg_at_100 - value: 50.81 + value: 52.632 - type: ndcg_at_1000 - value: 52.663000000000004 + value: 54.59 + - type: ndcg_at_20 + value: 50.012 - type: ndcg_at_3 - value: 42.18 + value: 43.996 - type: ndcg_at_5 - value: 43.974000000000004 + value: 46.122 - type: precision_at_1 - value: 38.408 + value: 40.127 - type: precision_at_10 - value: 8.656 + value: 9.051 - type: precision_at_100 - value: 1.3860000000000001 + value: 1.465 - type: precision_at_1000 - value: 0.184 + value: 0.193 + - type: precision_at_20 + value: 5.35 - type: precision_at_3 - value: 20.276 + value: 21.104 - type: precision_at_5 - value: 14.241999999999999 + value: 15.146 - type: recall_at_1 - value: 30.613 + value: 32.272 - type: recall_at_10 - value: 56.44 + value: 57.870999999999995 - type: recall_at_100 - value: 75.044 + value: 76.211 - type: recall_at_1000 - value: 86.426 + value: 88.389 + - type: recall_at_20 + value: 64.354 - type: recall_at_3 - value: 43.766 + value: 45.426 - type: recall_at_5 - value: 48.998000000000005 + value: 51.23799999999999 - task: type: Retrieval dataset: - type: BeIR/cqadupstack + type: mteb/cqadupstack-gaming name: MTEB CQADupstackGamingRetrieval config: default split: test - revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db + revision: 4885aa143210c98657558c04aaf3dc47cfb54340 metrics: - type: map_at_1 - value: 37.370999999999995 + value: 40.261 - type: map_at_10 - value: 49.718 + value: 53.400000000000006 - type: map_at_100 - value: 50.737 + value: 54.42399999999999 - type: map_at_1000 - value: 50.79 + value: 54.473000000000006 + - type: map_at_20 + value: 54.052 - type: map_at_3 - value: 46.231 + value: 49.763000000000005 - type: map_at_5 - value: 48.329 + value: 51.878 - type: mrr_at_1 - value: 42.884 + value: 46.019 - type: mrr_at_10 - value: 53.176 + value: 56.653 - type: mrr_at_100 - value: 53.81700000000001 + value: 57.28 - type: mrr_at_1000 - value: 53.845 + value: 57.303000000000004 + - type: mrr_at_20 + value: 57.057 - type: mrr_at_3 - value: 50.199000000000005 + value: 53.971000000000004 - type: mrr_at_5 - value: 52.129999999999995 + value: 55.632000000000005 - type: ndcg_at_1 - value: 42.884 + value: 46.019 - type: ndcg_at_10 - value: 55.826 + value: 59.597 - type: ndcg_at_100 - value: 59.93000000000001 + value: 63.452 - type: ndcg_at_1000 - value: 61.013 + value: 64.434 + - type: ndcg_at_20 + value: 61.404 - type: ndcg_at_3 - value: 49.764 + value: 53.620999999999995 - type: ndcg_at_5 - value: 53.025999999999996 + value: 56.688 - type: precision_at_1 - value: 42.884 + value: 46.019 - type: precision_at_10 - value: 9.046999999999999 + value: 9.748999999999999 - type: precision_at_100 - value: 1.212 + value: 1.261 - type: precision_at_1000 - value: 0.135 + value: 0.13799999999999998 + - type: precision_at_20 + value: 5.436 - type: precision_at_3 - value: 22.131999999999998 + value: 24.075 - type: precision_at_5 - value: 15.524 + value: 16.715 - type: recall_at_1 - value: 37.370999999999995 + value: 40.261 - type: recall_at_10 - value: 70.482 + value: 74.522 - type: recall_at_100 - value: 88.425 + value: 91.014 - type: recall_at_1000 - value: 96.03399999999999 + value: 98.017 + - type: recall_at_20 + value: 81.186 - type: recall_at_3 - value: 54.43 + value: 58.72500000000001 - type: recall_at_5 - value: 62.327999999999996 + value: 66.23599999999999 - task: type: Retrieval dataset: - type: BeIR/cqadupstack + type: mteb/cqadupstack-gis name: MTEB CQADupstackGisRetrieval config: default split: test - revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db + revision: 5003b3064772da1887988e05400cf3806fe491f2 metrics: - type: map_at_1 - value: 22.875999999999998 + value: 27.666 - type: map_at_10 - value: 31.715 + value: 36.744 - type: map_at_100 - value: 32.847 + value: 37.794 - type: map_at_1000 - value: 32.922000000000004 + value: 37.865 + - type: map_at_20 + value: 37.336999999999996 - type: map_at_3 - value: 29.049999999999997 + value: 33.833999999999996 - type: map_at_5 - value: 30.396 + value: 35.61 - type: mrr_at_1 - value: 24.52 + value: 29.944 - type: mrr_at_10 - value: 33.497 + value: 38.838 - type: mrr_at_100 - value: 34.455000000000005 + value: 39.765 - type: mrr_at_1000 - value: 34.510000000000005 + value: 39.818999999999996 + - type: mrr_at_20 + value: 39.373000000000005 - type: mrr_at_3 - value: 30.791 + value: 36.234 - type: mrr_at_5 - value: 32.175 + value: 37.844 - type: ndcg_at_1 - value: 24.52 + value: 29.944 - type: ndcg_at_10 - value: 36.95 + value: 41.986000000000004 - type: ndcg_at_100 - value: 42.238 + value: 47.05 - type: ndcg_at_1000 - value: 44.147999999999996 + value: 48.897 + - type: ndcg_at_20 + value: 43.989 - type: ndcg_at_3 - value: 31.435000000000002 + value: 36.452 - type: ndcg_at_5 - value: 33.839000000000006 + value: 39.395 - type: precision_at_1 - value: 24.52 + value: 29.944 - type: precision_at_10 - value: 5.9319999999999995 + value: 6.4750000000000005 - type: precision_at_100 - value: 0.901 + value: 0.946 - type: precision_at_1000 - value: 0.11 + value: 0.11399999999999999 + - type: precision_at_20 + value: 3.6839999999999997 - type: precision_at_3 - value: 13.446 + value: 15.443000000000001 - type: precision_at_5 - value: 9.469 + value: 10.96 - type: recall_at_1 - value: 22.875999999999998 + value: 27.666 - type: recall_at_10 - value: 51.38 + value: 56.172999999999995 - type: recall_at_100 - value: 75.31099999999999 + value: 79.142 - type: recall_at_1000 - value: 89.718 + value: 93.013 + - type: recall_at_20 + value: 63.695 - type: recall_at_3 - value: 36.26 + value: 41.285 - type: recall_at_5 - value: 42.248999999999995 + value: 48.36 - task: type: Retrieval dataset: - type: BeIR/cqadupstack + type: mteb/cqadupstack-mathematica name: MTEB CQADupstackMathematicaRetrieval config: default split: test - revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db + revision: 90fceea13679c63fe563ded68f3b6f06e50061de metrics: - type: map_at_1 - value: 14.984 + value: 17.939 - type: map_at_10 - value: 23.457 + value: 27.301 - type: map_at_100 - value: 24.723 + value: 28.485 - type: map_at_1000 - value: 24.846 + value: 28.616000000000003 + - type: map_at_20 + value: 27.843 - type: map_at_3 - value: 20.873 + value: 24.342 - type: map_at_5 - value: 22.357 + value: 26.259 - type: mrr_at_1 - value: 18.159 + value: 22.761 - type: mrr_at_10 - value: 27.431 + value: 32.391 - type: mrr_at_100 - value: 28.449 + value: 33.297 - type: mrr_at_1000 - value: 28.52 + value: 33.361000000000004 + - type: mrr_at_20 + value: 32.845 - type: mrr_at_3 - value: 24.979000000000003 + value: 29.498 - type: mrr_at_5 - value: 26.447 + value: 31.375999999999998 - type: ndcg_at_1 - value: 18.159 + value: 22.761 - type: ndcg_at_10 - value: 28.627999999999997 + value: 33.036 - type: ndcg_at_100 - value: 34.741 + value: 38.743 - type: ndcg_at_1000 - value: 37.516 + value: 41.568 + - type: ndcg_at_20 + value: 34.838 - type: ndcg_at_3 - value: 23.902 + value: 27.803 - type: ndcg_at_5 - value: 26.294 + value: 30.781 - type: precision_at_1 - value: 18.159 + value: 22.761 - type: precision_at_10 - value: 5.485 + value: 6.132 - type: precision_at_100 - value: 0.985 + value: 1.031 - type: precision_at_1000 - value: 0.136 + value: 0.14200000000000002 + - type: precision_at_20 + value: 3.582 - type: precision_at_3 - value: 11.774 + value: 13.474 - type: precision_at_5 - value: 8.731 + value: 10.123999999999999 - type: recall_at_1 - value: 14.984 + value: 17.939 - type: recall_at_10 - value: 40.198 + value: 45.515 - type: recall_at_100 - value: 67.11500000000001 + value: 70.56700000000001 - type: recall_at_1000 - value: 86.497 + value: 90.306 + - type: recall_at_20 + value: 51.946999999999996 - type: recall_at_3 - value: 27.639000000000003 + value: 31.459 - type: recall_at_5 - value: 33.595000000000006 + value: 39.007 - task: type: Retrieval dataset: - type: BeIR/cqadupstack + type: mteb/cqadupstack-physics name: MTEB CQADupstackPhysicsRetrieval config: default split: test - revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db + revision: 79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4 metrics: - type: map_at_1 - value: 29.067 + value: 31.156 - type: map_at_10 - value: 39.457 + value: 42.317 - type: map_at_100 - value: 40.83 + value: 43.742 - type: map_at_1000 - value: 40.94 + value: 43.852000000000004 + - type: map_at_20 + value: 43.147999999999996 - type: map_at_3 - value: 35.995 + value: 38.981 - type: map_at_5 - value: 38.159 + value: 40.827000000000005 - type: mrr_at_1 - value: 34.937000000000005 + value: 38.401999999999994 - type: mrr_at_10 - value: 44.755 + value: 48.141 - type: mrr_at_100 - value: 45.549 + value: 48.991 - type: mrr_at_1000 - value: 45.589 + value: 49.03 + - type: mrr_at_20 + value: 48.665000000000006 - type: mrr_at_3 - value: 41.947 + value: 45.684999999999995 - type: mrr_at_5 - value: 43.733 + value: 47.042 - type: ndcg_at_1 - value: 34.937000000000005 + value: 38.401999999999994 - type: ndcg_at_10 - value: 45.573 + value: 48.541000000000004 - type: ndcg_at_100 - value: 51.266999999999996 + value: 54.063 - type: ndcg_at_1000 - value: 53.184 + value: 56.005 + - type: ndcg_at_20 + value: 50.895999999999994 - type: ndcg_at_3 - value: 39.961999999999996 + value: 43.352000000000004 - type: ndcg_at_5 - value: 43.02 + value: 45.769 - type: precision_at_1 - value: 34.937000000000005 + value: 38.401999999999994 - type: precision_at_10 - value: 8.296000000000001 + value: 8.738999999999999 - type: precision_at_100 - value: 1.32 + value: 1.335 - type: precision_at_1000 - value: 0.167 + value: 0.16999999999999998 + - type: precision_at_20 + value: 5.164 - type: precision_at_3 - value: 18.8 + value: 20.468 - type: precision_at_5 - value: 13.763 + value: 14.437 - type: recall_at_1 - value: 29.067 + value: 31.156 - type: recall_at_10 - value: 58.298 + value: 61.172000000000004 - type: recall_at_100 - value: 82.25099999999999 + value: 83.772 - type: recall_at_1000 - value: 94.476 + value: 96.192 + - type: recall_at_20 + value: 69.223 - type: recall_at_3 - value: 42.984 + value: 46.628 - type: recall_at_5 - value: 50.658 + value: 53.032000000000004 - task: type: Retrieval dataset: - type: BeIR/cqadupstack + type: mteb/cqadupstack-programmers name: MTEB CQADupstackProgrammersRetrieval config: default split: test - revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db + revision: 6184bc1440d2dbc7612be22b50686b8826d22b32 metrics: - type: map_at_1 - value: 25.985999999999997 + value: 26.741999999999997 - type: map_at_10 - value: 35.746 + value: 36.937 - type: map_at_100 - value: 37.067 + value: 38.452 - type: map_at_1000 - value: 37.191 + value: 38.557 + - type: map_at_20 + value: 37.858999999999995 - type: map_at_3 - value: 32.599000000000004 + value: 33.579 - type: map_at_5 - value: 34.239000000000004 + value: 35.415 - type: mrr_at_1 - value: 31.735000000000003 + value: 32.991 - type: mrr_at_10 - value: 40.515 + value: 42.297000000000004 - type: mrr_at_100 - value: 41.459 + value: 43.282 - type: mrr_at_1000 - value: 41.516 + value: 43.332 + - type: mrr_at_20 + value: 42.95 - type: mrr_at_3 - value: 37.938 + value: 39.707 - type: mrr_at_5 - value: 39.25 + value: 41.162 - type: ndcg_at_1 - value: 31.735000000000003 + value: 32.991 - type: ndcg_at_10 - value: 41.484 + value: 43.004999999999995 - type: ndcg_at_100 - value: 47.047 + value: 49.053000000000004 - type: ndcg_at_1000 - value: 49.427 + value: 51.166999999999994 + - type: ndcg_at_20 + value: 45.785 - type: ndcg_at_3 - value: 36.254999999999995 + value: 37.589 - type: ndcg_at_5 - value: 38.375 + value: 40.007999999999996 - type: precision_at_1 - value: 31.735000000000003 + value: 32.991 - type: precision_at_10 - value: 7.66 + value: 8.025 - type: precision_at_100 - value: 1.234 + value: 1.268 - type: precision_at_1000 - value: 0.16 + value: 0.163 + - type: precision_at_20 + value: 4.846 - type: precision_at_3 - value: 17.427999999999997 + value: 17.922 - type: precision_at_5 - value: 12.328999999999999 + value: 13.059000000000001 - type: recall_at_1 - value: 25.985999999999997 + value: 26.741999999999997 - type: recall_at_10 - value: 53.761 + value: 55.635999999999996 - type: recall_at_100 - value: 77.149 + value: 80.798 - type: recall_at_1000 - value: 93.342 + value: 94.918 + - type: recall_at_20 + value: 65.577 - type: recall_at_3 - value: 39.068000000000005 + value: 40.658 - type: recall_at_5 - value: 44.693 + value: 46.812 - task: type: Retrieval dataset: - type: BeIR/cqadupstack + type: mteb/cqadupstack name: MTEB CQADupstackRetrieval config: default split: test - revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db + revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 metrics: - type: map_at_1 - value: 24.949749999999998 + value: 27.274583333333336 - type: map_at_10 - value: 34.04991666666667 + value: 37.04091666666666 - type: map_at_100 - value: 35.26825 + value: 38.27966666666667 - type: map_at_1000 - value: 35.38316666666667 + value: 38.39383333333334 + - type: map_at_20 + value: 37.721500000000006 - type: map_at_3 - value: 31.181333333333335 + value: 33.937999999999995 - type: map_at_5 - value: 32.77391666666667 + value: 35.67974999999999 - type: mrr_at_1 - value: 29.402833333333334 + value: 32.40525 - type: mrr_at_10 - value: 38.01633333333333 + value: 41.43925000000001 - type: mrr_at_100 - value: 38.88033333333334 + value: 42.271 - type: mrr_at_1000 - value: 38.938500000000005 + value: 42.32416666666667 + - type: mrr_at_20 + value: 41.92733333333334 - type: mrr_at_3 - value: 35.5175 + value: 38.84941666666666 - type: mrr_at_5 - value: 36.93808333333333 + value: 40.379583333333336 - type: ndcg_at_1 - value: 29.402833333333334 + value: 32.40525 - type: ndcg_at_10 - value: 39.403166666666664 + value: 42.73808333333334 - type: ndcg_at_100 - value: 44.66408333333333 + value: 47.88941666666667 - type: ndcg_at_1000 - value: 46.96283333333333 + value: 50.05008333333334 + - type: ndcg_at_20 + value: 44.74183333333334 - type: ndcg_at_3 - value: 34.46633333333334 + value: 37.51908333333334 - type: ndcg_at_5 - value: 36.78441666666667 + value: 40.01883333333333 - type: precision_at_1 - value: 29.402833333333334 + value: 32.40525 - type: precision_at_10 - value: 6.965833333333333 + value: 7.5361666666666665 - type: precision_at_100 - value: 1.1330833333333334 + value: 1.1934166666666666 - type: precision_at_1000 - value: 0.15158333333333335 + value: 0.1575 + - type: precision_at_20 + value: 4.429166666666667 - type: precision_at_3 - value: 15.886666666666665 + value: 17.24941666666667 - type: precision_at_5 - value: 11.360416666666667 + value: 12.362333333333336 - type: recall_at_1 - value: 24.949749999999998 + value: 27.274583333333336 - type: recall_at_10 - value: 51.29325 + value: 55.21358333333334 - type: recall_at_100 - value: 74.3695 + value: 77.60366666666667 - type: recall_at_1000 - value: 90.31299999999999 + value: 92.43691666666666 + - type: recall_at_20 + value: 62.474583333333335 - type: recall_at_3 - value: 37.580083333333334 + value: 40.79375 - type: recall_at_5 - value: 43.529666666666664 + value: 47.15158333333334 - task: type: Retrieval dataset: - type: BeIR/cqadupstack + type: mteb/cqadupstack-stats name: MTEB CQADupstackStatsRetrieval config: default split: test - revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db + revision: 65ac3a16b8e91f9cee4c9828cc7c335575432a2a metrics: - type: map_at_1 - value: 22.081999999999997 + value: 27.389999999999997 - type: map_at_10 - value: 29.215999999999998 + value: 34.107 - type: map_at_100 - value: 30.163 + value: 35.022999999999996 - type: map_at_1000 - value: 30.269000000000002 + value: 35.13 + - type: map_at_20 + value: 34.605999999999995 - type: map_at_3 - value: 26.942 + value: 32.021 - type: map_at_5 - value: 28.236 + value: 32.948 - type: mrr_at_1 - value: 24.847 + value: 30.982 - type: mrr_at_10 - value: 31.918999999999997 + value: 37.345 - type: mrr_at_100 - value: 32.817 + value: 38.096999999999994 - type: mrr_at_1000 - value: 32.897 + value: 38.179 + - type: mrr_at_20 + value: 37.769000000000005 - type: mrr_at_3 - value: 29.831000000000003 + value: 35.481 - type: mrr_at_5 - value: 31.019999999999996 + value: 36.293 - type: ndcg_at_1 - value: 24.847 + value: 30.982 - type: ndcg_at_10 - value: 33.4 + value: 38.223 - type: ndcg_at_100 - value: 38.354 + value: 42.686 - type: ndcg_at_1000 - value: 41.045 + value: 45.352 + - type: ndcg_at_20 + value: 39.889 - type: ndcg_at_3 - value: 29.236 + value: 34.259 - type: ndcg_at_5 - value: 31.258000000000003 + value: 35.664 - type: precision_at_1 - value: 24.847 + value: 30.982 - type: precision_at_10 - value: 5.353 + value: 5.7669999999999995 - type: precision_at_100 - value: 0.853 + value: 0.877 - type: precision_at_1000 - value: 0.116 + value: 0.11800000000000001 + - type: precision_at_20 + value: 3.3360000000000003 - type: precision_at_3 - value: 12.679000000000002 + value: 14.264 - type: precision_at_5 - value: 8.988 + value: 9.54 - type: recall_at_1 - value: 22.081999999999997 + value: 27.389999999999997 - type: recall_at_10 - value: 43.505 + value: 48.009 - type: recall_at_100 - value: 66.45400000000001 + value: 68.244 - type: recall_at_1000 - value: 86.378 + value: 87.943 + - type: recall_at_20 + value: 54.064 - type: recall_at_3 - value: 32.163000000000004 + value: 36.813 - type: recall_at_5 - value: 37.059999999999995 + value: 40.321 - task: type: Retrieval dataset: - type: BeIR/cqadupstack + type: mteb/cqadupstack-tex name: MTEB CQADupstackTexRetrieval config: default split: test - revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db + revision: 46989137a86843e03a6195de44b09deda022eec7 metrics: - type: map_at_1 - value: 15.540000000000001 + value: 18.249000000000002 - type: map_at_10 - value: 22.362000000000002 + value: 25.907000000000004 - type: map_at_100 - value: 23.435 + value: 27.105 - type: map_at_1000 - value: 23.564 + value: 27.233 + - type: map_at_20 + value: 26.541999999999998 - type: map_at_3 - value: 20.143 + value: 23.376 - type: map_at_5 - value: 21.324 + value: 24.673000000000002 - type: mrr_at_1 - value: 18.892 + value: 21.989 - type: mrr_at_10 - value: 25.942999999999998 + value: 29.846 - type: mrr_at_100 - value: 26.883000000000003 + value: 30.808999999999997 - type: mrr_at_1000 - value: 26.968999999999998 + value: 30.885 + - type: mrr_at_20 + value: 30.384 - type: mrr_at_3 - value: 23.727 + value: 27.46 - type: mrr_at_5 - value: 24.923000000000002 + value: 28.758 - type: ndcg_at_1 - value: 18.892 + value: 21.989 - type: ndcg_at_10 - value: 26.811 + value: 30.874000000000002 - type: ndcg_at_100 - value: 32.066 + value: 36.504999999999995 - type: ndcg_at_1000 - value: 35.166 + value: 39.314 + - type: ndcg_at_20 + value: 32.952999999999996 - type: ndcg_at_3 - value: 22.706 + value: 26.249 - type: ndcg_at_5 - value: 24.508 + value: 28.229 - type: precision_at_1 - value: 18.892 + value: 21.989 - type: precision_at_10 - value: 4.942 + value: 5.705 - type: precision_at_100 - value: 0.878 + value: 0.9990000000000001 - type: precision_at_1000 - value: 0.131 + value: 0.14100000000000001 + - type: precision_at_20 + value: 3.4459999999999997 - type: precision_at_3 - value: 10.748000000000001 + value: 12.377 - type: precision_at_5 - value: 7.784000000000001 + value: 8.961 - type: recall_at_1 - value: 15.540000000000001 + value: 18.249000000000002 - type: recall_at_10 - value: 36.742999999999995 + value: 41.824 - type: recall_at_100 - value: 60.525 + value: 67.071 - type: recall_at_1000 - value: 82.57600000000001 + value: 86.863 + - type: recall_at_20 + value: 49.573 - type: recall_at_3 - value: 25.252000000000002 + value: 28.92 - type: recall_at_5 - value: 29.872 + value: 34.003 - task: type: Retrieval dataset: - type: BeIR/cqadupstack + type: mteb/cqadupstack-unix name: MTEB CQADupstackUnixRetrieval config: default split: test - revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db + revision: 6c6430d3a6d36f8d2a829195bc5dc94d7e063e53 metrics: - type: map_at_1 - value: 24.453 + value: 26.602999999999998 - type: map_at_10 - value: 33.363 + value: 36.818 - type: map_at_100 - value: 34.579 + value: 37.894 - type: map_at_1000 - value: 34.686 + value: 37.991 + - type: map_at_20 + value: 37.389 - type: map_at_3 - value: 30.583 + value: 33.615 - type: map_at_5 - value: 32.118 + value: 35.432 - type: mrr_at_1 - value: 28.918 + value: 31.53 - type: mrr_at_10 - value: 37.675 + value: 41.144 - type: mrr_at_100 - value: 38.567 + value: 41.937999999999995 - type: mrr_at_1000 - value: 38.632 + value: 41.993 + - type: mrr_at_20 + value: 41.585 - type: mrr_at_3 - value: 35.260999999999996 + value: 38.385999999999996 - type: mrr_at_5 - value: 36.576 + value: 39.995000000000005 - type: ndcg_at_1 - value: 28.918 + value: 31.53 - type: ndcg_at_10 - value: 38.736 + value: 42.792 - type: ndcg_at_100 - value: 44.261 + value: 47.749 - type: ndcg_at_1000 - value: 46.72 + value: 49.946 + - type: ndcg_at_20 + value: 44.59 - type: ndcg_at_3 - value: 33.81 + value: 37.025000000000006 - type: ndcg_at_5 - value: 36.009 + value: 39.811 - type: precision_at_1 - value: 28.918 + value: 31.53 - type: precision_at_10 - value: 6.586 + value: 7.2669999999999995 - type: precision_at_100 - value: 1.047 + value: 1.109 - type: precision_at_1000 - value: 0.13699999999999998 + value: 0.14100000000000001 + - type: precision_at_20 + value: 4.184 - type: precision_at_3 - value: 15.360999999999999 + value: 16.791 - type: precision_at_5 - value: 10.857999999999999 + value: 12.09 - type: recall_at_1 - value: 24.453 + value: 26.602999999999998 - type: recall_at_10 - value: 50.885999999999996 + value: 56.730999999999995 - type: recall_at_100 - value: 75.03 + value: 78.119 - type: recall_at_1000 - value: 92.123 + value: 93.458 + - type: recall_at_20 + value: 63.00599999999999 - type: recall_at_3 - value: 37.138 + value: 41.306 - type: recall_at_5 - value: 42.864999999999995 + value: 48.004999999999995 - task: type: Retrieval dataset: - type: BeIR/cqadupstack + type: mteb/cqadupstack-webmasters name: MTEB CQADupstackWebmastersRetrieval config: default split: test - revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db + revision: 160c094312a0e1facb97e55eeddb698c0abe3571 metrics: - type: map_at_1 - value: 24.57 + value: 23.988 - type: map_at_10 - value: 33.672000000000004 + value: 33.650999999999996 - type: map_at_100 - value: 35.244 + value: 35.263 - type: map_at_1000 - value: 35.467 + value: 35.481 + - type: map_at_20 + value: 34.463 - type: map_at_3 - value: 30.712 + value: 30.330000000000002 - type: map_at_5 - value: 32.383 + value: 32.056000000000004 - type: mrr_at_1 value: 29.644 - type: mrr_at_10 - value: 38.344 + value: 38.987 - type: mrr_at_100 - value: 39.219 + value: 39.973 - type: mrr_at_1000 - value: 39.282000000000004 + value: 40.013 + - type: mrr_at_20 + value: 39.553 - type: mrr_at_3 - value: 35.771 + value: 36.001 - type: mrr_at_5 - value: 37.273 + value: 37.869 - type: ndcg_at_1 value: 29.644 - type: ndcg_at_10 - value: 39.567 + value: 40.156 - type: ndcg_at_100 - value: 45.097 + value: 46.244 - type: ndcg_at_1000 - value: 47.923 + value: 48.483 + - type: ndcg_at_20 + value: 42.311 - type: ndcg_at_3 - value: 34.768 + value: 34.492 - type: ndcg_at_5 - value: 37.122 + value: 37.118 - type: precision_at_1 value: 29.644 - type: precision_at_10 - value: 7.5889999999999995 + value: 7.925 - type: precision_at_100 - value: 1.478 + value: 1.5890000000000002 - type: precision_at_1000 - value: 0.23500000000000001 + value: 0.245 + - type: precision_at_20 + value: 4.97 - type: precision_at_3 - value: 16.337 + value: 16.469 - type: precision_at_5 - value: 12.055 + value: 12.174 - type: recall_at_1 - value: 24.57 + value: 23.988 - type: recall_at_10 - value: 51.00900000000001 + value: 52.844 - type: recall_at_100 - value: 75.423 + value: 80.143 - type: recall_at_1000 - value: 93.671 + value: 93.884 + - type: recall_at_20 + value: 61.050000000000004 - type: recall_at_3 - value: 36.925999999999995 + value: 36.720000000000006 - type: recall_at_5 - value: 43.245 + value: 43.614999999999995 - task: type: Retrieval dataset: - type: BeIR/cqadupstack + type: mteb/cqadupstack-wordpress name: MTEB CQADupstackWordpressRetrieval config: default split: test - revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db + revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 metrics: - type: map_at_1 - value: 21.356 + value: 21.947 - type: map_at_10 - value: 27.904 + value: 29.902 - type: map_at_100 - value: 28.938000000000002 + value: 30.916 - type: map_at_1000 - value: 29.036 + value: 31.016 + - type: map_at_20 + value: 30.497999999999998 - type: map_at_3 - value: 25.726 + value: 27.044 - type: map_at_5 - value: 26.935 + value: 28.786 - type: mrr_at_1 - value: 22.551 + value: 23.845 - type: mrr_at_10 - value: 29.259 + value: 32.073 - type: mrr_at_100 - value: 30.272 + value: 32.940999999999995 - type: mrr_at_1000 - value: 30.348000000000003 + value: 33.015 + - type: mrr_at_20 + value: 32.603 - type: mrr_at_3 - value: 27.295 + value: 29.205 - type: mrr_at_5 - value: 28.358 + value: 31.044 - type: ndcg_at_1 - value: 22.551 + value: 23.845 - type: ndcg_at_10 - value: 31.817 + value: 34.79 - type: ndcg_at_100 - value: 37.164 + value: 39.573 - type: ndcg_at_1000 - value: 39.82 + value: 42.163000000000004 + - type: ndcg_at_20 + value: 36.778 - type: ndcg_at_3 - value: 27.595999999999997 + value: 29.326 - type: ndcg_at_5 - value: 29.568 + value: 32.289 - type: precision_at_1 - value: 22.551 + value: 23.845 - type: precision_at_10 - value: 4.917 + value: 5.527 - type: precision_at_100 - value: 0.828 + value: 0.847 - type: precision_at_1000 - value: 0.11399999999999999 + value: 0.11900000000000001 + - type: precision_at_20 + value: 3.2439999999999998 - type: precision_at_3 - value: 11.583 + value: 12.384 - type: precision_at_5 - value: 8.133 + value: 9.205 - type: recall_at_1 - value: 21.356 + value: 21.947 - type: recall_at_10 - value: 42.489 + value: 47.713 - type: recall_at_100 - value: 67.128 + value: 69.299 - type: recall_at_1000 - value: 87.441 + value: 88.593 + - type: recall_at_20 + value: 55.032000000000004 - type: recall_at_3 - value: 31.165 + value: 33.518 - type: recall_at_5 - value: 35.853 + value: 40.427 - task: type: Retrieval dataset: - type: climate-fever + type: mteb/climate-fever name: MTEB ClimateFEVER config: default split: test - revision: 392b78eb68c07badcd7c2cd8f39af108375dfcce + revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380 metrics: - type: map_at_1 - value: 12.306000000000001 + value: 13.655999999999999 - type: map_at_10 - value: 21.523 + value: 23.954 - type: map_at_100 - value: 23.358 + value: 26.07 - type: map_at_1000 - value: 23.541 + value: 26.266000000000002 + - type: map_at_20 + value: 25.113000000000003 - type: map_at_3 - value: 17.809 + value: 19.85 - type: map_at_5 - value: 19.631 + value: 21.792 - type: mrr_at_1 - value: 27.948 + value: 31.075000000000003 - type: mrr_at_10 - value: 40.355000000000004 + value: 43.480000000000004 - type: mrr_at_100 - value: 41.166000000000004 + value: 44.39 - type: mrr_at_1000 - value: 41.203 + value: 44.42 + - type: mrr_at_20 + value: 44.06 - type: mrr_at_3 - value: 36.819 + value: 40.38 - type: mrr_at_5 - value: 38.958999999999996 + value: 42.138999999999996 - type: ndcg_at_1 - value: 27.948 + value: 31.075000000000003 - type: ndcg_at_10 - value: 30.462 + value: 33.129999999999995 - type: ndcg_at_100 - value: 37.473 + value: 40.794000000000004 - type: ndcg_at_1000 - value: 40.717999999999996 + value: 44.062 + - type: ndcg_at_20 + value: 36.223 - type: ndcg_at_3 - value: 24.646 + value: 27.224999999999998 - type: ndcg_at_5 - value: 26.642 + value: 28.969 - type: precision_at_1 - value: 27.948 + value: 31.075000000000003 - type: precision_at_10 - value: 9.648 + value: 10.476 - type: precision_at_100 - value: 1.7239999999999998 + value: 1.864 - type: precision_at_1000 - value: 0.232 + value: 0.247 + - type: precision_at_20 + value: 6.593 - type: precision_at_3 - value: 18.48 + value: 20.456 - type: precision_at_5 - value: 14.293 + value: 15.440000000000001 - type: recall_at_1 - value: 12.306000000000001 + value: 13.655999999999999 - type: recall_at_10 - value: 37.181 + value: 39.678000000000004 - type: recall_at_100 - value: 61.148 + value: 65.523 - type: recall_at_1000 - value: 79.401 + value: 83.59100000000001 + - type: recall_at_20 + value: 48.27 - type: recall_at_3 - value: 22.883 + value: 24.863 - type: recall_at_5 - value: 28.59 + value: 30.453999999999997 - task: type: Retrieval dataset: - type: dbpedia-entity + type: mteb/dbpedia name: MTEB DBPedia config: default split: test - revision: f097057d03ed98220bc7309ddb10b71a54d667d6 + revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659 metrics: - type: map_at_1 - value: 9.357 + value: 9.139 - type: map_at_10 - value: 18.849 + value: 20.366999999999997 - type: map_at_100 - value: 25.369000000000003 + value: 29.755 - type: map_at_1000 - value: 26.950000000000003 + value: 31.563999999999997 + - type: map_at_20 + value: 24.021 - type: map_at_3 - value: 13.625000000000002 + value: 14.395 - type: map_at_5 - value: 15.956999999999999 + value: 16.853 - type: mrr_at_1 - value: 67.75 + value: 69.0 - type: mrr_at_10 - value: 74.734 + value: 76.778 - type: mrr_at_100 - value: 75.1 + value: 77.116 - type: mrr_at_1000 - value: 75.10900000000001 + value: 77.12299999999999 + - type: mrr_at_20 + value: 77.046 - type: mrr_at_3 - value: 73.542 + value: 75.208 - type: mrr_at_5 - value: 74.167 + value: 76.146 - type: ndcg_at_1 - value: 55.375 + value: 57.125 - type: ndcg_at_10 - value: 39.873999999999995 + value: 42.84 - type: ndcg_at_100 - value: 43.098 + value: 48.686 - type: ndcg_at_1000 - value: 50.69200000000001 + value: 56.294 + - type: ndcg_at_20 + value: 42.717 - type: ndcg_at_3 - value: 44.856 + value: 46.842 - type: ndcg_at_5 - value: 42.138999999999996 + value: 44.248 - type: precision_at_1 - value: 67.75 + value: 69.0 - type: precision_at_10 - value: 31.1 + value: 34.625 - type: precision_at_100 - value: 9.303 + value: 11.468 - type: precision_at_1000 - value: 2.0060000000000002 + value: 2.17 + - type: precision_at_20 + value: 26.562 - type: precision_at_3 - value: 48.25 + value: 50.917 - type: precision_at_5 - value: 40.949999999999996 + value: 43.35 - type: recall_at_1 - value: 9.357 + value: 9.139 - type: recall_at_10 - value: 23.832 + value: 26.247999999999998 - type: recall_at_100 - value: 47.906 + value: 56.647000000000006 - type: recall_at_1000 - value: 71.309 + value: 80.784 + - type: recall_at_20 + value: 35.010999999999996 - type: recall_at_3 - value: 14.512 + value: 15.57 - type: recall_at_5 - value: 18.3 + value: 19.198 - task: type: Classification dataset: @@ -1253,219 +1523,249 @@ model-index: name: MTEB EmotionClassification config: default split: test - revision: 829147f8f75a25f005913200eb5ed41fae320aa1 + revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 metrics: - type: accuracy - value: 49.655 + value: 55.93 - type: f1 - value: 45.51976190938951 + value: 49.35314406745291 - task: type: Retrieval dataset: - type: fever + type: mteb/fever name: MTEB FEVER config: default split: test - revision: 1429cf27e393599b8b359b9b72c666f96b2525f9 + revision: bea83ef9e8fb933d90a2f1d5515737465d613e12 metrics: - type: map_at_1 - value: 62.739999999999995 + value: 73.198 - type: map_at_10 - value: 73.07000000000001 + value: 81.736 - type: map_at_100 - value: 73.398 + value: 82.02000000000001 - type: map_at_1000 - value: 73.41 + value: 82.03399999999999 + - type: map_at_20 + value: 81.937 - type: map_at_3 - value: 71.33800000000001 + value: 80.692 - type: map_at_5 - value: 72.423 + value: 81.369 - type: mrr_at_1 - value: 67.777 + value: 78.803 - type: mrr_at_10 - value: 77.873 + value: 86.144 - type: mrr_at_100 - value: 78.091 + value: 86.263 - type: mrr_at_1000 - value: 78.094 + value: 86.26599999999999 + - type: mrr_at_20 + value: 86.235 - type: mrr_at_3 - value: 76.375 + value: 85.464 - type: mrr_at_5 - value: 77.316 + value: 85.95 - type: ndcg_at_1 - value: 67.777 + value: 78.803 - type: ndcg_at_10 - value: 78.24 + value: 85.442 - type: ndcg_at_100 - value: 79.557 + value: 86.422 - type: ndcg_at_1000 - value: 79.814 + value: 86.68900000000001 + - type: ndcg_at_20 + value: 85.996 - type: ndcg_at_3 - value: 75.125 + value: 83.839 - type: ndcg_at_5 - value: 76.834 + value: 84.768 - type: precision_at_1 - value: 67.777 + value: 78.803 - type: precision_at_10 - value: 9.832 + value: 10.261000000000001 - type: precision_at_100 - value: 1.061 + value: 1.0959999999999999 - type: precision_at_1000 - value: 0.11 + value: 0.11399999999999999 + - type: precision_at_20 + value: 5.286 - type: precision_at_3 - value: 29.433 + value: 32.083 - type: precision_at_5 - value: 18.665000000000003 + value: 19.898 - type: recall_at_1 - value: 62.739999999999995 + value: 73.198 - type: recall_at_10 - value: 89.505 + value: 92.42099999999999 - type: recall_at_100 - value: 95.102 + value: 96.28 - type: recall_at_1000 - value: 96.825 + value: 97.995 + - type: recall_at_20 + value: 94.36 - type: recall_at_3 - value: 81.028 + value: 88.042 - type: recall_at_5 - value: 85.28099999999999 + value: 90.429 - task: type: Retrieval dataset: - type: fiqa + type: mteb/fiqa name: MTEB FiQA2018 config: default split: test - revision: 41b686a7f28c59bcaaa5791efd47c67c8ebe28be + revision: 27a168819829fe9bcd655c2df245fb19452e8e06 metrics: - type: map_at_1 - value: 18.467 + value: 21.583 - type: map_at_10 - value: 30.020999999999997 + value: 36.503 - type: map_at_100 - value: 31.739 + value: 38.529 - type: map_at_1000 - value: 31.934 + value: 38.701 + - type: map_at_20 + value: 37.69 - type: map_at_3 - value: 26.003 + value: 31.807000000000002 - type: map_at_5 - value: 28.338 + value: 34.424 - type: mrr_at_1 - value: 35.339999999999996 + value: 43.827 - type: mrr_at_10 - value: 44.108999999999995 + value: 53.528 - type: mrr_at_100 - value: 44.993 + value: 54.291 - type: mrr_at_1000 - value: 45.042 + value: 54.32599999999999 + - type: mrr_at_20 + value: 54.064 - type: mrr_at_3 - value: 41.667 + value: 51.25999999999999 - type: mrr_at_5 - value: 43.14 + value: 52.641000000000005 - type: ndcg_at_1 - value: 35.339999999999996 + value: 43.827 - type: ndcg_at_10 - value: 37.202 + value: 44.931 - type: ndcg_at_100 - value: 43.852999999999994 + value: 51.778999999999996 - type: ndcg_at_1000 - value: 47.235 + value: 54.532000000000004 + - type: ndcg_at_20 + value: 47.899 - type: ndcg_at_3 - value: 33.5 + value: 41.062 - type: ndcg_at_5 - value: 34.985 + value: 42.33 - type: precision_at_1 - value: 35.339999999999996 + value: 43.827 - type: precision_at_10 - value: 10.247 + value: 12.608 - type: precision_at_100 - value: 1.7149999999999999 + value: 1.974 - type: precision_at_1000 - value: 0.232 + value: 0.247 + - type: precision_at_20 + value: 7.585 - type: precision_at_3 - value: 22.222 + value: 27.778000000000002 - type: precision_at_5 - value: 16.573999999999998 + value: 20.308999999999997 - type: recall_at_1 - value: 18.467 + value: 21.583 - type: recall_at_10 - value: 44.080999999999996 + value: 52.332 - type: recall_at_100 - value: 68.72200000000001 + value: 77.256 - type: recall_at_1000 - value: 89.087 + value: 93.613 + - type: recall_at_20 + value: 61.413 - type: recall_at_3 - value: 30.567 + value: 37.477 - type: recall_at_5 - value: 36.982 + value: 44.184 - task: type: Retrieval dataset: - type: hotpotqa + type: mteb/hotpotqa name: MTEB HotpotQA config: default split: test - revision: 766870b35a1b9ca65e67a0d1913899973551fc6c + revision: ab518f4d6fcca38d87c25209f94beba119d02014 metrics: - type: map_at_1 - value: 35.726 + value: 39.845000000000006 - type: map_at_10 - value: 50.207 + value: 64.331 - type: map_at_100 - value: 51.05499999999999 + value: 65.202 - type: map_at_1000 - value: 51.12799999999999 + value: 65.261 + - type: map_at_20 + value: 64.833 - type: map_at_3 - value: 47.576 + value: 60.663 - type: map_at_5 - value: 49.172 + value: 62.94 - type: mrr_at_1 - value: 71.452 + value: 79.689 - type: mrr_at_10 - value: 77.41900000000001 + value: 85.299 - type: mrr_at_100 - value: 77.711 + value: 85.461 - type: mrr_at_1000 - value: 77.723 + value: 85.466 + - type: mrr_at_20 + value: 85.39099999999999 - type: mrr_at_3 - value: 76.39399999999999 + value: 84.396 - type: mrr_at_5 - value: 77.00099999999999 + value: 84.974 - type: ndcg_at_1 - value: 71.452 + value: 79.689 - type: ndcg_at_10 - value: 59.260999999999996 + value: 72.49 - type: ndcg_at_100 - value: 62.424 + value: 75.485 - type: ndcg_at_1000 - value: 63.951 + value: 76.563 + - type: ndcg_at_20 + value: 73.707 - type: ndcg_at_3 - value: 55.327000000000005 + value: 67.381 - type: ndcg_at_5 - value: 57.416999999999994 + value: 70.207 - type: precision_at_1 - value: 71.452 + value: 79.689 - type: precision_at_10 - value: 12.061 + value: 15.267 - type: precision_at_100 - value: 1.455 + value: 1.7610000000000001 - type: precision_at_1000 - value: 0.166 + value: 0.19 + - type: precision_at_20 + value: 8.024000000000001 - type: precision_at_3 - value: 34.36 + value: 43.363 - type: precision_at_5 - value: 22.266 + value: 28.248 - type: recall_at_1 - value: 35.726 + value: 39.845000000000006 - type: recall_at_10 - value: 60.304 + value: 76.334 - type: recall_at_100 - value: 72.75500000000001 + value: 88.042 - type: recall_at_1000 - value: 82.978 + value: 95.09100000000001 + - type: recall_at_20 + value: 80.243 - type: recall_at_3 - value: 51.54 + value: 65.044 - type: recall_at_5 - value: 55.665 + value: 70.621 - task: type: Classification dataset: @@ -1473,83 +1773,93 @@ model-index: name: MTEB ImdbClassification config: default split: test - revision: 8d743909f834c38949e8323a8a6ce8721ea6c7f4 + revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 metrics: - type: accuracy - value: 66.63759999999999 + value: 93.57079999999999 - type: ap - value: 61.48938261286748 + value: 90.50045924786099 - type: f1 - value: 66.35089269264965 + value: 93.56673497845476 - task: type: Retrieval dataset: - type: msmarco + type: mteb/msmarco name: MTEB MSMARCO config: default - split: validation - revision: e6838a846e2408f22cf5cc337ebc83e0bcf77849 + split: dev + revision: c5a29a104738b98a9e76336939199e264163d4a0 metrics: - type: map_at_1 - value: 20.842 + value: 22.212 - type: map_at_10 - value: 32.992 + value: 34.528 - type: map_at_100 - value: 34.236 + value: 35.69 - type: map_at_1000 - value: 34.286 + value: 35.74 + - type: map_at_20 + value: 35.251 - type: map_at_3 - value: 29.049000000000003 + value: 30.628 - type: map_at_5 - value: 31.391999999999996 + value: 32.903999999999996 - type: mrr_at_1 - value: 21.375 + value: 22.794 - type: mrr_at_10 - value: 33.581 + value: 35.160000000000004 - type: mrr_at_100 - value: 34.760000000000005 + value: 36.251 - type: mrr_at_1000 - value: 34.803 + value: 36.295 + - type: mrr_at_20 + value: 35.845 - type: mrr_at_3 - value: 29.704000000000004 + value: 31.328 - type: mrr_at_5 - value: 32.015 + value: 33.574 - type: ndcg_at_1 - value: 21.375 + value: 22.779 - type: ndcg_at_10 - value: 39.905 + value: 41.461 - type: ndcg_at_100 - value: 45.843 + value: 47.049 - type: ndcg_at_1000 - value: 47.083999999999996 + value: 48.254000000000005 + - type: ndcg_at_20 + value: 44.031 - type: ndcg_at_3 - value: 31.918999999999997 + value: 33.561 - type: ndcg_at_5 - value: 36.107 + value: 37.62 - type: precision_at_1 - value: 21.375 + value: 22.779 - type: precision_at_10 - value: 6.393 + value: 6.552 - type: precision_at_100 - value: 0.935 + value: 0.936 - type: precision_at_1000 value: 0.104 + - type: precision_at_20 + value: 3.8120000000000003 - type: precision_at_3 - value: 13.663 + value: 14.274000000000001 - type: precision_at_5 - value: 10.324 + value: 10.622 - type: recall_at_1 - value: 20.842 + value: 22.212 - type: recall_at_10 - value: 61.17 + value: 62.732 - type: recall_at_100 - value: 88.518 + value: 88.567 - type: recall_at_1000 - value: 97.993 + value: 97.727 + - type: recall_at_20 + value: 72.733 - type: recall_at_3 - value: 39.571 + value: 41.367 - type: recall_at_5 - value: 49.653999999999996 + value: 51.105999999999995 - task: type: Classification dataset: @@ -1557,531 +1867,2211 @@ model-index: name: MTEB MTOPDomainClassification (en) config: en split: test - revision: a7e2a951126a26fc8c6a69f835f33a346ba259e3 + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf metrics: - type: accuracy - value: 93.46557227542178 + value: 94.24988600091199 - type: f1 - value: 92.87345917772146 + value: 94.06064583085202 - task: type: Classification dataset: - type: mteb/mtop_intent - name: MTEB MTOPIntentClassification (en) - config: en + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (de) + config: de split: test - revision: 6299947a7777084cc2d4b64235bf7190381ce755 + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf metrics: - type: accuracy - value: 72.42134062927497 + value: 74.86052409129333 - type: f1 - value: 55.03624810959269 + value: 72.24661442078647 - task: type: Classification dataset: - type: mteb/amazon_massive_intent - name: MTEB MassiveIntentClassification (en) - config: en + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (es) + config: es split: test - revision: 072a486a144adf7f4479a4a0dddb2152e161e1ea + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf metrics: - type: accuracy - value: 70.3866845998655 + value: 77.09139426284189 - type: f1 - value: 68.9674519872921 + value: 76.3725044443502 - task: type: Classification dataset: - type: mteb/amazon_massive_scenario - name: MTEB MassiveScenarioClassification (en) - config: en + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (fr) + config: fr split: test - revision: 7d571f92784cd94a019292a1f45445077d0ef634 + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf metrics: - type: accuracy - value: 76.27774041694687 + value: 79.79956154087064 - type: f1 - value: 76.72936190462792 + value: 78.41859658401724 - task: - type: Clustering + type: Classification dataset: - type: mteb/medrxiv-clustering-p2p - name: MTEB MedrxivClusteringP2P - config: default + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (hi) + config: hi split: test - revision: dcefc037ef84348e49b0d29109e891c01067226b + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf metrics: - - type: v_measure - value: 31.511745925773337 + - type: accuracy + value: 32.785944783076374 + - type: f1 + value: 31.182237278594922 - task: - type: Clustering + type: Classification dataset: - type: mteb/medrxiv-clustering-s2s - name: MTEB MedrxivClusteringS2S - config: default + type: mteb/mtop_domain + name: MTEB MTOPDomainClassification (th) + config: th split: test - revision: 3cd0e71dfbe09d4de0f9e5ecba43e7ce280959dc + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf metrics: - - type: v_measure - value: 28.764235987575365 + - type: accuracy + value: 16.654611211573236 + - type: f1 + value: 12.088413093236642 - task: - type: Reranking + type: Classification dataset: - type: mteb/mind_small - name: MTEB MindSmallReranking - config: default + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (en) + config: en split: test - revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba metrics: - - type: map - value: 32.29353136386601 - - type: mrr - value: 33.536774455851685 + - type: accuracy + value: 67.51481988144094 + - type: f1 + value: 49.561420234732125 - task: - type: Retrieval + type: Classification dataset: - type: nfcorpus - name: MTEB NFCorpus - config: default + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (de) + config: de split: test - revision: 7eb63cc0c1eb59324d709ebed25fcab851fa7610 + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba metrics: - - type: map_at_1 - value: 5.702 - - type: map_at_10 - value: 13.642000000000001 - - type: map_at_100 - value: 17.503 - - type: map_at_1000 - value: 19.126 - - type: map_at_3 - value: 9.748 - - type: map_at_5 - value: 11.642 - - type: mrr_at_1 - value: 45.82 - - type: mrr_at_10 - value: 54.821 - - type: mrr_at_100 - value: 55.422000000000004 - - type: mrr_at_1000 - value: 55.452999999999996 - - type: mrr_at_3 - value: 52.373999999999995 - - type: mrr_at_5 - value: 53.937000000000005 - - type: ndcg_at_1 - value: 44.272 - - type: ndcg_at_10 - value: 36.213 - - type: ndcg_at_100 - value: 33.829 - - type: ndcg_at_1000 - value: 42.557 - - type: ndcg_at_3 - value: 40.814 - - type: ndcg_at_5 - value: 39.562000000000005 - - type: precision_at_1 - value: 45.511 - - type: precision_at_10 - value: 27.214 - - type: precision_at_100 - value: 8.941 - - type: precision_at_1000 - value: 2.1870000000000003 - - type: precision_at_3 - value: 37.874 - - type: precision_at_5 - value: 34.489 - - type: recall_at_1 - value: 5.702 - - type: recall_at_10 - value: 17.638 - - type: recall_at_100 - value: 34.419 - - type: recall_at_1000 - value: 66.41 - - type: recall_at_3 - value: 10.914 - - type: recall_at_5 - value: 14.032 + - type: accuracy + value: 42.36122851507467 + - type: f1 + value: 25.445030887504398 - task: - type: Retrieval + type: Classification dataset: - type: nq - name: MTEB NQ - config: default + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (es) + config: es split: test - revision: 6062aefc120bfe8ece5897809fb2e53bfe0d128c + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba metrics: - - type: map_at_1 - value: 30.567 - - type: map_at_10 - value: 45.01 - - type: map_at_100 - value: 46.091 - - type: map_at_1000 - value: 46.126 - - type: map_at_3 - value: 40.897 - - type: map_at_5 - value: 43.301 - - type: mrr_at_1 - value: 34.56 - - type: mrr_at_10 - value: 47.725 - - type: mrr_at_100 - value: 48.548 - - type: mrr_at_1000 - value: 48.571999999999996 - - type: mrr_at_3 - value: 44.361 - - type: mrr_at_5 - value: 46.351 - - type: ndcg_at_1 - value: 34.531 - - type: ndcg_at_10 - value: 52.410000000000004 - - type: ndcg_at_100 - value: 56.999 - - type: ndcg_at_1000 - value: 57.830999999999996 - - type: ndcg_at_3 - value: 44.734 - - type: ndcg_at_5 - value: 48.701 - - type: precision_at_1 - value: 34.531 - - type: precision_at_10 - value: 8.612 - - type: precision_at_100 - value: 1.118 - - type: precision_at_1000 - value: 0.12 - - type: precision_at_3 - value: 20.307 - - type: precision_at_5 - value: 14.519000000000002 - - type: recall_at_1 - value: 30.567 - - type: recall_at_10 - value: 72.238 - - type: recall_at_100 - value: 92.154 - - type: recall_at_1000 - value: 98.375 - - type: recall_at_3 - value: 52.437999999999995 - - type: recall_at_5 - value: 61.516999999999996 + - type: accuracy + value: 44.73315543695797 + - type: f1 + value: 28.42075153540265 - task: - type: Retrieval + type: Classification dataset: - type: quora - name: MTEB QuoraRetrieval - config: default + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (fr) + config: fr split: test - revision: 6205996560df11e3a3da9ab4f926788fc30a7db4 + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba metrics: - - type: map_at_1 - value: 65.98 - - type: map_at_10 - value: 80.05600000000001 - - type: map_at_100 - value: 80.76299999999999 - - type: map_at_1000 - value: 80.786 - - type: map_at_3 - value: 76.848 - - type: map_at_5 - value: 78.854 - - type: mrr_at_1 - value: 75.86 - - type: mrr_at_10 - value: 83.397 - - type: mrr_at_100 - value: 83.555 - - type: mrr_at_1000 - value: 83.557 - - type: mrr_at_3 - value: 82.033 - - type: mrr_at_5 - value: 82.97 - - type: ndcg_at_1 - value: 75.88000000000001 - - type: ndcg_at_10 - value: 84.58099999999999 - - type: ndcg_at_100 - value: 86.151 - - type: ndcg_at_1000 - value: 86.315 - - type: ndcg_at_3 - value: 80.902 - - type: ndcg_at_5 - value: 82.953 - - type: precision_at_1 - value: 75.88000000000001 - - type: precision_at_10 - value: 12.986 - - type: precision_at_100 - value: 1.5110000000000001 - - type: precision_at_1000 - value: 0.156 - - type: precision_at_3 - value: 35.382999999999996 - - type: precision_at_5 - value: 23.555999999999997 - - type: recall_at_1 - value: 65.98 - - type: recall_at_10 - value: 93.716 - - type: recall_at_100 - value: 99.21799999999999 - - type: recall_at_1000 - value: 99.97 - - type: recall_at_3 - value: 83.551 - - type: recall_at_5 - value: 88.998 + - type: accuracy + value: 38.96022549326651 + - type: f1 + value: 25.926979537146106 - task: - type: Clustering + type: Classification dataset: - type: mteb/reddit-clustering - name: MTEB RedditClustering - config: default + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (hi) + config: hi split: test - revision: b2805658ae38990172679479369a78b86de8c390 + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba metrics: - - type: v_measure - value: 40.45148482612238 + - type: accuracy + value: 13.578343492291141 + - type: f1 + value: 8.929295550931657 - task: - type: Clustering + type: Classification dataset: - type: mteb/reddit-clustering-p2p - name: MTEB RedditClusteringP2P - config: default + type: mteb/mtop_intent + name: MTEB MTOPIntentClassification (th) + config: th split: test - revision: 385e3cb46b4cfa89021f56c4380204149d0efe33 + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba metrics: - - type: v_measure - value: 55.749490673039126 + - type: accuracy + value: 5.396021699819168 + - type: f1 + value: 1.8587148785378742 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (af) + config: af + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 37.22259583053128 + - type: f1 + value: 34.63013680947778 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (am) + config: am + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 3.194351042367182 + - type: f1 + value: 1.2612010214639442 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ar) + config: ar + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 14.26361802286483 + - type: f1 + value: 13.70260406613821 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (az) + config: az + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 37.21923335574983 + - type: f1 + value: 36.33553913878251 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (bn) + config: bn + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 10.756556825823807 + - type: f1 + value: 9.676431920229374 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (cy) + config: cy + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 32.49831876260928 + - type: f1 + value: 30.818895782691868 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (da) + config: da + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 40.995292535305985 + - type: f1 + value: 37.68768183180129 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (de) + config: de + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 42.780766644250164 + - type: f1 + value: 37.82194830667135 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (el) + config: el + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 33.490248823133825 + - type: f1 + value: 29.71809045584527 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (en) + config: en + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 73.8836583725622 + - type: f1 + value: 72.16381047416814 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (es) + config: es + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 44.45191661062542 + - type: f1 + value: 43.46583297093683 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (fa) + config: fa + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 26.738399462004036 + - type: f1 + value: 24.11896530001951 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (fi) + config: fi + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 38.09683927370545 + - type: f1 + value: 35.34443269387154 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (fr) + config: fr + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 46.89307330195024 + - type: f1 + value: 43.47164092514292 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (he) + config: he + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 25.198386012104912 + - type: f1 + value: 22.446286736401916 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (hi) + config: hi + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 13.940820443846672 + - type: f1 + value: 13.257747189396213 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (hu) + config: hu + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 34.710827168796236 + - type: f1 + value: 32.036974696095996 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (hy) + config: hy + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 6.711499663752522 + - type: f1 + value: 5.439441019096591 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (id) + config: id + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 38.56758574310693 + - type: f1 + value: 36.83183505458304 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (is) + config: is + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 32.22595830531271 + - type: f1 + value: 30.10972675771159 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (it) + config: it + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 45.79690652320107 + - type: f1 + value: 44.37143784350453 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ja) + config: ja + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 29.189643577673163 + - type: f1 + value: 25.43718135312703 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (jv) + config: jv + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 34.21990585070612 + - type: f1 + value: 32.333592263041396 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ka) + config: ka + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 8.890383322125084 + - type: f1 + value: 7.294310113130201 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (km) + config: km + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 4.616677874915938 + - type: f1 + value: 1.5028537477535886 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (kn) + config: kn + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 3.170813718897109 + - type: f1 + value: 1.5771411815826382 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ko) + config: ko + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 15.026899798251513 + - type: f1 + value: 14.077395255366183 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (lv) + config: lv + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 36.0995292535306 + - type: f1 + value: 35.0877269083235 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ml) + config: ml + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 2.9959650302622727 + - type: f1 + value: 0.8064424547273695 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (mn) + config: mn + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 23.301950235373234 + - type: f1 + value: 22.477376205075853 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ms) + config: ms + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 36.13315400134499 + - type: f1 + value: 32.99623898888715 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (my) + config: my + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 3.813046402151983 + - type: f1 + value: 1.1769597223141248 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (nb) + config: nb + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 39.66711499663752 + - type: f1 + value: 35.921474753569214 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (nl) + config: nl + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 41.079354404841965 + - type: f1 + value: 37.57739961852201 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (pl) + config: pl + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 38.211163416274374 + - type: f1 + value: 34.89419275422068 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (pt) + config: pt + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 45.19838601210491 + - type: f1 + value: 42.71660225307043 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ro) + config: ro + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 39.48554135843981 + - type: f1 + value: 37.47402102847154 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ru) + config: ru + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 31.819098856758576 + - type: f1 + value: 30.120158288509725 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (sl) + config: sl + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 35.44720914593141 + - type: f1 + value: 33.74530063536304 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (sq) + config: sq + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 36.89307330195024 + - type: f1 + value: 34.46971619696105 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (sv) + config: sv + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 38.83322125084062 + - type: f1 + value: 36.050770344888264 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (sw) + config: sw + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 37.535305985205106 + - type: f1 + value: 35.21395700670493 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ta) + config: ta + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 7.905178211163418 + - type: f1 + value: 6.163513326325246 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (te) + config: te + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 2.8480161398789514 + - type: f1 + value: 1.0163931337986962 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (th) + config: th + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 10.501008742434433 + - type: f1 + value: 6.858549418430471 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (tl) + config: tl + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 39.46536650975118 + - type: f1 + value: 34.96292597328575 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (tr) + config: tr + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 37.50168123739071 + - type: f1 + value: 35.031097269820464 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (ur) + config: ur + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 16.109616677874918 + - type: f1 + value: 15.884609726192519 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (vi) + config: vi + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 36.11297915265636 + - type: f1 + value: 34.59918716321474 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (zh-CN) + config: zh-CN + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 18.850033624747812 + - type: f1 + value: 15.09584388649328 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_intent + name: MTEB MassiveIntentClassification (zh-TW) + config: zh-TW + split: test + revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 + metrics: + - type: accuracy + value: 17.219233355749832 + - type: f1 + value: 14.538046039008337 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (af) + config: af + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 47.79757901815736 + - type: f1 + value: 45.078250421193324 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (am) + config: am + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 7.078009414929388 + - type: f1 + value: 4.0122456300041645 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ar) + config: ar + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 22.831203765971754 + - type: f1 + value: 20.131610050816555 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (az) + config: az + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 44.952925353059854 + - type: f1 + value: 42.6865575762921 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (bn) + config: bn + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 16.593813046402154 + - type: f1 + value: 14.087144503044291 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (cy) + config: cy + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 37.91862811028917 + - type: f1 + value: 34.968402727911915 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (da) + config: da + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 51.923335574983184 + - type: f1 + value: 49.357147840776335 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (de) + config: de + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 58.73570948217889 + - type: f1 + value: 54.92084137819753 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (el) + config: el + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 42.995965030262276 + - type: f1 + value: 38.47512542753069 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (en) + config: en + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 77.42098184263618 + - type: f1 + value: 77.03413816048877 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (es) + config: es + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 54.46536650975118 + - type: f1 + value: 53.08520810835907 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (fa) + config: fa + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 30.578345662407525 + - type: f1 + value: 28.822998245702635 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (fi) + config: fi + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 43.567585743106925 + - type: f1 + value: 39.79216651714347 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (fr) + config: fr + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 56.98722259583053 + - type: f1 + value: 55.31168113501439 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (he) + config: he + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 28.076664425016812 + - type: f1 + value: 24.927348965627573 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (hi) + config: hi + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 18.096839273705445 + - type: f1 + value: 17.386603595777103 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (hu) + config: hu + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 41.73839946200403 + - type: f1 + value: 38.65545902563735 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (hy) + config: hy + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 11.536650975117688 + - type: f1 + value: 10.898336694524854 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (id) + config: id + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 46.9502353732347 + - type: f1 + value: 44.332561323528644 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (is) + config: is + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 42.777404169468724 + - type: f1 + value: 39.378117766055354 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (it) + config: it + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 54.6469401479489 + - type: f1 + value: 52.512025274851794 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ja) + config: ja + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 35.90114324142569 + - type: f1 + value: 34.90331274712605 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (jv) + config: jv + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 42.51176866173504 + - type: f1 + value: 39.417541845685676 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ka) + config: ka + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 13.799596503026226 + - type: f1 + value: 11.587556164962251 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (km) + config: km + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 9.44855413584398 + - type: f1 + value: 4.30711077076907 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (kn) + config: kn + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 8.157363819771351 + - type: f1 + value: 5.5588908736809515 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ko) + config: ko + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 19.909213180901144 + - type: f1 + value: 18.964761241087984 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (lv) + config: lv + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 40.47747141896436 + - type: f1 + value: 38.17159556642586 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ml) + config: ml + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 6.701412239408204 + - type: f1 + value: 3.621974155647488 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (mn) + config: mn + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 28.55413584398117 + - type: f1 + value: 26.582548923662753 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ms) + config: ms + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 46.617350369872234 + - type: f1 + value: 41.35397419267425 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (my) + config: my + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 9.976462676529927 + - type: f1 + value: 5.900764382768462 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (nb) + config: nb + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 50.894418291862806 + - type: f1 + value: 47.70929403771086 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (nl) + config: nl + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 51.761936785474106 + - type: f1 + value: 48.42797973062516 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (pl) + config: pl + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 46.21385339609952 + - type: f1 + value: 43.7081546200347 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (pt) + config: pt + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 55.59852051109617 + - type: f1 + value: 54.19610878409633 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ro) + config: ro + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 50.54135843981169 + - type: f1 + value: 47.79393938467311 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ru) + config: ru + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 37.73032952252858 + - type: f1 + value: 35.96450149708041 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (sl) + config: sl + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 41.67114996637525 + - type: f1 + value: 40.28283538885605 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (sq) + config: sq + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 47.38063214525891 + - type: f1 + value: 44.93264016007152 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (sv) + config: sv + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 49.28379287155347 + - type: f1 + value: 46.25486396570196 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (sw) + config: sw + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 44.18291862811029 + - type: f1 + value: 41.17519157172804 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ta) + config: ta + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 12.599193006052452 + - type: f1 + value: 11.129236666238377 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (te) + config: te + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 7.017484868863484 + - type: f1 + value: 3.9665415549749077 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (th) + config: th + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 19.788164088769335 + - type: f1 + value: 15.783384761347582 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (tl) + config: tl + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 50.35978480161398 + - type: f1 + value: 47.30586047800275 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (tr) + config: tr + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 45.484196368527236 + - type: f1 + value: 44.65101184252231 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (ur) + config: ur + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 23.681909885675857 + - type: f1 + value: 22.247817138937524 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (vi) + config: vi + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 41.63080026899798 + - type: f1 + value: 39.546896741744 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (zh-CN) + config: zh-CN + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 30.141223940820446 + - type: f1 + value: 28.177838960078123 + - task: + type: Classification + dataset: + type: mteb/amazon_massive_scenario + name: MTEB MassiveScenarioClassification (zh-TW) + config: zh-TW + split: test + revision: 7d571f92784cd94a019292a1f45445077d0ef634 + metrics: + - type: accuracy + value: 27.515131136516473 + - type: f1 + value: 26.514325837594654 + - task: + type: Clustering + dataset: + type: mteb/medrxiv-clustering-p2p + name: MTEB MedrxivClusteringP2P + config: default + split: test + revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 + metrics: + - type: v_measure + value: 33.70592767911301 + - task: + type: Clustering + dataset: + type: mteb/medrxiv-clustering-s2s + name: MTEB MedrxivClusteringS2S + config: default + split: test + revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 + metrics: + - type: v_measure + value: 31.80943770643908 + - task: + type: Reranking + dataset: + type: mteb/mind_small + name: MTEB MindSmallReranking + config: default + split: test + revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 + metrics: + - type: map + value: 32.66434973425713 + - type: mrr + value: 33.92240574935323 + - task: + type: Retrieval + dataset: + type: mteb/nfcorpus + name: MTEB NFCorpus + config: default + split: test + revision: ec0fa4fe99da2ff19ca1214b7966684033a58814 + metrics: + - type: map_at_1 + value: 6.561999999999999 + - type: map_at_10 + value: 14.854000000000001 + - type: map_at_100 + value: 19.187 + - type: map_at_1000 + value: 20.812 + - type: map_at_20 + value: 16.744 + - type: map_at_3 + value: 10.804 + - type: map_at_5 + value: 12.555 + - type: mrr_at_1 + value: 48.916 + - type: mrr_at_10 + value: 57.644 + - type: mrr_at_100 + value: 58.17 + - type: mrr_at_1000 + value: 58.206 + - type: mrr_at_20 + value: 57.969 + - type: mrr_at_3 + value: 55.36600000000001 + - type: mrr_at_5 + value: 56.729 + - type: ndcg_at_1 + value: 46.594 + - type: ndcg_at_10 + value: 37.897999999999996 + - type: ndcg_at_100 + value: 35.711 + - type: ndcg_at_1000 + value: 44.65 + - type: ndcg_at_20 + value: 35.989 + - type: ndcg_at_3 + value: 42.869 + - type: ndcg_at_5 + value: 40.373 + - type: precision_at_1 + value: 48.297000000000004 + - type: precision_at_10 + value: 28.297 + - type: precision_at_100 + value: 9.099 + - type: precision_at_1000 + value: 2.229 + - type: precision_at_20 + value: 21.455 + - type: precision_at_3 + value: 40.248 + - type: precision_at_5 + value: 34.675 + - type: recall_at_1 + value: 6.561999999999999 + - type: recall_at_10 + value: 19.205 + - type: recall_at_100 + value: 36.742999999999995 + - type: recall_at_1000 + value: 69.119 + - type: recall_at_20 + value: 23.787 + - type: recall_at_3 + value: 11.918 + - type: recall_at_5 + value: 14.860000000000001 + - task: + type: Retrieval + dataset: + type: mteb/nq + name: MTEB NQ + config: default + split: test + revision: b774495ed302d8c44a3a7ea25c90dbce03968f31 + metrics: + - type: map_at_1 + value: 30.306 + - type: map_at_10 + value: 46.916999999999994 + - type: map_at_100 + value: 47.899 + - type: map_at_1000 + value: 47.925000000000004 + - type: map_at_20 + value: 47.583 + - type: map_at_3 + value: 42.235 + - type: map_at_5 + value: 45.118 + - type: mrr_at_1 + value: 34.327999999999996 + - type: mrr_at_10 + value: 49.248999999999995 + - type: mrr_at_100 + value: 49.96 + - type: mrr_at_1000 + value: 49.977 + - type: mrr_at_20 + value: 49.738 + - type: mrr_at_3 + value: 45.403999999999996 + - type: mrr_at_5 + value: 47.786 + - type: ndcg_at_1 + value: 34.327999999999996 + - type: ndcg_at_10 + value: 55.123999999999995 + - type: ndcg_at_100 + value: 59.136 + - type: ndcg_at_1000 + value: 59.71300000000001 + - type: ndcg_at_20 + value: 57.232000000000006 + - type: ndcg_at_3 + value: 46.48 + - type: ndcg_at_5 + value: 51.237 + - type: precision_at_1 + value: 34.327999999999996 + - type: precision_at_10 + value: 9.261 + - type: precision_at_100 + value: 1.1520000000000001 + - type: precision_at_1000 + value: 0.121 + - type: precision_at_20 + value: 5.148 + - type: precision_at_3 + value: 21.523999999999997 + - type: precision_at_5 + value: 15.659999999999998 + - type: recall_at_1 + value: 30.306 + - type: recall_at_10 + value: 77.65100000000001 + - type: recall_at_100 + value: 94.841 + - type: recall_at_1000 + value: 99.119 + - type: recall_at_20 + value: 85.37599999999999 + - type: recall_at_3 + value: 55.562 + - type: recall_at_5 + value: 66.5 + - task: + type: Retrieval + dataset: + type: mteb/quora + name: MTEB QuoraRetrieval + config: default + split: test + revision: e4e08e0b7dbe3c8700f0daef558ff32256715259 + metrics: + - type: map_at_1 + value: 71.516 + - type: map_at_10 + value: 85.48400000000001 + - type: map_at_100 + value: 86.11 + - type: map_at_1000 + value: 86.124 + - type: map_at_20 + value: 85.895 + - type: map_at_3 + value: 82.606 + - type: map_at_5 + value: 84.395 + - type: mrr_at_1 + value: 82.38 + - type: mrr_at_10 + value: 88.31099999999999 + - type: mrr_at_100 + value: 88.407 + - type: mrr_at_1000 + value: 88.407 + - type: mrr_at_20 + value: 88.385 + - type: mrr_at_3 + value: 87.42699999999999 + - type: mrr_at_5 + value: 88.034 + - type: ndcg_at_1 + value: 82.39999999999999 + - type: ndcg_at_10 + value: 89.07300000000001 + - type: ndcg_at_100 + value: 90.23400000000001 + - type: ndcg_at_1000 + value: 90.304 + - type: ndcg_at_20 + value: 89.714 + - type: ndcg_at_3 + value: 86.42699999999999 + - type: ndcg_at_5 + value: 87.856 + - type: precision_at_1 + value: 82.39999999999999 + - type: precision_at_10 + value: 13.499 + - type: precision_at_100 + value: 1.536 + - type: precision_at_1000 + value: 0.157 + - type: precision_at_20 + value: 7.155 + - type: precision_at_3 + value: 37.846999999999994 + - type: precision_at_5 + value: 24.778 + - type: recall_at_1 + value: 71.516 + - type: recall_at_10 + value: 95.831 + - type: recall_at_100 + value: 99.714 + - type: recall_at_1000 + value: 99.979 + - type: recall_at_20 + value: 97.87599999999999 + - type: recall_at_3 + value: 88.08 + - type: recall_at_5 + value: 92.285 + - task: + type: Clustering + dataset: + type: mteb/reddit-clustering + name: MTEB RedditClustering + config: default + split: test + revision: 24640382cdbf8abc73003fb0fa6d111a705499eb + metrics: + - type: v_measure + value: 61.3760407207699 + - task: + type: Clustering + dataset: + type: mteb/reddit-clustering-p2p + name: MTEB RedditClusteringP2P + config: default + split: test + revision: 385e3cb46b4cfa89021f56c4380204149d0efe33 + metrics: + - type: v_measure + value: 65.28621066626943 - task: type: Retrieval dataset: - type: scidocs + type: mteb/scidocs name: MTEB SCIDOCS config: default split: test - revision: 5c59ef3e437a0a9651c8fe6fde943e7dce59fba5 + revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88 metrics: - type: map_at_1 - value: 4.903 + value: 5.163 - type: map_at_10 - value: 11.926 + value: 14.377 - type: map_at_100 - value: 13.916999999999998 + value: 17.177 - type: map_at_1000 - value: 14.215 + value: 17.588 + - type: map_at_20 + value: 15.827 - type: map_at_3 - value: 8.799999999999999 + value: 9.879 - type: map_at_5 - value: 10.360999999999999 + value: 12.133 - type: mrr_at_1 - value: 24.099999999999998 + value: 25.5 - type: mrr_at_10 - value: 34.482 + value: 38.435 - type: mrr_at_100 - value: 35.565999999999995 + value: 39.573 - type: mrr_at_1000 - value: 35.619 + value: 39.606 + - type: mrr_at_20 + value: 39.134 - type: mrr_at_3 - value: 31.433 + value: 34.666999999999994 - type: mrr_at_5 - value: 33.243 + value: 37.117 - type: ndcg_at_1 - value: 24.099999999999998 + value: 25.5 - type: ndcg_at_10 - value: 19.872999999999998 + value: 23.688000000000002 - type: ndcg_at_100 - value: 27.606 + value: 33.849000000000004 - type: ndcg_at_1000 - value: 32.811 + value: 39.879 + - type: ndcg_at_20 + value: 27.36 - type: ndcg_at_3 - value: 19.497999999999998 + value: 22.009999999999998 - type: ndcg_at_5 - value: 16.813 + value: 19.691 - type: precision_at_1 - value: 24.099999999999998 + value: 25.5 - type: precision_at_10 - value: 10.08 + value: 12.540000000000001 - type: precision_at_100 - value: 2.122 + value: 2.721 - type: precision_at_1000 - value: 0.337 + value: 0.415 + - type: precision_at_20 + value: 8.385 - type: precision_at_3 - value: 18.2 + value: 21.099999999999998 - type: precision_at_5 - value: 14.62 + value: 17.84 - type: recall_at_1 - value: 4.903 + value: 5.163 - type: recall_at_10 - value: 20.438000000000002 + value: 25.405 - type: recall_at_100 - value: 43.043 + value: 55.213 - type: recall_at_1000 - value: 68.41000000000001 + value: 84.243 + - type: recall_at_20 + value: 34.003 - type: recall_at_3 - value: 11.068 + value: 12.837000000000002 - type: recall_at_5 - value: 14.818000000000001 + value: 18.096999999999998 + - task: + type: STS + dataset: + type: mteb/sickr-sts + name: MTEB SICK-R + config: default + split: test + revision: 20a6d6f312dd54037fe07a32d58e5e168867909d + metrics: + - type: cos_sim_pearson + value: 87.64406884822948 + - type: cos_sim_spearman + value: 83.00239648251724 + - type: euclidean_pearson + value: 85.03347205351844 + - type: euclidean_spearman + value: 83.00240733538445 + - type: manhattan_pearson + value: 85.0312758694447 + - type: manhattan_spearman + value: 82.99430696077589 + - task: + type: STS + dataset: + type: mteb/sts12-sts + name: MTEB STS12 + config: default + split: test + revision: a0d554a64d88156834ff5ae9920b964011b16384 + metrics: + - type: cos_sim_pearson + value: 87.68832340658764 + - type: cos_sim_spearman + value: 79.21679373212476 + - type: euclidean_pearson + value: 85.17094885886415 + - type: euclidean_spearman + value: 79.21421345946399 + - type: manhattan_pearson + value: 85.17409319145995 + - type: manhattan_spearman + value: 79.20992207976401 + - task: + type: STS + dataset: + type: mteb/sts13-sts + name: MTEB STS13 + config: default + split: test + revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca + metrics: + - type: cos_sim_pearson + value: 88.43733084958856 + - type: cos_sim_spearman + value: 89.43082089321751 + - type: euclidean_pearson + value: 88.63286785416938 + - type: euclidean_spearman + value: 89.43082081372343 + - type: manhattan_pearson + value: 88.62969346368385 + - type: manhattan_spearman + value: 89.43131586189746 + - task: + type: STS + dataset: + type: mteb/sts14-sts + name: MTEB STS14 + config: default + split: test + revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 + metrics: + - type: cos_sim_pearson + value: 86.62185532014894 + - type: cos_sim_spearman + value: 84.7923120886599 + - type: euclidean_pearson + value: 85.99786490539253 + - type: euclidean_spearman + value: 84.79231064318844 + - type: manhattan_pearson + value: 85.97647892920392 + - type: manhattan_spearman + value: 84.76865232132103 + - task: + type: STS + dataset: + type: mteb/sts15-sts + name: MTEB STS15 + config: default + split: test + revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 + metrics: + - type: cos_sim_pearson + value: 88.39303997282114 + - type: cos_sim_spearman + value: 89.54273264876765 + - type: euclidean_pearson + value: 88.8848627924181 + - type: euclidean_spearman + value: 89.54275013645078 + - type: manhattan_pearson + value: 88.86926987108802 + - type: manhattan_spearman + value: 89.53259197721715 + - task: + type: STS + dataset: + type: mteb/sts16-sts + name: MTEB STS16 + config: default + split: test + revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 + metrics: + - type: cos_sim_pearson + value: 85.21814352466886 + - type: cos_sim_spearman + value: 86.68505223422434 + - type: euclidean_pearson + value: 86.07422446469991 + - type: euclidean_spearman + value: 86.68505161067375 + - type: manhattan_pearson + value: 86.05114200797293 + - type: manhattan_spearman + value: 86.6587670422703 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (ko-ko) + config: ko-ko + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 39.17871768366095 + - type: cos_sim_spearman + value: 39.78510424960567 + - type: euclidean_pearson + value: 41.65680175653682 + - type: euclidean_spearman + value: 39.78538944779548 + - type: manhattan_pearson + value: 41.567603690394755 + - type: manhattan_spearman + value: 39.71393388259443 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (ar-ar) + config: ar-ar + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 49.26766904195114 + - type: cos_sim_spearman + value: 46.79722787057151 + - type: euclidean_pearson + value: 51.2329334717446 + - type: euclidean_spearman + value: 46.7920623095072 + - type: manhattan_pearson + value: 51.26488560860826 + - type: manhattan_spearman + value: 47.00400318665492 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (en-ar) + config: en-ar + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 1.6821294132202447 + - type: cos_sim_spearman + value: -0.7813676799492025 + - type: euclidean_pearson + value: 1.9197388753860283 + - type: euclidean_spearman + value: -0.7813676799492025 + - type: manhattan_pearson + value: 2.209862430499871 + - type: manhattan_spearman + value: -0.863014010062456 - task: type: STS dataset: - type: mteb/sickr-sts - name: MTEB SICK-R - config: default + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (en-de) + config: en-de split: test - revision: 20a6d6f312dd54037fe07a32d58e5e168867909d + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d metrics: - type: cos_sim_pearson - value: 78.58086597995997 + value: 48.76382428941107 - type: cos_sim_spearman - value: 69.63214182814991 + value: 47.50280322999196 - type: euclidean_pearson - value: 72.76175489042691 + value: 48.73919143974209 - type: euclidean_spearman - value: 67.84965161872971 + value: 47.50280322999196 - type: manhattan_pearson - value: 72.73812689782592 + value: 48.76291223862666 - type: manhattan_spearman - value: 67.83610439531277 + value: 47.51318193687094 - task: type: STS dataset: - type: mteb/sts12-sts - name: MTEB STS12 - config: default + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (en-en) + config: en-en split: test - revision: fdf84275bb8ce4b49c971d02e84dd1abc677a50f + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d metrics: - type: cos_sim_pearson - value: 75.13970861325006 + value: 89.6579390263212 - type: cos_sim_spearman - value: 67.5020551515597 + value: 89.64423556388047 - type: euclidean_pearson - value: 66.33415412418276 + value: 90.1160733522703 - type: euclidean_spearman - value: 66.82145056673268 + value: 89.64423556388047 - type: manhattan_pearson - value: 66.55489484006415 + value: 90.1528407376387 - type: manhattan_spearman - value: 66.95147433279057 + value: 89.61290724496793 - task: type: STS dataset: - type: mteb/sts13-sts - name: MTEB STS13 - config: default + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (en-tr) + config: en-tr split: test - revision: 1591bfcbe8c69d4bf7fe2a16e2451017832cafb9 + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d metrics: - type: cos_sim_pearson - value: 78.85850536483447 + value: 6.717092266815236 - type: cos_sim_spearman - value: 79.1633350177206 + value: 4.180543503488665 - type: euclidean_pearson - value: 72.74090561408477 + value: 7.120267092048099 - type: euclidean_spearman - value: 73.57374448302961 + value: 4.180543503488665 - type: manhattan_pearson - value: 72.92980654233226 + value: 6.396237465828514 - type: manhattan_spearman - value: 73.72777155112588 + value: 3.61244941411957 - task: type: STS dataset: - type: mteb/sts14-sts - name: MTEB STS14 - config: default + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (es-en) + config: es-en split: test - revision: e2125984e7df8b7871f6ae9949cf6b6795e7c54b + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d metrics: - type: cos_sim_pearson - value: 79.51125593897028 + value: 44.36476614938953 - type: cos_sim_spearman - value: 74.46048326701329 + value: 44.265723809500685 - type: euclidean_pearson - value: 70.87726087052985 + value: 44.61551298711104 - type: euclidean_spearman - value: 67.7721470654411 + value: 44.265723809500685 - type: manhattan_pearson - value: 71.05892792135637 + value: 44.54302374682193 - type: manhattan_spearman - value: 67.93472619779037 + value: 44.08642490624185 - task: type: STS dataset: - type: mteb/sts15-sts - name: MTEB STS15 - config: default + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (es-es) + config: es-es split: test - revision: 1cd7298cac12a96a373b6a2f18738bb3e739a9b6 + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d metrics: - type: cos_sim_pearson - value: 83.8299348880489 + value: 79.64871991975828 - type: cos_sim_spearman - value: 84.47194637929275 + value: 79.21979030014373 - type: euclidean_pearson - value: 78.68768462480418 + value: 81.8672798988218 - type: euclidean_spearman - value: 79.80526323901917 + value: 79.21950130108661 - type: manhattan_pearson - value: 78.6810718151946 + value: 82.02131606326583 - type: manhattan_spearman - value: 79.7820584821254 + value: 79.44848373553044 - task: type: STS dataset: - type: mteb/sts16-sts - name: MTEB STS16 - config: default + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (fr-en) + config: fr-en split: test - revision: 360a0b2dff98700d09e634a01e1cc1624d3e42cd + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d metrics: - type: cos_sim_pearson - value: 79.99206664843005 + value: 48.73898658957231 - type: cos_sim_spearman - value: 80.96089203722137 + value: 47.15192605817168 - type: euclidean_pearson - value: 71.31216213716365 + value: 49.11990573381456 - type: euclidean_spearman - value: 71.45258140049407 + value: 47.15192605817168 - type: manhattan_pearson - value: 71.26140340402836 + value: 48.5694400358235 - type: manhattan_spearman - value: 71.3896894666943 + value: 46.651326429708135 - task: type: STS dataset: type: mteb/sts17-crosslingual-sts - name: MTEB STS17 (en-en) - config: en-en + name: MTEB STS17 (it-en) + config: it-en + split: test + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d + metrics: + - type: cos_sim_pearson + value: 44.42168074232218 + - type: cos_sim_spearman + value: 42.64799010889372 + - type: euclidean_pearson + value: 44.41376048324183 + - type: euclidean_spearman + value: 42.64799010889372 + - type: manhattan_pearson + value: 44.724522621427546 + - type: manhattan_spearman + value: 42.60912761758016 + - task: + type: STS + dataset: + type: mteb/sts17-crosslingual-sts + name: MTEB STS17 (nl-en) + config: nl-en split: test - revision: 9fc37e8c632af1c87a3d23e685d49552a02582a0 + revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d metrics: - type: cos_sim_pearson - value: 87.35697089594868 + value: 40.55050173163197 - type: cos_sim_spearman - value: 87.78202647220289 + value: 36.59720399843921 - type: euclidean_pearson - value: 84.20969668786667 + value: 41.49402389245919 - type: euclidean_spearman - value: 83.91876425459982 + value: 36.59720399843921 - type: manhattan_pearson - value: 84.24429755612542 + value: 41.877514420153666 - type: manhattan_spearman - value: 83.98826315103398 + value: 36.782790653297695 - task: type: STS dataset: @@ -2089,20 +4079,377 @@ model-index: name: MTEB STS22 (en) config: en split: test - revision: 2de6ce8c1921b71a755b262c6b57fef195dd7906 + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 69.44405106094861 + - type: cos_sim_spearman + value: 70.25621893108706 + - type: euclidean_pearson + value: 71.15726637696066 + - type: euclidean_spearman + value: 70.25621893108706 + - type: manhattan_pearson + value: 71.28565265298322 + - type: manhattan_spearman + value: 70.30317892414027 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (de) + config: de + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 34.56638014500804 + - type: cos_sim_spearman + value: 39.48672765878819 + - type: euclidean_pearson + value: 31.61811391543846 + - type: euclidean_spearman + value: 39.48672765878819 + - type: manhattan_pearson + value: 31.839117286689977 + - type: manhattan_spearman + value: 39.71519891403971 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (es) + config: es + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 53.72389957326714 + - type: cos_sim_spearman + value: 59.47018781803598 + - type: euclidean_pearson + value: 57.02101112722141 + - type: euclidean_spearman + value: 59.47018781803598 + - type: manhattan_pearson + value: 57.16531255049132 + - type: manhattan_spearman + value: 59.57320508684436 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (pl) + config: pl + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 24.14602533311477 + - type: cos_sim_spearman + value: 35.38039329704056 + - type: euclidean_pearson + value: 13.540543553763765 + - type: euclidean_spearman + value: 35.38039329704056 + - type: manhattan_pearson + value: 13.566377379303256 + - type: manhattan_spearman + value: 35.88351047224126 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (tr) + config: tr + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 39.07697432450346 + - type: cos_sim_spearman + value: 45.65479772235109 + - type: euclidean_pearson + value: 41.68913259791294 + - type: euclidean_spearman + value: 45.65479772235109 + - type: manhattan_pearson + value: 41.58872552392231 + - type: manhattan_spearman + value: 45.462070534023404 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (ar) + config: ar + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 23.917322166825183 + - type: cos_sim_spearman + value: 25.06042767518008 + - type: euclidean_pearson + value: 24.29850435278771 + - type: euclidean_spearman + value: 25.06042767518008 + - type: manhattan_pearson + value: 24.461400062927154 + - type: manhattan_spearman + value: 25.285239684773046 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (ru) + config: ru + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 20.39987623162105 + - type: cos_sim_spearman + value: 30.62427846964406 + - type: euclidean_pearson + value: 20.817950942480323 + - type: euclidean_spearman + value: 30.618700916425222 + - type: manhattan_pearson + value: 20.756787430880788 + - type: manhattan_spearman + value: 30.813116243628436 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (zh) + config: zh + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 43.838363041373974 + - type: cos_sim_spearman + value: 54.17598089882719 + - type: euclidean_pearson + value: 47.51044033919419 + - type: euclidean_spearman + value: 54.17598089882719 + - type: manhattan_pearson + value: 47.54911083403354 + - type: manhattan_spearman + value: 54.2562151204606 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (fr) + config: fr + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 77.69372699157654 + - type: cos_sim_spearman + value: 79.88201388457435 + - type: euclidean_pearson + value: 78.81259581302578 + - type: euclidean_spearman + value: 79.88201388457435 + - type: manhattan_pearson + value: 78.85098508555477 + - type: manhattan_spearman + value: 80.20154858554835 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (de-en) + config: de-en + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 51.83713469138834 + - type: cos_sim_spearman + value: 54.2205845288082 + - type: euclidean_pearson + value: 54.14828396506985 + - type: euclidean_spearman + value: 54.2205845288082 + - type: manhattan_pearson + value: 54.10701855179347 + - type: manhattan_spearman + value: 54.30261135461622 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (es-en) + config: es-en + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 61.59147752554915 + - type: cos_sim_spearman + value: 66.65350021824162 + - type: euclidean_pearson + value: 62.577915098325434 + - type: euclidean_spearman + value: 66.65350021824162 + - type: manhattan_pearson + value: 62.22817675366819 + - type: manhattan_spearman + value: 66.35054389546214 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (it) + config: it + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 65.23775897743552 + - type: cos_sim_spearman + value: 68.1509652709288 + - type: euclidean_pearson + value: 66.17577980319408 + - type: euclidean_spearman + value: 68.1509652709288 + - type: manhattan_pearson + value: 66.40051933918704 + - type: manhattan_spearman + value: 68.37138808382802 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (pl-en) + config: pl-en + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 61.943863830043725 + - type: cos_sim_spearman + value: 62.699440972016774 + - type: euclidean_pearson + value: 62.810366501196 + - type: euclidean_spearman + value: 62.699440972016774 + - type: manhattan_pearson + value: 63.13065659868621 + - type: manhattan_spearman + value: 63.314141373703215 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (zh-en) + config: zh-en + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 48.1108866326284 + - type: cos_sim_spearman + value: 49.25274096772371 + - type: euclidean_pearson + value: 47.87203797435136 + - type: euclidean_spearman + value: 49.25274096772371 + - type: manhattan_pearson + value: 47.39927722979605 + - type: manhattan_spearman + value: 48.76629586560382 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (es-it) + config: es-it + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 58.58401639298775 + - type: cos_sim_spearman + value: 64.37272828346495 + - type: euclidean_pearson + value: 61.03680632288844 + - type: euclidean_spearman + value: 64.37272828346495 + - type: manhattan_pearson + value: 61.381331848220675 + - type: manhattan_spearman + value: 65.01053960017909 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (de-fr) + config: de-fr + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 44.374682063416735 + - type: cos_sim_spearman + value: 48.907776246550185 + - type: euclidean_pearson + value: 45.473260322201284 + - type: euclidean_spearman + value: 48.907776246550185 + - type: manhattan_pearson + value: 46.051779591771854 + - type: manhattan_spearman + value: 49.69297213757249 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (de-pl) + config: de-pl + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f + metrics: + - type: cos_sim_pearson + value: 31.55497030143048 + - type: cos_sim_spearman + value: 33.042073055100396 + - type: euclidean_pearson + value: 33.548707962408955 + - type: euclidean_spearman + value: 33.042073055100396 + - type: manhattan_pearson + value: 31.704989941561873 + - type: manhattan_spearman + value: 31.56395608711827 + - task: + type: STS + dataset: + type: mteb/sts22-crosslingual-sts + name: MTEB STS22 (fr-pl) + config: fr-pl + split: test + revision: eea2b4fe26a775864c896887d910b76a8098ad3f metrics: - type: cos_sim_pearson - value: 69.06962775868384 + value: 51.253093232573036 - type: cos_sim_spearman - value: 69.34889515492327 + value: 39.440531887330785 - type: euclidean_pearson - value: 69.28108180412313 + value: 51.42758694144294 - type: euclidean_spearman - value: 69.6437114853659 + value: 39.440531887330785 - type: manhattan_pearson - value: 69.39974983734993 + value: 49.623915715149394 - type: manhattan_spearman - value: 69.69057284482079 + value: 39.440531887330785 - task: type: STS dataset: @@ -2110,20 +4457,20 @@ model-index: name: MTEB STSBenchmark config: default split: test - revision: 8913289635987208e6e7c72789e4be2fe94b6abd + revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 metrics: - type: cos_sim_pearson - value: 82.42553734213958 + value: 87.61260941646887 - type: cos_sim_spearman - value: 81.38977341532744 + value: 88.96384726759047 - type: euclidean_pearson - value: 76.47494587945522 + value: 88.72268994912045 - type: euclidean_spearman - value: 75.92794860531089 + value: 88.96384726759047 - type: manhattan_pearson - value: 76.4768777169467 + value: 88.72080954591475 - type: manhattan_spearman - value: 75.9252673228599 + value: 88.92379960545995 - task: type: Reranking dataset: @@ -2131,81 +4478,91 @@ model-index: name: MTEB SciDocsRR config: default split: test - revision: 56a6d0140cf6356659e2a7c1413286a774468d44 + revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab metrics: - type: map - value: 80.78825425914722 + value: 87.64768404690723 - type: mrr - value: 94.60017197762296 + value: 96.25675341361615 - task: type: Retrieval dataset: - type: scifact + type: mteb/scifact name: MTEB SciFact config: default split: test - revision: a75ae049398addde9b70f6b268875f5cbce99089 + revision: 0228b52cf27578f30900b9e5271d331663a030d7 metrics: - type: map_at_1 - value: 60.633 + value: 61.194 - type: map_at_10 - value: 70.197 + value: 70.62899999999999 - type: map_at_100 - value: 70.758 + value: 71.119 - type: map_at_1000 - value: 70.765 + value: 71.14200000000001 + - type: map_at_20 + value: 71.033 - type: map_at_3 - value: 67.082 + value: 67.51899999999999 - type: map_at_5 - value: 69.209 + value: 69.215 - type: mrr_at_1 - value: 63.333 + value: 63.666999999999994 - type: mrr_at_10 - value: 71.17 + value: 71.456 - type: mrr_at_100 - value: 71.626 + value: 71.844 - type: mrr_at_1000 - value: 71.633 + value: 71.866 + - type: mrr_at_20 + value: 71.769 - type: mrr_at_3 - value: 68.833 + value: 69.167 - type: mrr_at_5 - value: 70.6 + value: 70.39999999999999 - type: ndcg_at_1 - value: 63.333 + value: 63.666999999999994 - type: ndcg_at_10 - value: 74.697 + value: 75.14 - type: ndcg_at_100 - value: 76.986 + value: 77.071 - type: ndcg_at_1000 - value: 77.225 + value: 77.55199999999999 + - type: ndcg_at_20 + value: 76.491 - type: ndcg_at_3 - value: 69.527 + value: 69.836 - type: ndcg_at_5 - value: 72.816 + value: 72.263 - type: precision_at_1 - value: 63.333 + value: 63.666999999999994 - type: precision_at_10 - value: 9.9 + value: 10.0 - type: precision_at_100 - value: 1.103 + value: 1.093 - type: precision_at_1000 - value: 0.11199999999999999 + value: 0.11299999999999999 + - type: precision_at_20 + value: 5.3 - type: precision_at_3 - value: 26.889000000000003 + value: 27.0 - type: precision_at_5 - value: 18.2 + value: 17.867 - type: recall_at_1 - value: 60.633 + value: 61.194 - type: recall_at_10 - value: 87.36699999999999 + value: 88.156 - type: recall_at_100 - value: 97.333 + value: 96.5 - type: recall_at_1000 - value: 99.333 + value: 100.0 + - type: recall_at_20 + value: 93.389 - type: recall_at_3 - value: 73.656 + value: 73.839 - type: recall_at_5 - value: 82.083 + value: 79.828 - task: type: PairClassification dataset: @@ -2213,54 +4570,54 @@ model-index: name: MTEB SprintDuplicateQuestions config: default split: test - revision: 5a8256d0dff9c4bd3be3ba3e67e4e70173f802ea + revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 metrics: - type: cos_sim_accuracy - value: 99.76633663366337 + value: 99.87425742574257 - type: cos_sim_ap - value: 93.84024096781063 + value: 96.97141655369937 - type: cos_sim_f1 - value: 88.08080808080808 + value: 93.6910084451068 - type: cos_sim_precision - value: 88.9795918367347 + value: 93.0898321816387 - type: cos_sim_recall - value: 87.2 + value: 94.3 - type: dot_accuracy - value: 99.46336633663367 + value: 99.87425742574257 - type: dot_ap - value: 75.78127156965245 + value: 96.97141655369938 - type: dot_f1 - value: 71.41403865717193 + value: 93.6910084451068 - type: dot_precision - value: 72.67080745341616 + value: 93.0898321816387 - type: dot_recall - value: 70.19999999999999 + value: 94.3 - type: euclidean_accuracy - value: 99.67524752475248 + value: 99.87425742574257 - type: euclidean_ap - value: 88.61274955249769 + value: 96.97141655369938 - type: euclidean_f1 - value: 82.30852211434735 + value: 93.6910084451068 - type: euclidean_precision - value: 89.34426229508196 + value: 93.0898321816387 - type: euclidean_recall - value: 76.3 + value: 94.3 - type: manhattan_accuracy - value: 99.67722772277227 + value: 99.87425742574257 - type: manhattan_ap - value: 88.77516158012779 + value: 96.98252972861131 - type: manhattan_f1 - value: 82.36536430834212 + value: 93.68473396320238 - type: manhattan_precision - value: 87.24832214765101 + value: 93.17507418397626 - type: manhattan_recall - value: 78.0 + value: 94.19999999999999 - type: max_accuracy - value: 99.76633663366337 + value: 99.87425742574257 - type: max_ap - value: 93.84024096781063 + value: 96.98252972861131 - type: max_f1 - value: 88.08080808080808 + value: 93.6910084451068 - task: type: Clustering dataset: @@ -2268,10 +4625,10 @@ model-index: name: MTEB StackExchangeClustering config: default split: test - revision: 70a89468f6dccacc6aa2b12a6eac54e74328f235 + revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 metrics: - type: v_measure - value: 59.20812266121527 + value: 66.5976926394361 - task: type: Clustering dataset: @@ -2279,10 +4636,10 @@ model-index: name: MTEB StackExchangeClusteringP2P config: default split: test - revision: d88009ab563dd0b16cfaf4436abaf97fa3550cf0 + revision: 815ca46b2622cec33ccafc3735d572c266efdb44 metrics: - type: v_measure - value: 33.954248554638056 + value: 36.3221929214798 - task: type: Reranking dataset: @@ -2290,12 +4647,12 @@ model-index: name: MTEB StackOverflowDupQuestions config: default split: test - revision: ef807ea29a75ec4f91b50fd4191cb4ee4589a9f9 + revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 metrics: - type: map - value: 51.52800990025549 + value: 55.28322662897131 - type: mrr - value: 52.360394915541974 + value: 56.223620129870135 - task: type: Summarization dataset: @@ -2303,154 +4660,174 @@ model-index: name: MTEB SummEval config: default split: test - revision: 8753c2788d36c01fc6f05d03fe3f7268d63f9122 + revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c metrics: - type: cos_sim_pearson - value: 30.737881131277356 + value: 31.176396304511282 - type: cos_sim_spearman - value: 31.45979323917254 + value: 32.11989671564906 - type: dot_pearson - value: 26.24686017962023 + value: 31.17639740597169 - type: dot_spearman - value: 25.006732878791743 + value: 32.145586989831564 - task: type: Retrieval dataset: - type: trec-covid + type: mteb/trec-covid name: MTEB TRECCOVID config: default split: test - revision: 2c8041b2c07a79b6f7ba8fe6acc72e5d9f92d217 + revision: bb9466bac8153a0349341eb1b22e06409e78ef4e metrics: - type: map_at_1 - value: 0.253 + value: 0.186 - type: map_at_10 - value: 2.1399999999999997 + value: 1.659 - type: map_at_100 - value: 12.873000000000001 + value: 9.224 - type: map_at_1000 - value: 31.002000000000002 + value: 22.506999999999998 + - type: map_at_20 + value: 2.937 - type: map_at_3 - value: 0.711 + value: 0.5539999999999999 - type: map_at_5 - value: 1.125 + value: 0.8920000000000001 - type: mrr_at_1 - value: 96.0 + value: 72.0 - type: mrr_at_10 - value: 98.0 + value: 82.633 - type: mrr_at_100 - value: 98.0 + value: 82.633 - type: mrr_at_1000 - value: 98.0 + value: 82.633 + - type: mrr_at_20 + value: 82.633 - type: mrr_at_3 - value: 98.0 + value: 80.333 - type: mrr_at_5 - value: 98.0 + value: 82.633 - type: ndcg_at_1 - value: 94.0 + value: 69.0 - type: ndcg_at_10 - value: 84.881 + value: 67.327 - type: ndcg_at_100 - value: 64.694 + value: 51.626000000000005 - type: ndcg_at_1000 - value: 56.85 + value: 47.396 + - type: ndcg_at_20 + value: 63.665000000000006 - type: ndcg_at_3 - value: 90.061 + value: 68.95 - type: ndcg_at_5 - value: 87.155 + value: 69.241 - type: precision_at_1 - value: 96.0 + value: 72.0 - type: precision_at_10 - value: 88.8 + value: 71.6 - type: precision_at_100 - value: 65.7 + value: 53.22 - type: precision_at_1000 - value: 25.080000000000002 + value: 20.721999999999998 + - type: precision_at_20 + value: 67.30000000000001 - type: precision_at_3 - value: 92.667 + value: 72.667 - type: precision_at_5 - value: 90.0 + value: 74.0 - type: recall_at_1 - value: 0.253 + value: 0.186 - type: recall_at_10 - value: 2.292 + value: 1.932 - type: recall_at_100 - value: 15.78 + value: 12.883 - type: recall_at_1000 - value: 53.015 + value: 44.511 + - type: recall_at_20 + value: 3.583 - type: recall_at_3 - value: 0.7270000000000001 + value: 0.601 - type: recall_at_5 - value: 1.162 + value: 1.0 - task: type: Retrieval dataset: - type: webis-touche2020 + type: mteb/touche2020 name: MTEB Touche2020 config: default split: test - revision: 527b7d77e16e343303e68cb6af11d6e18b9f7b3b + revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f metrics: - type: map_at_1 - value: 2.116 + value: 2.308 - type: map_at_10 - value: 9.625 + value: 9.744 - type: map_at_100 - value: 15.641 + value: 15.859000000000002 - type: map_at_1000 - value: 17.127 + value: 17.396 + - type: map_at_20 + value: 12.49 - type: map_at_3 - value: 4.316 + value: 4.848 - type: map_at_5 - value: 6.208 + value: 6.912999999999999 - type: mrr_at_1 value: 32.653 - type: mrr_at_10 - value: 48.083999999999996 + value: 47.207 - type: mrr_at_100 - value: 48.631 + value: 48.116 - type: mrr_at_1000 - value: 48.649 + value: 48.116 + - type: mrr_at_20 + value: 47.735 - type: mrr_at_3 value: 42.857 - type: mrr_at_5 - value: 46.224 + value: 44.285999999999994 - type: ndcg_at_1 - value: 29.592000000000002 + value: 28.571 - type: ndcg_at_10 - value: 25.430999999999997 + value: 24.421 - type: ndcg_at_100 - value: 36.344 + value: 35.961 - type: ndcg_at_1000 - value: 47.676 + value: 47.541 + - type: ndcg_at_20 + value: 25.999 - type: ndcg_at_3 - value: 26.144000000000002 + value: 25.333 - type: ndcg_at_5 - value: 26.304 + value: 25.532 - type: precision_at_1 value: 32.653 - type: precision_at_10 - value: 24.082 + value: 22.448999999999998 - type: precision_at_100 - value: 7.714 + value: 7.571 - type: precision_at_1000 value: 1.5310000000000001 + - type: precision_at_20 + value: 17.959 - type: precision_at_3 value: 26.531 - type: precision_at_5 - value: 26.939 + value: 26.122 - type: recall_at_1 - value: 2.116 + value: 2.308 - type: recall_at_10 - value: 16.794 + value: 16.075 - type: recall_at_100 - value: 47.452 + value: 47.357 - type: recall_at_1000 - value: 82.312 + value: 82.659 + - type: recall_at_20 + value: 24.554000000000002 - type: recall_at_3 - value: 5.306 + value: 5.909 - type: recall_at_5 - value: 9.306000000000001 + value: 9.718 - task: type: Classification dataset: @@ -2461,11 +4838,11 @@ model-index: revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de metrics: - type: accuracy - value: 67.709 + value: 67.2998046875 - type: ap - value: 13.541535578501716 + value: 12.796222498684031 - type: f1 - value: 52.569619919446794 + value: 51.7465070845071 - task: type: Classification dataset: @@ -2473,12 +4850,12 @@ model-index: name: MTEB TweetSentimentExtractionClassification config: default split: test - revision: 62146448f05be9e52a36b8ee9936447ea787eede + revision: d604517c81ca91fe16a244d1248fc021f9ecee7a metrics: - type: accuracy - value: 56.850594227504246 + value: 61.76004527447652 - type: f1 - value: 57.233377364910574 + value: 61.88985723942393 - task: type: Clustering dataset: @@ -2486,10 +4863,10 @@ model-index: name: MTEB TwentyNewsgroupsClustering config: default split: test - revision: 091a54f9a36281ce7d6590ec8c75dd485e7e01d4 + revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 metrics: - type: v_measure - value: 39.463722986090474 + value: 52.69229715788263 - task: type: PairClassification dataset: @@ -2500,51 +4877,51 @@ model-index: revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 metrics: - type: cos_sim_accuracy - value: 84.09131549144662 + value: 87.42325803182929 - type: cos_sim_ap - value: 66.86677647503386 + value: 78.29203513753492 - type: cos_sim_f1 - value: 62.94631710362049 + value: 71.33160557818093 - type: cos_sim_precision - value: 59.73933649289099 + value: 67.00672385810341 - type: cos_sim_recall - value: 66.51715039577837 + value: 76.2532981530343 - type: dot_accuracy - value: 80.27656911247541 + value: 87.42325803182929 - type: dot_ap - value: 54.291720398612085 + value: 78.29208368244002 - type: dot_f1 - value: 54.77150537634409 + value: 71.33160557818093 - type: dot_precision - value: 47.58660957571039 + value: 67.00672385810341 - type: dot_recall - value: 64.5118733509235 + value: 76.2532981530343 - type: euclidean_accuracy - value: 82.76211480002385 + value: 87.42325803182929 - type: euclidean_ap - value: 62.430397690753296 + value: 78.29202838891078 - type: euclidean_f1 - value: 59.191590539356774 + value: 71.33160557818093 - type: euclidean_precision - value: 56.296119971435374 + value: 67.00672385810341 - type: euclidean_recall - value: 62.401055408970976 + value: 76.2532981530343 - type: manhattan_accuracy - value: 82.7561542588067 + value: 87.42325803182929 - type: manhattan_ap - value: 62.41882051995577 + value: 78.23964459648822 - type: manhattan_f1 - value: 59.32101002778785 + value: 71.1651728553137 - type: manhattan_precision - value: 54.71361711611321 + value: 69.12935323383084 - type: manhattan_recall - value: 64.77572559366754 + value: 73.3245382585752 - type: max_accuracy - value: 84.09131549144662 + value: 87.42325803182929 - type: max_ap - value: 66.86677647503386 + value: 78.29208368244002 - type: max_f1 - value: 62.94631710362049 + value: 71.33160557818093 - task: type: PairClassification dataset: @@ -2555,49 +4932,49 @@ model-index: revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf metrics: - type: cos_sim_accuracy - value: 88.79574649745798 + value: 89.00725734466566 - type: cos_sim_ap - value: 85.28960532524223 + value: 86.1594112416402 - type: cos_sim_f1 - value: 77.98460043358001 + value: 78.544568993303 - type: cos_sim_precision - value: 75.78090948714224 + value: 73.42484097756947 - type: cos_sim_recall - value: 80.32029565753002 + value: 84.43178318447798 - type: dot_accuracy - value: 85.5939767920208 + value: 89.00725734466566 - type: dot_ap - value: 76.14131706694056 + value: 86.15940795129771 - type: dot_f1 - value: 72.70246298696868 + value: 78.544568993303 - type: dot_precision - value: 65.27012127894156 + value: 73.42484097756947 - type: dot_recall - value: 82.04496458269172 + value: 84.43178318447798 - type: euclidean_accuracy - value: 86.72332828812046 + value: 89.00725734466566 - type: euclidean_ap - value: 80.84854809178995 + value: 86.15939689541806 - type: euclidean_f1 - value: 72.47657499809551 + value: 78.544568993303 - type: euclidean_precision - value: 71.71717171717171 + value: 73.42484097756947 - type: euclidean_recall - value: 73.25223283030489 + value: 84.43178318447798 - type: manhattan_accuracy - value: 86.7563162184189 + value: 88.97426941436721 - type: manhattan_ap - value: 80.87598895575626 + value: 86.14154348065739 - type: manhattan_f1 - value: 72.54617892068092 + value: 78.53991175290814 - type: manhattan_precision - value: 68.49268225960881 + value: 74.60339452719086 - type: manhattan_recall - value: 77.10963966738528 + value: 82.91499846011703 - type: max_accuracy - value: 88.79574649745798 + value: 89.00725734466566 - type: max_ap - value: 85.28960532524223 + value: 86.1594112416402 - type: max_f1 - value: 77.98460043358001 + value: 78.544568993303 --- \ No newline at end of file