metadata
model-index:
- name: ternary-weight-embedding
results:
- dataset:
config: default
name: MTEB AFQMC (default)
revision: b44c3b011063adb25877c13823db83bb193913c4
split: validation
type: C-MTEB/AFQMC
metrics:
- type: cosine_pearson
value: 54.178784977880845
- type: cosine_spearman
value: 58.24983603546464
- type: euclidean_pearson
value: 56.12063302915049
- type: euclidean_spearman
value: 57.55891180444177
- type: main_score
value: 58.24983603546464
- type: manhattan_pearson
value: 56.14231021933658
- type: manhattan_spearman
value: 57.57073312857721
task:
type: STS
- dataset:
config: default
name: MTEB ATEC (default)
revision: 0f319b1142f28d00e055a6770f3f726ae9b7d865
split: test
type: C-MTEB/ATEC
metrics:
- type: cosine_pearson
value: 54.15496422617569
- type: cosine_spearman
value: 57.14812321292837
- type: euclidean_pearson
value: 60.12883177451991
- type: euclidean_spearman
value: 56.79823039497166
- type: main_score
value: 57.14812321292837
- type: manhattan_pearson
value: 60.12467197534511
- type: manhattan_spearman
value: 56.79279245167105
task:
type: STS
- dataset:
config: default
name: MTEB ATEC (default)
revision: 0f319b1142f28d00e055a6770f3f726ae9b7d865
split: validation
type: C-MTEB/ATEC
metrics:
- type: cosine_pearson
value: 54.24370907266313
- type: cosine_spearman
value: 57.79884664537865
- type: euclidean_pearson
value: 60.05093414384327
- type: euclidean_spearman
value: 57.47352051181076
- type: main_score
value: 57.79884664537865
- type: manhattan_pearson
value: 60.03579846842669
- type: manhattan_spearman
value: 57.46213161281322
task:
type: STS
- dataset:
config: en
name: MTEB AmazonReviewsClassification (en)
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
split: test
type: mteb/amazon_reviews_multi
metrics:
- type: accuracy
value: 37.554
- type: f1
value: 37.076043043891794
- type: f1_weighted
value: 37.076043043891794
- type: main_score
value: 37.554
task:
type: Classification
- dataset:
config: de
name: MTEB AmazonReviewsClassification (de)
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
split: test
type: mteb/amazon_reviews_multi
metrics:
- type: accuracy
value: 26.883999999999997
- type: f1
value: 26.766904450030015
- type: f1_weighted
value: 26.766904450030015
- type: main_score
value: 26.883999999999997
task:
type: Classification
- dataset:
config: es
name: MTEB AmazonReviewsClassification (es)
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
split: test
type: mteb/amazon_reviews_multi
metrics:
- type: accuracy
value: 29.028
- type: f1
value: 28.780140874309517
- type: f1_weighted
value: 28.780140874309517
- type: main_score
value: 29.028
task:
type: Classification
- dataset:
config: fr
name: MTEB AmazonReviewsClassification (fr)
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
split: test
type: mteb/amazon_reviews_multi
metrics:
- type: accuracy
value: 26.72
- type: f1
value: 26.480495303330265
- type: f1_weighted
value: 26.480495303330265
- type: main_score
value: 26.72
task:
type: Classification
- dataset:
config: ja
name: MTEB AmazonReviewsClassification (ja)
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
split: test
type: mteb/amazon_reviews_multi
metrics:
- type: accuracy
value: 28.518
- type: f1
value: 28.438382784401405
- type: f1_weighted
value: 28.438382784401405
- type: main_score
value: 28.518
task:
type: Classification
- dataset:
config: zh
name: MTEB AmazonReviewsClassification (zh)
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
split: test
type: mteb/amazon_reviews_multi
metrics:
- type: accuracy
value: 46.26200000000001
- type: f1
value: 45.27125595644993
- type: f1_weighted
value: 45.271255956449934
- type: main_score
value: 46.26200000000001
task:
type: Classification
- dataset:
config: en
name: MTEB AmazonReviewsClassification (en)
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
split: validation
type: mteb/amazon_reviews_multi
metrics:
- type: accuracy
value: 37.258
- type: f1
value: 36.810536650483364
- type: f1_weighted
value: 36.810536650483364
- type: main_score
value: 37.258
task:
type: Classification
- dataset:
config: de
name: MTEB AmazonReviewsClassification (de)
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
split: validation
type: mteb/amazon_reviews_multi
metrics:
- type: accuracy
value: 27.168
- type: f1
value: 27.110076496076186
- type: f1_weighted
value: 27.110076496076186
- type: main_score
value: 27.168
task:
type: Classification
- dataset:
config: es
name: MTEB AmazonReviewsClassification (es)
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
split: validation
type: mteb/amazon_reviews_multi
metrics:
- type: accuracy
value: 28.582
- type: f1
value: 28.380427955500743
- type: f1_weighted
value: 28.380427955500743
- type: main_score
value: 28.582
task:
type: Classification
- dataset:
config: fr
name: MTEB AmazonReviewsClassification (fr)
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
split: validation
type: mteb/amazon_reviews_multi
metrics:
- type: accuracy
value: 26.284000000000006
- type: f1
value: 26.020390218699436
- type: f1_weighted
value: 26.020390218699436
- type: main_score
value: 26.284000000000006
task:
type: Classification
- dataset:
config: ja
name: MTEB AmazonReviewsClassification (ja)
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
split: validation
type: mteb/amazon_reviews_multi
metrics:
- type: accuracy
value: 28.994000000000003
- type: f1
value: 28.882137331326792
- type: f1_weighted
value: 28.882137331326796
- type: main_score
value: 28.994000000000003
task:
type: Classification
- dataset:
config: zh
name: MTEB AmazonReviewsClassification (zh)
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
split: validation
type: mteb/amazon_reviews_multi
metrics:
- type: accuracy
value: 45.64799999999999
- type: f1
value: 44.72730068810682
- type: f1_weighted
value: 44.72730068810682
- type: main_score
value: 45.64799999999999
task:
type: Classification
- dataset:
config: default
name: MTEB BQ (default)
revision: e3dda5e115e487b39ec7e618c0c6a29137052a55
split: test
type: C-MTEB/BQ
metrics:
- type: cosine_pearson
value: 72.11546348999653
- type: cosine_spearman
value: 74.28360304728436
- type: euclidean_pearson
value: 72.68784320632835
- type: euclidean_spearman
value: 73.8988695267803
- type: main_score
value: 74.28360304728436
- type: manhattan_pearson
value: 72.67994924531479
- type: manhattan_spearman
value: 73.89602896299826
task:
type: STS
- dataset:
config: default
name: MTEB BQ (default)
revision: e3dda5e115e487b39ec7e618c0c6a29137052a55
split: validation
type: C-MTEB/BQ
metrics:
- type: cosine_pearson
value: 74.80134578588282
- type: cosine_spearman
value: 76.88519187053075
- type: euclidean_pearson
value: 75.45155138775218
- type: euclidean_spearman
value: 76.44577057706553
- type: main_score
value: 76.88519187053075
- type: manhattan_pearson
value: 75.44260208073607
- type: manhattan_spearman
value: 76.43697175914832
task:
type: STS
- dataset:
config: default
name: MTEB CLSClusteringP2P (default)
revision: 4b6227591c6c1a73bc76b1055f3b7f3588e72476
split: test
type: C-MTEB/CLSClusteringP2P
metrics:
- type: main_score
value: 54.09910844643998
- type: v_measure
value: 54.09910844643998
- type: v_measure_std
value: 1.4947511735406993
task:
type: Clustering
- dataset:
config: default
name: MTEB CLSClusteringP2P.v2 (default)
revision: 4b6227591c6c1a73bc76b1055f3b7f3588e72476
split: test
type: C-MTEB/CLSClusteringP2P
metrics:
- type: main_score
value: 54.68251196895576
- type: v_measure
value: 54.68251196895576
- type: v_measure_std
value: 1.8827992663564201
task:
type: Clustering
- dataset:
config: default
name: MTEB CLSClusteringS2S (default)
revision: e458b3f5414b62b7f9f83499ac1f5497ae2e869f
split: test
type: C-MTEB/CLSClusteringS2S
metrics:
- type: main_score
value: 46.50249324360948
- type: v_measure
value: 46.50249324360948
- type: v_measure_std
value: 1.575706238905038
task:
type: Clustering
- dataset:
config: default
name: MTEB CLSClusteringS2S.v2 (default)
revision: e458b3f5414b62b7f9f83499ac1f5497ae2e869f
split: test
type: C-MTEB/CLSClusteringS2S
metrics:
- type: main_score
value: 48.5426747048895
- type: v_measure
value: 48.5426747048895
- type: v_measure_std
value: 1.8768930779321003
task:
type: Clustering
- dataset:
config: default
name: MTEB CMedQAv1
revision: 8d7f1e942507dac42dc58017c1a001c3717da7df
split: test
type: C-MTEB/CMedQAv1-reranking
metrics:
- type: main_score
value: 88.74015900697846
- type: map
value: 88.74015900697846
- type: mrr
value: 90.6736507936508
- type: nAUC_map_diff1
value: 57.22083174189404
- type: nAUC_map_max
value: 55.77168783483574
- type: nAUC_map_std
value: 13.021507960473016
- type: nAUC_mrr_diff1
value: 62.952041755231626
- type: nAUC_mrr_max
value: 64.05253602193952
- type: nAUC_mrr_std
value: 23.213961052744555
task:
type: Reranking
- dataset:
config: default
name: MTEB CMedQAv2
revision: 23d186750531a14a0357ca22cd92d712fd512ea0
split: test
type: C-MTEB/CMedQAv2-reranking
metrics:
- type: main_score
value: 88.38113834645159
- type: map
value: 88.38113834645159
- type: mrr
value: 90.52234126984126
- type: nAUC_map_diff1
value: 59.755818934870234
- type: nAUC_map_max
value: 62.11664250496217
- type: nAUC_map_std
value: 5.676411532480005
- type: nAUC_mrr_diff1
value: 67.07181219755715
- type: nAUC_mrr_max
value: 75.28581323486274
- type: nAUC_mrr_std
value: 17.997453037644533
task:
type: Reranking
- dataset:
config: default
name: MTEB CmedqaRetrieval (default)
revision: cd540c506dae1cf9e9a59c3e06f42030d54e7301
split: dev
type: C-MTEB/CmedqaRetrieval
metrics:
- type: main_score
value: 45.113
- type: map_at_1
value: 25.974999999999998
- type: map_at_10
value: 38.567
- type: map_at_100
value: 40.407
- type: map_at_1000
value: 40.52
- type: map_at_20
value: 39.564
- type: map_at_3
value: 34.410000000000004
- type: map_at_5
value: 36.689
- type: mrr_at_1
value: 39.65991497874469
- type: mrr_at_10
value: 47.631104204622524
- type: mrr_at_100
value: 48.59652256415256
- type: mrr_at_1000
value: 48.64137820297102
- type: mrr_at_20
value: 48.193915296968996
- type: mrr_at_3
value: 45.24047678586305
- type: mrr_at_5
value: 46.595815620571805
- type: nauc_map_at_1000_diff1
value: 47.78866266244895
- type: nauc_map_at_1000_max
value: 41.6785411481451
- type: nauc_map_at_1000_std
value: -13.089914559064795
- type: nauc_map_at_100_diff1
value: 47.74420073204668
- type: nauc_map_at_100_max
value: 41.63744109954874
- type: nauc_map_at_100_std
value: -13.115455589619943
- type: nauc_map_at_10_diff1
value: 47.60785675353292
- type: nauc_map_at_10_max
value: 40.81808736337173
- type: nauc_map_at_10_std
value: -14.010361731187555
- type: nauc_map_at_1_diff1
value: 52.623450404334235
- type: nauc_map_at_1_max
value: 32.48612623845417
- type: nauc_map_at_1_std
value: -14.301701958402347
- type: nauc_map_at_20_diff1
value: 47.623803967273446
- type: nauc_map_at_20_max
value: 41.19261840063425
- type: nauc_map_at_20_std
value: -13.695407747959843
- type: nauc_map_at_3_diff1
value: 48.343168244250215
- type: nauc_map_at_3_max
value: 37.915780566548726
- type: nauc_map_at_3_std
value: -14.422861700351183
- type: nauc_map_at_5_diff1
value: 47.94643138191748
- type: nauc_map_at_5_max
value: 39.59735520213133
- type: nauc_map_at_5_std
value: -14.23393731727717
- type: nauc_mrr_at_1000_diff1
value: 53.86065646960032
- type: nauc_mrr_at_1000_max
value: 49.296644570372415
- type: nauc_mrr_at_1000_std
value: -8.960076833549836
- type: nauc_mrr_at_100_diff1
value: 53.83949971297237
- type: nauc_mrr_at_100_max
value: 49.283394247434124
- type: nauc_mrr_at_100_std
value: -8.938461521660448
- type: nauc_mrr_at_10_diff1
value: 53.771434307862776
- type: nauc_mrr_at_10_max
value: 49.232816072077185
- type: nauc_mrr_at_10_std
value: -9.187019034523127
- type: nauc_mrr_at_1_diff1
value: 59.00104676791166
- type: nauc_mrr_at_1_max
value: 51.260502472764436
- type: nauc_mrr_at_1_std
value: -9.593234514978924
- type: nauc_mrr_at_20_diff1
value: 53.78230872607412
- type: nauc_mrr_at_20_max
value: 49.21546854521065
- type: nauc_mrr_at_20_std
value: -9.163123771380729
- type: nauc_mrr_at_3_diff1
value: 54.55765283386248
- type: nauc_mrr_at_3_max
value: 49.60566476180881
- type: nauc_mrr_at_3_std
value: -9.148690932678257
- type: nauc_mrr_at_5_diff1
value: 53.97605291263774
- type: nauc_mrr_at_5_max
value: 49.40659705825686
- type: nauc_mrr_at_5_std
value: -9.161744382503885
- type: nauc_ndcg_at_1000_diff1
value: 48.31934043697359
- type: nauc_ndcg_at_1000_max
value: 44.71283713093145
- type: nauc_ndcg_at_1000_std
value: -9.913668236760554
- type: nauc_ndcg_at_100_diff1
value: 47.38673504958784
- type: nauc_ndcg_at_100_max
value: 44.25750824625488
- type: nauc_ndcg_at_100_std
value: -9.48225516354158
- type: nauc_ndcg_at_10_diff1
value: 46.90328760292496
- type: nauc_ndcg_at_10_max
value: 42.41986408299954
- type: nauc_ndcg_at_10_std
value: -12.909824341781825
- type: nauc_ndcg_at_1_diff1
value: 59.00104676791166
- type: nauc_ndcg_at_1_max
value: 51.260502472764436
- type: nauc_ndcg_at_1_std
value: -9.593234514978924
- type: nauc_ndcg_at_20_diff1
value: 46.77258181931362
- type: nauc_ndcg_at_20_max
value: 42.68119586804985
- type: nauc_ndcg_at_20_std
value: -12.423629313301388
- type: nauc_ndcg_at_3_diff1
value: 48.58977327180224
- type: nauc_ndcg_at_3_max
value: 43.61141480978654
- type: nauc_ndcg_at_3_std
value: -11.641555503967986
- type: nauc_ndcg_at_5_diff1
value: 47.67463895870823
- type: nauc_ndcg_at_5_max
value: 42.49711123538069
- type: nauc_ndcg_at_5_std
value: -12.452049013694403
- type: nauc_precision_at_1000_diff1
value: 3.4806441777617567
- type: nauc_precision_at_1000_max
value: 27.678000825789933
- type: nauc_precision_at_1000_std
value: 17.698677421147963
- type: nauc_precision_at_100_diff1
value: 7.875973729853397
- type: nauc_precision_at_100_max
value: 34.08812146111534
- type: nauc_precision_at_100_std
value: 16.524389567188365
- type: nauc_precision_at_10_diff1
value: 21.272214137016185
- type: nauc_precision_at_10_max
value: 44.00343281584008
- type: nauc_precision_at_10_std
value: 0.9945485334597843
- type: nauc_precision_at_1_diff1
value: 59.00104676791166
- type: nauc_precision_at_1_max
value: 51.260502472764436
- type: nauc_precision_at_1_std
value: -9.593234514978924
- type: nauc_precision_at_20_diff1
value: 16.396372302736623
- type: nauc_precision_at_20_max
value: 40.20456764221775
- type: nauc_precision_at_20_std
value: 4.91733314934679
- type: nauc_precision_at_3_diff1
value: 34.03262918273604
- type: nauc_precision_at_3_max
value: 48.260349668115424
- type: nauc_precision_at_3_std
value: -3.812770245046173
- type: nauc_precision_at_5_diff1
value: 28.088076219897022
- type: nauc_precision_at_5_max
value: 46.962465223407946
- type: nauc_precision_at_5_std
value: -1.663412582138589
- type: nauc_recall_at_1000_diff1
value: 29.003325559730303
- type: nauc_recall_at_1000_max
value: 51.11704447020827
- type: nauc_recall_at_1000_std
value: 52.358021602871176
- type: nauc_recall_at_100_diff1
value: 26.944312318666174
- type: nauc_recall_at_100_max
value: 35.1360457491317
- type: nauc_recall_at_100_std
value: 8.995107524344007
- type: nauc_recall_at_10_diff1
value: 34.767375834324746
- type: nauc_recall_at_10_max
value: 33.15737890896935
- type: nauc_recall_at_10_std
value: -13.876362538877132
- type: nauc_recall_at_1_diff1
value: 52.623450404334235
- type: nauc_recall_at_1_max
value: 32.48612623845417
- type: nauc_recall_at_1_std
value: -14.301701958402347
- type: nauc_recall_at_20_diff1
value: 31.796685961276754
- type: nauc_recall_at_20_max
value: 31.645635578789094
- type: nauc_recall_at_20_std
value: -12.896907066742678
- type: nauc_recall_at_3_diff1
value: 40.9513328277203
- type: nauc_recall_at_3_max
value: 33.2576092320736
- type: nauc_recall_at_3_std
value: -13.677012060621971
- type: nauc_recall_at_5_diff1
value: 38.17560293253476
- type: nauc_recall_at_5_max
value: 33.66607334519376
- type: nauc_recall_at_5_std
value: -13.710743011608292
- type: ndcg_at_1
value: 39.660000000000004
- type: ndcg_at_10
value: 45.113
- type: ndcg_at_100
value: 52.257
- type: ndcg_at_1000
value: 54.25
- type: ndcg_at_20
value: 47.857
- type: ndcg_at_3
value: 40.022999999999996
- type: ndcg_at_5
value: 42.076
- type: precision_at_1
value: 39.660000000000004
- type: precision_at_10
value: 10.025
- type: precision_at_100
value: 1.5789999999999997
- type: precision_at_1000
value: 0.183
- type: precision_at_20
value: 5.915
- type: precision_at_3
value: 22.639
- type: precision_at_5
value: 16.359
- type: recall_at_1
value: 25.974999999999998
- type: recall_at_10
value: 55.332
- type: recall_at_100
value: 84.899
- type: recall_at_1000
value: 98.312
- type: recall_at_20
value: 64.702
- type: recall_at_3
value: 39.800999999999995
- type: recall_at_5
value: 46.378
task:
type: Retrieval
- dataset:
config: default
name: MTEB Cmnli (default)
revision: 41bc36f332156f7adc9e38f53777c959b2ae9766
split: validation
type: C-MTEB/CMNLI
metrics:
- type: cosine_accuracy
value: 85.13529765484064
- type: cosine_accuracy_threshold
value: 86.13102436065674
- type: cosine_ap
value: 92.19355954725854
- type: cosine_f1
value: 86.10154427285858
- type: cosine_f1_threshold
value: 84.71119403839111
- type: cosine_precision
value: 82.02794242167654
- type: cosine_recall
value: 90.6008884732289
- type: dot_accuracy
value: 74.91280817799158
- type: dot_accuracy_threshold
value: 63842.00439453125
- type: dot_ap
value: 83.44759720338214
- type: dot_f1
value: 77.27227076428642
- type: dot_f1_threshold
value: 60020.81298828125
- type: dot_precision
value: 67.72887323943661
- type: dot_recall
value: 89.94622398877719
- type: euclidean_accuracy
value: 84.47384245339748
- type: euclidean_accuracy_threshold
value: 1469.5334434509277
- type: euclidean_ap
value: 91.3556506449566
- type: euclidean_f1
value: 85.43989402803841
- type: euclidean_f1_threshold
value: 1538.3913040161133
- type: euclidean_precision
value: 80.92848180677541
- type: euclidean_recall
value: 90.48398410100538
- type: main_score
value: 85.13529765484064
- type: manhattan_accuracy
value: 84.41371016235718
- type: manhattan_accuracy_threshold
value: 49906.90002441406
- type: manhattan_ap
value: 91.3493016502162
- type: manhattan_f1
value: 85.49062844542446
- type: manhattan_f1_threshold
value: 51897.528076171875
- type: manhattan_precision
value: 80.88879616106823
- type: manhattan_recall
value: 90.6476502221183
- type: max_accuracy
value: 85.13529765484064
- type: max_ap
value: 92.19355954725854
- type: max_f1
value: 86.10154427285858
- type: max_precision
value: 82.02794242167654
- type: max_recall
value: 90.6476502221183
- type: similarity_accuracy
value: 85.13529765484064
- type: similarity_accuracy_threshold
value: 86.13102436065674
- type: similarity_ap
value: 92.17394345753924
- type: similarity_f1
value: 86.10154427285858
- type: similarity_f1_threshold
value: 84.71119403839111
- type: similarity_precision
value: 82.02794242167654
- type: similarity_recall
value: 90.6008884732289
task:
type: PairClassification
- dataset:
config: default
name: MTEB CovidRetrieval (default)
revision: 1271c7809071a13532e05f25fb53511ffce77117
split: dev
type: C-MTEB/CovidRetrieval
metrics:
- type: main_score
value: 82.819
- type: map_at_1
value: 69.494
- type: map_at_10
value: 78.789
- type: map_at_100
value: 78.988
- type: map_at_1000
value: 78.993
- type: map_at_20
value: 78.917
- type: map_at_3
value: 77.011
- type: map_at_5
value: 78.17699999999999
- type: mrr_at_1
value: 69.6522655426765
- type: mrr_at_10
value: 78.76729054811248
- type: mrr_at_100
value: 78.95793499668491
- type: mrr_at_1000
value: 78.96216519091867
- type: mrr_at_20
value: 78.88679240958568
- type: mrr_at_3
value: 77.04601334738325
- type: mrr_at_5
value: 78.22620302072359
- type: nauc_map_at_1000_diff1
value: 80.05214569065625
- type: nauc_map_at_1000_max
value: 36.21145092927336
- type: nauc_map_at_1000_std
value: -36.16531861028991
- type: nauc_map_at_100_diff1
value: 80.05020798715684
- type: nauc_map_at_100_max
value: 36.22261208348821
- type: nauc_map_at_100_std
value: -36.14332748625607
- type: nauc_map_at_10_diff1
value: 79.93946377542417
- type: nauc_map_at_10_max
value: 36.20456984006017
- type: nauc_map_at_10_std
value: -36.398588779081294
- type: nauc_map_at_1_diff1
value: 82.0602604309234
- type: nauc_map_at_1_max
value: 35.19249651707143
- type: nauc_map_at_1_std
value: -34.21839088932163
- type: nauc_map_at_20_diff1
value: 80.02697105208125
- type: nauc_map_at_20_max
value: 36.18004342047142
- type: nauc_map_at_20_std
value: -36.17886432477489
- type: nauc_map_at_3_diff1
value: 80.16978634467844
- type: nauc_map_at_3_max
value: 35.83327596692132
- type: nauc_map_at_3_std
value: -38.37503788050143
- type: nauc_map_at_5_diff1
value: 79.81622895096287
- type: nauc_map_at_5_max
value: 36.08883221963992
- type: nauc_map_at_5_std
value: -37.25482423232233
- type: nauc_mrr_at_1000_diff1
value: 80.2097372109321
- type: nauc_mrr_at_1000_max
value: 36.42174288518451
- type: nauc_mrr_at_1000_std
value: -35.85189499872029
- type: nauc_mrr_at_100_diff1
value: 80.20776581779481
- type: nauc_mrr_at_100_max
value: 36.432844978272236
- type: nauc_mrr_at_100_std
value: -35.82999739661053
- type: nauc_mrr_at_10_diff1
value: 80.09236275670942
- type: nauc_mrr_at_10_max
value: 36.380671611318604
- type: nauc_mrr_at_10_std
value: -36.15989339661995
- type: nauc_mrr_at_1_diff1
value: 81.87621839783066
- type: nauc_mrr_at_1_max
value: 35.88505228929411
- type: nauc_mrr_at_1_std
value: -32.92079357076708
- type: nauc_mrr_at_20_diff1
value: 80.18395501536016
- type: nauc_mrr_at_20_max
value: 36.38952241095837
- type: nauc_mrr_at_20_std
value: -35.866686376779114
- type: nauc_mrr_at_3_diff1
value: 80.23254549571189
- type: nauc_mrr_at_3_max
value: 36.05935731319566
- type: nauc_mrr_at_3_std
value: -37.866428822755864
- type: nauc_mrr_at_5_diff1
value: 79.88253909007965
- type: nauc_mrr_at_5_max
value: 36.457475361699146
- type: nauc_mrr_at_5_std
value: -36.49596735742553
- type: nauc_ndcg_at_1000_diff1
value: 79.7618117697378
- type: nauc_ndcg_at_1000_max
value: 36.629676985491514
- type: nauc_ndcg_at_1000_std
value: -35.86456326352086
- type: nauc_ndcg_at_100_diff1
value: 79.65891920529748
- type: nauc_ndcg_at_100_max
value: 36.97516050762293
- type: nauc_ndcg_at_100_std
value: -35.2001931520788
- type: nauc_ndcg_at_10_diff1
value: 79.17817241151019
- type: nauc_ndcg_at_10_max
value: 36.86254607903196
- type: nauc_ndcg_at_10_std
value: -36.235782359106615
- type: nauc_ndcg_at_1_diff1
value: 81.6552730132908
- type: nauc_ndcg_at_1_max
value: 35.6139365961136
- type: nauc_ndcg_at_1_std
value: -33.34605824156571
- type: nauc_ndcg_at_20_diff1
value: 79.53353610120962
- type: nauc_ndcg_at_20_max
value: 36.83633652729889
- type: nauc_ndcg_at_20_std
value: -35.223937945126934
- type: nauc_ndcg_at_3_diff1
value: 79.66142161189495
- type: nauc_ndcg_at_3_max
value: 36.201048838264995
- type: nauc_ndcg_at_3_std
value: -40.303435032913235
- type: nauc_ndcg_at_5_diff1
value: 78.84195807786423
- type: nauc_ndcg_at_5_max
value: 36.82613815363756
- type: nauc_ndcg_at_5_std
value: -38.073937317858494
- type: nauc_precision_at_1000_diff1
value: -52.61992538101203
- type: nauc_precision_at_1000_max
value: 2.876218848436758
- type: nauc_precision_at_1000_std
value: 54.38129782759948
- type: nauc_precision_at_100_diff1
value: -15.670131055377192
- type: nauc_precision_at_100_max
value: 27.850990893649424
- type: nauc_precision_at_100_std
value: 53.31709331697566
- type: nauc_precision_at_10_diff1
value: 38.84116900921712
- type: nauc_precision_at_10_max
value: 34.79084147327868
- type: nauc_precision_at_10_std
value: -6.829876002907609
- type: nauc_precision_at_1_diff1
value: 81.6552730132908
- type: nauc_precision_at_1_max
value: 35.6139365961136
- type: nauc_precision_at_1_std
value: -33.34605824156571
- type: nauc_precision_at_20_diff1
value: 25.845188821845383
- type: nauc_precision_at_20_max
value: 31.18865106000545
- type: nauc_precision_at_20_std
value: 14.759241268473561
- type: nauc_precision_at_3_diff1
value: 70.8201683277703
- type: nauc_precision_at_3_max
value: 36.6701970406286
- type: nauc_precision_at_3_std
value: -42.35040269825674
- type: nauc_precision_at_5_diff1
value: 59.02227125330898
- type: nauc_precision_at_5_max
value: 38.69410968250484
- type: nauc_precision_at_5_std
value: -30.30854393285351
- type: nauc_recall_at_1000_diff1
value: .nan
- type: nauc_recall_at_1000_max
value: .nan
- type: nauc_recall_at_1000_std
value: .nan
- type: nauc_recall_at_100_diff1
value: 64.60431422737145
- type: nauc_recall_at_100_max
value: 82.10937063790381
- type: nauc_recall_at_100_std
value: 51.005079723478254
- type: nauc_recall_at_10_diff1
value: 70.9120103945018
- type: nauc_recall_at_10_max
value: 44.17151180123427
- type: nauc_recall_at_10_std
value: -30.974453318671753
- type: nauc_recall_at_1_diff1
value: 82.0602604309234
- type: nauc_recall_at_1_max
value: 35.19249651707143
- type: nauc_recall_at_1_std
value: -34.21839088932163
- type: nauc_recall_at_20_diff1
value: 72.87073930480337
- type: nauc_recall_at_20_max
value: 48.156799565062514
- type: nauc_recall_at_20_std
value: -8.986967985015431
- type: nauc_recall_at_3_diff1
value: 77.48100030181683
- type: nauc_recall_at_3_max
value: 37.72633639333108
- type: nauc_recall_at_3_std
value: -48.48712826839765
- type: nauc_recall_at_5_diff1
value: 72.79074903006723
- type: nauc_recall_at_5_max
value: 41.41902376036034
- type: nauc_recall_at_5_std
value: -42.76726369093494
- type: ndcg_at_1
value: 69.758
- type: ndcg_at_10
value: 82.819
- type: ndcg_at_100
value: 83.705
- type: ndcg_at_1000
value: 83.814
- type: ndcg_at_20
value: 83.26599999999999
- type: ndcg_at_3
value: 79.296
- type: ndcg_at_5
value: 81.406
- type: precision_at_1
value: 69.758
- type: precision_at_10
value: 9.621
- type: precision_at_100
value: 1.002
- type: precision_at_1000
value: 0.101
- type: precision_at_20
value: 4.9
- type: precision_at_3
value: 28.767
- type: precision_at_5
value: 18.335
- type: recall_at_1
value: 69.494
- type: recall_at_10
value: 95.205
- type: recall_at_100
value: 99.157
- type: recall_at_1000
value: 100
- type: recall_at_20
value: 96.944
- type: recall_at_3
value: 85.827
- type: recall_at_5
value: 90.938
task:
type: Retrieval
- dataset:
config: default
name: MTEB DuRetrieval (default)
revision: a1a333e290fe30b10f3f56498e3a0d911a693ced
split: dev
type: C-MTEB/DuRetrieval
metrics:
- type: main_score
value: 87.612
- type: map_at_1
value: 26.430999999999997
- type: map_at_10
value: 80.56400000000001
- type: map_at_100
value: 83.41199999999999
- type: map_at_1000
value: 83.447
- type: map_at_20
value: 82.76
- type: map_at_3
value: 56.21000000000001
- type: map_at_5
value: 70.743
- type: mrr_at_1
value: 91
- type: mrr_at_10
value: 93.58273809523806
- type: mrr_at_100
value: 93.66391788982004
- type: mrr_at_1000
value: 93.66580315421686
- type: mrr_at_20
value: 93.63874828087282
- type: mrr_at_3
value: 93.31666666666662
- type: mrr_at_5
value: 93.46916666666664
- type: nauc_map_at_1000_diff1
value: 6.40109926511269
- type: nauc_map_at_1000_max
value: 45.156518205790604
- type: nauc_map_at_1000_std
value: 18.593211884073153
- type: nauc_map_at_100_diff1
value: 6.448711557202101
- type: nauc_map_at_100_max
value: 45.18486159645747
- type: nauc_map_at_100_std
value: 18.616651795860722
- type: nauc_map_at_10_diff1
value: 9.84371320616581
- type: nauc_map_at_10_max
value: 40.01693268761593
- type: nauc_map_at_10_std
value: 6.908031365560747
- type: nauc_map_at_1_diff1
value: 44.953791181056026
- type: nauc_map_at_1_max
value: -11.387509242343764
- type: nauc_map_at_1_std
value: -29.012460300786657
- type: nauc_map_at_20_diff1
value: 7.38580064647179
- type: nauc_map_at_20_max
value: 44.371332746028116
- type: nauc_map_at_20_std
value: 15.833956559472313
- type: nauc_map_at_3_diff1
value: 29.427952587601474
- type: nauc_map_at_3_max
value: 3.8473270447259185
- type: nauc_map_at_3_std
value: -23.647521458336257
- type: nauc_map_at_5_diff1
value: 18.713364613154774
- type: nauc_map_at_5_max
value: 19.953262401680806
- type: nauc_map_at_5_std
value: -11.913720371911865
- type: nauc_mrr_at_1000_diff1
value: 38.040195739817776
- type: nauc_mrr_at_1000_max
value: 75.65224176632694
- type: nauc_mrr_at_1000_std
value: 39.40257100387963
- type: nauc_mrr_at_100_diff1
value: 38.03778676125757
- type: nauc_mrr_at_100_max
value: 75.65948630024249
- type: nauc_mrr_at_100_std
value: 39.41815341603381
- type: nauc_mrr_at_10_diff1
value: 37.97394065068133
- type: nauc_mrr_at_10_max
value: 75.81910017199222
- type: nauc_mrr_at_10_std
value: 39.61696874072677
- type: nauc_mrr_at_1_diff1
value: 38.83675692499215
- type: nauc_mrr_at_1_max
value: 69.99014420583036
- type: nauc_mrr_at_1_std
value: 31.8093163191202
- type: nauc_mrr_at_20_diff1
value: 38.073680057072046
- type: nauc_mrr_at_20_max
value: 75.67883974175976
- type: nauc_mrr_at_20_std
value: 39.40528722492238
- type: nauc_mrr_at_3_diff1
value: 38.28768415096703
- type: nauc_mrr_at_3_max
value: 76.28320654945267
- type: nauc_mrr_at_3_std
value: 40.02109572008348
- type: nauc_mrr_at_5_diff1
value: 38.216868985695534
- type: nauc_mrr_at_5_max
value: 76.08316007275711
- type: nauc_mrr_at_5_std
value: 39.972457019046104
- type: nauc_ndcg_at_1000_diff1
value: 10.58897864933073
- type: nauc_ndcg_at_1000_max
value: 55.5870887191566
- type: nauc_ndcg_at_1000_std
value: 30.15493957024938
- type: nauc_ndcg_at_100_diff1
value: 10.868069465772917
- type: nauc_ndcg_at_100_max
value: 55.83351050391704
- type: nauc_ndcg_at_100_std
value: 31.072236695619615
- type: nauc_ndcg_at_10_diff1
value: 9.27127335978071
- type: nauc_ndcg_at_10_max
value: 49.307727646759396
- type: nauc_ndcg_at_10_std
value: 21.552518764523654
- type: nauc_ndcg_at_1_diff1
value: 38.83675692499215
- type: nauc_ndcg_at_1_max
value: 69.99014420583036
- type: nauc_ndcg_at_1_std
value: 31.8093163191202
- type: nauc_ndcg_at_20_diff1
value: 10.99181537977003
- type: nauc_ndcg_at_20_max
value: 53.47605191227965
- type: nauc_ndcg_at_20_std
value: 26.382045731517312
- type: nauc_ndcg_at_3_diff1
value: 6.5534137252176565
- type: nauc_ndcg_at_3_max
value: 51.11114538632076
- type: nauc_ndcg_at_3_std
value: 22.189612114146588
- type: nauc_ndcg_at_5_diff1
value: 7.719135315891541
- type: nauc_ndcg_at_5_max
value: 44.360770591119504
- type: nauc_ndcg_at_5_std
value: 18.743172873708048
- type: nauc_precision_at_1000_diff1
value: -31.034822128380522
- type: nauc_precision_at_1000_max
value: 20.332934368138066
- type: nauc_precision_at_1000_std
value: 44.338803511915614
- type: nauc_precision_at_100_diff1
value: -30.811245735858193
- type: nauc_precision_at_100_max
value: 22.198372768725562
- type: nauc_precision_at_100_std
value: 46.00454797611091
- type: nauc_precision_at_10_diff1
value: -31.274194785864424
- type: nauc_precision_at_10_max
value: 32.71187220917922
- type: nauc_precision_at_10_std
value: 42.875968635394166
- type: nauc_precision_at_1_diff1
value: 38.83675692499215
- type: nauc_precision_at_1_max
value: 69.99014420583036
- type: nauc_precision_at_1_std
value: 31.8093163191202
- type: nauc_precision_at_20_diff1
value: -30.67860376845899
- type: nauc_precision_at_20_max
value: 26.74661379366211
- type: nauc_precision_at_20_std
value: 45.59437091314149
- type: nauc_precision_at_3_diff1
value: -27.743818205448356
- type: nauc_precision_at_3_max
value: 49.61474868721054
- type: nauc_precision_at_3_std
value: 37.18101217122168
- type: nauc_precision_at_5_diff1
value: -32.80762892713753
- type: nauc_precision_at_5_max
value: 40.44209817083512
- type: nauc_precision_at_5_std
value: 40.96210133419465
- type: nauc_recall_at_1000_diff1
value: 6.1482636348882735
- type: nauc_recall_at_1000_max
value: 73.7437459367813
- type: nauc_recall_at_1000_std
value: 74.5949419078317
- type: nauc_recall_at_100_diff1
value: 11.893958942544847
- type: nauc_recall_at_100_max
value: 65.83194149698096
- type: nauc_recall_at_100_std
value: 64.9416138646916
- type: nauc_recall_at_10_diff1
value: 10.763264138044141
- type: nauc_recall_at_10_max
value: 37.42063013254842
- type: nauc_recall_at_10_std
value: 5.538711799036518
- type: nauc_recall_at_1_diff1
value: 44.953791181056026
- type: nauc_recall_at_1_max
value: -11.387509242343764
- type: nauc_recall_at_1_std
value: -29.012460300786657
- type: nauc_recall_at_20_diff1
value: 10.112135571654077
- type: nauc_recall_at_20_max
value: 48.8731227722893
- type: nauc_recall_at_20_std
value: 27.875300064543712
- type: nauc_recall_at_3_diff1
value: 29.035019905697286
- type: nauc_recall_at_3_max
value: -0.8253741490097173
- type: nauc_recall_at_3_std
value: -26.374385040717073
- type: nauc_recall_at_5_diff1
value: 18.90308787689296
- type: nauc_recall_at_5_max
value: 12.373785172914031
- type: nauc_recall_at_5_std
value: -16.43244911732844
- type: ndcg_at_1
value: 91
- type: ndcg_at_10
value: 87.612
- type: ndcg_at_100
value: 90.36699999999999
- type: ndcg_at_1000
value: 90.712
- type: ndcg_at_20
value: 89.184
- type: ndcg_at_3
value: 86.875
- type: ndcg_at_5
value: 85.724
- type: precision_at_1
value: 91
- type: precision_at_10
value: 41.805
- type: precision_at_100
value: 4.805000000000001
- type: precision_at_1000
value: 0.48900000000000005
- type: precision_at_20
value: 22.793
- type: precision_at_3
value: 77.85
- type: precision_at_5
value: 65.73
- type: recall_at_1
value: 26.430999999999997
- type: recall_at_10
value: 88.35499999999999
- type: recall_at_100
value: 97.366
- type: recall_at_1000
value: 99.235
- type: recall_at_20
value: 93.525
- type: recall_at_3
value: 58.226
- type: recall_at_5
value: 74.919
task:
type: Retrieval
- dataset:
config: default
name: MTEB EcomRetrieval (default)
revision: 687de13dc7294d6fd9be10c6945f9e8fec8166b9
split: dev
type: C-MTEB/EcomRetrieval
metrics:
- type: main_score
value: 67.161
- type: map_at_1
value: 52.300000000000004
- type: map_at_10
value: 62.166999999999994
- type: map_at_100
value: 62.778999999999996
- type: map_at_1000
value: 62.792
- type: map_at_20
value: 62.602000000000004
- type: map_at_3
value: 59.516999999999996
- type: map_at_5
value: 61.212
- type: mrr_at_1
value: 52.300000000000004
- type: mrr_at_10
value: 62.16682539682541
- type: mrr_at_100
value: 62.77902948698046
- type: mrr_at_1000
value: 62.792087634956715
- type: mrr_at_20
value: 62.60165070567552
- type: mrr_at_3
value: 59.516666666666666
- type: mrr_at_5
value: 61.21166666666669
- type: nauc_map_at_1000_diff1
value: 65.16885595814446
- type: nauc_map_at_1000_max
value: 29.496662567956776
- type: nauc_map_at_1000_std
value: -12.7407402615998
- type: nauc_map_at_100_diff1
value: 65.16617941092908
- type: nauc_map_at_100_max
value: 29.502786614031272
- type: nauc_map_at_100_std
value: -12.730774193607482
- type: nauc_map_at_10_diff1
value: 65.06166340128496
- type: nauc_map_at_10_max
value: 29.354270783663335
- type: nauc_map_at_10_std
value: -13.094010608834502
- type: nauc_map_at_1_diff1
value: 67.95208901030078
- type: nauc_map_at_1_max
value: 27.603263383543116
- type: nauc_map_at_1_std
value: -13.626768587136143
- type: nauc_map_at_20_diff1
value: 65.14110896910226
- type: nauc_map_at_20_max
value: 29.517296694609605
- type: nauc_map_at_20_std
value: -12.764608204706748
- type: nauc_map_at_3_diff1
value: 65.35395305214332
- type: nauc_map_at_3_max
value: 28.576381656915512
- type: nauc_map_at_3_std
value: -14.869794329585687
- type: nauc_map_at_5_diff1
value: 65.08475092137822
- type: nauc_map_at_5_max
value: 29.10210463237133
- type: nauc_map_at_5_std
value: -13.358219612661818
- type: nauc_mrr_at_1000_diff1
value: 65.16885595814446
- type: nauc_mrr_at_1000_max
value: 29.496662567956776
- type: nauc_mrr_at_1000_std
value: -12.7407402615998
- type: nauc_mrr_at_100_diff1
value: 65.16617941092908
- type: nauc_mrr_at_100_max
value: 29.502786614031272
- type: nauc_mrr_at_100_std
value: -12.730774193607482
- type: nauc_mrr_at_10_diff1
value: 65.06166340128496
- type: nauc_mrr_at_10_max
value: 29.354270783663335
- type: nauc_mrr_at_10_std
value: -13.094010608834502
- type: nauc_mrr_at_1_diff1
value: 67.95208901030078
- type: nauc_mrr_at_1_max
value: 27.603263383543116
- type: nauc_mrr_at_1_std
value: -13.626768587136143
- type: nauc_mrr_at_20_diff1
value: 65.14110896910226
- type: nauc_mrr_at_20_max
value: 29.517296694609605
- type: nauc_mrr_at_20_std
value: -12.764608204706748
- type: nauc_mrr_at_3_diff1
value: 65.35395305214332
- type: nauc_mrr_at_3_max
value: 28.576381656915512
- type: nauc_mrr_at_3_std
value: -14.869794329585687
- type: nauc_mrr_at_5_diff1
value: 65.08475092137822
- type: nauc_mrr_at_5_max
value: 29.10210463237133
- type: nauc_mrr_at_5_std
value: -13.358219612661818
- type: nauc_ndcg_at_1000_diff1
value: 64.33394397420086
- type: nauc_ndcg_at_1000_max
value: 30.864795296588355
- type: nauc_ndcg_at_1000_std
value: -10.533294101922223
- type: nauc_ndcg_at_100_diff1
value: 64.23470966841133
- type: nauc_ndcg_at_100_max
value: 31.15266740134302
- type: nauc_ndcg_at_100_std
value: -10.011060738308094
- type: nauc_ndcg_at_10_diff1
value: 63.857719978347525
- type: nauc_ndcg_at_10_max
value: 30.617530021869715
- type: nauc_ndcg_at_10_std
value: -11.653834629145123
- type: nauc_ndcg_at_1_diff1
value: 67.95208901030078
- type: nauc_ndcg_at_1_max
value: 27.603263383543116
- type: nauc_ndcg_at_1_std
value: -13.626768587136143
- type: nauc_ndcg_at_20_diff1
value: 64.1331572787928
- type: nauc_ndcg_at_20_max
value: 31.25635677077675
- type: nauc_ndcg_at_20_std
value: -10.276309085700548
- type: nauc_ndcg_at_3_diff1
value: 64.48544321575491
- type: nauc_ndcg_at_3_max
value: 28.830444576799763
- type: nauc_ndcg_at_3_std
value: -15.353961858062192
- type: nauc_ndcg_at_5_diff1
value: 63.939873109118736
- type: nauc_ndcg_at_5_max
value: 29.829976475635444
- type: nauc_ndcg_at_5_std
value: -12.501228923097408
- type: nauc_precision_at_1000_diff1
value: 37.252308330738565
- type: nauc_precision_at_1000_max
value: 83.39039319431379
- type: nauc_precision_at_1000_std
value: 88.40906733063514
- type: nauc_precision_at_100_diff1
value: 49.082892416225164
- type: nauc_precision_at_100_max
value: 64.97665732959823
- type: nauc_precision_at_100_std
value: 52.973337483141194
- type: nauc_precision_at_10_diff1
value: 57.595510418530985
- type: nauc_precision_at_10_max
value: 37.97668604684725
- type: nauc_precision_at_10_std
value: -2.7592219444461974
- type: nauc_precision_at_1_diff1
value: 67.95208901030078
- type: nauc_precision_at_1_max
value: 27.603263383543116
- type: nauc_precision_at_1_std
value: -13.626768587136143
- type: nauc_precision_at_20_diff1
value: 56.98568570908991
- type: nauc_precision_at_20_max
value: 46.84055109587015
- type: nauc_precision_at_20_std
value: 13.665255154616707
- type: nauc_precision_at_3_diff1
value: 61.609690960979556
- type: nauc_precision_at_3_max
value: 29.644530943175635
- type: nauc_precision_at_3_std
value: -16.983269477487482
- type: nauc_precision_at_5_diff1
value: 59.36708969559955
- type: nauc_precision_at_5_max
value: 32.83509607326578
- type: nauc_precision_at_5_std
value: -8.585997498785966
- type: nauc_recall_at_1000_diff1
value: 37.25230833073967
- type: nauc_recall_at_1000_max
value: 83.39039319431508
- type: nauc_recall_at_1000_std
value: 88.40906733063599
- type: nauc_recall_at_100_diff1
value: 49.08289241622588
- type: nauc_recall_at_100_max
value: 64.97665732959842
- type: nauc_recall_at_100_std
value: 52.973337483141805
- type: nauc_recall_at_10_diff1
value: 57.59551041853113
- type: nauc_recall_at_10_max
value: 37.97668604684728
- type: nauc_recall_at_10_std
value: -2.7592219444460553
- type: nauc_recall_at_1_diff1
value: 67.95208901030078
- type: nauc_recall_at_1_max
value: 27.603263383543116
- type: nauc_recall_at_1_std
value: -13.626768587136143
- type: nauc_recall_at_20_diff1
value: 56.98568570908995
- type: nauc_recall_at_20_max
value: 46.84055109587016
- type: nauc_recall_at_20_std
value: 13.665255154616771
- type: nauc_recall_at_3_diff1
value: 61.60969096097951
- type: nauc_recall_at_3_max
value: 29.644530943175585
- type: nauc_recall_at_3_std
value: -16.983269477487518
- type: nauc_recall_at_5_diff1
value: 59.36708969559964
- type: nauc_recall_at_5_max
value: 32.83509607326576
- type: nauc_recall_at_5_std
value: -8.585997498785925
- type: ndcg_at_1
value: 52.300000000000004
- type: ndcg_at_10
value: 67.161
- type: ndcg_at_100
value: 69.947
- type: ndcg_at_1000
value: 70.28999999999999
- type: ndcg_at_20
value: 68.734
- type: ndcg_at_3
value: 61.827
- type: ndcg_at_5
value: 64.857
- type: precision_at_1
value: 52.300000000000004
- type: precision_at_10
value: 8.290000000000001
- type: precision_at_100
value: 0.955
- type: precision_at_1000
value: 0.098
- type: precision_at_20
value: 4.455
- type: precision_at_3
value: 22.833000000000002
- type: precision_at_5
value: 15.160000000000002
- type: recall_at_1
value: 52.300000000000004
- type: recall_at_10
value: 82.89999999999999
- type: recall_at_100
value: 95.5
- type: recall_at_1000
value: 98.2
- type: recall_at_20
value: 89.1
- type: recall_at_3
value: 68.5
- type: recall_at_5
value: 75.8
task:
type: Retrieval
- dataset:
config: default
name: MTEB IFlyTek (default)
revision: 421605374b29664c5fc098418fe20ada9bd55f8a
split: validation
type: C-MTEB/IFlyTek-classification
metrics:
- type: accuracy
value: 46.20238553289727
- type: f1
value: 38.90517287997349
- type: f1_weighted
value: 47.72465908202853
- type: main_score
value: 46.20238553289727
task:
type: Classification
- dataset:
config: default
name: MTEB JDReview (default)
revision: b7c64bd89eb87f8ded463478346f76731f07bf8b
split: test
type: C-MTEB/JDReview-classification
metrics:
- type: accuracy
value: 86.86679174484053
- type: ap
value: 55.4190333000728
- type: ap_weighted
value: 55.4190333000728
- type: f1
value: 81.51249564649538
- type: f1_weighted
value: 87.76253254123183
- type: main_score
value: 86.86679174484053
task:
type: Classification
- dataset:
config: default
name: MTEB LCQMC (default)
revision: 17f9b096f80380fce5ed12a9be8be7784b337daf
split: test
type: C-MTEB/LCQMC
metrics:
- type: cosine_pearson
value: 73.38528996257187
- type: cosine_spearman
value: 79.5838761369882
- type: euclidean_pearson
value: 79.46435441303319
- type: euclidean_spearman
value: 79.43632868036107
- type: main_score
value: 79.5838761369882
- type: manhattan_pearson
value: 79.4611643611669
- type: manhattan_spearman
value: 79.43277700673374
task:
type: STS
- dataset:
config: default
name: MTEB MMarcoReranking (default)
revision: 8e0c766dbe9e16e1d221116a3f36795fbade07f6
split: dev
type: C-MTEB/Mmarco-reranking
metrics:
- type: main_score
value: 35.03765306725339
- type: map
value: 35.03765306725339
- type: mrr
value: 34.97817460317461
- type: nAUC_map_diff1
value: 19.818725573084713
- type: nAUC_map_max
value: 1.0198000256681778
- type: nAUC_map_std
value: -17.624562992793454
- type: nAUC_mrr_diff1
value: 20.024047042722497
- type: nAUC_mrr_max
value: 0.7457000534749312
- type: nAUC_mrr_std
value: -17.90188283839233
task:
type: Reranking
- dataset:
config: default
name: MTEB MMarcoRetrieval (default)
revision: 539bbde593d947e2a124ba72651aafc09eb33fc2
split: dev
type: C-MTEB/MMarcoRetrieval
metrics:
- type: main_score
value: 78.377
- type: map_at_1
value: 64.496
- type: map_at_10
value: 74.316
- type: map_at_100
value: 74.67399999999999
- type: map_at_1000
value: 74.682
- type: map_at_20
value: 74.553
- type: map_at_3
value: 72.27799999999999
- type: map_at_5
value: 73.601
- type: mrr_at_1
value: 66.71919770773638
- type: mrr_at_10
value: 74.91401168872505
- type: mrr_at_100
value: 75.22834586029683
- type: mrr_at_1000
value: 75.23576223918033
- type: mrr_at_20
value: 75.1182142204938
- type: mrr_at_3
value: 73.12320916905448
- type: mrr_at_5
value: 74.29512893982785
- type: nauc_map_at_1000_diff1
value: 76.44188987176878
- type: nauc_map_at_1000_max
value: 41.96583590277497
- type: nauc_map_at_1000_std
value: -11.893947475776763
- type: nauc_map_at_100_diff1
value: 76.43873298246776
- type: nauc_map_at_100_max
value: 41.97615263718138
- type: nauc_map_at_100_std
value: -11.876568525374811
- type: nauc_map_at_10_diff1
value: 76.28512812077162
- type: nauc_map_at_10_max
value: 42.111303574942454
- type: nauc_map_at_10_std
value: -11.92840846101945
- type: nauc_map_at_1_diff1
value: 78.56803879890037
- type: nauc_map_at_1_max
value: 35.78389741877027
- type: nauc_map_at_1_std
value: -16.393221992175597
- type: nauc_map_at_20_diff1
value: 76.38231793080924
- type: nauc_map_at_20_max
value: 42.05893602716135
- type: nauc_map_at_20_std
value: -11.826119522321708
- type: nauc_map_at_3_diff1
value: 76.19646582467323
- type: nauc_map_at_3_max
value: 40.85705983970697
- type: nauc_map_at_3_std
value: -13.612451556499103
- type: nauc_map_at_5_diff1
value: 76.27491857077754
- type: nauc_map_at_5_max
value: 41.85172393299732
- type: nauc_map_at_5_std
value: -12.431459301392097
- type: nauc_mrr_at_1000_diff1
value: 76.92019548266093
- type: nauc_mrr_at_1000_max
value: 42.48811285027811
- type: nauc_mrr_at_1000_std
value: -11.317598684822896
- type: nauc_mrr_at_100_diff1
value: 76.91765336636446
- type: nauc_mrr_at_100_max
value: 42.49675674263938
- type: nauc_mrr_at_100_std
value: -11.300949737082554
- type: nauc_mrr_at_10_diff1
value: 76.79353999065357
- type: nauc_mrr_at_10_max
value: 42.69909380024054
- type: nauc_mrr_at_10_std
value: -11.270497173030472
- type: nauc_mrr_at_1_diff1
value: 79.7117803522604
- type: nauc_mrr_at_1_max
value: 39.57572084455486
- type: nauc_mrr_at_1_std
value: -15.17966940645922
- type: nauc_mrr_at_20_diff1
value: 76.86280745329786
- type: nauc_mrr_at_20_max
value: 42.58380455810906
- type: nauc_mrr_at_20_std
value: -11.229046706936792
- type: nauc_mrr_at_3_diff1
value: 76.67262469325809
- type: nauc_mrr_at_3_max
value: 41.88964824896713
- type: nauc_mrr_at_3_std
value: -12.50874971662575
- type: nauc_mrr_at_5_diff1
value: 76.7796936981817
- type: nauc_mrr_at_5_max
value: 42.526924542005716
- type: nauc_mrr_at_5_std
value: -11.587665730732638
- type: nauc_ndcg_at_1000_diff1
value: 75.97798533105214
- type: nauc_ndcg_at_1000_max
value: 43.27769851746166
- type: nauc_ndcg_at_1000_std
value: -9.42928648077032
- type: nauc_ndcg_at_100_diff1
value: 75.88448755710286
- type: nauc_ndcg_at_100_max
value: 43.60463027736502
- type: nauc_ndcg_at_100_std
value: -8.788723172592231
- type: nauc_ndcg_at_10_diff1
value: 75.15087803753813
- type: nauc_ndcg_at_10_max
value: 44.53805050283006
- type: nauc_ndcg_at_10_std
value: -8.665523258131715
- type: nauc_ndcg_at_1_diff1
value: 79.7117803522604
- type: nauc_ndcg_at_1_max
value: 39.57572084455486
- type: nauc_ndcg_at_1_std
value: -15.17966940645922
- type: nauc_ndcg_at_20_diff1
value: 75.46054281314423
- type: nauc_ndcg_at_20_max
value: 44.27762687558626
- type: nauc_ndcg_at_20_std
value: -8.193908784135246
- type: nauc_ndcg_at_3_diff1
value: 75.11224084416658
- type: nauc_ndcg_at_3_max
value: 42.17726543068363
- type: nauc_ndcg_at_3_std
value: -12.105416322156225
- type: nauc_ndcg_at_5_diff1
value: 75.185399674847
- type: nauc_ndcg_at_5_max
value: 43.84929152627948
- type: nauc_ndcg_at_5_std
value: -10.009405007276152
- type: nauc_precision_at_1000_diff1
value: -17.09549485267675
- type: nauc_precision_at_1000_max
value: 14.73979745312348
- type: nauc_precision_at_1000_std
value: 24.533810348714972
- type: nauc_precision_at_100_diff1
value: -6.236032711925149
- type: nauc_precision_at_100_max
value: 23.110915607656956
- type: nauc_precision_at_100_std
value: 28.206485316450536
- type: nauc_precision_at_10_diff1
value: 23.03478697515758
- type: nauc_precision_at_10_max
value: 39.203686218447196
- type: nauc_precision_at_10_std
value: 17.452696042464027
- type: nauc_precision_at_1_diff1
value: 79.7117803522604
- type: nauc_precision_at_1_max
value: 39.57572084455486
- type: nauc_precision_at_1_std
value: -15.17966940645922
- type: nauc_precision_at_20_diff1
value: 12.09320340381941
- type: nauc_precision_at_20_max
value: 34.16449285853928
- type: nauc_precision_at_20_std
value: 24.122439998396754
- type: nauc_precision_at_3_diff1
value: 47.52910178931779
- type: nauc_precision_at_3_max
value: 40.82046536615527
- type: nauc_precision_at_3_std
value: -0.8258153652756941
- type: nauc_precision_at_5_diff1
value: 36.2587080465967
- type: nauc_precision_at_5_max
value: 41.54352007897181
- type: nauc_precision_at_5_std
value: 8.545492145033677
- type: nauc_recall_at_1000_diff1
value: 62.82235268536862
- type: nauc_recall_at_1000_max
value: 67.88265991328028
- type: nauc_recall_at_1000_std
value: 78.62591611987277
- type: nauc_recall_at_100_diff1
value: 65.84511669611754
- type: nauc_recall_at_100_max
value: 68.3187288143991
- type: nauc_recall_at_100_std
value: 55.49540536103681
- type: nauc_recall_at_10_diff1
value: 66.07985276674994
- type: nauc_recall_at_10_max
value: 59.09892833450687
- type: nauc_recall_at_10_std
value: 12.173481401831848
- type: nauc_recall_at_1_diff1
value: 78.56803879890037
- type: nauc_recall_at_1_max
value: 35.78389741877027
- type: nauc_recall_at_1_std
value: -16.393221992175597
- type: nauc_recall_at_20_diff1
value: 65.60243952156914
- type: nauc_recall_at_20_max
value: 63.1161484484425
- type: nauc_recall_at_20_std
value: 26.62753581692267
- type: nauc_recall_at_3_diff1
value: 70.07831205118282
- type: nauc_recall_at_3_max
value: 45.268817233496385
- type: nauc_recall_at_3_std
value: -8.031502210695518
- type: nauc_recall_at_5_diff1
value: 68.75996804285091
- type: nauc_recall_at_5_max
value: 51.89436269116051
- type: nauc_recall_at_5_std
value: 0.2288841275558609
- type: ndcg_at_1
value: 66.719
- type: ndcg_at_10
value: 78.377
- type: ndcg_at_100
value: 79.964
- type: ndcg_at_1000
value: 80.185
- type: ndcg_at_20
value: 79.183
- type: ndcg_at_3
value: 74.533
- type: ndcg_at_5
value: 76.782
- type: precision_at_1
value: 66.719
- type: precision_at_10
value: 9.596
- type: precision_at_100
value: 1.038
- type: precision_at_1000
value: 0.106
- type: precision_at_20
value: 4.971
- type: precision_at_3
value: 28.247
- type: precision_at_5
value: 18.146
- type: recall_at_1
value: 64.496
- type: recall_at_10
value: 90.275
- type: recall_at_100
value: 97.41300000000001
- type: recall_at_1000
value: 99.128
- type: recall_at_20
value: 93.378
- type: recall_at_3
value: 80.19
- type: recall_at_5
value: 85.531
task:
type: Retrieval
- dataset:
config: ur
name: MTEB MassiveIntentClassification (ur)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 9.731002017484869
- type: f1
value: 9.069748597280716
- type: f1_weighted
value: 10.419815856589645
- type: main_score
value: 9.731002017484869
task:
type: Classification
- dataset:
config: id
name: MTEB MassiveIntentClassification (id)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 43.6079354404842
- type: f1
value: 43.25295087060459
- type: f1_weighted
value: 43.98296829713196
- type: main_score
value: 43.6079354404842
task:
type: Classification
- dataset:
config: fa
name: MTEB MassiveIntentClassification (fa)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 12.814391392064561
- type: f1
value: 11.570863357381153
- type: f1_weighted
value: 13.101944439649085
- type: main_score
value: 12.814391392064561
task:
type: Classification
- dataset:
config: mn
name: MTEB MassiveIntentClassification (mn)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 23.97108271687962
- type: f1
value: 24.053292531614012
- type: f1_weighted
value: 23.851839245854936
- type: main_score
value: 23.97108271687962
task:
type: Classification
- dataset:
config: en
name: MTEB MassiveIntentClassification (en)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 62.81102891728313
- type: f1
value: 59.6018863604308
- type: f1_weighted
value: 62.99690622554801
- type: main_score
value: 62.81102891728313
task:
type: Classification
- dataset:
config: fi
name: MTEB MassiveIntentClassification (fi)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 41.570275722932074
- type: f1
value: 39.46488297245669
- type: f1_weighted
value: 42.502239816443456
- type: main_score
value: 41.570275722932074
task:
type: Classification
- dataset:
config: cy
name: MTEB MassiveIntentClassification (cy)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 39.8453261600538
- type: f1
value: 38.317101603964545
- type: f1_weighted
value: 40.72395492044339
- type: main_score
value: 39.8453261600538
task:
type: Classification
- dataset:
config: af
name: MTEB MassiveIntentClassification (af)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 38.671822461331544
- type: f1
value: 37.58007811317987
- type: f1_weighted
value: 39.10181248428257
- type: main_score
value: 38.671822461331544
task:
type: Classification
- dataset:
config: el
name: MTEB MassiveIntentClassification (el)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 30.067249495628783
- type: f1
value: 27.556355974187912
- type: f1_weighted
value: 30.963659893012967
- type: main_score
value: 30.067249495628783
task:
type: Classification
- dataset:
config: km
name: MTEB MassiveIntentClassification (km)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 4.673839946200404
- type: f1
value: 2.5494965639802367
- type: f1_weighted
value: 2.8145341830526283
- type: main_score
value: 4.673839946200404
task:
type: Classification
- dataset:
config: it
name: MTEB MassiveIntentClassification (it)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 46.96032279757902
- type: f1
value: 46.65651797980574
- type: f1_weighted
value: 46.975519849473244
- type: main_score
value: 46.96032279757902
task:
type: Classification
- dataset:
config: sw
name: MTEB MassiveIntentClassification (sw)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 40.292535305985204
- type: f1
value: 39.17628651352315
- type: f1_weighted
value: 40.839902493979025
- type: main_score
value: 40.292535305985204
task:
type: Classification
- dataset:
config: te
name: MTEB MassiveIntentClassification (te)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 2.8513786146603897
- type: f1
value: 1.89175192605774
- type: f1_weighted
value: 2.5364563808332528
- type: main_score
value: 2.8513786146603897
task:
type: Classification
- dataset:
config: is
name: MTEB MassiveIntentClassification (is)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 34.451916610625425
- type: f1
value: 33.53074927467555
- type: f1_weighted
value: 35.258319919642446
- type: main_score
value: 34.451916610625425
task:
type: Classification
- dataset:
config: lv
name: MTEB MassiveIntentClassification (lv)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 43.197713517148614
- type: f1
value: 42.60125486508949
- type: f1_weighted
value: 43.64379570892065
- type: main_score
value: 43.197713517148614
task:
type: Classification
- dataset:
config: zh-CN
name: MTEB MassiveIntentClassification (zh-CN)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 76.01882985877604
- type: f1
value: 72.3776579790032
- type: f1_weighted
value: 76.50838946568814
- type: main_score
value: 76.01882985877604
task:
type: Classification
- dataset:
config: az
name: MTEB MassiveIntentClassification (az)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 32.75386684599866
- type: f1
value: 31.456766708589896
- type: f1_weighted
value: 34.0366091215231
- type: main_score
value: 32.75386684599866
task:
type: Classification
- dataset:
config: nl
name: MTEB MassiveIntentClassification (nl)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 42.57565568258238
- type: f1
value: 40.287327106843634
- type: f1_weighted
value: 43.24716333400812
- type: main_score
value: 42.57565568258238
task:
type: Classification
- dataset:
config: he
name: MTEB MassiveIntentClassification (he)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 2.589105581708137
- type: f1
value: 0.9290630374280033
- type: f1_weighted
value: 1.4161878562981063
- type: main_score
value: 2.589105581708137
task:
type: Classification
- dataset:
config: ko
name: MTEB MassiveIntentClassification (ko)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 25.131136516476126
- type: f1
value: 25.27815061140736
- type: f1_weighted
value: 25.192290174357346
- type: main_score
value: 25.131136516476126
task:
type: Classification
- dataset:
config: pt
name: MTEB MassiveIntentClassification (pt)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 46.946872898453265
- type: f1
value: 45.91259775330914
- type: f1_weighted
value: 47.398746034653406
- type: main_score
value: 46.946872898453265
task:
type: Classification
- dataset:
config: sl
name: MTEB MassiveIntentClassification (sl)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 42.017484868863484
- type: f1
value: 40.32799351790967
- type: f1_weighted
value: 42.5095745023745
- type: main_score
value: 42.017484868863484
task:
type: Classification
- dataset:
config: sq
name: MTEB MassiveIntentClassification (sq)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 44.16274377942165
- type: f1
value: 42.59183982340306
- type: f1_weighted
value: 45.008212136093455
- type: main_score
value: 44.16274377942165
task:
type: Classification
- dataset:
config: fr
name: MTEB MassiveIntentClassification (fr)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 43.62474781439139
- type: f1
value: 43.380784365848676
- type: f1_weighted
value: 44.048384678055065
- type: main_score
value: 43.62474781439139
task:
type: Classification
- dataset:
config: ar
name: MTEB MassiveIntentClassification (ar)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 14.862138533960998
- type: f1
value: 13.123956008770515
- type: f1_weighted
value: 15.175944552935434
- type: main_score
value: 14.862138533960998
task:
type: Classification
- dataset:
config: pl
name: MTEB MassiveIntentClassification (pl)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 42.59246805648957
- type: f1
value: 39.235549164523064
- type: f1_weighted
value: 43.725796046895724
- type: main_score
value: 42.59246805648957
task:
type: Classification
- dataset:
config: es
name: MTEB MassiveIntentClassification (es)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 43.35911230665771
- type: f1
value: 43.58305920517007
- type: f1_weighted
value: 43.45163273620528
- type: main_score
value: 43.35911230665771
task:
type: Classification
- dataset:
config: bn
name: MTEB MassiveIntentClassification (bn)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 3.5743106926698043
- type: f1
value: 1.9170031165168782
- type: f1_weighted
value: 2.876028356151793
- type: main_score
value: 3.5743106926698043
task:
type: Classification
- dataset:
config: jv
name: MTEB MassiveIntentClassification (jv)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 39.3207800941493
- type: f1
value: 37.84945159101641
- type: f1_weighted
value: 39.76328887450172
- type: main_score
value: 39.3207800941493
task:
type: Classification
- dataset:
config: sv
name: MTEB MassiveIntentClassification (sv)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 43.17417619367855
- type: f1
value: 41.51452421502344
- type: f1_weighted
value: 43.575740588663436
- type: main_score
value: 43.17417619367855
task:
type: Classification
- dataset:
config: my
name: MTEB MassiveIntentClassification (my)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 3.940820443846671
- type: f1
value: 1.9024958528487437
- type: f1_weighted
value: 2.457367277110865
- type: main_score
value: 3.940820443846671
task:
type: Classification
- dataset:
config: kn
name: MTEB MassiveIntentClassification (kn)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 3.702084734364493
- type: f1
value: 2.4577796241626655
- type: f1_weighted
value: 2.718182564884992
- type: main_score
value: 3.702084734364493
task:
type: Classification
- dataset:
config: ta
name: MTEB MassiveIntentClassification (ta)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 3.338937457969065
- type: f1
value: 1.1915267419026427
- type: f1_weighted
value: 2.050367350794539
- type: main_score
value: 3.338937457969065
task:
type: Classification
- dataset:
config: th
name: MTEB MassiveIntentClassification (th)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 7.108271687962341
- type: f1
value: 6.499099733722208
- type: f1_weighted
value: 6.067679182650555
- type: main_score
value: 7.108271687962341
task:
type: Classification
- dataset:
config: am
name: MTEB MassiveIntentClassification (am)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 3.1069266980497647
- type: f1
value: 1.7074940472292803
- type: f1_weighted
value: 1.91368359274973
- type: main_score
value: 3.1069266980497647
task:
type: Classification
- dataset:
config: ja
name: MTEB MassiveIntentClassification (ja)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 51.04909213180902
- type: f1
value: 49.6308815392496
- type: f1_weighted
value: 51.14123088715286
- type: main_score
value: 51.04909213180902
task:
type: Classification
- dataset:
config: nb
name: MTEB MassiveIntentClassification (nb)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 40.154673839946206
- type: f1
value: 39.60433103070425
- type: f1_weighted
value: 40.82227593141525
- type: main_score
value: 40.154673839946206
task:
type: Classification
- dataset:
config: de
name: MTEB MassiveIntentClassification (de)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 42.10827168796234
- type: f1
value: 39.80603638685329
- type: f1_weighted
value: 42.64856677255116
- type: main_score
value: 42.10827168796234
task:
type: Classification
- dataset:
config: hu
name: MTEB MassiveIntentClassification (hu)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 41.4862138533961
- type: f1
value: 39.857700066184535
- type: f1_weighted
value: 41.81361990540168
- type: main_score
value: 41.4862138533961
task:
type: Classification
- dataset:
config: ml
name: MTEB MassiveIntentClassification (ml)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 2.8513786146603897
- type: f1
value: 1.0396796629394838
- type: f1_weighted
value: 1.679462787118112
- type: main_score
value: 2.8513786146603897
task:
type: Classification
- dataset:
config: tr
name: MTEB MassiveIntentClassification (tr)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 33.046402151983855
- type: f1
value: 32.071046557110236
- type: f1_weighted
value: 34.06498014118775
- type: main_score
value: 33.046402151983855
task:
type: Classification
- dataset:
config: vi
name: MTEB MassiveIntentClassification (vi)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 39.81842636180228
- type: f1
value: 38.62179673846196
- type: f1_weighted
value: 40.75093127009189
- type: main_score
value: 39.81842636180228
task:
type: Classification
- dataset:
config: hy
name: MTEB MassiveIntentClassification (hy)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 3.533960995292535
- type: f1
value: 1.52098804123016
- type: f1_weighted
value: 2.4587393644122364
- type: main_score
value: 3.533960995292535
task:
type: Classification
- dataset:
config: hi
name: MTEB MassiveIntentClassification (hi)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 3.4868863483523875
- type: f1
value: 2.463127466064235
- type: f1_weighted
value: 2.814989313283245
- type: main_score
value: 3.4868863483523875
task:
type: Classification
- dataset:
config: ms
name: MTEB MassiveIntentClassification (ms)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 40.534633490248815
- type: f1
value: 39.582513269336324
- type: f1_weighted
value: 41.20588357370901
- type: main_score
value: 40.534633490248815
task:
type: Classification
- dataset:
config: zh-TW
name: MTEB MassiveIntentClassification (zh-TW)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 70.70275722932078
- type: f1
value: 69.33141153617231
- type: f1_weighted
value: 71.00260120362651
- type: main_score
value: 70.70275722932078
task:
type: Classification
- dataset:
config: ro
name: MTEB MassiveIntentClassification (ro)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 43.67854741089442
- type: f1
value: 42.5674825712523
- type: f1_weighted
value: 44.03833175704378
- type: main_score
value: 43.67854741089442
task:
type: Classification
- dataset:
config: ru
name: MTEB MassiveIntentClassification (ru)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 25.793544048419637
- type: f1
value: 25.149981562366037
- type: f1_weighted
value: 26.24170939636344
- type: main_score
value: 25.793544048419637
task:
type: Classification
- dataset:
config: ka
name: MTEB MassiveIntentClassification (ka)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 2.7336919973100207
- type: f1
value: 1.580154047083068
- type: f1_weighted
value: 1.4503728058594556
- type: main_score
value: 2.7336919973100207
task:
type: Classification
- dataset:
config: da
name: MTEB MassiveIntentClassification (da)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 44.9831876260928
- type: f1
value: 42.431462813005005
- type: f1_weighted
value: 45.83858201062737
- type: main_score
value: 44.9831876260928
task:
type: Classification
- dataset:
config: tl
name: MTEB MassiveIntentClassification (tl)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: test
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 41.62071284465366
- type: f1
value: 39.31648308450773
- type: f1_weighted
value: 42.262157671126744
- type: main_score
value: 41.62071284465366
task:
type: Classification
- dataset:
config: ur
name: MTEB MassiveIntentClassification (ur)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 9.380226266601081
- type: f1
value: 8.990917585770982
- type: f1_weighted
value: 10.555950181353865
- type: main_score
value: 9.380226266601081
task:
type: Classification
- dataset:
config: id
name: MTEB MassiveIntentClassification (id)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 43.30054107230693
- type: f1
value: 42.639087356331615
- type: f1_weighted
value: 43.69592952307332
- type: main_score
value: 43.30054107230693
task:
type: Classification
- dataset:
config: fa
name: MTEB MassiveIntentClassification (fa)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 12.700442695523856
- type: f1
value: 11.392101752675561
- type: f1_weighted
value: 13.02878956397257
- type: main_score
value: 12.700442695523856
task:
type: Classification
- dataset:
config: mn
name: MTEB MassiveIntentClassification (mn)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 24.176094441711754
- type: f1
value: 24.441961354848978
- type: f1_weighted
value: 23.943329604222363
- type: main_score
value: 24.176094441711754
task:
type: Classification
- dataset:
config: en
name: MTEB MassiveIntentClassification (en)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 63.42351205115592
- type: f1
value: 60.1409077218653
- type: f1_weighted
value: 63.23590096985862
- type: main_score
value: 63.42351205115592
task:
type: Classification
- dataset:
config: fi
name: MTEB MassiveIntentClassification (fi)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 41.44121987211018
- type: f1
value: 39.88179246641776
- type: f1_weighted
value: 41.9912976137676
- type: main_score
value: 41.44121987211018
task:
type: Classification
- dataset:
config: cy
name: MTEB MassiveIntentClassification (cy)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 38.88342351205115
- type: f1
value: 38.016148507410676
- type: f1_weighted
value: 39.67623313324099
- type: main_score
value: 38.88342351205115
task:
type: Classification
- dataset:
config: af
name: MTEB MassiveIntentClassification (af)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 38.93753074274471
- type: f1
value: 38.21819811332998
- type: f1_weighted
value: 39.30475730087287
- type: main_score
value: 38.93753074274471
task:
type: Classification
- dataset:
config: el
name: MTEB MassiveIntentClassification (el)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 29.80324643384162
- type: f1
value: 27.995696304231167
- type: f1_weighted
value: 30.733913113361595
- type: main_score
value: 29.80324643384162
task:
type: Classification
- dataset:
config: km
name: MTEB MassiveIntentClassification (km)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 4.240039350713232
- type: f1
value: 2.0292251621717106
- type: f1_weighted
value: 2.276215235518406
- type: main_score
value: 4.240039350713232
task:
type: Classification
- dataset:
config: it
name: MTEB MassiveIntentClassification (it)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 46.36989670437776
- type: f1
value: 46.86880494144779
- type: f1_weighted
value: 46.363803235010955
- type: main_score
value: 46.36989670437776
task:
type: Classification
- dataset:
config: sw
name: MTEB MassiveIntentClassification (sw)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 40.40826364977865
- type: f1
value: 38.863056374899564
- type: f1_weighted
value: 41.12559214755415
- type: main_score
value: 40.40826364977865
task:
type: Classification
- dataset:
config: te
name: MTEB MassiveIntentClassification (te)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 3.0545991146089517
- type: f1
value: 1.5592797896550983
- type: f1_weighted
value: 2.4262056804247747
- type: main_score
value: 3.0545991146089517
task:
type: Classification
- dataset:
config: is
name: MTEB MassiveIntentClassification (is)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 33.571077225774715
- type: f1
value: 33.55240297777297
- type: f1_weighted
value: 34.24069482813271
- type: main_score
value: 33.571077225774715
task:
type: Classification
- dataset:
config: lv
name: MTEB MassiveIntentClassification (lv)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 41.1952779144122
- type: f1
value: 40.80253122677525
- type: f1_weighted
value: 41.393629514985236
- type: main_score
value: 41.1952779144122
task:
type: Classification
- dataset:
config: zh-CN
name: MTEB MassiveIntentClassification (zh-CN)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 76.91096901131333
- type: f1
value: 74.15538363364416
- type: f1_weighted
value: 76.88270352222386
- type: main_score
value: 76.91096901131333
task:
type: Classification
- dataset:
config: az
name: MTEB MassiveIntentClassification (az)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 32.52336448598131
- type: f1
value: 31.168524555924627
- type: f1_weighted
value: 33.63759944502999
- type: main_score
value: 32.52336448598131
task:
type: Classification
- dataset:
config: nl
name: MTEB MassiveIntentClassification (nl)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 40.944417117560256
- type: f1
value: 40.09456965856331
- type: f1_weighted
value: 41.32329455781106
- type: main_score
value: 40.944417117560256
task:
type: Classification
- dataset:
config: he
name: MTEB MassiveIntentClassification (he)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 2.857845548450566
- type: f1
value: 0.9974462765950423
- type: f1_weighted
value: 1.4711644445043064
- type: main_score
value: 2.857845548450566
task:
type: Classification
- dataset:
config: ko
name: MTEB MassiveIntentClassification (ko)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 24.97786522380718
- type: f1
value: 25.402064400694012
- type: f1_weighted
value: 24.952618227938256
- type: main_score
value: 24.97786522380718
task:
type: Classification
- dataset:
config: pt
name: MTEB MassiveIntentClassification (pt)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 46.43384161337924
- type: f1
value: 45.85033610327831
- type: f1_weighted
value: 46.896780452483746
- type: main_score
value: 46.43384161337924
task:
type: Classification
- dataset:
config: sl
name: MTEB MassiveIntentClassification (sl)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 41.0231185440236
- type: f1
value: 39.514332596107806
- type: f1_weighted
value: 41.48978371983357
- type: main_score
value: 41.0231185440236
task:
type: Classification
- dataset:
config: sq
name: MTEB MassiveIntentClassification (sq)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 44.402361042793906
- type: f1
value: 43.502050301829684
- type: f1_weighted
value: 45.11202804620879
- type: main_score
value: 44.402361042793906
task:
type: Classification
- dataset:
config: fr
name: MTEB MassiveIntentClassification (fr)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 43.871126414166255
- type: f1
value: 43.69727486044771
- type: f1_weighted
value: 44.23366783802563
- type: main_score
value: 43.871126414166255
task:
type: Classification
- dataset:
config: ar
name: MTEB MassiveIntentClassification (ar)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 14.490900147565174
- type: f1
value: 12.97222226579436
- type: f1_weighted
value: 15.100344259005311
- type: main_score
value: 14.490900147565174
task:
type: Classification
- dataset:
config: pl
name: MTEB MassiveIntentClassification (pl)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 42.434825381210025
- type: f1
value: 39.31393032873764
- type: f1_weighted
value: 43.40696936550348
- type: main_score
value: 42.434825381210025
task:
type: Classification
- dataset:
config: es
name: MTEB MassiveIntentClassification (es)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 43.531726512543045
- type: f1
value: 43.87114148531123
- type: f1_weighted
value: 43.48862520255502
- type: main_score
value: 43.531726512543045
task:
type: Classification
- dataset:
config: bn
name: MTEB MassiveIntentClassification (bn)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 3.3202164289227745
- type: f1
value: 2.1671008894013344
- type: f1_weighted
value: 2.565626865416203
- type: main_score
value: 3.3202164289227745
task:
type: Classification
- dataset:
config: jv
name: MTEB MassiveIntentClassification (jv)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 39.66060009837678
- type: f1
value: 38.09804291983762
- type: f1_weighted
value: 40.20764763707242
- type: main_score
value: 39.66060009837678
task:
type: Classification
- dataset:
config: sv
name: MTEB MassiveIntentClassification (sv)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 42.34628627643876
- type: f1
value: 41.351901052262484
- type: f1_weighted
value: 42.74617846152216
- type: main_score
value: 42.34628627643876
task:
type: Classification
- dataset:
config: my
name: MTEB MassiveIntentClassification (my)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 4.520413182488932
- type: f1
value: 2.492518890768375
- type: f1_weighted
value: 2.8083959445441664
- type: main_score
value: 4.520413182488932
task:
type: Classification
- dataset:
config: kn
name: MTEB MassiveIntentClassification (kn)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 3.8416133792424993
- type: f1
value: 2.249913702782843
- type: f1_weighted
value: 2.8461656650048237
- type: main_score
value: 3.8416133792424993
task:
type: Classification
- dataset:
config: ta
name: MTEB MassiveIntentClassification (ta)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 3.207083128381702
- type: f1
value: 1.1165194517745034
- type: f1_weighted
value: 2.03797254022123
- type: main_score
value: 3.207083128381702
task:
type: Classification
- dataset:
config: th
name: MTEB MassiveIntentClassification (th)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 7.19626168224299
- type: f1
value: 6.231436588286175
- type: f1_weighted
value: 6.082174205390991
- type: main_score
value: 7.19626168224299
task:
type: Classification
- dataset:
config: am
name: MTEB MassiveIntentClassification (am)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 3.216920806689622
- type: f1
value: 1.5706785264445453
- type: f1_weighted
value: 1.8826764274890193
- type: main_score
value: 3.216920806689622
task:
type: Classification
- dataset:
config: ja
name: MTEB MassiveIntentClassification (ja)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 50.90506640432858
- type: f1
value: 49.286571027358264
- type: f1_weighted
value: 51.19695384060615
- type: main_score
value: 50.90506640432858
task:
type: Classification
- dataset:
config: nb
name: MTEB MassiveIntentClassification (nb)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 40.39842597147073
- type: f1
value: 39.932402097909794
- type: f1_weighted
value: 40.80662785740894
- type: main_score
value: 40.39842597147073
task:
type: Classification
- dataset:
config: de
name: MTEB MassiveIntentClassification (de)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 41.03787506148549
- type: f1
value: 38.80554261518457
- type: f1_weighted
value: 41.656448662726575
- type: main_score
value: 41.03787506148549
task:
type: Classification
- dataset:
config: hu
name: MTEB MassiveIntentClassification (hu)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 39.685194294146584
- type: f1
value: 38.336045001667216
- type: f1_weighted
value: 40.09559058068552
- type: main_score
value: 39.685194294146584
task:
type: Classification
- dataset:
config: ml
name: MTEB MassiveIntentClassification (ml)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 2.9611411706837187
- type: f1
value: 1.069101410866621
- type: f1_weighted
value: 1.505398164168353
- type: main_score
value: 2.9611411706837187
task:
type: Classification
- dataset:
config: tr
name: MTEB MassiveIntentClassification (tr)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 33.295622233152976
- type: f1
value: 32.679303624594866
- type: f1_weighted
value: 34.08394987338718
- type: main_score
value: 33.295622233152976
task:
type: Classification
- dataset:
config: vi
name: MTEB MassiveIntentClassification (vi)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 39.621249385145106
- type: f1
value: 37.53378817519671
- type: f1_weighted
value: 40.291091761567074
- type: main_score
value: 39.621249385145106
task:
type: Classification
- dataset:
config: hy
name: MTEB MassiveIntentClassification (hy)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 3.2857845548450557
- type: f1
value: 1.5889657377830335
- type: f1_weighted
value: 2.151424756030313
- type: main_score
value: 3.2857845548450557
task:
type: Classification
- dataset:
config: hi
name: MTEB MassiveIntentClassification (hi)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 3.4825381210034436
- type: f1
value: 2.757982434342728
- type: f1_weighted
value: 2.758836281053027
- type: main_score
value: 3.4825381210034436
task:
type: Classification
- dataset:
config: ms
name: MTEB MassiveIntentClassification (ms)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 40.11805213969503
- type: f1
value: 39.5525331645839
- type: f1_weighted
value: 40.68556481585343
- type: main_score
value: 40.11805213969503
task:
type: Classification
- dataset:
config: zh-TW
name: MTEB MassiveIntentClassification (zh-TW)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 70.70339399901624
- type: f1
value: 69.56397933117364
- type: f1_weighted
value: 70.636837176592
- type: main_score
value: 70.70339399901624
task:
type: Classification
- dataset:
config: ro
name: MTEB MassiveIntentClassification (ro)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 42.636497786522376
- type: f1
value: 41.835721061508046
- type: f1_weighted
value: 43.09527195252628
- type: main_score
value: 42.636497786522376
task:
type: Classification
- dataset:
config: ru
name: MTEB MassiveIntentClassification (ru)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 25.528775209050664
- type: f1
value: 24.590196538466525
- type: f1_weighted
value: 25.7679465276038
- type: main_score
value: 25.528775209050664
task:
type: Classification
- dataset:
config: ka
name: MTEB MassiveIntentClassification (ka)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 2.4053123462862764
- type: f1
value: 1.0039182333347725
- type: f1_weighted
value: 1.1504849663162338
- type: main_score
value: 2.4053123462862764
task:
type: Classification
- dataset:
config: da
name: MTEB MassiveIntentClassification (da)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 43.94490900147566
- type: f1
value: 42.06122028437199
- type: f1_weighted
value: 44.65360841137625
- type: main_score
value: 43.94490900147566
task:
type: Classification
- dataset:
config: tl
name: MTEB MassiveIntentClassification (tl)
revision: 4672e20407010da34463acc759c162ca9734bca6
split: validation
type: mteb/amazon_massive_intent
metrics:
- type: accuracy
value: 40.57550418101328
- type: f1
value: 38.55684696220655
- type: f1_weighted
value: 41.32491799752021
- type: main_score
value: 40.57550418101328
task:
type: Classification
- dataset:
config: is
name: MTEB MassiveScenarioClassification (is)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 36.267652992602564
- type: f1
value: 34.39028052609626
- type: f1_weighted
value: 37.125977056069445
- type: main_score
value: 36.267652992602564
task:
type: Classification
- dataset:
config: el
name: MTEB MassiveScenarioClassification (el)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 31.146603900470748
- type: f1
value: 28.930820125296385
- type: f1_weighted
value: 32.25487660975725
- type: main_score
value: 31.146603900470748
task:
type: Classification
- dataset:
config: az
name: MTEB MassiveScenarioClassification (az)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 32.59919300605246
- type: f1
value: 31.37679773609919
- type: f1_weighted
value: 33.440069864146615
- type: main_score
value: 32.59919300605246
task:
type: Classification
- dataset:
config: it
name: MTEB MassiveScenarioClassification (it)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 46.99058507061197
- type: f1
value: 44.84473711276094
- type: f1_weighted
value: 47.46949378039814
- type: main_score
value: 46.99058507061197
task:
type: Classification
- dataset:
config: hy
name: MTEB MassiveScenarioClassification (hy)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 8.611297915265634
- type: f1
value: 5.503073257951686
- type: f1_weighted
value: 6.564944192746709
- type: main_score
value: 8.611297915265634
task:
type: Classification
- dataset:
config: fr
name: MTEB MassiveScenarioClassification (fr)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 46.41560188298588
- type: f1
value: 44.668711805596985
- type: f1_weighted
value: 46.9398356896547
- type: main_score
value: 46.41560188298588
task:
type: Classification
- dataset:
config: te
name: MTEB MassiveScenarioClassification (te)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 7.3537323470074
- type: f1
value: 4.683418420407853
- type: f1_weighted
value: 5.5457976920254515
- type: main_score
value: 7.3537323470074
task:
type: Classification
- dataset:
config: tr
name: MTEB MassiveScenarioClassification (tr)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 33.843308675184936
- type: f1
value: 34.20551221429157
- type: f1_weighted
value: 34.45819700010564
- type: main_score
value: 33.843308675184936
task:
type: Classification
- dataset:
config: am
name: MTEB MassiveScenarioClassification (am)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 7.330195023537326
- type: f1
value: 4.686351443642142
- type: f1_weighted
value: 5.028446849221205
- type: main_score
value: 7.330195023537326
task:
type: Classification
- dataset:
config: nl
name: MTEB MassiveScenarioClassification (nl)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 45.09414929388031
- type: f1
value: 41.76347483758408
- type: f1_weighted
value: 46.25240119606559
- type: main_score
value: 45.09414929388031
task:
type: Classification
- dataset:
config: da
name: MTEB MassiveScenarioClassification (da)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 46.20040349697378
- type: f1
value: 43.19389996393894
- type: f1_weighted
value: 47.33543539294865
- type: main_score
value: 46.20040349697378
task:
type: Classification
- dataset:
config: nb
name: MTEB MassiveScenarioClassification (nb)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 39.95965030262273
- type: f1
value: 38.05830932392177
- type: f1_weighted
value: 41.02697348660353
- type: main_score
value: 39.95965030262273
task:
type: Classification
- dataset:
config: id
name: MTEB MassiveScenarioClassification (id)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 43.409549428379286
- type: f1
value: 41.24992542864968
- type: f1_weighted
value: 44.215059864156004
- type: main_score
value: 43.409549428379286
task:
type: Classification
- dataset:
config: ru
name: MTEB MassiveScenarioClassification (ru)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 26.186953597848017
- type: f1
value: 25.217093636667503
- type: f1_weighted
value: 26.708313663056355
- type: main_score
value: 26.186953597848017
task:
type: Classification
- dataset:
config: tl
name: MTEB MassiveScenarioClassification (tl)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 41.355077336919976
- type: f1
value: 39.36825529822422
- type: f1_weighted
value: 42.55595439584406
- type: main_score
value: 41.355077336919976
task:
type: Classification
- dataset:
config: ka
name: MTEB MassiveScenarioClassification (ka)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 6.940147948890384
- type: f1
value: 4.163887735005802
- type: f1_weighted
value: 4.523934747194612
- type: main_score
value: 6.940147948890384
task:
type: Classification
- dataset:
config: en
name: MTEB MassiveScenarioClassification (en)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 69.54270342972427
- type: f1
value: 67.2450361370488
- type: f1_weighted
value: 69.95238321926432
- type: main_score
value: 69.54270342972427
task:
type: Classification
- dataset:
config: ms
name: MTEB MassiveScenarioClassification (ms)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 42.72696704774715
- type: f1
value: 40.6887727140475
- type: f1_weighted
value: 43.8995830956876
- type: main_score
value: 42.72696704774715
task:
type: Classification
- dataset:
config: my
name: MTEB MassiveScenarioClassification (my)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 10.242098184263618
- type: f1
value: 6.484592667006297
- type: f1_weighted
value: 7.305871266507618
- type: main_score
value: 10.242098184263618
task:
type: Classification
- dataset:
config: ml
name: MTEB MassiveScenarioClassification (ml)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 6.546738399462004
- type: f1
value: 3.918317398893017
- type: f1_weighted
value: 4.158485825583739
- type: main_score
value: 6.546738399462004
task:
type: Classification
- dataset:
config: sv
name: MTEB MassiveScenarioClassification (sv)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 42.888365837256224
- type: f1
value: 40.10274962871748
- type: f1_weighted
value: 43.37964867963827
- type: main_score
value: 42.888365837256224
task:
type: Classification
- dataset:
config: hu
name: MTEB MassiveScenarioClassification (hu)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 42.55884330867518
- type: f1
value: 40.38029399273437
- type: f1_weighted
value: 43.398374319784985
- type: main_score
value: 42.55884330867518
task:
type: Classification
- dataset:
config: lv
name: MTEB MassiveScenarioClassification (lv)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 41.11970410221923
- type: f1
value: 38.79855099321214
- type: f1_weighted
value: 41.871902773554794
- type: main_score
value: 41.11970410221923
task:
type: Classification
- dataset:
config: vi
name: MTEB MassiveScenarioClassification (vi)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 38.9004707464694
- type: f1
value: 38.49724424945353
- type: f1_weighted
value: 39.6441441466175
- type: main_score
value: 38.9004707464694
task:
type: Classification
- dataset:
config: pl
name: MTEB MassiveScenarioClassification (pl)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 40.76664425016812
- type: f1
value: 38.92958333060731
- type: f1_weighted
value: 41.85931316638676
- type: main_score
value: 40.76664425016812
task:
type: Classification
- dataset:
config: sl
name: MTEB MassiveScenarioClassification (sl)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 41.51647612642905
- type: f1
value: 39.840898472140985
- type: f1_weighted
value: 42.10983379134603
- type: main_score
value: 41.51647612642905
task:
type: Classification
- dataset:
config: hi
name: MTEB MassiveScenarioClassification (hi)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 8.463349024882314
- type: f1
value: 6.687885838127879
- type: f1_weighted
value: 6.665248063087772
- type: main_score
value: 8.463349024882314
task:
type: Classification
- dataset:
config: th
name: MTEB MassiveScenarioClassification (th)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 13.3591123066577
- type: f1
value: 12.221561665549492
- type: f1_weighted
value: 11.984542743854307
- type: main_score
value: 13.3591123066577
task:
type: Classification
- dataset:
config: sw
name: MTEB MassiveScenarioClassification (sw)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 42.663080026899806
- type: f1
value: 40.728981841584826
- type: f1_weighted
value: 43.23457106933713
- type: main_score
value: 42.663080026899806
task:
type: Classification
- dataset:
config: fa
name: MTEB MassiveScenarioClassification (fa)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 15.494283792871554
- type: f1
value: 15.006092058084446
- type: f1_weighted
value: 15.27733810236863
- type: main_score
value: 15.494283792871554
task:
type: Classification
- dataset:
config: af
name: MTEB MassiveScenarioClassification (af)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 40.44720914593141
- type: f1
value: 37.751297424074835
- type: f1_weighted
value: 41.61089365114751
- type: main_score
value: 40.44720914593141
task:
type: Classification
- dataset:
config: jv
name: MTEB MassiveScenarioClassification (jv)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 41.87626092804304
- type: f1
value: 40.023561485491356
- type: f1_weighted
value: 42.18306723227648
- type: main_score
value: 41.87626092804304
task:
type: Classification
- dataset:
config: ko
name: MTEB MassiveScenarioClassification (ko)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 25.783456624075317
- type: f1
value: 24.81316657558517
- type: f1_weighted
value: 26.453345232329223
- type: main_score
value: 25.783456624075317
task:
type: Classification
- dataset:
config: bn
name: MTEB MassiveScenarioClassification (bn)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 7.572293207800943
- type: f1
value: 5.122318304085453
- type: f1_weighted
value: 5.918371344281876
- type: main_score
value: 7.572293207800943
task:
type: Classification
- dataset:
config: kn
name: MTEB MassiveScenarioClassification (kn)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 7.915265635507733
- type: f1
value: 5.835330719935933
- type: f1_weighted
value: 6.015782878022624
- type: main_score
value: 7.915265635507733
task:
type: Classification
- dataset:
config: cy
name: MTEB MassiveScenarioClassification (cy)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 41.788836583725626
- type: f1
value: 38.88966219885448
- type: f1_weighted
value: 42.8721422217295
- type: main_score
value: 41.788836583725626
task:
type: Classification
- dataset:
config: zh-CN
name: MTEB MassiveScenarioClassification (zh-CN)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 82.76395427034296
- type: f1
value: 82.41363804132969
- type: f1_weighted
value: 83.00514475868802
- type: main_score
value: 82.76395427034296
task:
type: Classification
- dataset:
config: km
name: MTEB MassiveScenarioClassification (km)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 9.640215198386013
- type: f1
value: 5.2564140077599655
- type: f1_weighted
value: 7.164883387181156
- type: main_score
value: 9.640215198386013
task:
type: Classification
- dataset:
config: pt
name: MTEB MassiveScenarioClassification (pt)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 46.64761264290519
- type: f1
value: 45.13961173395828
- type: f1_weighted
value: 47.36627661666301
- type: main_score
value: 46.64761264290519
task:
type: Classification
- dataset:
config: sq
name: MTEB MassiveScenarioClassification (sq)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 44.061869535978474
- type: f1
value: 42.22172641814783
- type: f1_weighted
value: 45.208537616148384
- type: main_score
value: 44.061869535978474
task:
type: Classification
- dataset:
config: ta
name: MTEB MassiveScenarioClassification (ta)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 7.528581035642231
- type: f1
value: 4.055954812609527
- type: f1_weighted
value: 5.346908730375917
- type: main_score
value: 7.528581035642231
task:
type: Classification
- dataset:
config: es
name: MTEB MassiveScenarioClassification (es)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 44.70410221923336
- type: f1
value: 43.287928645786955
- type: f1_weighted
value: 45.12367939775564
- type: main_score
value: 44.70410221923336
task:
type: Classification
- dataset:
config: mn
name: MTEB MassiveScenarioClassification (mn)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 25.363147276395427
- type: f1
value: 24.336469645147293
- type: f1_weighted
value: 25.552654529491054
- type: main_score
value: 25.363147276395427
task:
type: Classification
- dataset:
config: zh-TW
name: MTEB MassiveScenarioClassification (zh-TW)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 78.58776059179557
- type: f1
value: 78.7265027141245
- type: f1_weighted
value: 78.79726292257465
- type: main_score
value: 78.58776059179557
task:
type: Classification
- dataset:
config: de
name: MTEB MassiveScenarioClassification (de)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 44.546065904505724
- type: f1
value: 41.38065337271016
- type: f1_weighted
value: 45.261964994908595
- type: main_score
value: 44.546065904505724
task:
type: Classification
- dataset:
config: ar
name: MTEB MassiveScenarioClassification (ar)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 18.80295897780767
- type: f1
value: 17.010531525930894
- type: f1_weighted
value: 19.333227207926747
- type: main_score
value: 18.80295897780767
task:
type: Classification
- dataset:
config: ro
name: MTEB MassiveScenarioClassification (ro)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 45.268997982515124
- type: f1
value: 43.011099614102136
- type: f1_weighted
value: 46.00264255471957
- type: main_score
value: 45.268997982515124
task:
type: Classification
- dataset:
config: fi
name: MTEB MassiveScenarioClassification (fi)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 39.179556153328846
- type: f1
value: 36.72504541129902
- type: f1_weighted
value: 40.271736849507
- type: main_score
value: 39.179556153328846
task:
type: Classification
- dataset:
config: ja
name: MTEB MassiveScenarioClassification (ja)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 53.26832548755884
- type: f1
value: 53.061278209142884
- type: f1_weighted
value: 53.17435324984633
- type: main_score
value: 53.26832548755884
task:
type: Classification
- dataset:
config: ur
name: MTEB MassiveScenarioClassification (ur)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 14.579690652320107
- type: f1
value: 14.206144389074717
- type: f1_weighted
value: 14.748527351475005
- type: main_score
value: 14.579690652320107
task:
type: Classification
- dataset:
config: he
name: MTEB MassiveScenarioClassification (he)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: test
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 7.874915938130464
- type: f1
value: 4.118787513270843
- type: f1_weighted
value: 5.817041463583794
- type: main_score
value: 7.874915938130464
task:
type: Classification
- dataset:
config: is
name: MTEB MassiveScenarioClassification (is)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 35.99606492867683
- type: f1
value: 35.20584439264268
- type: f1_weighted
value: 36.67345255137978
- type: main_score
value: 35.99606492867683
task:
type: Classification
- dataset:
config: el
name: MTEB MassiveScenarioClassification (el)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 29.98524348253812
- type: f1
value: 28.765943623633927
- type: f1_weighted
value: 30.75476241135753
- type: main_score
value: 29.98524348253812
task:
type: Classification
- dataset:
config: az
name: MTEB MassiveScenarioClassification (az)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 31.908509591736355
- type: f1
value: 31.68840089466658
- type: f1_weighted
value: 32.614271948745056
- type: main_score
value: 31.908509591736355
task:
type: Classification
- dataset:
config: it
name: MTEB MassiveScenarioClassification (it)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 45.60255779636006
- type: f1
value: 44.91130039504065
- type: f1_weighted
value: 45.901808454025705
- type: main_score
value: 45.60255779636006
task:
type: Classification
- dataset:
config: hy
name: MTEB MassiveScenarioClassification (hy)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 8.480078701426464
- type: f1
value: 5.33881465273221
- type: f1_weighted
value: 6.299851933533558
- type: main_score
value: 8.480078701426464
task:
type: Classification
- dataset:
config: fr
name: MTEB MassiveScenarioClassification (fr)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 45.61239547466798
- type: f1
value: 44.796398022321725
- type: f1_weighted
value: 46.050421712295055
- type: main_score
value: 45.61239547466798
task:
type: Classification
- dataset:
config: te
name: MTEB MassiveScenarioClassification (te)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 7.333989178553861
- type: f1
value: 4.666258475314794
- type: f1_weighted
value: 5.362484645743675
- type: main_score
value: 7.333989178553861
task:
type: Classification
- dataset:
config: tr
name: MTEB MassiveScenarioClassification (tr)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 33.561239547466805
- type: f1
value: 34.0795945686752
- type: f1_weighted
value: 33.96219218340987
- type: main_score
value: 33.561239547466805
task:
type: Classification
- dataset:
config: am
name: MTEB MassiveScenarioClassification (am)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 7.604525332021645
- type: f1
value: 4.536275730289927
- type: f1_weighted
value: 5.166653156052664
- type: main_score
value: 7.604525332021645
task:
type: Classification
- dataset:
config: nl
name: MTEB MassiveScenarioClassification (nl)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 42.749631087063456
- type: f1
value: 40.97034342282828
- type: f1_weighted
value: 43.80143496413575
- type: main_score
value: 42.749631087063456
task:
type: Classification
- dataset:
config: da
name: MTEB MassiveScenarioClassification (da)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 44.00393507132317
- type: f1
value: 42.542091357589484
- type: f1_weighted
value: 45.08395452421817
- type: main_score
value: 44.00393507132317
task:
type: Classification
- dataset:
config: nb
name: MTEB MassiveScenarioClassification (nb)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 39.527791441219875
- type: f1
value: 38.877669437208
- type: f1_weighted
value: 40.440029463255314
- type: main_score
value: 39.527791441219875
task:
type: Classification
- dataset:
config: id
name: MTEB MassiveScenarioClassification (id)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 42.882439744220356
- type: f1
value: 41.74248841008145
- type: f1_weighted
value: 43.49099459057078
- type: main_score
value: 42.882439744220356
task:
type: Classification
- dataset:
config: ru
name: MTEB MassiveScenarioClassification (ru)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 25.976389572061
- type: f1
value: 25.526996864828078
- type: f1_weighted
value: 26.323888911031062
- type: main_score
value: 25.976389572061
task:
type: Classification
- dataset:
config: tl
name: MTEB MassiveScenarioClassification (tl)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 40.23610427939007
- type: f1
value: 39.5110278526761
- type: f1_weighted
value: 41.1833957480028
- type: main_score
value: 40.23610427939007
task:
type: Classification
- dataset:
config: ka
name: MTEB MassiveScenarioClassification (ka)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 6.714215445154944
- type: f1
value: 3.9946938970745207
- type: f1_weighted
value: 4.251073566584421
- type: main_score
value: 6.714215445154944
task:
type: Classification
- dataset:
config: en
name: MTEB MassiveScenarioClassification (en)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 68.92769306443678
- type: f1
value: 66.90869048636226
- type: f1_weighted
value: 69.3297020719043
- type: main_score
value: 68.92769306443678
task:
type: Classification
- dataset:
config: ms
name: MTEB MassiveScenarioClassification (ms)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 42.18396458435809
- type: f1
value: 41.2623335611438
- type: f1_weighted
value: 43.127995422526844
- type: main_score
value: 42.18396458435809
task:
type: Classification
- dataset:
config: my
name: MTEB MassiveScenarioClassification (my)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 11.082144613871126
- type: f1
value: 7.425778448776913
- type: f1_weighted
value: 7.789515570254026
- type: main_score
value: 11.082144613871126
task:
type: Classification
- dataset:
config: ml
name: MTEB MassiveScenarioClassification (ml)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 6.5027053615346775
- type: f1
value: 3.7466133746195633
- type: f1_weighted
value: 3.973214419588327
- type: main_score
value: 6.5027053615346775
task:
type: Classification
- dataset:
config: sv
name: MTEB MassiveScenarioClassification (sv)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 41.81013280865716
- type: f1
value: 40.48354280118566
- type: f1_weighted
value: 42.13905635564818
- type: main_score
value: 41.81013280865716
task:
type: Classification
- dataset:
config: hu
name: MTEB MassiveScenarioClassification (hu)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 39.621249385145106
- type: f1
value: 38.77569615351878
- type: f1_weighted
value: 40.408850998703294
- type: main_score
value: 39.621249385145106
task:
type: Classification
- dataset:
config: lv
name: MTEB MassiveScenarioClassification (lv)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 39.51795376291195
- type: f1
value: 38.33840469203745
- type: f1_weighted
value: 40.14217235774783
- type: main_score
value: 39.51795376291195
task:
type: Classification
- dataset:
config: vi
name: MTEB MassiveScenarioClassification (vi)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 39.11460895228726
- type: f1
value: 38.90884303194236
- type: f1_weighted
value: 39.61233724806451
- type: main_score
value: 39.11460895228726
task:
type: Classification
- dataset:
config: pl
name: MTEB MassiveScenarioClassification (pl)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 39.47860304968028
- type: f1
value: 38.456455709925585
- type: f1_weighted
value: 40.42759387176258
- type: main_score
value: 39.47860304968028
task:
type: Classification
- dataset:
config: sl
name: MTEB MassiveScenarioClassification (sl)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 40.78209542547959
- type: f1
value: 39.80360672965863
- type: f1_weighted
value: 41.3388040455552
- type: main_score
value: 40.78209542547959
task:
type: Classification
- dataset:
config: hi
name: MTEB MassiveScenarioClassification (hi)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 8.52926709296606
- type: f1
value: 6.39881808638407
- type: f1_weighted
value: 6.616654119148606
- type: main_score
value: 8.52926709296606
task:
type: Classification
- dataset:
config: th
name: MTEB MassiveScenarioClassification (th)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 13.753074274471224
- type: f1
value: 12.326492805202069
- type: f1_weighted
value: 12.151551505564898
- type: main_score
value: 13.753074274471224
task:
type: Classification
- dataset:
config: sw
name: MTEB MassiveScenarioClassification (sw)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 41.72651254303984
- type: f1
value: 40.35106230585296
- type: f1_weighted
value: 42.504608904683664
- type: main_score
value: 41.72651254303984
task:
type: Classification
- dataset:
config: fa
name: MTEB MassiveScenarioClassification (fa)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 15.31234628627644
- type: f1
value: 15.057814723718712
- type: f1_weighted
value: 14.825818910096011
- type: main_score
value: 15.31234628627644
task:
type: Classification
- dataset:
config: af
name: MTEB MassiveScenarioClassification (af)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 40.9640924741761
- type: f1
value: 39.415962035771635
- type: f1_weighted
value: 41.880993077133425
- type: main_score
value: 40.9640924741761
task:
type: Classification
- dataset:
config: jv
name: MTEB MassiveScenarioClassification (jv)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 40.850959173635026
- type: f1
value: 40.083644344440664
- type: f1_weighted
value: 41.0320978528804
- type: main_score
value: 40.850959173635026
task:
type: Classification
- dataset:
config: ko
name: MTEB MassiveScenarioClassification (ko)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 25.336940482046238
- type: f1
value: 24.385419734710172
- type: f1_weighted
value: 25.75871028873885
- type: main_score
value: 25.336940482046238
task:
type: Classification
- dataset:
config: bn
name: MTEB MassiveScenarioClassification (bn)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 7.496310870634529
- type: f1
value: 4.99273826850258
- type: f1_weighted
value: 5.802198772868682
- type: main_score
value: 7.496310870634529
task:
type: Classification
- dataset:
config: kn
name: MTEB MassiveScenarioClassification (kn)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 8.53418593212002
- type: f1
value: 6.456521102839466
- type: f1_weighted
value: 6.396377755145225
- type: main_score
value: 8.53418593212002
task:
type: Classification
- dataset:
config: cy
name: MTEB MassiveScenarioClassification (cy)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 40.20167240531235
- type: f1
value: 38.24662692870761
- type: f1_weighted
value: 41.01478059789378
- type: main_score
value: 40.20167240531235
task:
type: Classification
- dataset:
config: zh-CN
name: MTEB MassiveScenarioClassification (zh-CN)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 82.36104279390064
- type: f1
value: 81.67294237294372
- type: f1_weighted
value: 82.57942325045582
- type: main_score
value: 82.36104279390064
task:
type: Classification
- dataset:
config: km
name: MTEB MassiveScenarioClassification (km)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 9.24741760944417
- type: f1
value: 5.058356544468532
- type: f1_weighted
value: 6.903761892494975
- type: main_score
value: 9.24741760944417
task:
type: Classification
- dataset:
config: pt
name: MTEB MassiveScenarioClassification (pt)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 44.92867683226759
- type: f1
value: 44.483430371315194
- type: f1_weighted
value: 45.52549319038087
- type: main_score
value: 44.92867683226759
task:
type: Classification
- dataset:
config: sq
name: MTEB MassiveScenarioClassification (sq)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 43.059517953762914
- type: f1
value: 42.65977402358004
- type: f1_weighted
value: 43.97266713539029
- type: main_score
value: 43.059517953762914
task:
type: Classification
- dataset:
config: ta
name: MTEB MassiveScenarioClassification (ta)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 7.530742744712249
- type: f1
value: 4.0701142041780844
- type: f1_weighted
value: 5.372399660680421
- type: main_score
value: 7.530742744712249
task:
type: Classification
- dataset:
config: es
name: MTEB MassiveScenarioClassification (es)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 42.75454992621741
- type: f1
value: 42.45294276737515
- type: f1_weighted
value: 42.97139601656561
- type: main_score
value: 42.75454992621741
task:
type: Classification
- dataset:
config: mn
name: MTEB MassiveScenarioClassification (mn)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 24.84997540580423
- type: f1
value: 24.120781457641122
- type: f1_weighted
value: 24.975936906386913
- type: main_score
value: 24.84997540580423
task:
type: Classification
- dataset:
config: zh-TW
name: MTEB MassiveScenarioClassification (zh-TW)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 79.05558288243975
- type: f1
value: 78.9835268881931
- type: f1_weighted
value: 79.28165747769849
- type: main_score
value: 79.05558288243975
task:
type: Classification
- dataset:
config: de
name: MTEB MassiveScenarioClassification (de)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 42.58730939498278
- type: f1
value: 40.70852558786616
- type: f1_weighted
value: 43.251635086069804
- type: main_score
value: 42.58730939498278
task:
type: Classification
- dataset:
config: ar
name: MTEB MassiveScenarioClassification (ar)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 18.017707820954254
- type: f1
value: 16.687990492609256
- type: f1_weighted
value: 18.46136988933461
- type: main_score
value: 18.017707820954254
task:
type: Classification
- dataset:
config: ro
name: MTEB MassiveScenarioClassification (ro)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 43.78750614854894
- type: f1
value: 43.35116606491994
- type: f1_weighted
value: 44.31377810377738
- type: main_score
value: 43.78750614854894
task:
type: Classification
- dataset:
config: fi
name: MTEB MassiveScenarioClassification (fi)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 38.25381210034432
- type: f1
value: 36.84507866986097
- type: f1_weighted
value: 39.15201046891089
- type: main_score
value: 38.25381210034432
task:
type: Classification
- dataset:
config: ja
name: MTEB MassiveScenarioClassification (ja)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 53.29562223315298
- type: f1
value: 53.13848950509313
- type: f1_weighted
value: 53.26552991076052
- type: main_score
value: 53.29562223315298
task:
type: Classification
- dataset:
config: ur
name: MTEB MassiveScenarioClassification (ur)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 14.594195769798329
- type: f1
value: 14.397328781753599
- type: f1_weighted
value: 14.660528748357581
- type: main_score
value: 14.594195769798329
task:
type: Classification
- dataset:
config: he
name: MTEB MassiveScenarioClassification (he)
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
split: validation
type: mteb/amazon_massive_scenario
metrics:
- type: accuracy
value: 7.879980324643385
- type: f1
value: 4.038729452806328
- type: f1_weighted
value: 5.6615870271046065
- type: main_score
value: 7.879980324643385
task:
type: Classification
- dataset:
config: default
name: MTEB MedicalRetrieval (default)
revision: 2039188fb5800a9803ba5048df7b76e6fb151fc6
split: dev
type: C-MTEB/MedicalRetrieval
metrics:
- type: main_score
value: 63.67699999999999
- type: map_at_1
value: 53.800000000000004
- type: map_at_10
value: 60.431999999999995
- type: map_at_100
value: 61.029999999999994
- type: map_at_1000
value: 61.071
- type: map_at_20
value: 60.777
- type: map_at_3
value: 58.75
- type: map_at_5
value: 59.81999999999999
- type: mrr_at_1
value: 53.800000000000004
- type: mrr_at_10
value: 60.431507936507934
- type: mrr_at_100
value: 61.02980943818268
- type: mrr_at_1000
value: 61.07101618078645
- type: mrr_at_20
value: 60.77656156107394
- type: mrr_at_3
value: 58.750000000000014
- type: mrr_at_5
value: 59.81999999999999
- type: nauc_map_at_1000_diff1
value: 78.00629416107189
- type: nauc_map_at_1000_max
value: 52.07420225671461
- type: nauc_map_at_1000_std
value: 7.235435225393745
- type: nauc_map_at_100_diff1
value: 77.9857887699591
- type: nauc_map_at_100_max
value: 52.055541763889416
- type: nauc_map_at_100_std
value: 7.236124232996337
- type: nauc_map_at_10_diff1
value: 77.91480569228007
- type: nauc_map_at_10_max
value: 52.04826888831761
- type: nauc_map_at_10_std
value: 7.006421025182576
- type: nauc_map_at_1_diff1
value: 82.20582404755503
- type: nauc_map_at_1_max
value: 54.4721454376076
- type: nauc_map_at_1_std
value: 4.496959883774145
- type: nauc_map_at_20_diff1
value: 77.95162276147624
- type: nauc_map_at_20_max
value: 52.023771967662434
- type: nauc_map_at_20_std
value: 7.196031931418907
- type: nauc_map_at_3_diff1
value: 79.03105924086732
- type: nauc_map_at_3_max
value: 53.07038869379984
- type: nauc_map_at_3_std
value: 6.793908288837009
- type: nauc_map_at_5_diff1
value: 78.21124356618412
- type: nauc_map_at_5_max
value: 52.395049895165236
- type: nauc_map_at_5_std
value: 6.899017664263203
- type: nauc_mrr_at_1000_diff1
value: 78.00601481881016
- type: nauc_mrr_at_1000_max
value: 51.9762321296784
- type: nauc_mrr_at_1000_std
value: 7.350305802489671
- type: nauc_mrr_at_100_diff1
value: 77.98550974774857
- type: nauc_mrr_at_100_max
value: 51.957702695554374
- type: nauc_mrr_at_100_std
value: 7.3508410873853
- type: nauc_mrr_at_10_diff1
value: 77.91480569228007
- type: nauc_mrr_at_10_max
value: 51.951920429485554
- type: nauc_mrr_at_10_std
value: 7.118827560486609
- type: nauc_mrr_at_1_diff1
value: 82.20582404755503
- type: nauc_mrr_at_1_max
value: 54.29747391710586
- type: nauc_mrr_at_1_std
value: 4.700743324359466
- type: nauc_mrr_at_20_diff1
value: 77.95162276147624
- type: nauc_mrr_at_20_max
value: 51.92663048569973
- type: nauc_mrr_at_20_std
value: 7.30936366037528
- type: nauc_mrr_at_3_diff1
value: 79.03105924086732
- type: nauc_mrr_at_3_max
value: 52.97714066514463
- type: nauc_mrr_at_3_std
value: 6.902697655601455
- type: nauc_mrr_at_5_diff1
value: 78.21124356618412
- type: nauc_mrr_at_5_max
value: 52.300015316154855
- type: nauc_mrr_at_5_std
value: 7.0098913397753355
- type: nauc_ndcg_at_1000_diff1
value: 76.68028623834495
- type: nauc_ndcg_at_1000_max
value: 51.004333240246936
- type: nauc_ndcg_at_1000_std
value: 8.698515324279041
- type: nauc_ndcg_at_100_diff1
value: 76.05356190504821
- type: nauc_ndcg_at_100_max
value: 50.35277313216198
- type: nauc_ndcg_at_100_std
value: 9.249854230791188
- type: nauc_ndcg_at_10_diff1
value: 75.6902794635337
- type: nauc_ndcg_at_10_max
value: 50.299670065322886
- type: nauc_ndcg_at_10_std
value: 8.187845234344188
- type: nauc_ndcg_at_1_diff1
value: 82.20582404755503
- type: nauc_ndcg_at_1_max
value: 54.4721454376076
- type: nauc_ndcg_at_1_std
value: 4.496959883774145
- type: nauc_ndcg_at_20_diff1
value: 75.79556771472184
- type: nauc_ndcg_at_20_max
value: 50.21494338026765
- type: nauc_ndcg_at_20_std
value: 8.945573827175862
- type: nauc_ndcg_at_3_diff1
value: 78.05339525168148
- type: nauc_ndcg_at_3_max
value: 52.459350220474875
- type: nauc_ndcg_at_3_std
value: 7.638500728186775
- type: nauc_ndcg_at_5_diff1
value: 76.45369105380743
- type: nauc_ndcg_at_5_max
value: 51.19789390012578
- type: nauc_ndcg_at_5_std
value: 7.86689928703537
- type: nauc_precision_at_1000_diff1
value: 59.591503267973344
- type: nauc_precision_at_1000_max
value: 39.813258636787445
- type: nauc_precision_at_1000_std
value: 43.9635854341733
- type: nauc_precision_at_100_diff1
value: 61.11499109543613
- type: nauc_precision_at_100_max
value: 35.64662848762093
- type: nauc_precision_at_100_std
value: 28.753011837832272
- type: nauc_precision_at_10_diff1
value: 66.60448251399139
- type: nauc_precision_at_10_max
value: 42.93455303203327
- type: nauc_precision_at_10_std
value: 12.939019781028152
- type: nauc_precision_at_1_diff1
value: 82.20582404755503
- type: nauc_precision_at_1_max
value: 54.4721454376076
- type: nauc_precision_at_1_std
value: 4.496959883774145
- type: nauc_precision_at_20_diff1
value: 65.68865023744912
- type: nauc_precision_at_20_max
value: 41.36253272803797
- type: nauc_precision_at_20_std
value: 17.955772857309167
- type: nauc_precision_at_3_diff1
value: 75.01623067257978
- type: nauc_precision_at_3_max
value: 50.493541645806374
- type: nauc_precision_at_3_std
value: 10.313986802282976
- type: nauc_precision_at_5_diff1
value: 70.23831710178575
- type: nauc_precision_at_5_max
value: 46.894174569451266
- type: nauc_precision_at_5_std
value: 11.210506044454311
- type: nauc_recall_at_1000_diff1
value: 59.59150326797431
- type: nauc_recall_at_1000_max
value: 39.81325863678807
- type: nauc_recall_at_1000_std
value: 43.96358543417411
- type: nauc_recall_at_100_diff1
value: 61.11499109543585
- type: nauc_recall_at_100_max
value: 35.64662848762088
- type: nauc_recall_at_100_std
value: 28.753011837832233
- type: nauc_recall_at_10_diff1
value: 66.60448251399139
- type: nauc_recall_at_10_max
value: 42.934553032033186
- type: nauc_recall_at_10_std
value: 12.939019781028199
- type: nauc_recall_at_1_diff1
value: 82.20582404755503
- type: nauc_recall_at_1_max
value: 54.4721454376076
- type: nauc_recall_at_1_std
value: 4.496959883774145
- type: nauc_recall_at_20_diff1
value: 65.68865023744912
- type: nauc_recall_at_20_max
value: 41.36253272803804
- type: nauc_recall_at_20_std
value: 17.955772857309242
- type: nauc_recall_at_3_diff1
value: 75.01623067257968
- type: nauc_recall_at_3_max
value: 50.49354164580636
- type: nauc_recall_at_3_std
value: 10.313986802282942
- type: nauc_recall_at_5_diff1
value: 70.23831710178574
- type: nauc_recall_at_5_max
value: 46.894174569451344
- type: nauc_recall_at_5_std
value: 11.210506044454428
- type: ndcg_at_1
value: 53.800000000000004
- type: ndcg_at_10
value: 63.67699999999999
- type: ndcg_at_100
value: 66.81899999999999
- type: ndcg_at_1000
value: 67.93299999999999
- type: ndcg_at_20
value: 64.90599999999999
- type: ndcg_at_3
value: 60.260999999999996
- type: ndcg_at_5
value: 62.193
- type: precision_at_1
value: 53.800000000000004
- type: precision_at_10
value: 7.39
- type: precision_at_100
value: 0.8920000000000001
- type: precision_at_1000
value: 0.098
- type: precision_at_20
value: 3.9350000000000005
- type: precision_at_3
value: 21.532999999999998
- type: precision_at_5
value: 13.86
- type: recall_at_1
value: 53.800000000000004
- type: recall_at_10
value: 73.9
- type: recall_at_100
value: 89.2
- type: recall_at_1000
value: 98
- type: recall_at_20
value: 78.7
- type: recall_at_3
value: 64.60000000000001
- type: recall_at_5
value: 69.3
task:
type: Retrieval
- dataset:
config: default
name: MTEB MultilingualSentiment (default)
revision: 46958b007a63fdbf239b7672c25d0bea67b5ea1a
split: test
type: C-MTEB/MultilingualSentiment-classification
metrics:
- type: accuracy
value: 75.08333333333333
- type: f1
value: 74.89816720884292
- type: f1_weighted
value: 74.89816720884292
- type: main_score
value: 75.08333333333333
task:
type: Classification
- dataset:
config: default
name: MTEB MultilingualSentiment (default)
revision: 46958b007a63fdbf239b7672c25d0bea67b5ea1a
split: validation
type: C-MTEB/MultilingualSentiment-classification
metrics:
- type: accuracy
value: 74.05999999999999
- type: f1
value: 73.86707495511577
- type: f1_weighted
value: 73.86707495511577
- type: main_score
value: 74.05999999999999
task:
type: Classification
- dataset:
config: default
name: MTEB Ocnli (default)
revision: 66e76a618a34d6d565d5538088562851e6daa7ec
split: validation
type: C-MTEB/OCNLI
metrics:
- type: cosine_accuracy
value: 83.21602598808879
- type: cosine_accuracy_threshold
value: 84.68668460845947
- type: cosine_ap
value: 88.4718001921049
- type: cosine_f1
value: 84.60774577954318
- type: cosine_f1_threshold
value: 84.68668460845947
- type: cosine_precision
value: 79.85004686035614
- type: cosine_recall
value: 89.96832101372756
- type: dot_accuracy
value: 74.12019491066594
- type: dot_accuracy_threshold
value: 61163.34228515625
- type: dot_ap
value: 82.21843913674356
- type: dot_f1
value: 77.42230347349177
- type: dot_f1_threshold
value: 58876.74560546875
- type: dot_precision
value: 68.25141015310233
- type: dot_recall
value: 89.44033790918691
- type: euclidean_accuracy
value: 81.64591229020033
- type: euclidean_accuracy_threshold
value: 1478.951644897461
- type: euclidean_ap
value: 86.55778250928002
- type: euclidean_f1
value: 83.37326305701964
- type: euclidean_f1_threshold
value: 1558.5298538208008
- type: euclidean_precision
value: 76.31578947368422
- type: euclidean_recall
value: 91.86906019007391
- type: main_score
value: 83.21602598808879
- type: manhattan_accuracy
value: 81.48348673524634
- type: manhattan_accuracy_threshold
value: 51344.219970703125
- type: manhattan_ap
value: 86.53822978018471
- type: manhattan_f1
value: 83.26139088729016
- type: manhattan_f1_threshold
value: 52366.3330078125
- type: manhattan_precision
value: 76.27416520210897
- type: manhattan_recall
value: 91.65786694825766
- type: max_accuracy
value: 83.21602598808879
- type: max_ap
value: 88.4718001921049
- type: max_f1
value: 84.60774577954318
- type: max_precision
value: 79.85004686035614
- type: max_recall
value: 91.86906019007391
- type: similarity_accuracy
value: 83.21602598808879
- type: similarity_accuracy_threshold
value: 84.686678647995
- type: similarity_ap
value: 88.4718001921049
- type: similarity_f1
value: 84.60774577954318
- type: similarity_f1_threshold
value: 84.686678647995
- type: similarity_precision
value: 79.85004686035614
- type: similarity_recall
value: 89.96832101372756
task:
type: PairClassification
- dataset:
config: default
name: MTEB OnlineShopping (default)
revision: e610f2ebd179a8fda30ae534c3878750a96db120
split: test
type: C-MTEB/OnlineShopping-classification
metrics:
- type: accuracy
value: 93.11
- type: ap
value: 90.79657759750567
- type: ap_weighted
value: 90.79657759750567
- type: f1
value: 93.09242477250346
- type: f1_weighted
value: 93.10745591896395
- type: main_score
value: 93.11
task:
type: Classification
- dataset:
config: default
name: MTEB PAWSX (default)
revision: 9c6a90e430ac22b5779fb019a23e820b11a8b5e1
split: test
type: C-MTEB/PAWSX
metrics:
- type: cosine_pearson
value: 40.85294247983506
- type: cosine_spearman
value: 46.26470443016128
- type: euclidean_pearson
value: 45.71637535133479
- type: euclidean_spearman
value: 45.97506398282988
- type: main_score
value: 46.26470443016128
- type: manhattan_pearson
value: 45.729052797800186
- type: manhattan_spearman
value: 45.985167052566766
task:
type: STS
- dataset:
config: default
name: MTEB QBQTC (default)
revision: 790b0510dc52b1553e8c49f3d2afb48c0e5c48b7
split: test
type: C-MTEB/QBQTC
metrics:
- type: cosine_pearson
value: 39.61472967042291
- type: cosine_spearman
value: 41.89107113069583
- type: euclidean_pearson
value: 33.90266552269117
- type: euclidean_spearman
value: 38.34637073395628
- type: main_score
value: 41.89107113069583
- type: manhattan_pearson
value: 33.91591969583774
- type: manhattan_spearman
value: 38.3574043430055
task:
type: STS
- dataset:
config: it
name: MTEB STS22 (it)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 41.19325498554315
- type: cosine_spearman
value: 50.231173886602534
- type: euclidean_pearson
value: 46.345868108079905
- type: euclidean_spearman
value: 50.45422800948126
- type: main_score
value: 50.231173886602534
- type: manhattan_pearson
value: 46.504629255328354
- type: manhattan_spearman
value: 50.68265515693354
- type: pearson
value: 41.193261234261946
- type: spearman
value: 50.231173886602534
task:
type: STS
- dataset:
config: de-en
name: MTEB STS22 (de-en)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 32.89821953127283
- type: cosine_spearman
value: 38.539988618106655
- type: euclidean_pearson
value: 34.49033573399129
- type: euclidean_spearman
value: 38.11416536645878
- type: main_score
value: 38.539988618106655
- type: manhattan_pearson
value: 34.448528477994145
- type: manhattan_spearman
value: 37.65048357982259
- type: pearson
value: 32.89820801840136
- type: spearman
value: 38.539988618106655
task:
type: STS
- dataset:
config: tr
name: MTEB STS22 (tr)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 24.305368866867557
- type: cosine_spearman
value: 37.92726109087382
- type: euclidean_pearson
value: 30.799738296378283
- type: euclidean_spearman
value: 38.287216177996605
- type: main_score
value: 37.92726109087382
- type: manhattan_pearson
value: 30.78126726940549
- type: manhattan_spearman
value: 38.19740860271776
- type: pearson
value: 24.30535329212899
- type: spearman
value: 37.92726109087382
task:
type: STS
- dataset:
config: es
name: MTEB STS22 (es)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 18.19997923279834
- type: cosine_spearman
value: 48.77210511608037
- type: euclidean_pearson
value: 31.136177249632553
- type: euclidean_spearman
value: 48.41077369782945
- type: main_score
value: 48.77210511608037
- type: manhattan_pearson
value: 31.15147622246907
- type: manhattan_spearman
value: 48.45673863744031
- type: pearson
value: 18.19996287259861
- type: spearman
value: 48.77210511608037
task:
type: STS
- dataset:
config: pl
name: MTEB STS22 (pl)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: -2.108270346091053
- type: cosine_spearman
value: 12.387180120208471
- type: euclidean_pearson
value: -5.143877670933451
- type: euclidean_spearman
value: 11.732310436300734
- type: main_score
value: 12.387180120208471
- type: manhattan_pearson
value: -4.879466616861115
- type: manhattan_spearman
value: 11.837089585725971
- type: pearson
value: -2.1082730668298817
- type: spearman
value: 12.17651413548943
task:
type: STS
- dataset:
config: en
name: MTEB STS22 (en)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 55.197301705940106
- type: cosine_spearman
value: 60.735304152341016
- type: euclidean_pearson
value: 58.01165360549454
- type: euclidean_spearman
value: 59.382742411350954
- type: main_score
value: 60.735304152341016
- type: manhattan_pearson
value: 57.83223965160013
- type: manhattan_spearman
value: 59.21305162189526
- type: pearson
value: 55.197304101079325
- type: spearman
value: 60.735304152341016
task:
type: STS
- dataset:
config: ar
name: MTEB STS22 (ar)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 24.954116820344225
- type: cosine_spearman
value: 27.716439696864427
- type: euclidean_pearson
value: 28.648374663088948
- type: euclidean_spearman
value: 27.971284844826314
- type: main_score
value: 27.716439696864427
- type: manhattan_pearson
value: 28.59989248858511
- type: manhattan_spearman
value: 27.885463405824424
- type: pearson
value: 24.95413634470457
- type: spearman
value: 27.69571464401494
task:
type: STS
- dataset:
config: zh-en
name: MTEB STS22 (zh-en)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 59.628323211697996
- type: cosine_spearman
value: 64.69426903014181
- type: euclidean_pearson
value: 63.02177505481432
- type: euclidean_spearman
value: 64.38122549439275
- type: main_score
value: 64.69426903014181
- type: manhattan_pearson
value: 62.89174360491565
- type: manhattan_spearman
value: 64.21033497242632
- type: pearson
value: 59.62832002315839
- type: spearman
value: 64.69426903014181
task:
type: STS
- dataset:
config: fr-pl
name: MTEB STS22 (fr-pl)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 72.48621899222191
- type: cosine_spearman
value: 84.51542547285167
- type: euclidean_pearson
value: 79.91610128578839
- type: euclidean_spearman
value: 84.51542547285167
- type: main_score
value: 84.51542547285167
- type: manhattan_pearson
value: 78.02332046266785
- type: manhattan_spearman
value: 84.51542547285167
- type: pearson
value: 72.48643781257383
- type: spearman
value: 84.51542547285167
task:
type: STS
- dataset:
config: es-it
name: MTEB STS22 (es-it)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 27.18583867128202
- type: cosine_spearman
value: 35.63056944816928
- type: euclidean_pearson
value: 31.075483547268558
- type: euclidean_spearman
value: 34.46346668406813
- type: main_score
value: 35.63056944816928
- type: manhattan_pearson
value: 31.131467056697325
- type: manhattan_spearman
value: 34.4668541493314
- type: pearson
value: 27.185822556247537
- type: spearman
value: 35.63056944816928
task:
type: STS
- dataset:
config: de-pl
name: MTEB STS22 (de-pl)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 12.399198871280888
- type: cosine_spearman
value: 7.26740842591211
- type: euclidean_pearson
value: 4.516712707783254
- type: euclidean_spearman
value: 6.466761734921793
- type: main_score
value: 7.26740842591211
- type: manhattan_pearson
value: 4.192789002097003
- type: manhattan_spearman
value: 6.805496873417696
- type: pearson
value: 12.39933152364361
- type: spearman
value: 7.26740842591211
task:
type: STS
- dataset:
config: fr
name: MTEB STS22 (fr)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 41.39868462834059
- type: cosine_spearman
value: 55.038606223576956
- type: euclidean_pearson
value: 49.25365386033138
- type: euclidean_spearman
value: 54.672553472547804
- type: main_score
value: 55.038606223576956
- type: manhattan_pearson
value: 49.20067830976173
- type: manhattan_spearman
value: 54.6387725076268
- type: pearson
value: 41.398676979588934
- type: spearman
value: 55.038606223576956
task:
type: STS
- dataset:
config: de-fr
name: MTEB STS22 (de-fr)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 25.608150018172793
- type: cosine_spearman
value: 33.90331453592282
- type: euclidean_pearson
value: 29.30906861776585
- type: euclidean_spearman
value: 33.826599994845935
- type: main_score
value: 33.90331453592282
- type: manhattan_pearson
value: 28.909395620689825
- type: manhattan_spearman
value: 34.59825802567819
- type: pearson
value: 25.60814265156774
- type: spearman
value: 33.90331453592282
task:
type: STS
- dataset:
config: ru
name: MTEB STS22 (ru)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 5.7754167331012205
- type: cosine_spearman
value: 21.92465228528612
- type: euclidean_pearson
value: 13.675536468172805
- type: euclidean_spearman
value: 22.21141895175971
- type: main_score
value: 21.92465228528612
- type: manhattan_pearson
value: 13.800372687665202
- type: manhattan_spearman
value: 22.351737531079902
- type: pearson
value: 5.775448495602957
- type: spearman
value: 21.93020607531918
task:
type: STS
- dataset:
config: zh
name: MTEB STS22 (zh)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 68.28512697903551
- type: cosine_spearman
value: 68.69300952790282
- type: euclidean_pearson
value: 67.64215814935483
- type: euclidean_spearman
value: 68.4668380232344
- type: main_score
value: 68.69300952790282
- type: manhattan_pearson
value: 67.5660533943303
- type: manhattan_spearman
value: 68.42998433131551
- type: pearson
value: 68.2851240441332
- type: spearman
value: 68.69300952790282
task:
type: STS
- dataset:
config: pl-en
name: MTEB STS22 (pl-en)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 40.99563262228948
- type: cosine_spearman
value: 37.611329662509235
- type: euclidean_pearson
value: 39.60891222145106
- type: euclidean_spearman
value: 35.74639115569783
- type: main_score
value: 37.611329662509235
- type: manhattan_pearson
value: 40.68243822309343
- type: manhattan_spearman
value: 37.710306845831624
- type: pearson
value: 40.995650280936935
- type: spearman
value: 37.611329662509235
task:
type: STS
- dataset:
config: de
name: MTEB STS22 (de)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 14.610716018581904
- type: cosine_spearman
value: 20.259656537989873
- type: euclidean_pearson
value: 13.39968577970463
- type: euclidean_spearman
value: 19.999373521862104
- type: main_score
value: 20.259656537989873
- type: manhattan_pearson
value: 13.403453329892018
- type: manhattan_spearman
value: 20.035611306496893
- type: pearson
value: 14.610694852334966
- type: spearman
value: 20.265485348016593
task:
type: STS
- dataset:
config: es-en
name: MTEB STS22 (es-en)
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 43.612676183806045
- type: cosine_spearman
value: 47.5304556170413
- type: euclidean_pearson
value: 43.21510166675375
- type: euclidean_spearman
value: 46.4219875335409
- type: main_score
value: 47.5304556170413
- type: manhattan_pearson
value: 43.102337861589625
- type: manhattan_spearman
value: 46.34374314971102
- type: pearson
value: 43.61267091853167
- type: spearman
value: 47.5304556170413
task:
type: STS
- dataset:
config: pl-en
name: MTEB STS22.v2 (pl-en)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 43.59038612153097
- type: cosine_spearman
value: 39.81437610772274
- type: euclidean_pearson
value: 40.891702223169865
- type: euclidean_spearman
value: 37.26643328502484
- type: main_score
value: 39.81437610772274
- type: manhattan_pearson
value: 41.986843698898646
- type: manhattan_spearman
value: 39.32140456159206
- type: pearson
value: 43.59041415960545
- type: spearman
value: 39.81437610772274
task:
type: STS
- dataset:
config: ar
name: MTEB STS22.v2 (ar)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 33.5428416005511
- type: cosine_spearman
value: 30.37853891189232
- type: euclidean_pearson
value: 34.60992317007351
- type: euclidean_spearman
value: 30.63949361401844
- type: main_score
value: 30.37853891189232
- type: manhattan_pearson
value: 34.5473591619201
- type: manhattan_spearman
value: 30.55163856425125
- type: pearson
value: 33.54285209728335
- type: spearman
value: 30.37853891189232
task:
type: STS
- dataset:
config: en
name: MTEB STS22.v2 (en)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 55.197301705940106
- type: cosine_spearman
value: 60.735304152341016
- type: euclidean_pearson
value: 58.01165360549454
- type: euclidean_spearman
value: 59.382742411350954
- type: main_score
value: 60.735304152341016
- type: manhattan_pearson
value: 57.83223965160013
- type: manhattan_spearman
value: 59.21305162189526
- type: pearson
value: 55.197304101079325
- type: spearman
value: 60.735304152341016
task:
type: STS
- dataset:
config: es
name: MTEB STS22.v2 (es)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 46.72328050183308
- type: cosine_spearman
value: 53.40518194954016
- type: euclidean_pearson
value: 49.16877625976702
- type: euclidean_spearman
value: 53.00258608772506
- type: main_score
value: 53.40518194954016
- type: manhattan_pearson
value: 49.01410667940025
- type: manhattan_spearman
value: 53.062750442834506
- type: pearson
value: 46.7232634869361
- type: spearman
value: 53.40518194954016
task:
type: STS
- dataset:
config: tr
name: MTEB STS22.v2 (tr)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 32.0893349414057
- type: cosine_spearman
value: 39.88878359477052
- type: euclidean_pearson
value: 36.428586715032374
- type: euclidean_spearman
value: 40.255813965229805
- type: main_score
value: 39.88878359477052
- type: manhattan_pearson
value: 36.445628185483436
- type: manhattan_spearman
value: 40.164916492530764
- type: pearson
value: 32.08931820630885
- type: spearman
value: 39.88878359477052
task:
type: STS
- dataset:
config: ru
name: MTEB STS22.v2 (ru)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 25.70940596089177
- type: cosine_spearman
value: 24.851779346905516
- type: euclidean_pearson
value: 26.662807591660304
- type: euclidean_spearman
value: 25.1585901925807
- type: main_score
value: 24.851779346905516
- type: manhattan_pearson
value: 26.78879644774409
- type: manhattan_spearman
value: 25.30589435887287
- type: pearson
value: 25.70945811392969
- type: spearman
value: 24.851779346905516
task:
type: STS
- dataset:
config: pl
name: MTEB STS22.v2 (pl)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 9.765885880324584
- type: cosine_spearman
value: 16.56405546386883
- type: euclidean_pearson
value: 1.266714500111099
- type: euclidean_spearman
value: 15.873350818474066
- type: main_score
value: 16.56405546386883
- type: manhattan_pearson
value: 1.5996037603404067
- type: manhattan_spearman
value: 15.984627455042919
- type: pearson
value: 9.765881253247253
- type: spearman
value: 16.37790963582666
task:
type: STS
- dataset:
config: fr-pl
name: MTEB STS22.v2 (fr-pl)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 72.48621899222191
- type: cosine_spearman
value: 84.51542547285167
- type: euclidean_pearson
value: 79.91610128578839
- type: euclidean_spearman
value: 84.51542547285167
- type: main_score
value: 84.51542547285167
- type: manhattan_pearson
value: 78.02332046266785
- type: manhattan_spearman
value: 84.51542547285167
- type: pearson
value: 72.48643781257383
- type: spearman
value: 84.51542547285167
task:
type: STS
- dataset:
config: it
name: MTEB STS22.v2 (it)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 42.88595613594089
- type: cosine_spearman
value: 50.00902534757776
- type: euclidean_pearson
value: 46.575636873141114
- type: euclidean_spearman
value: 50.233303173324586
- type: main_score
value: 50.00902534757776
- type: manhattan_pearson
value: 46.74155972808209
- type: manhattan_spearman
value: 50.463048251775234
- type: pearson
value: 42.88596484422846
- type: spearman
value: 50.00902534757776
task:
type: STS
- dataset:
config: es-it
name: MTEB STS22.v2 (es-it)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 34.232754484342784
- type: cosine_spearman
value: 35.46570858579162
- type: euclidean_pearson
value: 35.3961638859008
- type: euclidean_spearman
value: 34.24608020799141
- type: main_score
value: 35.46570858579162
- type: manhattan_pearson
value: 35.350258241347966
- type: manhattan_spearman
value: 34.26016474055034
- type: pearson
value: 34.23272177114215
- type: spearman
value: 35.46570858579162
task:
type: STS
- dataset:
config: de-pl
name: MTEB STS22.v2 (de-pl)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 12.399198871280888
- type: cosine_spearman
value: 7.26740842591211
- type: euclidean_pearson
value: 4.516712707783254
- type: euclidean_spearman
value: 6.466761734921793
- type: main_score
value: 7.26740842591211
- type: manhattan_pearson
value: 4.192789002097003
- type: manhattan_spearman
value: 6.805496873417696
- type: pearson
value: 12.39933152364361
- type: spearman
value: 7.26740842591211
task:
type: STS
- dataset:
config: de-en
name: MTEB STS22.v2 (de-en)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 36.68189817510892
- type: cosine_spearman
value: 42.40215334784587
- type: euclidean_pearson
value: 37.445161615466496
- type: euclidean_spearman
value: 41.662219127863445
- type: main_score
value: 42.40215334784587
- type: manhattan_pearson
value: 37.428330543761284
- type: manhattan_spearman
value: 41.15139706997196
- type: pearson
value: 36.68189189374063
- type: spearman
value: 42.40215334784587
task:
type: STS
- dataset:
config: de
name: MTEB STS22.v2 (de)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 14.610716018581904
- type: cosine_spearman
value: 20.259656537989873
- type: euclidean_pearson
value: 13.39968577970463
- type: euclidean_spearman
value: 19.999373521862104
- type: main_score
value: 20.259656537989873
- type: manhattan_pearson
value: 13.403453329892018
- type: manhattan_spearman
value: 20.035611306496893
- type: pearson
value: 14.610694852334966
- type: spearman
value: 20.265485348016593
task:
type: STS
- dataset:
config: zh
name: MTEB STS22.v2 (zh)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 73.90500579546007
- type: cosine_spearman
value: 73.22277115520708
- type: euclidean_pearson
value: 72.26966368799292
- type: euclidean_spearman
value: 72.94244978403874
- type: main_score
value: 73.22277115520708
- type: manhattan_pearson
value: 72.16998184519838
- type: manhattan_spearman
value: 72.90099748660055
- type: pearson
value: 73.90500331010179
- type: spearman
value: 73.22277115520708
task:
type: STS
- dataset:
config: fr
name: MTEB STS22.v2 (fr)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 44.47331432475519
- type: cosine_spearman
value: 53.744106729883455
- type: euclidean_pearson
value: 50.300639054319085
- type: euclidean_spearman
value: 53.34089424965791
- type: main_score
value: 53.744106729883455
- type: manhattan_pearson
value: 50.179266864566344
- type: manhattan_spearman
value: 53.30577964927791
- type: pearson
value: 44.473302379385146
- type: spearman
value: 53.744106729883455
task:
type: STS
- dataset:
config: zh-en
name: MTEB STS22.v2 (zh-en)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 59.792988838605446
- type: cosine_spearman
value: 65.17573588512339
- type: euclidean_pearson
value: 63.23291280267431
- type: euclidean_spearman
value: 65.05898841612739
- type: main_score
value: 65.17573588512339
- type: manhattan_pearson
value: 63.14148457672645
- type: manhattan_spearman
value: 64.86625240506501
- type: pearson
value: 59.79298501419738
- type: spearman
value: 65.17573588512339
task:
type: STS
- dataset:
config: es-en
name: MTEB STS22.v2 (es-en)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 44.85365072983659
- type: cosine_spearman
value: 48.95861510766423
- type: euclidean_pearson
value: 44.12298881847811
- type: euclidean_spearman
value: 47.73736578240132
- type: main_score
value: 48.95861510766423
- type: manhattan_pearson
value: 43.96292551331725
- type: manhattan_spearman
value: 47.58103280136259
- type: pearson
value: 44.853649047286616
- type: spearman
value: 48.95861510766423
task:
type: STS
- dataset:
config: de-fr
name: MTEB STS22.v2 (de-fr)
revision: d31f33a128469b20e357535c39b82fb3c3f6f2bd
split: test
type: mteb/sts22-crosslingual-sts
metrics:
- type: cosine_pearson
value: 22.860434001017367
- type: cosine_spearman
value: 32.320901954587185
- type: euclidean_pearson
value: 26.992993163046002
- type: euclidean_spearman
value: 32.24246077908339
- type: main_score
value: 32.320901954587185
- type: manhattan_pearson
value: 26.552931446274357
- type: manhattan_spearman
value: 33.0347166516718
- type: pearson
value: 22.860425946102264
- type: spearman
value: 32.320901954587185
task:
type: STS
- dataset:
config: default
name: MTEB STSB (default)
revision: 0cde68302b3541bb8b3c340dc0644b0b745b3dc0
split: test
type: C-MTEB/STSB
metrics:
- type: cosine_pearson
value: 79.54312520554313
- type: cosine_spearman
value: 81.46931930954155
- type: euclidean_pearson
value: 80.05389266606164
- type: euclidean_spearman
value: 80.50696783023315
- type: main_score
value: 81.46931930954155
- type: manhattan_pearson
value: 80.04970274541007
- type: manhattan_spearman
value: 80.50864360601699
task:
type: STS
- dataset:
config: default
name: MTEB STSB (default)
revision: 0cde68302b3541bb8b3c340dc0644b0b745b3dc0
split: validation
type: C-MTEB/STSB
metrics:
- type: cosine_pearson
value: 83.48618236826155
- type: cosine_spearman
value: 85.3132611044822
- type: euclidean_pearson
value: 82.93001035409826
- type: euclidean_spearman
value: 83.10105109108397
- type: main_score
value: 85.3132611044822
- type: manhattan_pearson
value: 82.88533878745255
- type: manhattan_spearman
value: 83.04890918186679
task:
type: STS
- dataset:
config: default
name: MTEB T2Reranking (default)
revision: 76631901a18387f85eaa53e5450019b87ad58ef9
split: dev
type: C-MTEB/T2Reranking
metrics:
- type: main_score
value: 68.38732925233572
- type: map
value: 68.38732925233572
- type: mrr
value: 79.11475293118828
- type: nAUC_map_diff1
value: -11.253110368505121
- type: nAUC_map_max
value: 38.51256776553827
- type: nAUC_map_std
value: -1.9662536573348381
- type: nAUC_mrr_diff1
value: -10.630308274175526
- type: nAUC_mrr_max
value: 32.327491874763126
- type: nAUC_mrr_std
value: -1.7352995694761828
task:
type: Reranking
- dataset:
config: default
name: MTEB T2Retrieval (default)
revision: 8731a845f1bf500a4f111cf1070785c793d10e64
split: dev
type: C-MTEB/T2Retrieval
metrics:
- type: main_score
value: 82.36099999999999
- type: map_at_1
value: 26.255
- type: map_at_10
value: 74.149
- type: map_at_100
value: 77.973
- type: map_at_1000
value: 78.044
- type: map_at_20
value: 77.015
- type: map_at_3
value: 51.812000000000005
- type: map_at_5
value: 63.827999999999996
- type: mrr_at_1
value: 87.51972645975802
- type: mrr_at_10
value: 90.78435813509476
- type: mrr_at_100
value: 90.89422964077919
- type: mrr_at_1000
value: 90.89770903884691
- type: mrr_at_20
value: 90.85502538725395
- type: mrr_at_3
value: 90.25001461219234
- type: mrr_at_5
value: 90.60859781401618
- type: nauc_map_at_1000_diff1
value: 16.021743379862592
- type: nauc_map_at_1000_max
value: 40.378025084637784
- type: nauc_map_at_1000_std
value: 18.32476707192497
- type: nauc_map_at_100_diff1
value: 16.02545540308598
- type: nauc_map_at_100_max
value: 40.265093943279986
- type: nauc_map_at_100_std
value: 18.292534126250082
- type: nauc_map_at_10_diff1
value: 19.617650276071352
- type: nauc_map_at_10_max
value: 28.371544730138975
- type: nauc_map_at_10_std
value: 5.6371012355859955
- type: nauc_map_at_1_diff1
value: 47.33700373223127
- type: nauc_map_at_1_max
value: -25.333610801205936
- type: nauc_map_at_1_std
value: -29.566785599172384
- type: nauc_map_at_20_diff1
value: 16.614949908223352
- type: nauc_map_at_20_max
value: 37.6796912032617
- type: nauc_map_at_20_std
value: 15.427406349207578
- type: nauc_map_at_3_diff1
value: 35.44856017189094
- type: nauc_map_at_3_max
value: -13.397520313065106
- type: nauc_map_at_3_std
value: -25.733491121718828
- type: nauc_map_at_5_diff1
value: 29.102390100969412
- type: nauc_map_at_5_max
value: 1.9850020715468955
- type: nauc_map_at_5_std
value: -15.925386927929594
- type: nauc_mrr_at_1000_diff1
value: 44.97733258856923
- type: nauc_mrr_at_1000_max
value: 70.12348371687067
- type: nauc_mrr_at_1000_std
value: 36.125835138518404
- type: nauc_mrr_at_100_diff1
value: 44.978054094169536
- type: nauc_mrr_at_100_max
value: 70.12938961836127
- type: nauc_mrr_at_100_std
value: 36.13929863715235
- type: nauc_mrr_at_10_diff1
value: 44.95657675119858
- type: nauc_mrr_at_10_max
value: 70.23157682911028
- type: nauc_mrr_at_10_std
value: 36.186025142743325
- type: nauc_mrr_at_1_diff1
value: 45.53039470469623
- type: nauc_mrr_at_1_max
value: 67.19215396968853
- type: nauc_mrr_at_1_std
value: 31.73945035410656
- type: nauc_mrr_at_20_diff1
value: 44.99711921466835
- type: nauc_mrr_at_20_max
value: 70.16747553191094
- type: nauc_mrr_at_20_std
value: 36.19983013748518
- type: nauc_mrr_at_3_diff1
value: 44.92504582719811
- type: nauc_mrr_at_3_max
value: 70.17206471786271
- type: nauc_mrr_at_3_std
value: 35.86538182968299
- type: nauc_mrr_at_5_diff1
value: 44.88240202293795
- type: nauc_mrr_at_5_max
value: 70.25570239429001
- type: nauc_mrr_at_5_std
value: 36.12414563662779
- type: nauc_ndcg_at_1000_diff1
value: 20.085183872723004
- type: nauc_ndcg_at_1000_max
value: 52.54091309895082
- type: nauc_ndcg_at_1000_std
value: 29.182787889425697
- type: nauc_ndcg_at_100_diff1
value: 19.70087837544669
- type: nauc_ndcg_at_100_max
value: 51.41291578231909
- type: nauc_ndcg_at_100_std
value: 29.391058739202926
- type: nauc_ndcg_at_10_diff1
value: 19.61155631423253
- type: nauc_ndcg_at_10_max
value: 41.69125197676835
- type: nauc_ndcg_at_10_std
value: 18.485185354348886
- type: nauc_ndcg_at_1_diff1
value: 45.53039470469623
- type: nauc_ndcg_at_1_max
value: 67.19215396968853
- type: nauc_ndcg_at_1_std
value: 31.73945035410656
- type: nauc_ndcg_at_20_diff1
value: 19.884673845091573
- type: nauc_ndcg_at_20_max
value: 45.77756113362076
- type: nauc_ndcg_at_20_std
value: 23.65628062983877
- type: nauc_ndcg_at_3_diff1
value: 16.396492503030576
- type: nauc_ndcg_at_3_max
value: 57.36632449538558
- type: nauc_ndcg_at_3_std
value: 28.110468173360108
- type: nauc_ndcg_at_5_diff1
value: 15.990139806075096
- type: nauc_ndcg_at_5_max
value: 49.87917678198262
- type: nauc_ndcg_at_5_std
value: 23.77870723184885
- type: nauc_precision_at_1000_diff1
value: -29.64669603533363
- type: nauc_precision_at_1000_max
value: 51.500364584604654
- type: nauc_precision_at_1000_std
value: 50.16891173815387
- type: nauc_precision_at_100_diff1
value: -29.466626146744606
- type: nauc_precision_at_100_max
value: 52.73740649971535
- type: nauc_precision_at_100_std
value: 51.846137080440656
- type: nauc_precision_at_10_diff1
value: -27.896447661643325
- type: nauc_precision_at_10_max
value: 55.722284162663684
- type: nauc_precision_at_10_std
value: 47.727742997832934
- type: nauc_precision_at_1_diff1
value: 45.53039470469623
- type: nauc_precision_at_1_max
value: 67.19215396968853
- type: nauc_precision_at_1_std
value: 31.73945035410656
- type: nauc_precision_at_20_diff1
value: -28.744448426184704
- type: nauc_precision_at_20_max
value: 54.491376214009925
- type: nauc_precision_at_20_std
value: 50.901916245689236
- type: nauc_precision_at_3_diff1
value: -19.456295360120368
- type: nauc_precision_at_3_max
value: 62.40625587462132
- type: nauc_precision_at_3_std
value: 40.87948253365838
- type: nauc_precision_at_5_diff1
value: -25.9612067834916
- type: nauc_precision_at_5_max
value: 59.139116299790494
- type: nauc_precision_at_5_std
value: 44.29380766917937
- type: nauc_recall_at_1000_diff1
value: 5.392030054530998
- type: nauc_recall_at_1000_max
value: 53.602628435557854
- type: nauc_recall_at_1000_std
value: 64.0692631857673
- type: nauc_recall_at_100_diff1
value: 8.764716008636329
- type: nauc_recall_at_100_max
value: 42.289617268904074
- type: nauc_recall_at_100_std
value: 46.008944199137
- type: nauc_recall_at_10_diff1
value: 17.70428741242271
- type: nauc_recall_at_10_max
value: 16.82055621929404
- type: nauc_recall_at_10_std
value: 0.9443257188294529
- type: nauc_recall_at_1_diff1
value: 47.33700373223127
- type: nauc_recall_at_1_max
value: -25.333610801205936
- type: nauc_recall_at_1_std
value: -29.566785599172384
- type: nauc_recall_at_20_diff1
value: 13.37892799655899
- type: nauc_recall_at_20_max
value: 30.092126871647647
- type: nauc_recall_at_20_std
value: 20.023192649193746
- type: nauc_recall_at_3_diff1
value: 33.201540040096866
- type: nauc_recall_at_3_max
value: -18.541554905715042
- type: nauc_recall_at_3_std
value: -28.22641082741867
- type: nauc_recall_at_5_diff1
value: 27.11037328467852
- type: nauc_recall_at_5_max
value: -7.787633276369825
- type: nauc_recall_at_5_std
value: -21.13477331887689
- type: ndcg_at_1
value: 87.52
- type: ndcg_at_10
value: 82.36099999999999
- type: ndcg_at_100
value: 86.447
- type: ndcg_at_1000
value: 87.14
- type: ndcg_at_20
value: 84.358
- type: ndcg_at_3
value: 83.48400000000001
- type: ndcg_at_5
value: 82.15299999999999
- type: precision_at_1
value: 87.52
- type: precision_at_10
value: 41.223
- type: precision_at_100
value: 4.993
- type: precision_at_1000
value: 0.516
- type: precision_at_20
value: 23.044
- type: precision_at_3
value: 73.156
- type: precision_at_5
value: 61.45
- type: recall_at_1
value: 26.255
- type: recall_at_10
value: 81.81
- type: recall_at_100
value: 94.973
- type: recall_at_1000
value: 98.488
- type: recall_at_20
value: 88.306
- type: recall_at_3
value: 53.913999999999994
- type: recall_at_5
value: 67.95400000000001
task:
type: Retrieval
- dataset:
config: default
name: MTEB TNews (default)
revision: 317f262bf1e6126357bbe89e875451e4b0938fe4
split: validation
type: C-MTEB/TNews-classification
metrics:
- type: accuracy
value: 50.16399999999999
- type: f1
value: 48.68459760388684
- type: f1_weighted
value: 50.19811019404876
- type: main_score
value: 50.16399999999999
task:
type: Classification
- dataset:
config: default
name: MTEB ThuNewsClusteringP2P (default)
revision: 5798586b105c0434e4f0fe5e767abe619442cf93
split: test
type: C-MTEB/ThuNewsClusteringP2P
metrics:
- type: main_score
value: 75.41012311158184
- type: v_measure
value: 75.41012311158184
- type: v_measure_std
value: 1.600085445536508
task:
type: Clustering
- dataset:
config: default
name: MTEB ThuNewsClusteringP2P.v2 (default)
revision: 5798586b105c0434e4f0fe5e767abe619442cf93
split: test
type: C-MTEB/ThuNewsClusteringP2P
metrics:
- type: main_score
value: 77.21421760551837
- type: v_measure
value: 77.21421760551837
- type: v_measure_std
value: 2.0725968692052548
task:
type: Clustering
- dataset:
config: default
name: MTEB ThuNewsClusteringS2S (default)
revision: 8a8b2caeda43f39e13c4bc5bea0f8a667896e10d
split: test
type: C-MTEB/ThuNewsClusteringS2S
metrics:
- type: main_score
value: 66.99801222525865
- type: v_measure
value: 66.99801222525865
- type: v_measure_std
value: 2.268490312605906
task:
type: Clustering
- dataset:
config: default
name: MTEB ThuNewsClusteringS2S.v2 (default)
revision: 8a8b2caeda43f39e13c4bc5bea0f8a667896e10d
split: test
type: C-MTEB/ThuNewsClusteringS2S
metrics:
- type: main_score
value: 67.40189244236278
- type: v_measure
value: 67.40189244236278
- type: v_measure_std
value: 2.1667814876460776
task:
type: Clustering
- dataset:
config: default
name: MTEB VideoRetrieval (default)
revision: 58c2597a5943a2ba48f4668c3b90d796283c5639
split: dev
type: C-MTEB/VideoRetrieval
metrics:
- type: main_score
value: 74.18
- type: map_at_1
value: 58.3
- type: map_at_10
value: 69.202
- type: map_at_100
value: 69.606
- type: map_at_1000
value: 69.612
- type: map_at_20
value: 69.465
- type: map_at_3
value: 67.133
- type: map_at_5
value: 68.28800000000001
- type: mrr_at_1
value: 58.3
- type: mrr_at_10
value: 69.201626984127
- type: mrr_at_100
value: 69.60581565029914
- type: mrr_at_1000
value: 69.61192749722464
- type: mrr_at_20
value: 69.46514532809812
- type: mrr_at_3
value: 67.13333333333334
- type: mrr_at_5
value: 68.28833333333336
- type: nauc_map_at_1000_diff1
value: 59.34478083408511
- type: nauc_map_at_1000_max
value: 13.389346875878058
- type: nauc_map_at_1000_std
value: -36.85885637076865
- type: nauc_map_at_100_diff1
value: 59.34807111517757
- type: nauc_map_at_100_max
value: 13.403390284190857
- type: nauc_map_at_100_std
value: -36.84487623810107
- type: nauc_map_at_10_diff1
value: 59.27462774183162
- type: nauc_map_at_10_max
value: 13.127942709744026
- type: nauc_map_at_10_std
value: -37.37951203716279
- type: nauc_map_at_1_diff1
value: 62.06374419623817
- type: nauc_map_at_1_max
value: 12.216836842404124
- type: nauc_map_at_1_std
value: -32.376437563970576
- type: nauc_map_at_20_diff1
value: 59.358042547082235
- type: nauc_map_at_20_max
value: 13.425901648847269
- type: nauc_map_at_20_std
value: -36.950481605718885
- type: nauc_map_at_3_diff1
value: 58.504179878593376
- type: nauc_map_at_3_max
value: 13.717981132556927
- type: nauc_map_at_3_std
value: -37.61571792082351
- type: nauc_map_at_5_diff1
value: 58.850653009668854
- type: nauc_map_at_5_max
value: 12.895882710197466
- type: nauc_map_at_5_std
value: -38.15921238754432
- type: nauc_mrr_at_1000_diff1
value: 59.34478083408511
- type: nauc_mrr_at_1000_max
value: 13.389346875878058
- type: nauc_mrr_at_1000_std
value: -36.85885637076865
- type: nauc_mrr_at_100_diff1
value: 59.34807111517757
- type: nauc_mrr_at_100_max
value: 13.403390284190857
- type: nauc_mrr_at_100_std
value: -36.84487623810107
- type: nauc_mrr_at_10_diff1
value: 59.27462774183162
- type: nauc_mrr_at_10_max
value: 13.127942709744026
- type: nauc_mrr_at_10_std
value: -37.37951203716279
- type: nauc_mrr_at_1_diff1
value: 62.06374419623817
- type: nauc_mrr_at_1_max
value: 12.216836842404124
- type: nauc_mrr_at_1_std
value: -32.376437563970576
- type: nauc_mrr_at_20_diff1
value: 59.358042547082235
- type: nauc_mrr_at_20_max
value: 13.425901648847269
- type: nauc_mrr_at_20_std
value: -36.950481605718885
- type: nauc_mrr_at_3_diff1
value: 58.504179878593376
- type: nauc_mrr_at_3_max
value: 13.717981132556927
- type: nauc_mrr_at_3_std
value: -37.61571792082351
- type: nauc_mrr_at_5_diff1
value: 58.850653009668854
- type: nauc_mrr_at_5_max
value: 12.895882710197466
- type: nauc_mrr_at_5_std
value: -38.15921238754432
- type: nauc_ndcg_at_1000_diff1
value: 59.13898931852901
- type: nauc_ndcg_at_1000_max
value: 13.869168802179367
- type: nauc_ndcg_at_1000_std
value: -36.57313702589791
- type: nauc_ndcg_at_100_diff1
value: 59.25286696443839
- type: nauc_ndcg_at_100_max
value: 14.329031676206943
- type: nauc_ndcg_at_100_std
value: -36.068844762404865
- type: nauc_ndcg_at_10_diff1
value: 58.952556848610506
- type: nauc_ndcg_at_10_max
value: 13.058643821489968
- type: nauc_ndcg_at_10_std
value: -38.637915329187656
- type: nauc_ndcg_at_1_diff1
value: 62.06374419623817
- type: nauc_ndcg_at_1_max
value: 12.216836842404124
- type: nauc_ndcg_at_1_std
value: -32.376437563970576
- type: nauc_ndcg_at_20_diff1
value: 59.265051921114534
- type: nauc_ndcg_at_20_max
value: 14.424145412434584
- type: nauc_ndcg_at_20_std
value: -36.67703874681748
- type: nauc_ndcg_at_3_diff1
value: 57.2145236748128
- type: nauc_ndcg_at_3_max
value: 14.163756082395205
- type: nauc_ndcg_at_3_std
value: -39.40414075321231
- type: nauc_ndcg_at_5_diff1
value: 57.809804336288416
- type: nauc_ndcg_at_5_max
value: 12.527159951596808
- type: nauc_ndcg_at_5_std
value: -40.552009922696094
- type: nauc_precision_at_1000_diff1
value: 56.125116713349286
- type: nauc_precision_at_1000_max
value: 48.571428571425365
- type: nauc_precision_at_1000_std
value: 59.68253968253704
- type: nauc_precision_at_100_diff1
value: 68.61646619432061
- type: nauc_precision_at_100_max
value: 65.6533201515904
- type: nauc_precision_at_100_std
value: 38.515406162464885
- type: nauc_precision_at_10_diff1
value: 58.1383709608438
- type: nauc_precision_at_10_max
value: 12.102681699471365
- type: nauc_precision_at_10_std
value: -45.96425614006378
- type: nauc_precision_at_1_diff1
value: 62.06374419623817
- type: nauc_precision_at_1_max
value: 12.216836842404124
- type: nauc_precision_at_1_std
value: -32.376437563970576
- type: nauc_precision_at_20_diff1
value: 61.35243841126161
- type: nauc_precision_at_20_max
value: 29.105077928606924
- type: nauc_precision_at_20_std
value: -24.708037060978505
- type: nauc_precision_at_3_diff1
value: 52.33167085174655
- type: nauc_precision_at_3_max
value: 15.808744545343355
- type: nauc_precision_at_3_std
value: -46.12751133667663
- type: nauc_precision_at_5_diff1
value: 53.2268623802632
- type: nauc_precision_at_5_max
value: 10.356224013966232
- type: nauc_precision_at_5_std
value: -51.58816300277973
- type: nauc_recall_at_1000_diff1
value: 56.125116713353194
- type: nauc_recall_at_1000_max
value: 48.57142857143081
- type: nauc_recall_at_1000_std
value: 59.682539682538724
- type: nauc_recall_at_100_diff1
value: 68.61646619432132
- type: nauc_recall_at_100_max
value: 65.65332015158995
- type: nauc_recall_at_100_std
value: 38.515406162465176
- type: nauc_recall_at_10_diff1
value: 58.13837096084369
- type: nauc_recall_at_10_max
value: 12.10268169947128
- type: nauc_recall_at_10_std
value: -45.964256140063505
- type: nauc_recall_at_1_diff1
value: 62.06374419623817
- type: nauc_recall_at_1_max
value: 12.216836842404124
- type: nauc_recall_at_1_std
value: -32.376437563970576
- type: nauc_recall_at_20_diff1
value: 61.35243841126176
- type: nauc_recall_at_20_max
value: 29.10507792860751
- type: nauc_recall_at_20_std
value: -24.708037060978192
- type: nauc_recall_at_3_diff1
value: 52.33167085174654
- type: nauc_recall_at_3_max
value: 15.808744545343204
- type: nauc_recall_at_3_std
value: -46.12751133667685
- type: nauc_recall_at_5_diff1
value: 53.2268623802632
- type: nauc_recall_at_5_max
value: 10.356224013966395
- type: nauc_recall_at_5_std
value: -51.58816300277942
- type: ndcg_at_1
value: 58.3
- type: ndcg_at_10
value: 74.18
- type: ndcg_at_100
value: 76.067
- type: ndcg_at_1000
value: 76.223
- type: ndcg_at_20
value: 75.14
- type: ndcg_at_3
value: 69.907
- type: ndcg_at_5
value: 71.956
- type: precision_at_1
value: 58.3
- type: precision_at_10
value: 8.97
- type: precision_at_100
value: 0.983
- type: precision_at_1000
value: 0.1
- type: precision_at_20
value: 4.675
- type: precision_at_3
value: 25.967000000000002
- type: precision_at_5
value: 16.56
- type: recall_at_1
value: 58.3
- type: recall_at_10
value: 89.7
- type: recall_at_100
value: 98.3
- type: recall_at_1000
value: 99.5
- type: recall_at_20
value: 93.5
- type: recall_at_3
value: 77.9
- type: recall_at_5
value: 82.8
task:
type: Retrieval
- dataset:
config: default
name: MTEB Waimai (default)
revision: 339287def212450dcaa9df8c22bf93e9980c7023
split: test
type: C-MTEB/waimai-classification
metrics:
- type: accuracy
value: 86.19
- type: ap
value: 69.16016629543033
- type: ap_weighted
value: 69.16016629543033
- type: f1
value: 84.65946646369608
- type: f1_weighted
value: 86.35113290203732
- type: main_score
value: 86.19
task:
type: Classification
tags:
- mteb
ternary-weight-embedding
基于xiaobu-embedding-v2[1]，在nli-zh[2]和t2ranking[3]数据集文本上微调得到的三元权重text embedding模型。模型中所有Linear层的权重取值为1，0或-1。模型中所有Linear层的权重取值为1，0或-1。推理时间和存储空间可以达到全精度模型的0.37x（在A800上的测试结果）的和0.13x。
使用请安装BITBLAS，支持的GPU见[4]
pip install bitblas
初次运行可能会花一些时间
使用Sentence-Transformers进行测试
pip install -U sentence-transformers
import mteb
from sentence_transformers import SentenceTransformer
model = SentenceTransformer('malenia1/ternary-weight-embedding',trust_remote_code=True)
print(model)
tasks = mteb.get_tasks("OnlineShopping")
evaluation = mteb.MTEB(tasks=tasks)
results = evaluation.run(model, output_folder=f"results")