metadata
tags:
- mteb
model-index:
- name: bge-2048-15G
results:
- task:
type: STS
dataset:
type: C-MTEB/AFQMC
name: MTEB AFQMC
config: default
split: validation
revision: None
metrics:
- type: cos_sim_pearson
value: 35.144052273688665
- type: cos_sim_spearman
value: 36.03131554905942
- type: euclidean_pearson
value: 34.95481744048328
- type: euclidean_spearman
value: 36.0313155504039
- type: manhattan_pearson
value: 35.01398244390669
- type: manhattan_spearman
value: 36.07582645320379
- task:
type: STS
dataset:
type: C-MTEB/ATEC
name: MTEB ATEC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 42.12829268908657
- type: cos_sim_spearman
value: 42.59932326711113
- type: euclidean_pearson
value: 45.0728007358798
- type: euclidean_spearman
value: 42.59932101508718
- type: manhattan_pearson
value: 45.0740044376148
- type: manhattan_spearman
value: 42.59773917033953
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (zh)
config: zh
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 39.32600000000001
- type: f1
value: 37.552594883461666
- task:
type: STS
dataset:
type: C-MTEB/BQ
name: MTEB BQ
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 50.01347417982947
- type: cos_sim_spearman
value: 50.93232950078808
- type: euclidean_pearson
value: 50.18139878220082
- type: euclidean_spearman
value: 50.932329500482474
- type: manhattan_pearson
value: 50.239972396501244
- type: manhattan_spearman
value: 50.98304741370781
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringP2P
name: MTEB CLSClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 35.30752153686832
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringS2S
name: MTEB CLSClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 37.87520358227087
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv1-reranking
name: MTEB CMedQAv1
config: default
split: test
revision: None
metrics:
- type: map
value: 76.70424054530682
- type: mrr
value: 80.55007936507937
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv2-reranking
name: MTEB CMedQAv2
config: default
split: test
revision: None
metrics:
- type: map
value: 78.28031561021152
- type: mrr
value: 81.80051587301587
- task:
type: Retrieval
dataset:
type: C-MTEB/CmedqaRetrieval
name: MTEB CmedqaRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 19.296
- type: map_at_10
value: 28.706
- type: map_at_100
value: 30.462
- type: map_at_1000
value: 30.622
- type: map_at_3
value: 25.56
- type: map_at_5
value: 27.223999999999997
- type: mrr_at_1
value: 29.857
- type: mrr_at_10
value: 37.214999999999996
- type: mrr_at_100
value: 38.26
- type: mrr_at_1000
value: 38.330999999999996
- type: mrr_at_3
value: 34.959
- type: mrr_at_5
value: 36.18
- type: ndcg_at_1
value: 29.857
- type: ndcg_at_10
value: 34.509
- type: ndcg_at_100
value: 41.884
- type: ndcg_at_1000
value: 45.023
- type: ndcg_at_3
value: 30.288999999999998
- type: ndcg_at_5
value: 31.886
- type: precision_at_1
value: 29.857
- type: precision_at_10
value: 7.779
- type: precision_at_100
value: 1.383
- type: precision_at_1000
value: 0.178
- type: precision_at_3
value: 17.179
- type: precision_at_5
value: 12.443
- type: recall_at_1
value: 19.296
- type: recall_at_10
value: 43.221
- type: recall_at_100
value: 74.09700000000001
- type: recall_at_1000
value: 95.735
- type: recall_at_3
value: 30.436999999999998
- type: recall_at_5
value: 35.5
- task:
type: PairClassification
dataset:
type: C-MTEB/CMNLI
name: MTEB Cmnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 63.6440168370415
- type: cos_sim_ap
value: 68.79549034123916
- type: cos_sim_f1
value: 69.1282620766241
- type: cos_sim_precision
value: 57.20631030785726
- type: cos_sim_recall
value: 87.3275660509703
- type: dot_accuracy
value: 63.6440168370415
- type: dot_ap
value: 68.79977430584098
- type: dot_f1
value: 69.1282620766241
- type: dot_precision
value: 57.20631030785726
- type: dot_recall
value: 87.3275660509703
- type: euclidean_accuracy
value: 63.6440168370415
- type: euclidean_ap
value: 68.79548765330846
- type: euclidean_f1
value: 69.1282620766241
- type: euclidean_precision
value: 57.20631030785726
- type: euclidean_recall
value: 87.3275660509703
- type: manhattan_accuracy
value: 63.6440168370415
- type: manhattan_ap
value: 68.7809534749333
- type: manhattan_f1
value: 69.1069621999448
- type: manhattan_precision
value: 56.95876288659794
- type: manhattan_recall
value: 87.84194528875379
- type: max_accuracy
value: 63.6440168370415
- type: max_ap
value: 68.79977430584098
- type: max_f1
value: 69.1282620766241
- task:
type: Retrieval
dataset:
type: C-MTEB/CovidRetrieval
name: MTEB CovidRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 60.485
- type: map_at_10
value: 71.15599999999999
- type: map_at_100
value: 71.504
- type: map_at_1000
value: 71.514
- type: map_at_3
value: 69.06400000000001
- type: map_at_5
value: 70.297
- type: mrr_at_1
value: 60.695
- type: mrr_at_10
value: 71.218
- type: mrr_at_100
value: 71.557
- type: mrr_at_1000
value: 71.568
- type: mrr_at_3
value: 69.21300000000001
- type: mrr_at_5
value: 70.393
- type: ndcg_at_1
value: 60.695
- type: ndcg_at_10
value: 75.899
- type: ndcg_at_100
value: 77.524
- type: ndcg_at_1000
value: 77.775
- type: ndcg_at_3
value: 71.72500000000001
- type: ndcg_at_5
value: 73.90299999999999
- type: precision_at_1
value: 60.695
- type: precision_at_10
value: 9.136
- type: precision_at_100
value: 0.991
- type: precision_at_1000
value: 0.101
- type: precision_at_3
value: 26.589000000000002
- type: precision_at_5
value: 17.028
- type: recall_at_1
value: 60.485
- type: recall_at_10
value: 90.49000000000001
- type: recall_at_100
value: 97.99799999999999
- type: recall_at_1000
value: 99.895
- type: recall_at_3
value: 79.215
- type: recall_at_5
value: 84.48400000000001
- task:
type: Retrieval
dataset:
type: C-MTEB/DuRetrieval
name: MTEB DuRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 20.649
- type: map_at_10
value: 59.345000000000006
- type: map_at_100
value: 63.31
- type: map_at_1000
value: 63.449
- type: map_at_3
value: 41.614000000000004
- type: map_at_5
value: 51.425
- type: mrr_at_1
value: 73.7
- type: mrr_at_10
value: 80.42200000000001
- type: mrr_at_100
value: 80.675
- type: mrr_at_1000
value: 80.685
- type: mrr_at_3
value: 79.292
- type: mrr_at_5
value: 79.952
- type: ndcg_at_1
value: 73.7
- type: ndcg_at_10
value: 69.685
- type: ndcg_at_100
value: 76.071
- type: ndcg_at_1000
value: 77.561
- type: ndcg_at_3
value: 68.43
- type: ndcg_at_5
value: 66.78
- type: precision_at_1
value: 73.7
- type: precision_at_10
value: 33.615
- type: precision_at_100
value: 4.466
- type: precision_at_1000
value: 0.483
- type: precision_at_3
value: 60.93300000000001
- type: precision_at_5
value: 50.62
- type: recall_at_1
value: 20.649
- type: recall_at_10
value: 71.678
- type: recall_at_100
value: 91.107
- type: recall_at_1000
value: 98.489
- type: recall_at_3
value: 44.757000000000005
- type: recall_at_5
value: 57.853
- task:
type: Retrieval
dataset:
type: C-MTEB/EcomRetrieval
name: MTEB EcomRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 33
- type: map_at_10
value: 43.988
- type: map_at_100
value: 44.798
- type: map_at_1000
value: 44.840999999999994
- type: map_at_3
value: 41.25
- type: map_at_5
value: 42.91
- type: mrr_at_1
value: 33
- type: mrr_at_10
value: 43.988
- type: mrr_at_100
value: 44.798
- type: mrr_at_1000
value: 44.840999999999994
- type: mrr_at_3
value: 41.25
- type: mrr_at_5
value: 42.91
- type: ndcg_at_1
value: 33
- type: ndcg_at_10
value: 49.544
- type: ndcg_at_100
value: 53.605999999999995
- type: ndcg_at_1000
value: 54.764
- type: ndcg_at_3
value: 43.887
- type: ndcg_at_5
value: 46.907
- type: precision_at_1
value: 33
- type: precision_at_10
value: 6.710000000000001
- type: precision_at_100
value: 0.864
- type: precision_at_1000
value: 0.096
- type: precision_at_3
value: 17.166999999999998
- type: precision_at_5
value: 11.78
- type: recall_at_1
value: 33
- type: recall_at_10
value: 67.10000000000001
- type: recall_at_100
value: 86.4
- type: recall_at_1000
value: 95.6
- type: recall_at_3
value: 51.5
- type: recall_at_5
value: 58.9
- task:
type: Classification
dataset:
type: C-MTEB/IFlyTek-classification
name: MTEB IFlyTek
config: default
split: validation
revision: 421605374b29664c5fc098418fe20ada9bd55f8a
metrics:
- type: accuracy
value: 47.741439015005774
- type: f1
value: 35.474522593244856
- task:
type: Classification
dataset:
type: C-MTEB/JDReview-classification
name: MTEB JDReview
config: default
split: test
revision: b7c64bd89eb87f8ded463478346f76731f07bf8b
metrics:
- type: accuracy
value: 84.18386491557224
- type: ap
value: 50.331823892307895
- type: f1
value: 78.46423899842871
- task:
type: STS
dataset:
type: C-MTEB/LCQMC
name: MTEB LCQMC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 61.27982183684614
- type: cos_sim_spearman
value: 66.68643494062978
- type: euclidean_pearson
value: 65.18446288753343
- type: euclidean_spearman
value: 66.68643272258043
- type: manhattan_pearson
value: 65.4169819947427
- type: manhattan_spearman
value: 66.92322208196136
- task:
type: Reranking
dataset:
type: C-MTEB/Mmarco-reranking
name: MTEB MMarcoReranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 23.220821016871753
- type: mrr
value: 21.65793650793651
- task:
type: Retrieval
dataset:
type: C-MTEB/MMarcoRetrieval
name: MTEB MMarcoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 48.725
- type: map_at_10
value: 59.06
- type: map_at_100
value: 59.638999999999996
- type: map_at_1000
value: 59.666
- type: map_at_3
value: 56.528999999999996
- type: map_at_5
value: 58.099999999999994
- type: mrr_at_1
value: 50.559
- type: mrr_at_10
value: 59.852000000000004
- type: mrr_at_100
value: 60.376
- type: mrr_at_1000
value: 60.4
- type: mrr_at_3
value: 57.555
- type: mrr_at_5
value: 58.967000000000006
- type: ndcg_at_1
value: 50.559
- type: ndcg_at_10
value: 63.931000000000004
- type: ndcg_at_100
value: 66.669
- type: ndcg_at_1000
value: 67.38199999999999
- type: ndcg_at_3
value: 59.03
- type: ndcg_at_5
value: 61.719
- type: precision_at_1
value: 50.559
- type: precision_at_10
value: 8.318
- type: precision_at_100
value: 0.97
- type: precision_at_1000
value: 0.10300000000000001
- type: precision_at_3
value: 22.975
- type: precision_at_5
value: 15.189
- type: recall_at_1
value: 48.725
- type: recall_at_10
value: 78.294
- type: recall_at_100
value: 90.947
- type: recall_at_1000
value: 96.58
- type: recall_at_3
value: 65.25
- type: recall_at_5
value: 71.64399999999999
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-CN)
config: zh-CN
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 61.506388702084735
- type: f1
value: 58.09296024607959
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-CN)
config: zh-CN
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 68.54068594485541
- type: f1
value: 67.10301600316001
- task:
type: Retrieval
dataset:
type: C-MTEB/MedicalRetrieval
name: MTEB MedicalRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 36.6
- type: map_at_10
value: 41.626000000000005
- type: map_at_100
value: 42.259
- type: map_at_1000
value: 42.325
- type: map_at_3
value: 39.95
- type: map_at_5
value: 40.925
- type: mrr_at_1
value: 36.6
- type: mrr_at_10
value: 41.642
- type: mrr_at_100
value: 42.276
- type: mrr_at_1000
value: 42.342
- type: mrr_at_3
value: 39.967000000000006
- type: mrr_at_5
value: 40.942
- type: ndcg_at_1
value: 36.6
- type: ndcg_at_10
value: 44.563
- type: ndcg_at_100
value: 48.051
- type: ndcg_at_1000
value: 50.126000000000005
- type: ndcg_at_3
value: 41.113
- type: ndcg_at_5
value: 42.856
- type: precision_at_1
value: 36.6
- type: precision_at_10
value: 5.4
- type: precision_at_100
value: 0.713
- type: precision_at_1000
value: 0.08800000000000001
- type: precision_at_3
value: 14.832999999999998
- type: precision_at_5
value: 9.74
- type: recall_at_1
value: 36.6
- type: recall_at_10
value: 54
- type: recall_at_100
value: 71.3
- type: recall_at_1000
value: 88.3
- type: recall_at_3
value: 44.5
- type: recall_at_5
value: 48.699999999999996
- task:
type: Classification
dataset:
type: C-MTEB/MultilingualSentiment-classification
name: MTEB MultilingualSentiment
config: default
split: validation
revision: 46958b007a63fdbf239b7672c25d0bea67b5ea1a
metrics:
- type: accuracy
value: 71.13666666666667
- type: f1
value: 70.70955352114072
- task:
type: PairClassification
dataset:
type: C-MTEB/OCNLI
name: MTEB Ocnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 57.552788305360046
- type: cos_sim_ap
value: 57.99925908040314
- type: cos_sim_f1
value: 68.03603603603602
- type: cos_sim_precision
value: 51.64113785557986
- type: cos_sim_recall
value: 99.68321013727561
- type: dot_accuracy
value: 57.552788305360046
- type: dot_ap
value: 57.99925908040314
- type: dot_f1
value: 68.03603603603602
- type: dot_precision
value: 51.64113785557986
- type: dot_recall
value: 99.68321013727561
- type: euclidean_accuracy
value: 57.552788305360046
- type: euclidean_ap
value: 57.99925908040314
- type: euclidean_f1
value: 68.03603603603602
- type: euclidean_precision
value: 51.64113785557986
- type: euclidean_recall
value: 99.68321013727561
- type: manhattan_accuracy
value: 57.82349756361668
- type: manhattan_ap
value: 58.06447658366083
- type: manhattan_f1
value: 68.0820439006837
- type: manhattan_precision
value: 51.637554585152834
- type: manhattan_recall
value: 99.89440337909187
- type: max_accuracy
value: 57.82349756361668
- type: max_ap
value: 58.06447658366083
- type: max_f1
value: 68.0820439006837
- task:
type: Classification
dataset:
type: C-MTEB/OnlineShopping-classification
name: MTEB OnlineShopping
config: default
split: test
revision: e610f2ebd179a8fda30ae534c3878750a96db120
metrics:
- type: accuracy
value: 89.78999999999999
- type: ap
value: 88.18580068282115
- type: f1
value: 89.78440880027107
- task:
type: STS
dataset:
type: C-MTEB/PAWSX
name: MTEB PAWSX
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 14.275914601577927
- type: cos_sim_spearman
value: 15.159517089386151
- type: euclidean_pearson
value: 16.947703922600706
- type: euclidean_spearman
value: 15.159778229648783
- type: manhattan_pearson
value: 16.949723597317618
- type: manhattan_spearman
value: 15.168661830776104
- task:
type: STS
dataset:
type: C-MTEB/QBQTC
name: MTEB QBQTC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 30.756080454040273
- type: cos_sim_spearman
value: 31.587419186134024
- type: euclidean_pearson
value: 30.260574537599428
- type: euclidean_spearman
value: 31.587396288861157
- type: manhattan_pearson
value: 30.17903086373953
- type: manhattan_spearman
value: 31.542451771249734
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh)
config: zh
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 61.62397999344316
- type: cos_sim_spearman
value: 65.18755757041059
- type: euclidean_pearson
value: 62.0244836285851
- type: euclidean_spearman
value: 65.18755757041059
- type: manhattan_pearson
value: 61.68364429686755
- type: manhattan_spearman
value: 64.78960487603719
- task:
type: STS
dataset:
type: C-MTEB/STSB
name: MTEB STSB
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 70.70527620167447
- type: cos_sim_spearman
value: 70.54535482359282
- type: euclidean_pearson
value: 70.19143313983818
- type: euclidean_spearman
value: 70.54531318718149
- type: manhattan_pearson
value: 70.22947491094558
- type: manhattan_spearman
value: 70.53618087989595
- task:
type: Reranking
dataset:
type: C-MTEB/T2Reranking
name: MTEB T2Reranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 61.87616616845465
- type: mrr
value: 69.01596973702593
- task:
type: Retrieval
dataset:
type: C-MTEB/T2Retrieval
name: MTEB T2Retrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 13.855
- type: map_at_10
value: 39.846
- type: map_at_100
value: 44.62
- type: map_at_1000
value: 44.884
- type: map_at_3
value: 26.775
- type: map_at_5
value: 33.164
- type: mrr_at_1
value: 52.025
- type: mrr_at_10
value: 60.812999999999995
- type: mrr_at_100
value: 61.388
- type: mrr_at_1000
value: 61.419999999999995
- type: mrr_at_3
value: 58.788
- type: mrr_at_5
value: 60.050000000000004
- type: ndcg_at_1
value: 52.025
- type: ndcg_at_10
value: 50.127
- type: ndcg_at_100
value: 59.163
- type: ndcg_at_1000
value: 62.277
- type: ndcg_at_3
value: 48.912
- type: ndcg_at_5
value: 48.337
- type: precision_at_1
value: 52.025
- type: precision_at_10
value: 26.828999999999997
- type: precision_at_100
value: 4.266
- type: precision_at_1000
value: 0.502
- type: precision_at_3
value: 43.53
- type: precision_at_5
value: 37.269000000000005
- type: recall_at_1
value: 13.855
- type: recall_at_10
value: 52.459
- type: recall_at_100
value: 79.671
- type: recall_at_1000
value: 95.248
- type: recall_at_3
value: 30.080000000000002
- type: recall_at_5
value: 39.735
- task:
type: Classification
dataset:
type: C-MTEB/TNews-classification
name: MTEB TNews
config: default
split: validation
revision: 317f262bf1e6126357bbe89e875451e4b0938fe4
metrics:
- type: accuracy
value: 52.721
- type: f1
value: 50.992066915421375
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringP2P
name: MTEB ThuNewsClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 56.17125156769023
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringS2S
name: MTEB ThuNewsClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 57.552647981278426
- task:
type: Retrieval
dataset:
type: C-MTEB/VideoRetrieval
name: MTEB VideoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 31.900000000000002
- type: map_at_10
value: 39.366
- type: map_at_100
value: 40.186
- type: map_at_1000
value: 40.239000000000004
- type: map_at_3
value: 36.933
- type: map_at_5
value: 38.378
- type: mrr_at_1
value: 31.900000000000002
- type: mrr_at_10
value: 39.366
- type: mrr_at_100
value: 40.186
- type: mrr_at_1000
value: 40.239000000000004
- type: mrr_at_3
value: 36.933
- type: mrr_at_5
value: 38.378
- type: ndcg_at_1
value: 31.900000000000002
- type: ndcg_at_10
value: 43.541999999999994
- type: ndcg_at_100
value: 47.697
- type: ndcg_at_1000
value: 49.306
- type: ndcg_at_3
value: 38.568999999999996
- type: ndcg_at_5
value: 41.168
- type: precision_at_1
value: 31.900000000000002
- type: precision_at_10
value: 5.6899999999999995
- type: precision_at_100
value: 0.767
- type: precision_at_1000
value: 0.09
- type: precision_at_3
value: 14.433000000000002
- type: precision_at_5
value: 9.92
- type: recall_at_1
value: 31.900000000000002
- type: recall_at_10
value: 56.89999999999999
- type: recall_at_100
value: 76.7
- type: recall_at_1000
value: 89.8
- type: recall_at_3
value: 43.3
- type: recall_at_5
value: 49.6
- task:
type: Classification
dataset:
type: C-MTEB/waimai-classification
name: MTEB Waimai
config: default
split: test
revision: 339287def212450dcaa9df8c22bf93e9980c7023
metrics:
- type: accuracy
value: 85.52
- type: ap
value: 67.91529494717969
- type: f1
value: 83.83738833971195