metadata
tags:
- mteb
model-index:
- name: data1
results:
- task:
type: STS
dataset:
type: C-MTEB/AFQMC
name: MTEB AFQMC
config: default
split: validation
revision: b44c3b011063adb25877c13823db83bb193913c4
metrics:
- type: cos_sim_pearson
value: 53.66919706568301
- type: cos_sim_spearman
value: 53.84074348656974
- type: euclidean_pearson
value: 53.58226184439896
- type: euclidean_spearman
value: 53.84074348656974
- type: manhattan_pearson
value: 53.64565834381205
- type: manhattan_spearman
value: 53.75526003581371
- task:
type: STS
dataset:
type: C-MTEB/ATEC
name: MTEB ATEC
config: default
split: test
revision: 0f319b1142f28d00e055a6770f3f726ae9b7d865
metrics:
- type: cos_sim_pearson
value: 58.123744893539495
- type: cos_sim_spearman
value: 54.44277675493291
- type: euclidean_pearson
value: 61.20550691770944
- type: euclidean_spearman
value: 54.44277225170509
- type: manhattan_pearson
value: 60.57835645653918
- type: manhattan_spearman
value: 54.46153709699013
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (zh)
config: zh
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 29.746
- type: f1
value: 29.039321522193585
- task:
type: STS
dataset:
type: C-MTEB/BQ
name: MTEB BQ
config: default
split: test
revision: e3dda5e115e487b39ec7e618c0c6a29137052a55
metrics:
- type: cos_sim_pearson
value: 70.7026320728244
- type: cos_sim_spearman
value: 70.57218534128499
- type: euclidean_pearson
value: 69.28488221289881
- type: euclidean_spearman
value: 70.57218534192015
- type: manhattan_pearson
value: 69.65344674392082
- type: manhattan_spearman
value: 70.64136691477553
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringP2P
name: MTEB CLSClusteringP2P
config: default
split: test
revision: 4b6227591c6c1a73bc76b1055f3b7f3588e72476
metrics:
- type: v_measure
value: 38.87791994762536
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringS2S
name: MTEB CLSClusteringS2S
config: default
split: test
revision: e458b3f5414b62b7f9f83499ac1f5497ae2e869f
metrics:
- type: v_measure
value: 39.09103599244803
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv1-reranking
name: MTEB CMedQAv1
config: default
split: test
revision: 8d7f1e942507dac42dc58017c1a001c3717da7df
metrics:
- type: map
value: 80.40249793910444
- type: mrr
value: 82.96805555555555
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv2-reranking
name: MTEB CMedQAv2
config: default
split: test
revision: 23d186750531a14a0357ca22cd92d712fd512ea0
metrics:
- type: map
value: 80.39046823499085
- type: mrr
value: 83.22674603174602
- task:
type: Retrieval
dataset:
type: C-MTEB/CmedqaRetrieval
name: MTEB CmedqaRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 15.715000000000002
- type: map_at_10
value: 24.651
- type: map_at_100
value: 26.478
- type: map_at_1000
value: 26.648
- type: map_at_3
value: 21.410999999999998
- type: map_at_5
value: 23.233
- type: mrr_at_1
value: 24.806
- type: mrr_at_10
value: 32.336
- type: mrr_at_100
value: 33.493
- type: mrr_at_1000
value: 33.568999999999996
- type: mrr_at_3
value: 29.807
- type: mrr_at_5
value: 31.294
- type: ndcg_at_1
value: 24.806
- type: ndcg_at_10
value: 30.341
- type: ndcg_at_100
value: 38.329
- type: ndcg_at_1000
value: 41.601
- type: ndcg_at_3
value: 25.655
- type: ndcg_at_5
value: 27.758
- type: precision_at_1
value: 24.806
- type: precision_at_10
value: 7.119000000000001
- type: precision_at_100
value: 1.3679999999999999
- type: precision_at_1000
value: 0.179
- type: precision_at_3
value: 14.787
- type: precision_at_5
value: 11.208
- type: recall_at_1
value: 15.715000000000002
- type: recall_at_10
value: 39.519999999999996
- type: recall_at_100
value: 73.307
- type: recall_at_1000
value: 95.611
- type: recall_at_3
value: 26.026
- type: recall_at_5
value: 32.027
- task:
type: PairClassification
dataset:
type: C-MTEB/CMNLI
name: MTEB Cmnli
config: default
split: validation
revision: 41bc36f332156f7adc9e38f53777c959b2ae9766
metrics:
- type: cos_sim_accuracy
value: 66.89116055321708
- type: cos_sim_ap
value: 75.66575745519994
- type: cos_sim_f1
value: 70.2448775612194
- type: cos_sim_precision
value: 61.347765363128495
- type: cos_sim_recall
value: 82.16039279869068
- type: dot_accuracy
value: 66.89116055321708
- type: dot_ap
value: 75.68262052264197
- type: dot_f1
value: 70.2448775612194
- type: dot_precision
value: 61.347765363128495
- type: dot_recall
value: 82.16039279869068
- type: euclidean_accuracy
value: 66.89116055321708
- type: euclidean_ap
value: 75.66576722188334
- type: euclidean_f1
value: 70.2448775612194
- type: euclidean_precision
value: 61.347765363128495
- type: euclidean_recall
value: 82.16039279869068
- type: manhattan_accuracy
value: 67.03547805171377
- type: manhattan_ap
value: 75.78816934864089
- type: manhattan_f1
value: 70.35407081416284
- type: manhattan_precision
value: 61.4752665617899
- type: manhattan_recall
value: 82.23053542202479
- type: max_accuracy
value: 67.03547805171377
- type: max_ap
value: 75.78816934864089
- type: max_f1
value: 70.35407081416284
- task:
type: Retrieval
dataset:
type: C-MTEB/CovidRetrieval
name: MTEB CovidRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 41.57
- type: map_at_10
value: 52.932
- type: map_at_100
value: 53.581999999999994
- type: map_at_1000
value: 53.61900000000001
- type: map_at_3
value: 50.066
- type: map_at_5
value: 51.735
- type: mrr_at_1
value: 41.623
- type: mrr_at_10
value: 52.964999999999996
- type: mrr_at_100
value: 53.6
- type: mrr_at_1000
value: 53.637
- type: mrr_at_3
value: 50.158
- type: mrr_at_5
value: 51.786
- type: ndcg_at_1
value: 41.623
- type: ndcg_at_10
value: 58.55200000000001
- type: ndcg_at_100
value: 61.824999999999996
- type: ndcg_at_1000
value: 62.854
- type: ndcg_at_3
value: 52.729000000000006
- type: ndcg_at_5
value: 55.696999999999996
- type: precision_at_1
value: 41.623
- type: precision_at_10
value: 7.692
- type: precision_at_100
value: 0.927
- type: precision_at_1000
value: 0.101
- type: precision_at_3
value: 20.162
- type: precision_at_5
value: 13.572000000000001
- type: recall_at_1
value: 41.57
- type: recall_at_10
value: 76.185
- type: recall_at_100
value: 91.728
- type: recall_at_1000
value: 99.895
- type: recall_at_3
value: 60.27400000000001
- type: recall_at_5
value: 67.46600000000001
- task:
type: Retrieval
dataset:
type: C-MTEB/DuRetrieval
name: MTEB DuRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 21.071
- type: map_at_10
value: 65.093
- type: map_at_100
value: 69.097
- type: map_at_1000
value: 69.172
- type: map_at_3
value: 44.568000000000005
- type: map_at_5
value: 56.016999999999996
- type: mrr_at_1
value: 76.35
- type: mrr_at_10
value: 83.721
- type: mrr_at_100
value: 83.899
- type: mrr_at_1000
value: 83.904
- type: mrr_at_3
value: 82.958
- type: mrr_at_5
value: 83.488
- type: ndcg_at_1
value: 76.35
- type: ndcg_at_10
value: 75.05199999999999
- type: ndcg_at_100
value: 80.596
- type: ndcg_at_1000
value: 81.394
- type: ndcg_at_3
value: 73.298
- type: ndcg_at_5
value: 72.149
- type: precision_at_1
value: 76.35
- type: precision_at_10
value: 36.96
- type: precision_at_100
value: 4.688
- type: precision_at_1000
value: 0.48700000000000004
- type: precision_at_3
value: 66.2
- type: precision_at_5
value: 55.81
- type: recall_at_1
value: 21.071
- type: recall_at_10
value: 77.459
- type: recall_at_100
value: 94.425
- type: recall_at_1000
value: 98.631
- type: recall_at_3
value: 48.335
- type: recall_at_5
value: 63.227999999999994
- task:
type: Retrieval
dataset:
type: C-MTEB/EcomRetrieval
name: MTEB EcomRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 36.3
- type: map_at_10
value: 46.888999999999996
- type: map_at_100
value: 47.789
- type: map_at_1000
value: 47.827999999999996
- type: map_at_3
value: 43.85
- type: map_at_5
value: 45.58
- type: mrr_at_1
value: 36.3
- type: mrr_at_10
value: 46.888999999999996
- type: mrr_at_100
value: 47.789
- type: mrr_at_1000
value: 47.827999999999996
- type: mrr_at_3
value: 43.85
- type: mrr_at_5
value: 45.58
- type: ndcg_at_1
value: 36.3
- type: ndcg_at_10
value: 52.539
- type: ndcg_at_100
value: 56.882
- type: ndcg_at_1000
value: 57.841
- type: ndcg_at_3
value: 46.303
- type: ndcg_at_5
value: 49.406
- type: precision_at_1
value: 36.3
- type: precision_at_10
value: 7.049999999999999
- type: precision_at_100
value: 0.907
- type: precision_at_1000
value: 0.098
- type: precision_at_3
value: 17.8
- type: precision_at_5
value: 12.18
- type: recall_at_1
value: 36.3
- type: recall_at_10
value: 70.5
- type: recall_at_100
value: 90.7
- type: recall_at_1000
value: 98.1
- type: recall_at_3
value: 53.400000000000006
- type: recall_at_5
value: 60.9
- task:
type: Classification
dataset:
type: C-MTEB/IFlyTek-classification
name: MTEB IFlyTek
config: default
split: validation
revision: 421605374b29664c5fc098418fe20ada9bd55f8a
metrics:
- type: accuracy
value: 50.927279722970376
- type: f1
value: 39.57514582425314
- task:
type: Classification
dataset:
type: C-MTEB/JDReview-classification
name: MTEB JDReview
config: default
split: test
revision: b7c64bd89eb87f8ded463478346f76731f07bf8b
metrics:
- type: accuracy
value: 84.93433395872421
- type: ap
value: 50.35046267230439
- type: f1
value: 78.76452515604298
- task:
type: STS
dataset:
type: C-MTEB/LCQMC
name: MTEB LCQMC
config: default
split: test
revision: 17f9b096f80380fce5ed12a9be8be7784b337daf
metrics:
- type: cos_sim_pearson
value: 67.40319768112933
- type: cos_sim_spearman
value: 74.9867527749418
- type: euclidean_pearson
value: 74.08762625643878
- type: euclidean_spearman
value: 74.98675720634276
- type: manhattan_pearson
value: 73.86303861791671
- type: manhattan_spearman
value: 75.0594224188492
- task:
type: Reranking
dataset:
type: C-MTEB/Mmarco-reranking
name: MTEB MMarcoReranking
config: default
split: dev
revision: 8e0c766dbe9e16e1d221116a3f36795fbade07f6
metrics:
- type: map
value: 18.860945903258536
- type: mrr
value: 17.686507936507937
- task:
type: Retrieval
dataset:
type: C-MTEB/MMarcoRetrieval
name: MTEB MMarcoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 49.16
- type: map_at_10
value: 57.992
- type: map_at_100
value: 58.638
- type: map_at_1000
value: 58.67
- type: map_at_3
value: 55.71
- type: map_at_5
value: 57.04900000000001
- type: mrr_at_1
value: 50.989
- type: mrr_at_10
value: 58.814
- type: mrr_at_100
value: 59.401
- type: mrr_at_1000
value: 59.431
- type: mrr_at_3
value: 56.726
- type: mrr_at_5
value: 57.955
- type: ndcg_at_1
value: 50.989
- type: ndcg_at_10
value: 62.259
- type: ndcg_at_100
value: 65.347
- type: ndcg_at_1000
value: 66.231
- type: ndcg_at_3
value: 57.78
- type: ndcg_at_5
value: 60.09100000000001
- type: precision_at_1
value: 50.989
- type: precision_at_10
value: 7.9479999999999995
- type: precision_at_100
value: 0.951
- type: precision_at_1000
value: 0.10200000000000001
- type: precision_at_3
value: 22.087
- type: precision_at_5
value: 14.479000000000001
- type: recall_at_1
value: 49.16
- type: recall_at_10
value: 74.792
- type: recall_at_100
value: 89.132
- type: recall_at_1000
value: 96.13199999999999
- type: recall_at_3
value: 62.783
- type: recall_at_5
value: 68.26100000000001
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-CN)
config: zh-CN
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 67.45796906523202
- type: f1
value: 65.97280169222601
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-CN)
config: zh-CN
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 71.59717552118359
- type: f1
value: 72.46681610207507
- task:
type: Retrieval
dataset:
type: C-MTEB/MedicalRetrieval
name: MTEB MedicalRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 40.5
- type: map_at_10
value: 46.892
- type: map_at_100
value: 47.579
- type: map_at_1000
value: 47.648
- type: map_at_3
value: 45.367000000000004
- type: map_at_5
value: 46.182
- type: mrr_at_1
value: 40.6
- type: mrr_at_10
value: 46.942
- type: mrr_at_100
value: 47.629
- type: mrr_at_1000
value: 47.698
- type: mrr_at_3
value: 45.417
- type: mrr_at_5
value: 46.232
- type: ndcg_at_1
value: 40.5
- type: ndcg_at_10
value: 50.078
- type: ndcg_at_100
value: 53.635999999999996
- type: ndcg_at_1000
value: 55.696999999999996
- type: ndcg_at_3
value: 46.847
- type: ndcg_at_5
value: 48.323
- type: precision_at_1
value: 40.5
- type: precision_at_10
value: 6.02
- type: precision_at_100
value: 0.773
- type: precision_at_1000
value: 0.094
- type: precision_at_3
value: 17.033
- type: precision_at_5
value: 10.94
- type: recall_at_1
value: 40.5
- type: recall_at_10
value: 60.199999999999996
- type: recall_at_100
value: 77.3
- type: recall_at_1000
value: 94
- type: recall_at_3
value: 51.1
- type: recall_at_5
value: 54.7
- task:
type: Retrieval
dataset:
type: Shitao/MLDR
name: MTEB MultiLongDocRetrieval (zh)
config: zh
split: test
revision: None
metrics:
- type: map_at_1
value: 7.000000000000001
- type: map_at_10
value: 10.020999999999999
- type: map_at_100
value: 10.511
- type: map_at_1000
value: 10.595
- type: map_at_3
value: 9.042
- type: map_at_5
value: 9.654
- type: mrr_at_1
value: 6.875000000000001
- type: mrr_at_10
value: 9.958
- type: mrr_at_100
value: 10.449
- type: mrr_at_1000
value: 10.532
- type: mrr_at_3
value: 8.979
- type: mrr_at_5
value: 9.592
- type: ndcg_at_1
value: 7.000000000000001
- type: ndcg_at_10
value: 11.651
- type: ndcg_at_100
value: 14.580000000000002
- type: ndcg_at_1000
value: 17.183
- type: ndcg_at_3
value: 9.646
- type: ndcg_at_5
value: 10.738
- type: precision_at_1
value: 7.000000000000001
- type: precision_at_10
value: 1.687
- type: precision_at_100
value: 0.319
- type: precision_at_1000
value: 0.053
- type: precision_at_3
value: 3.7920000000000003
- type: precision_at_5
value: 2.8000000000000003
- type: recall_at_1
value: 7.000000000000001
- type: recall_at_10
value: 16.875
- type: recall_at_100
value: 31.874999999999996
- type: recall_at_1000
value: 53.25
- type: recall_at_3
value: 11.375
- type: recall_at_5
value: 14.000000000000002
- task:
type: Classification
dataset:
type: C-MTEB/MultilingualSentiment-classification
name: MTEB MultilingualSentiment
config: default
split: validation
revision: 46958b007a63fdbf239b7672c25d0bea67b5ea1a
metrics:
- type: accuracy
value: 55.90333333333333
- type: f1
value: 55.291185234519546
- task:
type: PairClassification
dataset:
type: C-MTEB/OCNLI
name: MTEB Ocnli
config: default
split: validation
revision: 66e76a618a34d6d565d5538088562851e6daa7ec
metrics:
- type: cos_sim_accuracy
value: 59.01461829994585
- type: cos_sim_ap
value: 61.84829541140869
- type: cos_sim_f1
value: 67.94150731158605
- type: cos_sim_precision
value: 52.674418604651166
- type: cos_sim_recall
value: 95.67053854276664
- type: dot_accuracy
value: 59.01461829994585
- type: dot_ap
value: 61.84829541140869
- type: dot_f1
value: 67.94150731158605
- type: dot_precision
value: 52.674418604651166
- type: dot_recall
value: 95.67053854276664
- type: euclidean_accuracy
value: 59.01461829994585
- type: euclidean_ap
value: 61.84829541140869
- type: euclidean_f1
value: 67.94150731158605
- type: euclidean_precision
value: 52.674418604651166
- type: euclidean_recall
value: 95.67053854276664
- type: manhattan_accuracy
value: 59.06876015159719
- type: manhattan_ap
value: 61.91217952354554
- type: manhattan_f1
value: 67.89059572873735
- type: manhattan_precision
value: 52.613240418118465
- type: manhattan_recall
value: 95.67053854276664
- type: max_accuracy
value: 59.06876015159719
- type: max_ap
value: 61.91217952354554
- type: max_f1
value: 67.94150731158605
- task:
type: Classification
dataset:
type: C-MTEB/OnlineShopping-classification
name: MTEB OnlineShopping
config: default
split: test
revision: e610f2ebd179a8fda30ae534c3878750a96db120
metrics:
- type: accuracy
value: 82.53
- type: ap
value: 77.67591637020448
- type: f1
value: 82.39976599130478
- task:
type: STS
dataset:
type: C-MTEB/PAWSX
name: MTEB PAWSX
config: default
split: test
revision: 9c6a90e430ac22b5779fb019a23e820b11a8b5e1
metrics:
- type: cos_sim_pearson
value: 55.76388035743312
- type: cos_sim_spearman
value: 58.34768166139753
- type: euclidean_pearson
value: 57.971763429924074
- type: euclidean_spearman
value: 58.34750745303424
- type: manhattan_pearson
value: 58.044053497280245
- type: manhattan_spearman
value: 58.61627719613188
- task:
type: PairClassification
dataset:
type: paws-x
name: MTEB PawsX (zh)
config: zh
split: test
revision: 8a04d940a42cd40658986fdd8e3da561533a3646
metrics:
- type: cos_sim_accuracy
value: 75.75
- type: cos_sim_ap
value: 78.80617392926526
- type: cos_sim_f1
value: 75.92417061611374
- type: cos_sim_precision
value: 65.87171052631578
- type: cos_sim_recall
value: 89.59731543624162
- type: dot_accuracy
value: 75.75
- type: dot_ap
value: 78.83768586994135
- type: dot_f1
value: 75.92417061611374
- type: dot_precision
value: 65.87171052631578
- type: dot_recall
value: 89.59731543624162
- type: euclidean_accuracy
value: 75.75
- type: euclidean_ap
value: 78.80617392926526
- type: euclidean_f1
value: 75.92417061611374
- type: euclidean_precision
value: 65.87171052631578
- type: euclidean_recall
value: 89.59731543624162
- type: manhattan_accuracy
value: 75.75
- type: manhattan_ap
value: 78.98640478955386
- type: manhattan_f1
value: 75.92954990215264
- type: manhattan_precision
value: 67.47826086956522
- type: manhattan_recall
value: 86.80089485458613
- type: max_accuracy
value: 75.75
- type: max_ap
value: 78.98640478955386
- type: max_f1
value: 75.92954990215264
- task:
type: STS
dataset:
type: C-MTEB/QBQTC
name: MTEB QBQTC
config: default
split: test
revision: 790b0510dc52b1553e8c49f3d2afb48c0e5c48b7
metrics:
- type: cos_sim_pearson
value: 74.40348414238575
- type: cos_sim_spearman
value: 71.452270332177
- type: euclidean_pearson
value: 72.62509231589097
- type: euclidean_spearman
value: 71.45228258458943
- type: manhattan_pearson
value: 73.03846856200839
- type: manhattan_spearman
value: 71.43673225319574
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh)
config: zh
split: test
revision: eea2b4fe26a775864c896887d910b76a8098ad3f
metrics:
- type: cos_sim_pearson
value: 75.38335474357001
- type: cos_sim_spearman
value: 74.92262892309807
- type: euclidean_pearson
value: 73.93451693251345
- type: euclidean_spearman
value: 74.92262892309807
- type: manhattan_pearson
value: 74.55911294300788
- type: manhattan_spearman
value: 74.89436791272614
- task:
type: STS
dataset:
type: C-MTEB/STSB
name: MTEB STSB
config: default
split: test
revision: 0cde68302b3541bb8b3c340dc0644b0b745b3dc0
metrics:
- type: cos_sim_pearson
value: 83.01687361650126
- type: cos_sim_spearman
value: 82.74413230806265
- type: euclidean_pearson
value: 81.50177295189083
- type: euclidean_spearman
value: 82.74413230806265
- type: manhattan_pearson
value: 81.90798387028589
- type: manhattan_spearman
value: 82.65064251275778
- task:
type: Reranking
dataset:
type: C-MTEB/T2Reranking
name: MTEB T2Reranking
config: default
split: dev
revision: 76631901a18387f85eaa53e5450019b87ad58ef9
metrics:
- type: map
value: 66.25459669294304
- type: mrr
value: 76.76845224661744
- task:
type: Retrieval
dataset:
type: C-MTEB/T2Retrieval
name: MTEB T2Retrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 22.515
- type: map_at_10
value: 63.63999999999999
- type: map_at_100
value: 67.67
- type: map_at_1000
value: 67.792
- type: map_at_3
value: 44.239
- type: map_at_5
value: 54.54599999999999
- type: mrr_at_1
value: 79.752
- type: mrr_at_10
value: 83.525
- type: mrr_at_100
value: 83.753
- type: mrr_at_1000
value: 83.763
- type: mrr_at_3
value: 82.65599999999999
- type: mrr_at_5
value: 83.192
- type: ndcg_at_1
value: 79.752
- type: ndcg_at_10
value: 72.699
- type: ndcg_at_100
value: 78.145
- type: ndcg_at_1000
value: 79.481
- type: ndcg_at_3
value: 74.401
- type: ndcg_at_5
value: 72.684
- type: precision_at_1
value: 79.752
- type: precision_at_10
value: 37.163000000000004
- type: precision_at_100
value: 4.769
- type: precision_at_1000
value: 0.508
- type: precision_at_3
value: 65.67399999999999
- type: precision_at_5
value: 55.105000000000004
- type: recall_at_1
value: 22.515
- type: recall_at_10
value: 71.816
- type: recall_at_100
value: 89.442
- type: recall_at_1000
value: 96.344
- type: recall_at_3
value: 46.208
- type: recall_at_5
value: 58.695
- task:
type: Classification
dataset:
type: C-MTEB/TNews-classification
name: MTEB TNews
config: default
split: validation
revision: 317f262bf1e6126357bbe89e875451e4b0938fe4
metrics:
- type: accuracy
value: 55.077999999999996
- type: f1
value: 53.2447237349446
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringP2P
name: MTEB ThuNewsClusteringP2P
config: default
split: test
revision: 5798586b105c0434e4f0fe5e767abe619442cf93
metrics:
- type: v_measure
value: 59.50582115422618
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringS2S
name: MTEB ThuNewsClusteringS2S
config: default
split: test
revision: 8a8b2caeda43f39e13c4bc5bea0f8a667896e10d
metrics:
- type: v_measure
value: 54.71907850412647
- task:
type: Retrieval
dataset:
type: C-MTEB/VideoRetrieval
name: MTEB VideoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 49.4
- type: map_at_10
value: 59.245999999999995
- type: map_at_100
value: 59.811
- type: map_at_1000
value: 59.836
- type: map_at_3
value: 56.733
- type: map_at_5
value: 58.348
- type: mrr_at_1
value: 49.4
- type: mrr_at_10
value: 59.245999999999995
- type: mrr_at_100
value: 59.811
- type: mrr_at_1000
value: 59.836
- type: mrr_at_3
value: 56.733
- type: mrr_at_5
value: 58.348
- type: ndcg_at_1
value: 49.4
- type: ndcg_at_10
value: 64.08
- type: ndcg_at_100
value: 67.027
- type: ndcg_at_1000
value: 67.697
- type: ndcg_at_3
value: 58.995
- type: ndcg_at_5
value: 61.891
- type: precision_at_1
value: 49.4
- type: precision_at_10
value: 7.93
- type: precision_at_100
value: 0.935
- type: precision_at_1000
value: 0.099
- type: precision_at_3
value: 21.833
- type: precision_at_5
value: 14.499999999999998
- type: recall_at_1
value: 49.4
- type: recall_at_10
value: 79.3
- type: recall_at_100
value: 93.5
- type: recall_at_1000
value: 98.8
- type: recall_at_3
value: 65.5
- type: recall_at_5
value: 72.5
- task:
type: Classification
dataset:
type: C-MTEB/waimai-classification
name: MTEB Waimai
config: default
split: test
revision: 339287def212450dcaa9df8c22bf93e9980c7023
metrics:
- type: accuracy
value: 81.16
- type: ap
value: 60.864524843400616
- type: f1
value: 79.41246877404483
ZNV Embedding utilizes a 6B LLM (Large Language Model) for embedding, achieving excellent embedding results.
In a single inference, we used two prompts to extract two different embeddings for a sentence, and then concatenated them.
Model usage method:
- Define ZNVEmbeddingModel
import os
from transformers import (
LlamaForCausalLM,
LlamaTokenizer, AutoConfig,
)
import torch
import torch.nn.functional as F
import numpy as np
class ZNVEmbeddingModel(torch.nn.Module):
def __init__(self, model_name_or_path):
super(ZNVEmbeddingModel, self).__init__()
self.prompt_prefix = "阅读下文，然后答题\n"
self.prompt_suffixes = ["\n1.一个字总结上文的意思是:",
"\n2.上文深层次的意思是:"]
self.hidden_size = 4096
self.model_name_or_path = model_name_or_path
self.linear_suffixes = torch.nn.ModuleList(
[torch.nn.Linear(self.hidden_size, self.hidden_size//len(self.prompt_suffixes))
for _ in range(len(self.prompt_suffixes))])
self.tokenizer, self.llama = self.load_llama()
self.tanh = torch.nn.Tanh()
self.suffixes_ids = []
self.suffixes_ids_len = []
self.suffixes_len = 0
for suffix in self.prompt_suffixes:
ids = self.tokenizer(suffix, return_tensors="pt")["input_ids"].tolist()[0]
self.suffixes_ids += ids
self.suffixes_ids_len.append(len(ids))
self.suffixes_len += len(ids)
self.suffixes_ones = torch.ones(self.suffixes_len)
self.suffixes_ids = torch.tensor(self.suffixes_ids)
linear_file = os.path.join(model_name_or_path, "linears")
load_layers = torch.load(linear_file)
model_state = self.state_dict()
model_state.update(load_layers)
self.load_state_dict(model_state, strict=False)
def load_llama(self):
llm_path = os.path.join(self.model_name_or_path)
config = AutoConfig.from_pretrained(llm_path)
tokenizer = LlamaTokenizer.from_pretrained(self.model_name_or_path)
tokenizer.padding_side = "left"
model = LlamaForCausalLM.from_pretrained(
llm_path,
config=config,
low_cpu_mem_usage=True
)
model.config.use_cache = False
return tokenizer, model
def forward(self, sentences):
prompts_embeddings = []
sentences = [self.prompt_prefix + s for s in sentences]
inputs = self.tokenizer(sentences, max_length=256, padding=True, truncation=True,
return_tensors='pt')
attention_mask = inputs["attention_mask"]
input_ids = inputs["input_ids"]
batch_size = len(sentences)
suffixes_ones = self.suffixes_ones.unsqueeze(0)
suffixes_ones = suffixes_ones.repeat(batch_size, 1)
device = next(self.parameters()).device
attention_mask = torch.cat([attention_mask, suffixes_ones], dim=-1).to(device)
suffixes_ids = self.suffixes_ids.unsqueeze(0)
suffixes_ids = suffixes_ids.repeat(batch_size, 1)
input_ids = torch.cat([input_ids, suffixes_ids], dim=-1).to(device)
last_hidden_state = self.llama.base_model.base_model(attention_mask=attention_mask, input_ids=input_ids).last_hidden_state
index = -1
for i in range(len(self.suffixes_ids_len)):
embedding = last_hidden_state[:, index, :]
embedding = self.linear_suffixes[i](embedding)
prompts_embeddings.append(embedding)
index -= self.suffixes_ids_len[-i-1]
output_embedding = torch.cat(prompts_embeddings, dim=-1)
output_embedding = self.tanh(output_embedding)
output_embedding = F.normalize(output_embedding, p=2, dim=1)
return output_embedding
def encode(self, sentences, batch_size=10, **kwargs):
size = len(sentences)
embeddings = None
handled = 0
while handled < size:
tokens = sentences[handled:handled + batch_size]
output_embeddings = self.forward(tokens)
result = output_embeddings.cpu().numpy()
handled += result.shape[0]
if embeddings is not None:
embeddings = np.concatenate((embeddings, result), axis=0)
else:
embeddings = result
return embeddings
- Use ZNVEmbeddingModel for Embedding.
znv_model = ZNVEmbeddingModel("your_model_path")
znv_model.eval()
with torch.no_grad():
output = znv_model(["请问你的电话号码是多少？","可以告诉我你的手机号吗？"])
cos_sim = F.cosine_similarity(output[0],output[1],dim=0)
print(cos_sim)