metadata
tags:
- mteb
model-index:
- name: stella-base-zh
results:
- task:
type: STS
dataset:
type: C-MTEB/AFQMC
name: MTEB AFQMC
config: default
split: validation
revision: None
metrics:
- type: cos_sim_pearson
value: 49.34825050234731
- type: cos_sim_spearman
value: 51.74726338428475
- type: euclidean_pearson
value: 50.14955499038012
- type: euclidean_spearman
value: 51.74730359287025
- type: manhattan_pearson
value: 50.016703594410615
- type: manhattan_spearman
value: 51.63936364317057
- task:
type: STS
dataset:
type: C-MTEB/ATEC
name: MTEB ATEC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 52.26876163587667
- type: cos_sim_spearman
value: 52.818410137444374
- type: euclidean_pearson
value: 55.24925286208574
- type: euclidean_spearman
value: 52.818404507964686
- type: manhattan_pearson
value: 55.21236977375391
- type: manhattan_spearman
value: 52.80289117015117
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (zh)
config: zh
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 40.245999999999995
- type: f1
value: 38.55443674287747
- task:
type: STS
dataset:
type: C-MTEB/BQ
name: MTEB BQ
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 61.553652835163255
- type: cos_sim_spearman
value: 63.29065064027392
- type: euclidean_pearson
value: 62.000329557485
- type: euclidean_spearman
value: 63.290650638944825
- type: manhattan_pearson
value: 62.02786936153664
- type: manhattan_spearman
value: 63.32720383880146
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringP2P
name: MTEB CLSClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 39.71224230526474
- task:
type: Clustering
dataset:
type: C-MTEB/CLSClusteringS2S
name: MTEB CLSClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 36.55705201882987
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv1-reranking
name: MTEB CMedQAv1
config: default
split: test
revision: None
metrics:
- type: map
value: 85.69418720521168
- type: mrr
value: 87.97444444444446
- task:
type: Reranking
dataset:
type: C-MTEB/CMedQAv2-reranking
name: MTEB CMedQAv2
config: default
split: test
revision: None
metrics:
- type: map
value: 86.46348358482606
- type: mrr
value: 88.81428571428572
- task:
type: Retrieval
dataset:
type: C-MTEB/CmedqaRetrieval
name: MTEB CmedqaRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 23.721
- type: map_at_10
value: 35.428
- type: map_at_100
value: 37.438
- type: map_at_1000
value: 37.557
- type: map_at_3
value: 31.589
- type: map_at_5
value: 33.647
- type: mrr_at_1
value: 36.709
- type: mrr_at_10
value: 44.590999999999994
- type: mrr_at_100
value: 45.684999999999995
- type: mrr_at_1000
value: 45.732
- type: mrr_at_3
value: 42.331
- type: mrr_at_5
value: 43.532
- type: ndcg_at_1
value: 36.709
- type: ndcg_at_10
value: 41.858000000000004
- type: ndcg_at_100
value: 49.775999999999996
- type: ndcg_at_1000
value: 51.844
- type: ndcg_at_3
value: 37.067
- type: ndcg_at_5
value: 38.875
- type: precision_at_1
value: 36.709
- type: precision_at_10
value: 9.411999999999999
- type: precision_at_100
value: 1.5709999999999997
- type: precision_at_1000
value: 0.183
- type: precision_at_3
value: 21.154999999999998
- type: precision_at_5
value: 15.184000000000001
- type: recall_at_1
value: 23.721
- type: recall_at_10
value: 51.714000000000006
- type: recall_at_100
value: 84.60600000000001
- type: recall_at_1000
value: 98.414
- type: recall_at_3
value: 37.091
- type: recall_at_5
value: 42.978
- task:
type: PairClassification
dataset:
type: C-MTEB/CMNLI
name: MTEB Cmnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 73.61395069152135
- type: cos_sim_ap
value: 81.65459344597652
- type: cos_sim_f1
value: 75.66718995290425
- type: cos_sim_precision
value: 68.4918529746116
- type: cos_sim_recall
value: 84.5218611176058
- type: dot_accuracy
value: 73.61395069152135
- type: dot_ap
value: 81.64596407363373
- type: dot_f1
value: 75.66718995290425
- type: dot_precision
value: 68.4918529746116
- type: dot_recall
value: 84.5218611176058
- type: euclidean_accuracy
value: 73.61395069152135
- type: euclidean_ap
value: 81.6546013070452
- type: euclidean_f1
value: 75.66718995290425
- type: euclidean_precision
value: 68.4918529746116
- type: euclidean_recall
value: 84.5218611176058
- type: manhattan_accuracy
value: 73.51773902585688
- type: manhattan_ap
value: 81.57345451483191
- type: manhattan_f1
value: 75.7393958530681
- type: manhattan_precision
value: 68.87442572741195
- type: manhattan_recall
value: 84.12438625204582
- type: max_accuracy
value: 73.61395069152135
- type: max_ap
value: 81.6546013070452
- type: max_f1
value: 75.7393958530681
- task:
type: Retrieval
dataset:
type: C-MTEB/CovidRetrieval
name: MTEB CovidRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 73.551
- type: map_at_10
value: 81.513
- type: map_at_100
value: 81.734
- type: map_at_1000
value: 81.73700000000001
- type: map_at_3
value: 80.27300000000001
- type: map_at_5
value: 81.017
- type: mrr_at_1
value: 73.762
- type: mrr_at_10
value: 81.479
- type: mrr_at_100
value: 81.699
- type: mrr_at_1000
value: 81.702
- type: mrr_at_3
value: 80.33
- type: mrr_at_5
value: 80.999
- type: ndcg_at_1
value: 73.867
- type: ndcg_at_10
value: 84.711
- type: ndcg_at_100
value: 85.714
- type: ndcg_at_1000
value: 85.803
- type: ndcg_at_3
value: 82.244
- type: ndcg_at_5
value: 83.514
- type: precision_at_1
value: 73.867
- type: precision_at_10
value: 9.557
- type: precision_at_100
value: 1.001
- type: precision_at_1000
value: 0.101
- type: precision_at_3
value: 29.505
- type: precision_at_5
value: 18.377
- type: recall_at_1
value: 73.551
- type: recall_at_10
value: 94.521
- type: recall_at_100
value: 99.05199999999999
- type: recall_at_1000
value: 99.789
- type: recall_at_3
value: 87.777
- type: recall_at_5
value: 90.83200000000001
- task:
type: Retrieval
dataset:
type: C-MTEB/DuRetrieval
name: MTEB DuRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 26.230999999999998
- type: map_at_10
value: 80.635
- type: map_at_100
value: 83.393
- type: map_at_1000
value: 83.431
- type: map_at_3
value: 55.717000000000006
- type: map_at_5
value: 70.387
- type: mrr_at_1
value: 90.75
- type: mrr_at_10
value: 93.569
- type: mrr_at_100
value: 93.648
- type: mrr_at_1000
value: 93.65
- type: mrr_at_3
value: 93.27499999999999
- type: mrr_at_5
value: 93.482
- type: ndcg_at_1
value: 90.75
- type: ndcg_at_10
value: 87.801
- type: ndcg_at_100
value: 90.44
- type: ndcg_at_1000
value: 90.776
- type: ndcg_at_3
value: 86.556
- type: ndcg_at_5
value: 85.468
- type: precision_at_1
value: 90.75
- type: precision_at_10
value: 42.08
- type: precision_at_100
value: 4.816
- type: precision_at_1000
value: 0.49
- type: precision_at_3
value: 77.60000000000001
- type: precision_at_5
value: 65.49000000000001
- type: recall_at_1
value: 26.230999999999998
- type: recall_at_10
value: 89.00200000000001
- type: recall_at_100
value: 97.866
- type: recall_at_1000
value: 99.569
- type: recall_at_3
value: 57.778
- type: recall_at_5
value: 74.895
- task:
type: Retrieval
dataset:
type: C-MTEB/EcomRetrieval
name: MTEB EcomRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 47.599999999999994
- type: map_at_10
value: 57.296
- type: map_at_100
value: 58.011
- type: map_at_1000
value: 58.028
- type: map_at_3
value: 54.300000000000004
- type: map_at_5
value: 56.21000000000001
- type: mrr_at_1
value: 47.599999999999994
- type: mrr_at_10
value: 57.296
- type: mrr_at_100
value: 58.011
- type: mrr_at_1000
value: 58.028
- type: mrr_at_3
value: 54.300000000000004
- type: mrr_at_5
value: 56.21000000000001
- type: ndcg_at_1
value: 47.599999999999994
- type: ndcg_at_10
value: 62.458000000000006
- type: ndcg_at_100
value: 65.589
- type: ndcg_at_1000
value: 66.059
- type: ndcg_at_3
value: 56.364000000000004
- type: ndcg_at_5
value: 59.815
- type: precision_at_1
value: 47.599999999999994
- type: precision_at_10
value: 7.89
- type: precision_at_100
value: 0.928
- type: precision_at_1000
value: 0.097
- type: precision_at_3
value: 20.767
- type: precision_at_5
value: 14.14
- type: recall_at_1
value: 47.599999999999994
- type: recall_at_10
value: 78.9
- type: recall_at_100
value: 92.80000000000001
- type: recall_at_1000
value: 96.6
- type: recall_at_3
value: 62.3
- type: recall_at_5
value: 70.7
- task:
type: Classification
dataset:
type: C-MTEB/IFlyTek-classification
name: MTEB IFlyTek
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 47.46440938822624
- type: f1
value: 34.587004997852524
- task:
type: Classification
dataset:
type: C-MTEB/JDReview-classification
name: MTEB JDReview
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 84.9906191369606
- type: ap
value: 52.31309789960497
- type: f1
value: 79.55556102310072
- task:
type: STS
dataset:
type: C-MTEB/LCQMC
name: MTEB LCQMC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 69.80872804636063
- type: cos_sim_spearman
value: 75.83290476813391
- type: euclidean_pearson
value: 74.09865882324753
- type: euclidean_spearman
value: 75.83290698376118
- type: manhattan_pearson
value: 74.0616102379577
- type: manhattan_spearman
value: 75.81278969865738
- task:
type: Retrieval
dataset:
type: C-MTEB/MMarcoRetrieval
name: MTEB MMarcoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 65.029
- type: map_at_10
value: 74.39
- type: map_at_100
value: 74.734
- type: map_at_1000
value: 74.74300000000001
- type: map_at_3
value: 72.52
- type: map_at_5
value: 73.724
- type: mrr_at_1
value: 67.192
- type: mrr_at_10
value: 74.95100000000001
- type: mrr_at_100
value: 75.25500000000001
- type: mrr_at_1000
value: 75.263
- type: mrr_at_3
value: 73.307
- type: mrr_at_5
value: 74.355
- type: ndcg_at_1
value: 67.192
- type: ndcg_at_10
value: 78.22200000000001
- type: ndcg_at_100
value: 79.76299999999999
- type: ndcg_at_1000
value: 80.018
- type: ndcg_at_3
value: 74.656
- type: ndcg_at_5
value: 76.697
- type: precision_at_1
value: 67.192
- type: precision_at_10
value: 9.513
- type: precision_at_100
value: 1.027
- type: precision_at_1000
value: 0.105
- type: precision_at_3
value: 28.204
- type: precision_at_5
value: 18.009
- type: recall_at_1
value: 65.029
- type: recall_at_10
value: 89.462
- type: recall_at_100
value: 96.418
- type: recall_at_1000
value: 98.409
- type: recall_at_3
value: 80.029
- type: recall_at_5
value: 84.882
- task:
type: Classification
dataset:
type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (zh-CN)
config: zh-CN
split: test
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
metrics:
- type: accuracy
value: 65.56489576328177
- type: f1
value: 63.37174551232159
- task:
type: Classification
dataset:
type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (zh-CN)
config: zh-CN
split: test
revision: 7d571f92784cd94a019292a1f45445077d0ef634
metrics:
- type: accuracy
value: 71.4862138533961
- type: f1
value: 71.171374964826
- task:
type: Retrieval
dataset:
type: C-MTEB/MedicalRetrieval
name: MTEB MedicalRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 48.6
- type: map_at_10
value: 54.92700000000001
- type: map_at_100
value: 55.528
- type: map_at_1000
value: 55.584
- type: map_at_3
value: 53.55
- type: map_at_5
value: 54.379999999999995
- type: mrr_at_1
value: 48.8
- type: mrr_at_10
value: 55.028999999999996
- type: mrr_at_100
value: 55.629
- type: mrr_at_1000
value: 55.684999999999995
- type: mrr_at_3
value: 53.65
- type: mrr_at_5
value: 54.48
- type: ndcg_at_1
value: 48.6
- type: ndcg_at_10
value: 57.965999999999994
- type: ndcg_at_100
value: 61.043000000000006
- type: ndcg_at_1000
value: 62.624
- type: ndcg_at_3
value: 55.132000000000005
- type: ndcg_at_5
value: 56.621
- type: precision_at_1
value: 48.6
- type: precision_at_10
value: 6.75
- type: precision_at_100
value: 0.823
- type: precision_at_1000
value: 0.095
- type: precision_at_3
value: 19.900000000000002
- type: precision_at_5
value: 12.659999999999998
- type: recall_at_1
value: 48.6
- type: recall_at_10
value: 67.5
- type: recall_at_100
value: 82.3
- type: recall_at_1000
value: 94.89999999999999
- type: recall_at_3
value: 59.699999999999996
- type: recall_at_5
value: 63.3
- task:
type: Reranking
dataset:
type: C-MTEB/Mmarco-reranking
name: MTEB MMarcoReranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 29.196130696027474
- type: mrr
value: 28.43730158730159
- task:
type: Classification
dataset:
type: C-MTEB/MultilingualSentiment-classification
name: MTEB MultilingualSentiment
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 72.48333333333333
- type: f1
value: 72.00258522357558
- task:
type: PairClassification
dataset:
type: C-MTEB/OCNLI
name: MTEB Ocnli
config: default
split: validation
revision: None
metrics:
- type: cos_sim_accuracy
value: 65.13264753654575
- type: cos_sim_ap
value: 70.52831936800807
- type: cos_sim_f1
value: 71.35353535353535
- type: cos_sim_precision
value: 57.787958115183244
- type: cos_sim_recall
value: 93.24181626187962
- type: dot_accuracy
value: 65.13264753654575
- type: dot_ap
value: 70.52828597418102
- type: dot_f1
value: 71.35353535353535
- type: dot_precision
value: 57.787958115183244
- type: dot_recall
value: 93.24181626187962
- type: euclidean_accuracy
value: 65.13264753654575
- type: euclidean_ap
value: 70.52828597418102
- type: euclidean_f1
value: 71.35353535353535
- type: euclidean_precision
value: 57.787958115183244
- type: euclidean_recall
value: 93.24181626187962
- type: manhattan_accuracy
value: 64.8077964266378
- type: manhattan_ap
value: 70.39954487476643
- type: manhattan_f1
value: 71.2270200940573
- type: manhattan_precision
value: 59.84195402298851
- type: manhattan_recall
value: 87.96198521647307
- type: max_accuracy
value: 65.13264753654575
- type: max_ap
value: 70.52831936800807
- type: max_f1
value: 71.35353535353535
- task:
type: Classification
dataset:
type: C-MTEB/OnlineShopping-classification
name: MTEB OnlineShopping
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 90.34
- type: ap
value: 87.79622626876444
- type: f1
value: 90.32357430051181
- task:
type: STS
dataset:
type: C-MTEB/PAWSX
name: MTEB PAWSX
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 27.9175458105215
- type: cos_sim_spearman
value: 32.024302491613014
- type: euclidean_pearson
value: 33.01780461609846
- type: euclidean_spearman
value: 32.024301939183374
- type: manhattan_pearson
value: 32.94874897942371
- type: manhattan_spearman
value: 31.902283210178012
- task:
type: STS
dataset:
type: C-MTEB/QBQTC
name: MTEB QBQTC
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 36.288219964332754
- type: cos_sim_spearman
value: 36.46838652731507
- type: euclidean_pearson
value: 35.11414028811812
- type: euclidean_spearman
value: 36.468386523814104
- type: manhattan_pearson
value: 35.20922826624027
- type: manhattan_spearman
value: 36.55349180906185
- task:
type: STS
dataset:
type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (zh)
config: zh
split: test
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
metrics:
- type: cos_sim_pearson
value: 66.18186265837434
- type: cos_sim_spearman
value: 67.52365178443915
- type: euclidean_pearson
value: 65.46342439169497
- type: euclidean_spearman
value: 67.52365178443915
- type: manhattan_pearson
value: 67.3476263677961
- type: manhattan_spearman
value: 69.09476240936812
- task:
type: STS
dataset:
type: C-MTEB/STSB
name: MTEB STSB
config: default
split: test
revision: None
metrics:
- type: cos_sim_pearson
value: 72.53864906415339
- type: cos_sim_spearman
value: 72.63037820118355
- type: euclidean_pearson
value: 72.42255276991672
- type: euclidean_spearman
value: 72.63037820118355
- type: manhattan_pearson
value: 72.36324244766192
- type: manhattan_spearman
value: 72.58609772740323
- task:
type: Reranking
dataset:
type: C-MTEB/T2Reranking
name: MTEB T2Reranking
config: default
split: dev
revision: None
metrics:
- type: map
value: 66.45708148192449
- type: mrr
value: 76.08372693469173
- task:
type: Retrieval
dataset:
type: C-MTEB/T2Retrieval
name: MTEB T2Retrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 26.436999999999998
- type: map_at_10
value: 74.516
- type: map_at_100
value: 78.29899999999999
- type: map_at_1000
value: 78.372
- type: map_at_3
value: 52.217
- type: map_at_5
value: 64.24
- type: mrr_at_1
value: 88.23
- type: mrr_at_10
value: 91.06400000000001
- type: mrr_at_100
value: 91.18
- type: mrr_at_1000
value: 91.184
- type: mrr_at_3
value: 90.582
- type: mrr_at_5
value: 90.88300000000001
- type: ndcg_at_1
value: 88.23
- type: ndcg_at_10
value: 82.511
- type: ndcg_at_100
value: 86.531
- type: ndcg_at_1000
value: 87.244
- type: ndcg_at_3
value: 83.987
- type: ndcg_at_5
value: 82.46900000000001
- type: precision_at_1
value: 88.23
- type: precision_at_10
value: 41.245
- type: precision_at_100
value: 4.987
- type: precision_at_1000
value: 0.515
- type: precision_at_3
value: 73.675
- type: precision_at_5
value: 61.71
- type: recall_at_1
value: 26.436999999999998
- type: recall_at_10
value: 81.547
- type: recall_at_100
value: 94.548
- type: recall_at_1000
value: 98.197
- type: recall_at_3
value: 54.056000000000004
- type: recall_at_5
value: 67.93
- task:
type: Classification
dataset:
type: C-MTEB/TNews-classification
name: MTEB TNews
config: default
split: validation
revision: None
metrics:
- type: accuracy
value: 50.784
- type: f1
value: 48.89471168071432
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringP2P
name: MTEB ThuNewsClusteringP2P
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 63.19039347990962
- task:
type: Clustering
dataset:
type: C-MTEB/ThuNewsClusteringS2S
name: MTEB ThuNewsClusteringS2S
config: default
split: test
revision: None
metrics:
- type: v_measure
value: 55.357378578603225
- task:
type: Retrieval
dataset:
type: C-MTEB/VideoRetrieval
name: MTEB VideoRetrieval
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 58.8
- type: map_at_10
value: 68.623
- type: map_at_100
value: 69.074
- type: map_at_1000
value: 69.085
- type: map_at_3
value: 66.767
- type: map_at_5
value: 67.972
- type: mrr_at_1
value: 58.699999999999996
- type: mrr_at_10
value: 68.573
- type: mrr_at_100
value: 69.024
- type: mrr_at_1000
value: 69.035
- type: mrr_at_3
value: 66.717
- type: mrr_at_5
value: 67.92200000000001
- type: ndcg_at_1
value: 58.8
- type: ndcg_at_10
value: 73.038
- type: ndcg_at_100
value: 75.16199999999999
- type: ndcg_at_1000
value: 75.422
- type: ndcg_at_3
value: 69.297
- type: ndcg_at_5
value: 71.475
- type: precision_at_1
value: 58.8
- type: precision_at_10
value: 8.67
- type: precision_at_100
value: 0.9650000000000001
- type: precision_at_1000
value: 0.099
- type: precision_at_3
value: 25.533
- type: precision_at_5
value: 16.38
- type: recall_at_1
value: 58.8
- type: recall_at_10
value: 86.7
- type: recall_at_100
value: 96.5
- type: recall_at_1000
value: 98.5
- type: recall_at_3
value: 76.6
- type: recall_at_5
value: 81.89999999999999
- task:
type: Classification
dataset:
type: C-MTEB/waimai-classification
name: MTEB Waimai
config: default
split: test
revision: None
metrics:
- type: accuracy
value: 86.61999999999999
- type: ap
value: 69.93149123197975
- type: f1
value: 84.99670691559903
stella model
stella是一个通用的中文文本编码模型，目前有两个版本：base 和 large，2个版本的模型均支持1024的输入长度。
完整的训练思路和训练过程已记录在博客，欢迎阅读讨论。
训练数据：
- 开源数据(wudao_base_200GB[1]、m3e[2]和simclue[3])，着重挑选了长度大于512的文本
- 在通用语料库上使用LLM构造一批(question, paragraph)和(sentence, paragraph)数据
训练方法：
- 对比学习损失函数
- 带有难负例的对比学习损失函数(分别基于bm25和vector构造了难负例)
- EWC(Elastic Weights Consolidation)[4]
- cosent loss[5]
- 每一种类型的数据一个迭代器，分别计算loss进行更新
初始权重：
stella-base-zh和stella-large-zh分别以piccolo-base-zh[6]和piccolo-large-zh作为基础模型，512-1024的position embedding使用层次分解位置编码[7]进行初始化。
感谢商汤科技研究院开源的piccolo系列模型。
stella is a general-purpose Chinese text encoding model, currently with two versions: base and large, both of them support input lengths of 1024.
The training data mainly includes:
- Open-source training data (wudao_base_200GB, m3e, and simclue), with a focus on selecting texts with lengths greater than 512.
- A batch of (question, paragraph) and (sentence, paragraph) data constructed on a general corpus using LLM.
The loss functions mainly include:
- Contrastive learning loss function
- Contrastive learning loss function with hard negative examples (based on bm25 and vector hard negatives)
- EWC (Elastic Weights Consolidation)
- cosent loss
Model weight initialization:
stella-base-zh and stella-large-zh use piccolo-base-zh and piccolo-large-zh as the base models, respectively, and the
512-1024 position embedding uses the initialization strategy of hierarchical decomposed position encoding.
Training strategy:
One iterator for each type of data, separately calculating the loss.
Metric
C-MTEB leaderboard
stella模型在C-MTEB[8]的结果，评测脚本请参见博客。
| Model Name | Model Size (GB) | Dimension | Sequence Length | Average (35) | Classification (9) | Clustering (4) | Pair Classification (2) | Reranking (4) | Retrieval (8) | STS (8) |
|---|---|---|---|---|---|---|---|---|---|---|
| stella-large-zh | 0.65 | 1024 | 1024 | 64.54 | 67.62 | 48.65 | 78.72 | 65.98 | 71.02 | 58.3 |
| stella-base-zh | 0.2 | 768 | 1024 | 64.16 | 67.77 | 48.7 | 76.09 | 66.95 | 71.07 | 56.54 |
| piccolo-large-zh | 0.65 | 1024 | 512 | 64.11 | 67.03 | 47.04 | 78.38 | 65.98 | 70.93 | 58.02 |
| bge-large-zh | 1.3 | 1024 | 512 | 63.96 | 68.32 | 48.39 | 78.94 | 65.11 | 71.52 | 54.98 |
| piccolo-base-zh | 0.2 | 768 | 512 | 63.66 | 66.98 | 47.12 | 76.61 | 66.68 | 71.2 | 55.9 |
| bge-large-zh-no-instruct | 1.3 | 1024 | 512 | 63.4 | 68.58 | 50.01 | 76.77 | 64.9 | 70.54 | 53 |
| [bge-base-zh | 0.41 | 768 | 512 | 62.8 | 67.07 | 47.64 | 77.5 | 64.91 | 69.53 | 54.12 |
Evaluation for long text
经过实际观察发现，C-MTEB的评测数据长度基本都是小于512的， 更致命的是那些长度大于512的文本，其重点都在前半部分 这里以CMRC2018的数据为例说明这个问题：
question: 《无双大蛇z》是谁旗下ω-force开发的动作游戏？
passage：《无双大蛇z》是光荣旗下ω-force开发的动作游戏，于2009年3月12日登陆索尼playstation3，并于2009年11月27日推......
passage长度为800多，大于512，但是对于这个question而言只需要前面40个字就足以检索，多的内容对于模型而言是一种噪声，反而降低了效果。
简言之，现有数据集的2个问题：
1）长度大于512的过少
2）即便大于512，对于检索而言也只需要前512的文本内容
导致无法准确评估模型的长文本编码能力。
为了解决这个问题，搜集了相关开源数据并使用规则进行过滤，最终整理了6份长文本测试集,他们分别是：
- CMRC2018，通用百科
- CAIL，法律阅读理解
- DRCD，繁体百科，已转简体
- Military，军工问答
- Squad，英文阅读理解，已转中文
- Multifieldqa_zh，清华的大模型长文本理解能力评测数据[9]
处理规则是选取答案在512长度之后的文本，短的测试数据会欠采样一下，长短文本占比约为1:2，所以模型既得理解短文本也得理解长文本。 除了Military数据集，我们提供了其他5个测试数据的下载地址：https://drive.google.com/file/d/1WC6EWaCbVgz-vPMDFH4TwAMkLyh5WNcN/view?usp=sharing
评测指标为Recall@5, 结果如下：
| Dataset | piccolo-base-zh | piccolo-large-zh | bge-base-zh | bge-large-zh | stella-base-zh | stella-large-zh |
|---|---|---|---|---|---|---|
| CMRC2018 | 94.34 | 93.82 | 91.56 | 93.12 | 96.08 | 95.56 |
| CAIL | 28.04 | 33.64 | 31.22 | 33.94 | 34.62 | 37.18 |
| DRCD | 78.25 | 77.9 | 78.34 | 80.26 | 86.14 | 84.58 |
| Military | 76.61 | 73.06 | 75.65 | 75.81 | 83.71 | 80.48 |
| Squad | 91.21 | 86.61 | 87.87 | 90.38 | 93.31 | 91.21 |
| Multifieldqa_zh | 81.41 | 83.92 | 83.92 | 83.42 | 79.9 | 80.4 |
| Average | 74.98 | 74.83 | 74.76 | 76.15 | 78.96 | 78.24 |
注意： 因为长文本评测数据数量稀少，所以构造时也使用了train部分，如果自行评测，请注意模型的训练数据以免数据泄露。
Usage
本模型是在piccolo基础上训练的，因此用法和piccolo完全一致。
注意：在stella中instruction里的冒号是英文冒号, 即查询: 和结果: 。
在sentence-transformer库中的使用方法：
# 对于短对短数据集，下面是通用的使用方式
from sentence_transformers import SentenceTransformer
sentences = ["数据1", "数据2"]
model = SentenceTransformer('infgrad/stella-base-zh')
print(model.max_seq_length)
embeddings_1 = model.encode(sentences, normalize_embeddings=True)
embeddings_2 = model.encode(sentences, normalize_embeddings=True)
similarity = embeddings_1 @ embeddings_2.T
print(similarity)
# 如果是短对长数据集，推荐添加instruction，来帮助模型更好地进行检索。
# 注意instruction里的是英文的冒号
直接使用transformers库：
from transformers import AutoModel, AutoTokenizer
from sklearn.preprocessing import normalize
model = AutoModel.from_pretrained('infgrad/stella-base-zh')
tokenizer = AutoTokenizer.from_pretrained('infgrad/stella-base-zh')
sentences = ["数据1", "数据ABCDEFGH"]
batch_data = tokenizer(
batch_text_or_text_pairs=sentences,
padding="longest",
return_tensors="pt",
max_length=1024,
truncation=True,
)
attention_mask = batch_data["attention_mask"]
model_output = model(**batch_data)
last_hidden = model_output.last_hidden_state.masked_fill(~attention_mask[..., None].bool(), 0.0)
vectors = last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
vectors = normalize(vectors, norm="l2", axis=1, )
print(vectors.shape) # 2,768
Training Detail
硬件： 单卡A100-80GB
环境： torch1.13.*; transformers-trainer + deepspeed + gradient-checkpointing
学习率： 1e-6
batch_size： base模型为1024，额外增加20%的难负例；large模型为768，额外增加20%的难负例
数据量： 约100万，其中用LLM构造的数据约有200K. LLM模型大小为13b
ToDoList
评测的稳定性：
评测过程中发现Clustering任务会和官方的结果不一致，大约有±0.0x的小差距，基本上可以忽略不计，不影响评测结论。
但是不完全一样还是比较难理解的，本人试了bge和piccolo系列的模型都存在这个问题，个人猜测可能和使用的库、batch_size等环境有关。
更高质量的长文本训练和测试数据： 训练数据多是用13b模型构造的，肯定会存在噪声。 测试数据基本都是从mrc数据整理来的，所以问题都是factoid类型，不符合真实分布。
OOD的性能： 虽然近期出现了很多向量编码模型，但是对于不是那么通用的domain，这一众模型包括stella、openai和cohere, 它们的效果均比不上BM25。
Reference
- https://www.scidb.cn/en/detail?dataSetId=c6a3fe684227415a9db8e21bac4a15ab
- https://github.com/wangyuxinwhy/uniem
- https://github.com/CLUEbenchmark/SimCLUE
- https://arxiv.org/abs/1612.00796
- https://kexue.fm/archives/8847
- https://huggingface.co/sensenova/piccolo-base-zh
- https://kexue.fm/archives/7947
- https://github.com/FlagOpen/FlagEmbedding
- https://github.com/THUDM/LongBench