|
--- |
|
pipeline_tag: sentence-similarity |
|
tags: |
|
- sentence-transformers |
|
- feature-extraction |
|
- sentence-similarity |
|
- mteb |
|
model-index: |
|
- name: stella-mrl-large-zh-v3.5-1792d |
|
results: |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/AFQMC |
|
name: MTEB AFQMC |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 54.33822814973567 |
|
- type: cos_sim_spearman |
|
value: 58.85457316132848 |
|
- type: euclidean_pearson |
|
value: 57.57048145477383 |
|
- type: euclidean_spearman |
|
value: 58.854593263425095 |
|
- type: manhattan_pearson |
|
value: 57.55884028558309 |
|
- type: manhattan_spearman |
|
value: 58.84474216217465 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/ATEC |
|
name: MTEB ATEC |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 54.219652875381875 |
|
- type: cos_sim_spearman |
|
value: 58.079506691583546 |
|
- type: euclidean_pearson |
|
value: 61.646366330471736 |
|
- type: euclidean_spearman |
|
value: 58.07951006894859 |
|
- type: manhattan_pearson |
|
value: 61.64460832085762 |
|
- type: manhattan_spearman |
|
value: 58.08054699349972 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_reviews_multi |
|
name: MTEB AmazonReviewsClassification (zh) |
|
config: zh |
|
split: test |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
metrics: |
|
- type: accuracy |
|
value: 46.593999999999994 |
|
- type: f1 |
|
value: 44.73150848183217 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/BQ |
|
name: MTEB BQ |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 69.16841007040091 |
|
- type: cos_sim_spearman |
|
value: 71.04760904227217 |
|
- type: euclidean_pearson |
|
value: 69.95126084376611 |
|
- type: euclidean_spearman |
|
value: 71.04760904184589 |
|
- type: manhattan_pearson |
|
value: 69.92512024129407 |
|
- type: manhattan_spearman |
|
value: 71.02613161257672 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/CLSClusteringP2P |
|
name: MTEB CLSClusteringP2P |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 43.032332399653306 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/CLSClusteringS2S |
|
name: MTEB CLSClusteringS2S |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 40.41603958793544 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/CMedQAv1-reranking |
|
name: MTEB CMedQAv1 |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 89.33487924447584 |
|
- type: mrr |
|
value: 91.34623015873017 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/CMedQAv2-reranking |
|
name: MTEB CMedQAv2 |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 89.17795270698021 |
|
- type: mrr |
|
value: 91.0956746031746 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/CmedqaRetrieval |
|
name: MTEB CmedqaRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 26.809 |
|
- type: map_at_10 |
|
value: 39.906000000000006 |
|
- type: map_at_100 |
|
value: 41.858000000000004 |
|
- type: map_at_1000 |
|
value: 41.954 |
|
- type: map_at_3 |
|
value: 35.435 |
|
- type: map_at_5 |
|
value: 37.978 |
|
- type: mrr_at_1 |
|
value: 40.660000000000004 |
|
- type: mrr_at_10 |
|
value: 48.787000000000006 |
|
- type: mrr_at_100 |
|
value: 49.796 |
|
- type: mrr_at_1000 |
|
value: 49.832 |
|
- type: mrr_at_3 |
|
value: 46.166000000000004 |
|
- type: mrr_at_5 |
|
value: 47.675 |
|
- type: ndcg_at_1 |
|
value: 40.660000000000004 |
|
- type: ndcg_at_10 |
|
value: 46.614 |
|
- type: ndcg_at_100 |
|
value: 54.037 |
|
- type: ndcg_at_1000 |
|
value: 55.654 |
|
- type: ndcg_at_3 |
|
value: 41.032000000000004 |
|
- type: ndcg_at_5 |
|
value: 43.464999999999996 |
|
- type: precision_at_1 |
|
value: 40.660000000000004 |
|
- type: precision_at_10 |
|
value: 10.35 |
|
- type: precision_at_100 |
|
value: 1.6340000000000001 |
|
- type: precision_at_1000 |
|
value: 0.184 |
|
- type: precision_at_3 |
|
value: 23.122 |
|
- type: precision_at_5 |
|
value: 16.944 |
|
- type: recall_at_1 |
|
value: 26.809 |
|
- type: recall_at_10 |
|
value: 57.474000000000004 |
|
- type: recall_at_100 |
|
value: 87.976 |
|
- type: recall_at_1000 |
|
value: 98.74199999999999 |
|
- type: recall_at_3 |
|
value: 40.819 |
|
- type: recall_at_5 |
|
value: 48.175000000000004 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: C-MTEB/CMNLI |
|
name: MTEB Cmnli |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 83.4996993385448 |
|
- type: cos_sim_ap |
|
value: 90.66238348446467 |
|
- type: cos_sim_f1 |
|
value: 84.39077936333699 |
|
- type: cos_sim_precision |
|
value: 79.53651975998345 |
|
- type: cos_sim_recall |
|
value: 89.87608136544307 |
|
- type: dot_accuracy |
|
value: 83.4996993385448 |
|
- type: dot_ap |
|
value: 90.64660919236363 |
|
- type: dot_f1 |
|
value: 84.39077936333699 |
|
- type: dot_precision |
|
value: 79.53651975998345 |
|
- type: dot_recall |
|
value: 89.87608136544307 |
|
- type: euclidean_accuracy |
|
value: 83.4996993385448 |
|
- type: euclidean_ap |
|
value: 90.66238269557765 |
|
- type: euclidean_f1 |
|
value: 84.39077936333699 |
|
- type: euclidean_precision |
|
value: 79.53651975998345 |
|
- type: euclidean_recall |
|
value: 89.87608136544307 |
|
- type: manhattan_accuracy |
|
value: 83.35538184004811 |
|
- type: manhattan_ap |
|
value: 90.6446013420276 |
|
- type: manhattan_f1 |
|
value: 84.37465196569775 |
|
- type: manhattan_precision |
|
value: 80.5614632071459 |
|
- type: manhattan_recall |
|
value: 88.56675239653963 |
|
- type: max_accuracy |
|
value: 83.4996993385448 |
|
- type: max_ap |
|
value: 90.66238348446467 |
|
- type: max_f1 |
|
value: 84.39077936333699 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/CovidRetrieval |
|
name: MTEB CovidRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 68.967 |
|
- type: map_at_10 |
|
value: 77.95299999999999 |
|
- type: map_at_100 |
|
value: 78.213 |
|
- type: map_at_1000 |
|
value: 78.21900000000001 |
|
- type: map_at_3 |
|
value: 76.30799999999999 |
|
- type: map_at_5 |
|
value: 77.316 |
|
- type: mrr_at_1 |
|
value: 69.125 |
|
- type: mrr_at_10 |
|
value: 77.886 |
|
- type: mrr_at_100 |
|
value: 78.141 |
|
- type: mrr_at_1000 |
|
value: 78.147 |
|
- type: mrr_at_3 |
|
value: 76.291 |
|
- type: mrr_at_5 |
|
value: 77.29700000000001 |
|
- type: ndcg_at_1 |
|
value: 69.231 |
|
- type: ndcg_at_10 |
|
value: 81.867 |
|
- type: ndcg_at_100 |
|
value: 82.982 |
|
- type: ndcg_at_1000 |
|
value: 83.12 |
|
- type: ndcg_at_3 |
|
value: 78.592 |
|
- type: ndcg_at_5 |
|
value: 80.39 |
|
- type: precision_at_1 |
|
value: 69.231 |
|
- type: precision_at_10 |
|
value: 9.494 |
|
- type: precision_at_100 |
|
value: 0.9990000000000001 |
|
- type: precision_at_1000 |
|
value: 0.101 |
|
- type: precision_at_3 |
|
value: 28.591 |
|
- type: precision_at_5 |
|
value: 18.061 |
|
- type: recall_at_1 |
|
value: 68.967 |
|
- type: recall_at_10 |
|
value: 93.941 |
|
- type: recall_at_100 |
|
value: 98.84100000000001 |
|
- type: recall_at_1000 |
|
value: 99.895 |
|
- type: recall_at_3 |
|
value: 85.142 |
|
- type: recall_at_5 |
|
value: 89.46300000000001 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/DuRetrieval |
|
name: MTEB DuRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 25.824 |
|
- type: map_at_10 |
|
value: 79.396 |
|
- type: map_at_100 |
|
value: 82.253 |
|
- type: map_at_1000 |
|
value: 82.295 |
|
- type: map_at_3 |
|
value: 54.83 |
|
- type: map_at_5 |
|
value: 69.536 |
|
- type: mrr_at_1 |
|
value: 89.7 |
|
- type: mrr_at_10 |
|
value: 92.929 |
|
- type: mrr_at_100 |
|
value: 93.013 |
|
- type: mrr_at_1000 |
|
value: 93.015 |
|
- type: mrr_at_3 |
|
value: 92.658 |
|
- type: mrr_at_5 |
|
value: 92.841 |
|
- type: ndcg_at_1 |
|
value: 89.7 |
|
- type: ndcg_at_10 |
|
value: 86.797 |
|
- type: ndcg_at_100 |
|
value: 89.652 |
|
- type: ndcg_at_1000 |
|
value: 90.047 |
|
- type: ndcg_at_3 |
|
value: 85.651 |
|
- type: ndcg_at_5 |
|
value: 84.747 |
|
- type: precision_at_1 |
|
value: 89.7 |
|
- type: precision_at_10 |
|
value: 41.61 |
|
- type: precision_at_100 |
|
value: 4.788 |
|
- type: precision_at_1000 |
|
value: 0.488 |
|
- type: precision_at_3 |
|
value: 76.833 |
|
- type: precision_at_5 |
|
value: 65.14 |
|
- type: recall_at_1 |
|
value: 25.824 |
|
- type: recall_at_10 |
|
value: 87.896 |
|
- type: recall_at_100 |
|
value: 97.221 |
|
- type: recall_at_1000 |
|
value: 99.29599999999999 |
|
- type: recall_at_3 |
|
value: 57.178 |
|
- type: recall_at_5 |
|
value: 74.348 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/EcomRetrieval |
|
name: MTEB EcomRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 52.5 |
|
- type: map_at_10 |
|
value: 63.04 |
|
- type: map_at_100 |
|
value: 63.548 |
|
- type: map_at_1000 |
|
value: 63.56 |
|
- type: map_at_3 |
|
value: 60.483 |
|
- type: map_at_5 |
|
value: 62.22800000000001 |
|
- type: mrr_at_1 |
|
value: 52.5 |
|
- type: mrr_at_10 |
|
value: 63.04 |
|
- type: mrr_at_100 |
|
value: 63.548 |
|
- type: mrr_at_1000 |
|
value: 63.56 |
|
- type: mrr_at_3 |
|
value: 60.483 |
|
- type: mrr_at_5 |
|
value: 62.22800000000001 |
|
- type: ndcg_at_1 |
|
value: 52.5 |
|
- type: ndcg_at_10 |
|
value: 68.099 |
|
- type: ndcg_at_100 |
|
value: 70.48400000000001 |
|
- type: ndcg_at_1000 |
|
value: 70.769 |
|
- type: ndcg_at_3 |
|
value: 63.01 |
|
- type: ndcg_at_5 |
|
value: 66.148 |
|
- type: precision_at_1 |
|
value: 52.5 |
|
- type: precision_at_10 |
|
value: 8.39 |
|
- type: precision_at_100 |
|
value: 0.9490000000000001 |
|
- type: precision_at_1000 |
|
value: 0.097 |
|
- type: precision_at_3 |
|
value: 23.433 |
|
- type: precision_at_5 |
|
value: 15.58 |
|
- type: recall_at_1 |
|
value: 52.5 |
|
- type: recall_at_10 |
|
value: 83.89999999999999 |
|
- type: recall_at_100 |
|
value: 94.89999999999999 |
|
- type: recall_at_1000 |
|
value: 97.1 |
|
- type: recall_at_3 |
|
value: 70.3 |
|
- type: recall_at_5 |
|
value: 77.9 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/IFlyTek-classification |
|
name: MTEB IFlyTek |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 50.742593305117346 |
|
- type: f1 |
|
value: 38.7451988564002 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/JDReview-classification |
|
name: MTEB JDReview |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 86.09756097560977 |
|
- type: ap |
|
value: 54.39255221143281 |
|
- type: f1 |
|
value: 80.8326851537251 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/LCQMC |
|
name: MTEB LCQMC |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 72.32408066246728 |
|
- type: cos_sim_spearman |
|
value: 78.25773378380241 |
|
- type: euclidean_pearson |
|
value: 77.87824677060661 |
|
- type: euclidean_spearman |
|
value: 78.25773599854358 |
|
- type: manhattan_pearson |
|
value: 77.86648277798515 |
|
- type: manhattan_spearman |
|
value: 78.24642917155661 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/Mmarco-reranking |
|
name: MTEB MMarcoReranking |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 28.846601097874608 |
|
- type: mrr |
|
value: 27.902777777777775 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/MMarcoRetrieval |
|
name: MTEB MMarcoRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 66.533 |
|
- type: map_at_10 |
|
value: 75.58399999999999 |
|
- type: map_at_100 |
|
value: 75.91 |
|
- type: map_at_1000 |
|
value: 75.921 |
|
- type: map_at_3 |
|
value: 73.847 |
|
- type: map_at_5 |
|
value: 74.929 |
|
- type: mrr_at_1 |
|
value: 68.854 |
|
- type: mrr_at_10 |
|
value: 76.20700000000001 |
|
- type: mrr_at_100 |
|
value: 76.498 |
|
- type: mrr_at_1000 |
|
value: 76.508 |
|
- type: mrr_at_3 |
|
value: 74.71600000000001 |
|
- type: mrr_at_5 |
|
value: 75.653 |
|
- type: ndcg_at_1 |
|
value: 68.854 |
|
- type: ndcg_at_10 |
|
value: 79.209 |
|
- type: ndcg_at_100 |
|
value: 80.67 |
|
- type: ndcg_at_1000 |
|
value: 80.95 |
|
- type: ndcg_at_3 |
|
value: 75.923 |
|
- type: ndcg_at_5 |
|
value: 77.74799999999999 |
|
- type: precision_at_1 |
|
value: 68.854 |
|
- type: precision_at_10 |
|
value: 9.547 |
|
- type: precision_at_100 |
|
value: 1.027 |
|
- type: precision_at_1000 |
|
value: 0.105 |
|
- type: precision_at_3 |
|
value: 28.582 |
|
- type: precision_at_5 |
|
value: 18.112000000000002 |
|
- type: recall_at_1 |
|
value: 66.533 |
|
- type: recall_at_10 |
|
value: 89.736 |
|
- type: recall_at_100 |
|
value: 96.34 |
|
- type: recall_at_1000 |
|
value: 98.52 |
|
- type: recall_at_3 |
|
value: 81.047 |
|
- type: recall_at_5 |
|
value: 85.38900000000001 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (zh-CN) |
|
config: zh-CN |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 73.27841291190316 |
|
- type: f1 |
|
value: 70.82287701665152 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (zh-CN) |
|
config: zh-CN |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 76.20040349697376 |
|
- type: f1 |
|
value: 75.92782428878164 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/MedicalRetrieval |
|
name: MTEB MedicalRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 56.39999999999999 |
|
- type: map_at_10 |
|
value: 62.122 |
|
- type: map_at_100 |
|
value: 62.692 |
|
- type: map_at_1000 |
|
value: 62.739 |
|
- type: map_at_3 |
|
value: 60.617 |
|
- type: map_at_5 |
|
value: 61.582 |
|
- type: mrr_at_1 |
|
value: 56.39999999999999 |
|
- type: mrr_at_10 |
|
value: 62.125 |
|
- type: mrr_at_100 |
|
value: 62.696 |
|
- type: mrr_at_1000 |
|
value: 62.742 |
|
- type: mrr_at_3 |
|
value: 60.617 |
|
- type: mrr_at_5 |
|
value: 61.602000000000004 |
|
- type: ndcg_at_1 |
|
value: 56.39999999999999 |
|
- type: ndcg_at_10 |
|
value: 64.986 |
|
- type: ndcg_at_100 |
|
value: 67.889 |
|
- type: ndcg_at_1000 |
|
value: 69.16499999999999 |
|
- type: ndcg_at_3 |
|
value: 61.951 |
|
- type: ndcg_at_5 |
|
value: 63.685 |
|
- type: precision_at_1 |
|
value: 56.39999999999999 |
|
- type: precision_at_10 |
|
value: 7.3999999999999995 |
|
- type: precision_at_100 |
|
value: 0.8789999999999999 |
|
- type: precision_at_1000 |
|
value: 0.098 |
|
- type: precision_at_3 |
|
value: 21.933 |
|
- type: precision_at_5 |
|
value: 14.000000000000002 |
|
- type: recall_at_1 |
|
value: 56.39999999999999 |
|
- type: recall_at_10 |
|
value: 74.0 |
|
- type: recall_at_100 |
|
value: 87.9 |
|
- type: recall_at_1000 |
|
value: 98.0 |
|
- type: recall_at_3 |
|
value: 65.8 |
|
- type: recall_at_5 |
|
value: 70.0 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/MultilingualSentiment-classification |
|
name: MTEB MultilingualSentiment |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 76.64 |
|
- type: f1 |
|
value: 76.5446299028248 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: C-MTEB/OCNLI |
|
name: MTEB Ocnli |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 82.34975636166757 |
|
- type: cos_sim_ap |
|
value: 85.51352392694149 |
|
- type: cos_sim_f1 |
|
value: 83.53057199211045 |
|
- type: cos_sim_precision |
|
value: 78.35337650323775 |
|
- type: cos_sim_recall |
|
value: 89.44033790918691 |
|
- type: dot_accuracy |
|
value: 82.34975636166757 |
|
- type: dot_ap |
|
value: 85.51347115601486 |
|
- type: dot_f1 |
|
value: 83.53057199211045 |
|
- type: dot_precision |
|
value: 78.35337650323775 |
|
- type: dot_recall |
|
value: 89.44033790918691 |
|
- type: euclidean_accuracy |
|
value: 82.34975636166757 |
|
- type: euclidean_ap |
|
value: 85.51352392694149 |
|
- type: euclidean_f1 |
|
value: 83.53057199211045 |
|
- type: euclidean_precision |
|
value: 78.35337650323775 |
|
- type: euclidean_recall |
|
value: 89.44033790918691 |
|
- type: manhattan_accuracy |
|
value: 82.34975636166757 |
|
- type: manhattan_ap |
|
value: 85.48313896880585 |
|
- type: manhattan_f1 |
|
value: 83.52414136386261 |
|
- type: manhattan_precision |
|
value: 79.00188323917138 |
|
- type: manhattan_recall |
|
value: 88.59556494192185 |
|
- type: max_accuracy |
|
value: 82.34975636166757 |
|
- type: max_ap |
|
value: 85.51352392694149 |
|
- type: max_f1 |
|
value: 83.53057199211045 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/OnlineShopping-classification |
|
name: MTEB OnlineShopping |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 93.39 |
|
- type: ap |
|
value: 91.62127505252761 |
|
- type: f1 |
|
value: 93.38126146765326 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/PAWSX |
|
name: MTEB PAWSX |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 39.69424895486595 |
|
- type: cos_sim_spearman |
|
value: 45.357868735202885 |
|
- type: euclidean_pearson |
|
value: 44.85027304963503 |
|
- type: euclidean_spearman |
|
value: 45.356945176162064 |
|
- type: manhattan_pearson |
|
value: 44.866080721344744 |
|
- type: manhattan_spearman |
|
value: 45.37053172312661 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/QBQTC |
|
name: MTEB QBQTC |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 37.03908089465844 |
|
- type: cos_sim_spearman |
|
value: 38.98314179826781 |
|
- type: euclidean_pearson |
|
value: 37.189386019789545 |
|
- type: euclidean_spearman |
|
value: 38.98311189555396 |
|
- type: manhattan_pearson |
|
value: 37.14695118899785 |
|
- type: manhattan_spearman |
|
value: 38.94957261261034 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (zh) |
|
config: zh |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 65.08396305098712 |
|
- type: cos_sim_spearman |
|
value: 66.26346934994216 |
|
- type: euclidean_pearson |
|
value: 65.56501615370941 |
|
- type: euclidean_spearman |
|
value: 66.26346934994216 |
|
- type: manhattan_pearson |
|
value: 65.47984748172154 |
|
- type: manhattan_spearman |
|
value: 66.25326746119808 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/STSB |
|
name: MTEB STSB |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 80.95965207330296 |
|
- type: cos_sim_spearman |
|
value: 82.96149593569953 |
|
- type: euclidean_pearson |
|
value: 82.67125448003975 |
|
- type: euclidean_spearman |
|
value: 82.96141174550262 |
|
- type: manhattan_pearson |
|
value: 82.64660468206361 |
|
- type: manhattan_spearman |
|
value: 82.91756025324656 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/T2Reranking |
|
name: MTEB T2Reranking |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 66.43391960680063 |
|
- type: mrr |
|
value: 76.078440855015 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/T2Retrieval |
|
name: MTEB T2Retrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 28.29 |
|
- type: map_at_10 |
|
value: 78.441 |
|
- type: map_at_100 |
|
value: 82.043 |
|
- type: map_at_1000 |
|
value: 82.10499999999999 |
|
- type: map_at_3 |
|
value: 55.448 |
|
- type: map_at_5 |
|
value: 67.982 |
|
- type: mrr_at_1 |
|
value: 91.18 |
|
- type: mrr_at_10 |
|
value: 93.498 |
|
- type: mrr_at_100 |
|
value: 93.57 |
|
- type: mrr_at_1000 |
|
value: 93.572 |
|
- type: mrr_at_3 |
|
value: 93.112 |
|
- type: mrr_at_5 |
|
value: 93.351 |
|
- type: ndcg_at_1 |
|
value: 91.18 |
|
- type: ndcg_at_10 |
|
value: 85.849 |
|
- type: ndcg_at_100 |
|
value: 89.32600000000001 |
|
- type: ndcg_at_1000 |
|
value: 89.9 |
|
- type: ndcg_at_3 |
|
value: 87.333 |
|
- type: ndcg_at_5 |
|
value: 85.91499999999999 |
|
- type: precision_at_1 |
|
value: 91.18 |
|
- type: precision_at_10 |
|
value: 42.315000000000005 |
|
- type: precision_at_100 |
|
value: 5.029 |
|
- type: precision_at_1000 |
|
value: 0.517 |
|
- type: precision_at_3 |
|
value: 76.12400000000001 |
|
- type: precision_at_5 |
|
value: 63.690000000000005 |
|
- type: recall_at_1 |
|
value: 28.29 |
|
- type: recall_at_10 |
|
value: 84.679 |
|
- type: recall_at_100 |
|
value: 95.952 |
|
- type: recall_at_1000 |
|
value: 98.821 |
|
- type: recall_at_3 |
|
value: 56.987 |
|
- type: recall_at_5 |
|
value: 71.15599999999999 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/TNews-classification |
|
name: MTEB TNews |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 53.09799999999999 |
|
- type: f1 |
|
value: 51.397192036892314 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/ThuNewsClusteringP2P |
|
name: MTEB ThuNewsClusteringP2P |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 70.59693805158501 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/ThuNewsClusteringS2S |
|
name: MTEB ThuNewsClusteringS2S |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 63.21127290121542 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/VideoRetrieval |
|
name: MTEB VideoRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 61.3 |
|
- type: map_at_10 |
|
value: 70.658 |
|
- type: map_at_100 |
|
value: 71.096 |
|
- type: map_at_1000 |
|
value: 71.108 |
|
- type: map_at_3 |
|
value: 69.15 |
|
- type: map_at_5 |
|
value: 70.125 |
|
- type: mrr_at_1 |
|
value: 61.3 |
|
- type: mrr_at_10 |
|
value: 70.658 |
|
- type: mrr_at_100 |
|
value: 71.096 |
|
- type: mrr_at_1000 |
|
value: 71.108 |
|
- type: mrr_at_3 |
|
value: 69.15 |
|
- type: mrr_at_5 |
|
value: 70.125 |
|
- type: ndcg_at_1 |
|
value: 61.3 |
|
- type: ndcg_at_10 |
|
value: 74.71 |
|
- type: ndcg_at_100 |
|
value: 76.783 |
|
- type: ndcg_at_1000 |
|
value: 77.09899999999999 |
|
- type: ndcg_at_3 |
|
value: 71.634 |
|
- type: ndcg_at_5 |
|
value: 73.399 |
|
- type: precision_at_1 |
|
value: 61.3 |
|
- type: precision_at_10 |
|
value: 8.72 |
|
- type: precision_at_100 |
|
value: 0.967 |
|
- type: precision_at_1000 |
|
value: 0.099 |
|
- type: precision_at_3 |
|
value: 26.267000000000003 |
|
- type: precision_at_5 |
|
value: 16.619999999999997 |
|
- type: recall_at_1 |
|
value: 61.3 |
|
- type: recall_at_10 |
|
value: 87.2 |
|
- type: recall_at_100 |
|
value: 96.7 |
|
- type: recall_at_1000 |
|
value: 99.2 |
|
- type: recall_at_3 |
|
value: 78.8 |
|
- type: recall_at_5 |
|
value: 83.1 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/waimai-classification |
|
name: MTEB Waimai |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 88.01 |
|
- type: ap |
|
value: 72.51537272974005 |
|
- type: f1 |
|
value: 86.49546025793478 |
|
--- |
|
|
|
|
|
**新闻 | News** |
|
|
|
**[2024-04-06]** 开源[puff](https://huggingface.co/infgrad/puff-base-v1)系列模型，**专门针对检索和语义匹配任务，更多的考虑泛化性和私有通用测试集效果，向量维度可变，中英双语**。 |
|
|
|
**[2024-02-27]** 开源stella-mrl-large-zh-v3.5-1792d模型，支持**向量可变维度**。 |
|
|
|
**[2024-02-17]** 开源stella v3系列、dialogue编码模型和相关训练数据。 |
|
|
|
**[2023-10-19]** 开源stella-base-en-v2 使用简单，**不需要任何前缀文本**。 |
|
|
|
**[2023-10-12]** 开源stella-base-zh-v2和stella-large-zh-v2, 效果更好且使用简单，**不需要任何前缀文本**。 |
|
|
|
**[2023-09-11]** 开源stella-base-zh和stella-large-zh |
|
|
|
欢迎去[本人主页](https://huggingface.co/infgrad)查看最新模型，并提出您的宝贵意见！ |
|
|
|
# 1 开源模型 |
|
|
|
本次开源stella-mrl-large-zh-v3.5-1792d模型， |
|
本模型是在stella-large-zh-v3-1792d的基础上使用[MRL](https://arxiv.org/abs/2205.13147)方法训练而成。 |
|
其主要特点是**可变的向量维度**。 |
|
|
|
# 2 使用方法 |
|
|
|
```python |
|
from sentence_transformers import SentenceTransformer |
|
from sklearn.preprocessing import normalize |
|
|
|
model = SentenceTransformer("infgrad/stella-mrl-large-zh-v3.5-1792d") |
|
# 注意先不要normalize! 选取前n维后再normalize |
|
vectors = model.encode(["text1", "text2"], normalize_embeddings=False) |
|
print(vectors.shape) # shape is [2,1792] |
|
# n_dims越大效果越好，但是时空消耗就越大。建议维度选取128的倍数，因为是这么训练的 |
|
n_dims = 768 |
|
cut_vecs = normalize(vectors[:, :n_dims]) |
|
|
|
``` |
|
|
|
# 3 不同向量维度的CMTEB得分 |
|
|
|
stella-mrl-large-zh-v3.5-1792d_1024 代表取前1024维。整体趋势是维度越大效果越好。 |
|
|
|
| Model | Retrieval | STS | PairClassification | Classification | Reranking | Clustering | CMTEB-Score | |
|
|:------------------------------------|:---------:|:-----:|:------------------:|:--------------:|:---------:|:----------:|:-----------:| |
|
| stella-mrl-large-zh-v3.5-1792d_128 | 70.01 | 62.17 | 87.99 | 70.67 | 66.77 | 53.55 | 67.16 | |
|
| stella-mrl-large-zh-v3.5-1792d_256 | 72.19 | 62.41 | 88.09 | 71.22 | 68.32 | 53.38 | 68.02 | |
|
| stella-mrl-large-zh-v3.5-1792d_384 | 72.77 | 62.43 | 88.26 | 71.34 | 68.31 | 53.87 | 68.25 | |
|
| stella-mrl-large-zh-v3.5-1792d_512 | 73.11 | 62.45 | 88.16 | 71.46 | 68.32 | 53.28 | 68.29 | |
|
| stella-mrl-large-zh-v3.5-1792d_640 | 73.27 | 62.49 | 88.21 | 71.46 | 68.69 | 53.63 | 68.42 | |
|
| stella-mrl-large-zh-v3.5-1792d_768 | 73.38 | 62.5 | 88.19 | 71.49 | 68.64 | 53.77 | 68.47 | |
|
| stella-mrl-large-zh-v3.5-1792d_896 | 73.37 | 62.5 | 88.14 | 71.51 | 68.44 | 54.13 | 68.49 | |
|
| stella-mrl-large-zh-v3.5-1792d_1024 | 73.43 | 62.51 | 88.16 | 71.52 | 68.59 | 53.43 | 68.44 | |
|
| stella-mrl-large-zh-v3.5-1792d_1152 | 73.46 | 62.49 | 88.16 | 71.57 | 68.55 | 53.67 | 68.49 | |
|
| stella-mrl-large-zh-v3.5-1792d_1280 | 73.48 | 62.51 | 88.12 | 71.55 | 68.44 | 53.74 | 68.48 | |
|
| stella-mrl-large-zh-v3.5-1792d_1408 | 73.48 | 62.51 | 88.14 | 71.58 | 68.46 | 53.69 | 68.48 | |
|
| stella-mrl-large-zh-v3.5-1792d_1536 | 73.49 | 62.5 | 88.11 | 71.55 | 68.5 | 54.06 | 68.52 | |
|
| stella-mrl-large-zh-v3.5-1792d_1664 | 73.56 | 62.49 | 88.06 | 71.56 | 68.47 | 54.28 | 68.56 | |
|
| stella-mrl-large-zh-v3.5-1792d_1792 | 73.51 | 62.48 | 88.09 | 71.56 | 68.45 | 54.39 | 68.56 | |
|
|
|
上述表格中stella-mrl-large-zh-v3.5-1792d_1792的得分为68.56和榜单68.55得分不一致，原因和权重类型有关，小差异请忽略不计。 |
