|
--- |
|
pipeline_tag: sentence-similarity |
|
tags: |
|
- sentence-transformers |
|
- feature-extraction |
|
- sentence-similarity |
|
- mteb |
|
model-index: |
|
- name: tao-8k-origin |
|
results: |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/AFQMC |
|
name: MTEB AFQMC |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 47.33644889578121 |
|
- type: cos_sim_spearman |
|
value: 49.93968642502866 |
|
- type: euclidean_pearson |
|
value: 48.12029792973887 |
|
- type: euclidean_spearman |
|
value: 49.939666315145494 |
|
- type: manhattan_pearson |
|
value: 48.07449594650583 |
|
- type: manhattan_spearman |
|
value: 49.892461433911166 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/ATEC |
|
name: MTEB ATEC |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 50.976148098905746 |
|
- type: cos_sim_spearman |
|
value: 53.11230114448237 |
|
- type: euclidean_pearson |
|
value: 55.119977161851054 |
|
- type: euclidean_spearman |
|
value: 53.11229776647941 |
|
- type: manhattan_pearson |
|
value: 55.096968162828034 |
|
- type: manhattan_spearman |
|
value: 53.107481302419465 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_reviews_multi |
|
name: MTEB AmazonReviewsClassification (zh) |
|
config: zh |
|
split: test |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
metrics: |
|
- type: accuracy |
|
value: 40.804 |
|
- type: f1 |
|
value: 39.01066543513968 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/BQ |
|
name: MTEB BQ |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 62.843816050026824 |
|
- type: cos_sim_spearman |
|
value: 65.54142642656706 |
|
- type: euclidean_pearson |
|
value: 64.08809634876388 |
|
- type: euclidean_spearman |
|
value: 65.54142642558392 |
|
- type: manhattan_pearson |
|
value: 64.09391522108272 |
|
- type: manhattan_spearman |
|
value: 65.55445491162718 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/CLSClusteringP2P |
|
name: MTEB CLSClusteringP2P |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 40.028061591547804 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/CLSClusteringS2S |
|
name: MTEB CLSClusteringS2S |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 38.1897102944254 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/CMedQAv1-reranking |
|
name: MTEB CMedQAv1 |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 85.34294439514511 |
|
- type: mrr |
|
value: 88.03849206349206 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/CMedQAv2-reranking |
|
name: MTEB CMedQAv2 |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 85.81294364673899 |
|
- type: mrr |
|
value: 88.52146825396825 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/CmedqaRetrieval |
|
name: MTEB CmedqaRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 23.982 |
|
- type: map_at_10 |
|
value: 36.21 |
|
- type: map_at_100 |
|
value: 38.072 |
|
- type: map_at_1000 |
|
value: 38.194 |
|
- type: map_at_3 |
|
value: 32.239000000000004 |
|
- type: map_at_5 |
|
value: 34.377 |
|
- type: mrr_at_1 |
|
value: 36.858999999999995 |
|
- type: mrr_at_10 |
|
value: 45.084999999999994 |
|
- type: mrr_at_100 |
|
value: 46.104 |
|
- type: mrr_at_1000 |
|
value: 46.154 |
|
- type: mrr_at_3 |
|
value: 42.623 |
|
- type: mrr_at_5 |
|
value: 43.995 |
|
- type: ndcg_at_1 |
|
value: 36.858999999999995 |
|
- type: ndcg_at_10 |
|
value: 42.735 |
|
- type: ndcg_at_100 |
|
value: 50.181 |
|
- type: ndcg_at_1000 |
|
value: 52.309000000000005 |
|
- type: ndcg_at_3 |
|
value: 37.728 |
|
- type: ndcg_at_5 |
|
value: 39.664 |
|
- type: precision_at_1 |
|
value: 36.858999999999995 |
|
- type: precision_at_10 |
|
value: 9.615 |
|
- type: precision_at_100 |
|
value: 1.564 |
|
- type: precision_at_1000 |
|
value: 0.183 |
|
- type: precision_at_3 |
|
value: 21.514 |
|
- type: precision_at_5 |
|
value: 15.568999999999999 |
|
- type: recall_at_1 |
|
value: 23.982 |
|
- type: recall_at_10 |
|
value: 53.04600000000001 |
|
- type: recall_at_100 |
|
value: 84.113 |
|
- type: recall_at_1000 |
|
value: 98.37 |
|
- type: recall_at_3 |
|
value: 37.824999999999996 |
|
- type: recall_at_5 |
|
value: 44.023 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: C-MTEB/CMNLI |
|
name: MTEB Cmnli |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 77.71497294046902 |
|
- type: cos_sim_ap |
|
value: 86.84526989595028 |
|
- type: cos_sim_f1 |
|
value: 79.31987247608926 |
|
- type: cos_sim_precision |
|
value: 72.70601987142022 |
|
- type: cos_sim_recall |
|
value: 87.2574234276362 |
|
- type: dot_accuracy |
|
value: 77.71497294046902 |
|
- type: dot_ap |
|
value: 86.83880734247957 |
|
- type: dot_f1 |
|
value: 79.31987247608926 |
|
- type: dot_precision |
|
value: 72.70601987142022 |
|
- type: dot_recall |
|
value: 87.2574234276362 |
|
- type: euclidean_accuracy |
|
value: 77.71497294046902 |
|
- type: euclidean_ap |
|
value: 86.84526869685902 |
|
- type: euclidean_f1 |
|
value: 79.31987247608926 |
|
- type: euclidean_precision |
|
value: 72.70601987142022 |
|
- type: euclidean_recall |
|
value: 87.2574234276362 |
|
- type: manhattan_accuracy |
|
value: 77.8111846061335 |
|
- type: manhattan_ap |
|
value: 86.81142881585656 |
|
- type: manhattan_f1 |
|
value: 79.4201671780764 |
|
- type: manhattan_precision |
|
value: 72.53575570158485 |
|
- type: manhattan_recall |
|
value: 87.74842179097499 |
|
- type: max_accuracy |
|
value: 77.8111846061335 |
|
- type: max_ap |
|
value: 86.84526989595028 |
|
- type: max_f1 |
|
value: 79.4201671780764 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/CovidRetrieval |
|
name: MTEB CovidRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 70.706 |
|
- type: map_at_10 |
|
value: 78.619 |
|
- type: map_at_100 |
|
value: 78.915 |
|
- type: map_at_1000 |
|
value: 78.918 |
|
- type: map_at_3 |
|
value: 76.967 |
|
- type: map_at_5 |
|
value: 77.922 |
|
- type: mrr_at_1 |
|
value: 70.917 |
|
- type: mrr_at_10 |
|
value: 78.64 |
|
- type: mrr_at_100 |
|
value: 78.935 |
|
- type: mrr_at_1000 |
|
value: 78.938 |
|
- type: mrr_at_3 |
|
value: 77.081 |
|
- type: mrr_at_5 |
|
value: 77.972 |
|
- type: ndcg_at_1 |
|
value: 70.917 |
|
- type: ndcg_at_10 |
|
value: 82.186 |
|
- type: ndcg_at_100 |
|
value: 83.487 |
|
- type: ndcg_at_1000 |
|
value: 83.589 |
|
- type: ndcg_at_3 |
|
value: 78.874 |
|
- type: ndcg_at_5 |
|
value: 80.548 |
|
- type: precision_at_1 |
|
value: 70.917 |
|
- type: precision_at_10 |
|
value: 9.431000000000001 |
|
- type: precision_at_100 |
|
value: 1.001 |
|
- type: precision_at_1000 |
|
value: 0.101 |
|
- type: precision_at_3 |
|
value: 28.275 |
|
- type: precision_at_5 |
|
value: 17.829 |
|
- type: recall_at_1 |
|
value: 70.706 |
|
- type: recall_at_10 |
|
value: 93.256 |
|
- type: recall_at_100 |
|
value: 99.05199999999999 |
|
- type: recall_at_1000 |
|
value: 99.895 |
|
- type: recall_at_3 |
|
value: 84.247 |
|
- type: recall_at_5 |
|
value: 88.251 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/DuRetrieval |
|
name: MTEB DuRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 25.989 |
|
- type: map_at_10 |
|
value: 80.882 |
|
- type: map_at_100 |
|
value: 83.63199999999999 |
|
- type: map_at_1000 |
|
value: 83.663 |
|
- type: map_at_3 |
|
value: 55.772 |
|
- type: map_at_5 |
|
value: 70.598 |
|
- type: mrr_at_1 |
|
value: 90.14999999999999 |
|
- type: mrr_at_10 |
|
value: 93.30000000000001 |
|
- type: mrr_at_100 |
|
value: 93.363 |
|
- type: mrr_at_1000 |
|
value: 93.366 |
|
- type: mrr_at_3 |
|
value: 93.083 |
|
- type: mrr_at_5 |
|
value: 93.206 |
|
- type: ndcg_at_1 |
|
value: 90.14999999999999 |
|
- type: ndcg_at_10 |
|
value: 88.016 |
|
- type: ndcg_at_100 |
|
value: 90.52900000000001 |
|
- type: ndcg_at_1000 |
|
value: 90.84400000000001 |
|
- type: ndcg_at_3 |
|
value: 86.529 |
|
- type: ndcg_at_5 |
|
value: 85.65899999999999 |
|
- type: precision_at_1 |
|
value: 90.14999999999999 |
|
- type: precision_at_10 |
|
value: 42.295 |
|
- type: precision_at_100 |
|
value: 4.826 |
|
- type: precision_at_1000 |
|
value: 0.48900000000000005 |
|
- type: precision_at_3 |
|
value: 77.717 |
|
- type: precision_at_5 |
|
value: 65.81 |
|
- type: recall_at_1 |
|
value: 25.989 |
|
- type: recall_at_10 |
|
value: 89.446 |
|
- type: recall_at_100 |
|
value: 97.832 |
|
- type: recall_at_1000 |
|
value: 99.568 |
|
- type: recall_at_3 |
|
value: 58.223 |
|
- type: recall_at_5 |
|
value: 75.411 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/EcomRetrieval |
|
name: MTEB EcomRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 49.6 |
|
- type: map_at_10 |
|
value: 59.512 |
|
- type: map_at_100 |
|
value: 60.059 |
|
- type: map_at_1000 |
|
value: 60.077999999999996 |
|
- type: map_at_3 |
|
value: 56.882999999999996 |
|
- type: map_at_5 |
|
value: 58.298 |
|
- type: mrr_at_1 |
|
value: 49.6 |
|
- type: mrr_at_10 |
|
value: 59.512 |
|
- type: mrr_at_100 |
|
value: 60.059 |
|
- type: mrr_at_1000 |
|
value: 60.077999999999996 |
|
- type: mrr_at_3 |
|
value: 56.882999999999996 |
|
- type: mrr_at_5 |
|
value: 58.298 |
|
- type: ndcg_at_1 |
|
value: 49.6 |
|
- type: ndcg_at_10 |
|
value: 64.71000000000001 |
|
- type: ndcg_at_100 |
|
value: 67.238 |
|
- type: ndcg_at_1000 |
|
value: 67.74 |
|
- type: ndcg_at_3 |
|
value: 59.275 |
|
- type: ndcg_at_5 |
|
value: 61.805 |
|
- type: precision_at_1 |
|
value: 49.6 |
|
- type: precision_at_10 |
|
value: 8.12 |
|
- type: precision_at_100 |
|
value: 0.927 |
|
- type: precision_at_1000 |
|
value: 0.097 |
|
- type: precision_at_3 |
|
value: 22.067 |
|
- type: precision_at_5 |
|
value: 14.46 |
|
- type: recall_at_1 |
|
value: 49.6 |
|
- type: recall_at_10 |
|
value: 81.2 |
|
- type: recall_at_100 |
|
value: 92.7 |
|
- type: recall_at_1000 |
|
value: 96.6 |
|
- type: recall_at_3 |
|
value: 66.2 |
|
- type: recall_at_5 |
|
value: 72.3 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/IFlyTek-classification |
|
name: MTEB IFlyTek |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 47.98768757214313 |
|
- type: f1 |
|
value: 35.24243089488371 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/JDReview-classification |
|
name: MTEB JDReview |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 87.01688555347093 |
|
- type: ap |
|
value: 56.39167630414159 |
|
- type: f1 |
|
value: 81.91756262306008 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/LCQMC |
|
name: MTEB LCQMC |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 71.17874301231225 |
|
- type: cos_sim_spearman |
|
value: 77.47936067899236 |
|
- type: euclidean_pearson |
|
value: 76.3241109984839 |
|
- type: euclidean_spearman |
|
value: 77.47936511149533 |
|
- type: manhattan_pearson |
|
value: 76.3334642249198 |
|
- type: manhattan_spearman |
|
value: 77.48889610190774 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/Mmarco-reranking |
|
name: MTEB MMarcoReranking |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 27.96872431410137 |
|
- type: mrr |
|
value: 26.92023809523809 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/MMarcoRetrieval |
|
name: MTEB MMarcoRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 66.83099999999999 |
|
- type: map_at_10 |
|
value: 75.945 |
|
- type: map_at_100 |
|
value: 76.259 |
|
- type: map_at_1000 |
|
value: 76.27000000000001 |
|
- type: map_at_3 |
|
value: 74.22999999999999 |
|
- type: map_at_5 |
|
value: 75.318 |
|
- type: mrr_at_1 |
|
value: 69.069 |
|
- type: mrr_at_10 |
|
value: 76.491 |
|
- type: mrr_at_100 |
|
value: 76.764 |
|
- type: mrr_at_1000 |
|
value: 76.775 |
|
- type: mrr_at_3 |
|
value: 75.01 |
|
- type: mrr_at_5 |
|
value: 75.934 |
|
- type: ndcg_at_1 |
|
value: 69.069 |
|
- type: ndcg_at_10 |
|
value: 79.557 |
|
- type: ndcg_at_100 |
|
value: 80.946 |
|
- type: ndcg_at_1000 |
|
value: 81.23700000000001 |
|
- type: ndcg_at_3 |
|
value: 76.31099999999999 |
|
- type: ndcg_at_5 |
|
value: 78.121 |
|
- type: precision_at_1 |
|
value: 69.069 |
|
- type: precision_at_10 |
|
value: 9.58 |
|
- type: precision_at_100 |
|
value: 1.027 |
|
- type: precision_at_1000 |
|
value: 0.105 |
|
- type: precision_at_3 |
|
value: 28.73 |
|
- type: precision_at_5 |
|
value: 18.201 |
|
- type: recall_at_1 |
|
value: 66.83099999999999 |
|
- type: recall_at_10 |
|
value: 90.118 |
|
- type: recall_at_100 |
|
value: 96.377 |
|
- type: recall_at_1000 |
|
value: 98.656 |
|
- type: recall_at_3 |
|
value: 81.516 |
|
- type: recall_at_5 |
|
value: 85.798 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (zh-CN) |
|
config: zh-CN |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 68.2649630127774 |
|
- type: f1 |
|
value: 65.96868218344183 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (zh-CN) |
|
config: zh-CN |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 73.13382649630127 |
|
- type: f1 |
|
value: 72.69980239148315 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/MedicalRetrieval |
|
name: MTEB MedicalRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 51.2 |
|
- type: map_at_10 |
|
value: 57.715 |
|
- type: map_at_100 |
|
value: 58.233999999999995 |
|
- type: map_at_1000 |
|
value: 58.289 |
|
- type: map_at_3 |
|
value: 56.483000000000004 |
|
- type: map_at_5 |
|
value: 57.193000000000005 |
|
- type: mrr_at_1 |
|
value: 51.2 |
|
- type: mrr_at_10 |
|
value: 57.714 |
|
- type: mrr_at_100 |
|
value: 58.233000000000004 |
|
- type: mrr_at_1000 |
|
value: 58.288 |
|
- type: mrr_at_3 |
|
value: 56.483000000000004 |
|
- type: mrr_at_5 |
|
value: 57.193000000000005 |
|
- type: ndcg_at_1 |
|
value: 51.2 |
|
- type: ndcg_at_10 |
|
value: 60.63499999999999 |
|
- type: ndcg_at_100 |
|
value: 63.458000000000006 |
|
- type: ndcg_at_1000 |
|
value: 64.992 |
|
- type: ndcg_at_3 |
|
value: 58.11300000000001 |
|
- type: ndcg_at_5 |
|
value: 59.391000000000005 |
|
- type: precision_at_1 |
|
value: 51.2 |
|
- type: precision_at_10 |
|
value: 6.97 |
|
- type: precision_at_100 |
|
value: 0.836 |
|
- type: precision_at_1000 |
|
value: 0.096 |
|
- type: precision_at_3 |
|
value: 20.933 |
|
- type: precision_at_5 |
|
value: 13.18 |
|
- type: recall_at_1 |
|
value: 51.2 |
|
- type: recall_at_10 |
|
value: 69.69999999999999 |
|
- type: recall_at_100 |
|
value: 83.6 |
|
- type: recall_at_1000 |
|
value: 95.8 |
|
- type: recall_at_3 |
|
value: 62.8 |
|
- type: recall_at_5 |
|
value: 65.9 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/MultilingualSentiment-classification |
|
name: MTEB MultilingualSentiment |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 73.39 |
|
- type: f1 |
|
value: 72.85739851837214 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: C-MTEB/OCNLI |
|
name: MTEB Ocnli |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 73.36220898754738 |
|
- type: cos_sim_ap |
|
value: 78.50045169678386 |
|
- type: cos_sim_f1 |
|
value: 75.3875968992248 |
|
- type: cos_sim_precision |
|
value: 69.65085049239033 |
|
- type: cos_sim_recall |
|
value: 82.15417106652588 |
|
- type: dot_accuracy |
|
value: 73.36220898754738 |
|
- type: dot_ap |
|
value: 78.50039148302838 |
|
- type: dot_f1 |
|
value: 75.3875968992248 |
|
- type: dot_precision |
|
value: 69.65085049239033 |
|
- type: dot_recall |
|
value: 82.15417106652588 |
|
- type: euclidean_accuracy |
|
value: 73.36220898754738 |
|
- type: euclidean_ap |
|
value: 78.50045169678386 |
|
- type: euclidean_f1 |
|
value: 75.3875968992248 |
|
- type: euclidean_precision |
|
value: 69.65085049239033 |
|
- type: euclidean_recall |
|
value: 82.15417106652588 |
|
- type: manhattan_accuracy |
|
value: 73.09149972929075 |
|
- type: manhattan_ap |
|
value: 78.40911589236852 |
|
- type: manhattan_f1 |
|
value: 75.3623188405797 |
|
- type: manhattan_precision |
|
value: 69.45681211041853 |
|
- type: manhattan_recall |
|
value: 82.36536430834214 |
|
- type: max_accuracy |
|
value: 73.36220898754738 |
|
- type: max_ap |
|
value: 78.50045169678386 |
|
- type: max_f1 |
|
value: 75.3875968992248 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/OnlineShopping-classification |
|
name: MTEB OnlineShopping |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 91.81000000000002 |
|
- type: ap |
|
value: 89.35809579688139 |
|
- type: f1 |
|
value: 91.79220350456818 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/PAWSX |
|
name: MTEB PAWSX |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 30.06960208048424 |
|
- type: cos_sim_spearman |
|
value: 36.21568893707218 |
|
- type: euclidean_pearson |
|
value: 36.3789158810154 |
|
- type: euclidean_spearman |
|
value: 36.21568740241203 |
|
- type: manhattan_pearson |
|
value: 36.318190228955935 |
|
- type: manhattan_spearman |
|
value: 36.16813420759451 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/QBQTC |
|
name: MTEB QBQTC |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 36.779942621488736 |
|
- type: cos_sim_spearman |
|
value: 38.73716529566492 |
|
- type: euclidean_pearson |
|
value: 37.134107612179605 |
|
- type: euclidean_spearman |
|
value: 38.737099842399545 |
|
- type: manhattan_pearson |
|
value: 37.17579625045808 |
|
- type: manhattan_spearman |
|
value: 38.746051563332315 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (zh) |
|
config: zh |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 65.97416499132073 |
|
- type: cos_sim_spearman |
|
value: 68.87894646940939 |
|
- type: euclidean_pearson |
|
value: 67.2366929400408 |
|
- type: euclidean_spearman |
|
value: 68.87894646940939 |
|
- type: manhattan_pearson |
|
value: 67.30590304353478 |
|
- type: manhattan_spearman |
|
value: 68.90546655032796 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/STSB |
|
name: MTEB STSB |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 78.99420906581649 |
|
- type: cos_sim_spearman |
|
value: 79.36553449000968 |
|
- type: euclidean_pearson |
|
value: 78.77734144763518 |
|
- type: euclidean_spearman |
|
value: 79.36545230850567 |
|
- type: manhattan_pearson |
|
value: 78.82512507141092 |
|
- type: manhattan_spearman |
|
value: 79.43977311125059 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/T2Reranking |
|
name: MTEB T2Reranking |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 66.38018284846501 |
|
- type: mrr |
|
value: 76.11180965277104 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/T2Retrieval |
|
name: MTEB T2Retrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 27.423 |
|
- type: map_at_10 |
|
value: 77.206 |
|
- type: map_at_100 |
|
value: 80.83500000000001 |
|
- type: map_at_1000 |
|
value: 80.9 |
|
- type: map_at_3 |
|
value: 54.190000000000005 |
|
- type: map_at_5 |
|
value: 66.662 |
|
- type: mrr_at_1 |
|
value: 90.049 |
|
- type: mrr_at_10 |
|
value: 92.48100000000001 |
|
- type: mrr_at_100 |
|
value: 92.567 |
|
- type: mrr_at_1000 |
|
value: 92.571 |
|
- type: mrr_at_3 |
|
value: 92.07 |
|
- type: mrr_at_5 |
|
value: 92.32900000000001 |
|
- type: ndcg_at_1 |
|
value: 90.049 |
|
- type: ndcg_at_10 |
|
value: 84.69 |
|
- type: ndcg_at_100 |
|
value: 88.254 |
|
- type: ndcg_at_1000 |
|
value: 88.89399999999999 |
|
- type: ndcg_at_3 |
|
value: 86.091 |
|
- type: ndcg_at_5 |
|
value: 84.685 |
|
- type: precision_at_1 |
|
value: 90.049 |
|
- type: precision_at_10 |
|
value: 42.141 |
|
- type: precision_at_100 |
|
value: 5.016 |
|
- type: precision_at_1000 |
|
value: 0.516 |
|
- type: precision_at_3 |
|
value: 75.352 |
|
- type: precision_at_5 |
|
value: 63.176 |
|
- type: recall_at_1 |
|
value: 27.423 |
|
- type: recall_at_10 |
|
value: 83.595 |
|
- type: recall_at_100 |
|
value: 95.21 |
|
- type: recall_at_1000 |
|
value: 98.503 |
|
- type: recall_at_3 |
|
value: 55.84400000000001 |
|
- type: recall_at_5 |
|
value: 69.987 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/TNews-classification |
|
name: MTEB TNews |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 51.927 |
|
- type: f1 |
|
value: 50.16838216110367 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/ThuNewsClusteringP2P |
|
name: MTEB ThuNewsClusteringP2P |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 60.85131720842154 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/ThuNewsClusteringS2S |
|
name: MTEB ThuNewsClusteringS2S |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 57.0921610946628 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/VideoRetrieval |
|
name: MTEB VideoRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 56.99999999999999 |
|
- type: map_at_10 |
|
value: 67.611 |
|
- type: map_at_100 |
|
value: 68.095 |
|
- type: map_at_1000 |
|
value: 68.10300000000001 |
|
- type: map_at_3 |
|
value: 65.75 |
|
- type: map_at_5 |
|
value: 66.93 |
|
- type: mrr_at_1 |
|
value: 56.89999999999999 |
|
- type: mrr_at_10 |
|
value: 67.561 |
|
- type: mrr_at_100 |
|
value: 68.045 |
|
- type: mrr_at_1000 |
|
value: 68.053 |
|
- type: mrr_at_3 |
|
value: 65.7 |
|
- type: mrr_at_5 |
|
value: 66.88 |
|
- type: ndcg_at_1 |
|
value: 56.99999999999999 |
|
- type: ndcg_at_10 |
|
value: 72.25200000000001 |
|
- type: ndcg_at_100 |
|
value: 74.542 |
|
- type: ndcg_at_1000 |
|
value: 74.725 |
|
- type: ndcg_at_3 |
|
value: 68.47 |
|
- type: ndcg_at_5 |
|
value: 70.583 |
|
- type: precision_at_1 |
|
value: 56.99999999999999 |
|
- type: precision_at_10 |
|
value: 8.66 |
|
- type: precision_at_100 |
|
value: 0.972 |
|
- type: precision_at_1000 |
|
value: 0.099 |
|
- type: precision_at_3 |
|
value: 25.433 |
|
- type: precision_at_5 |
|
value: 16.28 |
|
- type: recall_at_1 |
|
value: 56.99999999999999 |
|
- type: recall_at_10 |
|
value: 86.6 |
|
- type: recall_at_100 |
|
value: 97.2 |
|
- type: recall_at_1000 |
|
value: 98.6 |
|
- type: recall_at_3 |
|
value: 76.3 |
|
- type: recall_at_5 |
|
value: 81.39999999999999 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/waimai-classification |
|
name: MTEB Waimai |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 87.10000000000001 |
|
- type: ap |
|
value: 70.81766065881429 |
|
- type: f1 |
|
value: 85.5323306120456 |
|
license: apache-2.0 |
|
language: |
|
- zh |
|
--- |
|
|
|
A try for emebdding model: |
|
|
|
The method is the same as the stella-v2, I just extend the length of the context on tao.(I found if you want to use the fully-8k context, you maybe need to convert the model to float32). |
|
|
|
Now I'm working on the tao-v2, It will have a different sturcture. |
|
|
|
I will release tao-v2 as fast as I can. |
|
|
|
Thank you to the open source community. |
