|
--- |
|
pipeline_tag: sentence-similarity |
|
tags: |
|
- sentence-transformers |
|
- feature-extraction |
|
- sentence-similarity |
|
- mteb |
|
model-index: |
|
- name: cloudy-large-zh |
|
results: |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/CMedQAv1-reranking |
|
name: MTEB CMedQAv1 |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 86.10362876754219 |
|
- type: mrr |
|
value: 88.77880952380951 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/CMedQAv2-reranking |
|
name: MTEB CMedQAv2 |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 86.94664825874587 |
|
- type: mrr |
|
value: 89.47257936507937 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/CmedqaRetrieval |
|
name: MTEB CmedqaRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 25.296999999999997 |
|
- type: map_at_10 |
|
value: 37.159 |
|
- type: map_at_100 |
|
value: 39.016 |
|
- type: map_at_1000 |
|
value: 39.134 |
|
- type: map_at_3 |
|
value: 33.248 |
|
- type: map_at_5 |
|
value: 35.371 |
|
- type: mrr_at_1 |
|
value: 38.435 |
|
- type: mrr_at_10 |
|
value: 46.235 |
|
- type: mrr_at_100 |
|
value: 47.265 |
|
- type: mrr_at_1000 |
|
value: 47.308 |
|
- type: mrr_at_3 |
|
value: 43.828 |
|
- type: mrr_at_5 |
|
value: 45.21 |
|
- type: ndcg_at_1 |
|
value: 38.435 |
|
- type: ndcg_at_10 |
|
value: 43.578 |
|
- type: ndcg_at_100 |
|
value: 50.995000000000005 |
|
- type: ndcg_at_1000 |
|
value: 53.012 |
|
- type: ndcg_at_3 |
|
value: 38.667 |
|
- type: ndcg_at_5 |
|
value: 40.657 |
|
- type: precision_at_1 |
|
value: 38.435 |
|
- type: precision_at_10 |
|
value: 9.607000000000001 |
|
- type: precision_at_100 |
|
value: 1.557 |
|
- type: precision_at_1000 |
|
value: 0.182 |
|
- type: precision_at_3 |
|
value: 21.714 |
|
- type: precision_at_5 |
|
value: 15.634 |
|
- type: recall_at_1 |
|
value: 25.296999999999997 |
|
- type: recall_at_10 |
|
value: 53.408 |
|
- type: recall_at_100 |
|
value: 84.202 |
|
- type: recall_at_1000 |
|
value: 97.61 |
|
- type: recall_at_3 |
|
value: 38.533 |
|
- type: recall_at_5 |
|
value: 44.927 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/CovidRetrieval |
|
name: MTEB CovidRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 74.763 |
|
- type: map_at_10 |
|
value: 82.604 |
|
- type: map_at_100 |
|
value: 82.795 |
|
- type: map_at_1000 |
|
value: 82.798 |
|
- type: map_at_3 |
|
value: 81.437 |
|
- type: map_at_5 |
|
value: 82.097 |
|
- type: mrr_at_1 |
|
value: 74.816 |
|
- type: mrr_at_10 |
|
value: 82.601 |
|
- type: mrr_at_100 |
|
value: 82.787 |
|
- type: mrr_at_1000 |
|
value: 82.78999999999999 |
|
- type: mrr_at_3 |
|
value: 81.472 |
|
- type: mrr_at_5 |
|
value: 82.146 |
|
- type: ndcg_at_1 |
|
value: 74.921 |
|
- type: ndcg_at_10 |
|
value: 85.83 |
|
- type: ndcg_at_100 |
|
value: 86.655 |
|
- type: ndcg_at_1000 |
|
value: 86.748 |
|
- type: ndcg_at_3 |
|
value: 83.497 |
|
- type: ndcg_at_5 |
|
value: 84.696 |
|
- type: precision_at_1 |
|
value: 74.921 |
|
- type: precision_at_10 |
|
value: 9.663 |
|
- type: precision_at_100 |
|
value: 1.0030000000000001 |
|
- type: precision_at_1000 |
|
value: 0.101 |
|
- type: precision_at_3 |
|
value: 29.996000000000002 |
|
- type: precision_at_5 |
|
value: 18.609 |
|
- type: recall_at_1 |
|
value: 74.763 |
|
- type: recall_at_10 |
|
value: 95.627 |
|
- type: recall_at_100 |
|
value: 99.262 |
|
- type: recall_at_1000 |
|
value: 100.0 |
|
- type: recall_at_3 |
|
value: 89.357 |
|
- type: recall_at_5 |
|
value: 92.255 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/DuRetrieval |
|
name: MTEB DuRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 26.021 |
|
- type: map_at_10 |
|
value: 78.561 |
|
- type: map_at_100 |
|
value: 81.291 |
|
- type: map_at_1000 |
|
value: 81.34400000000001 |
|
- type: map_at_3 |
|
value: 54.55799999999999 |
|
- type: map_at_5 |
|
value: 68.804 |
|
- type: mrr_at_1 |
|
value: 89.8 |
|
- type: mrr_at_10 |
|
value: 92.905 |
|
- type: mrr_at_100 |
|
value: 92.976 |
|
- type: mrr_at_1000 |
|
value: 92.979 |
|
- type: mrr_at_3 |
|
value: 92.608 |
|
- type: mrr_at_5 |
|
value: 92.783 |
|
- type: ndcg_at_1 |
|
value: 89.8 |
|
- type: ndcg_at_10 |
|
value: 86.203 |
|
- type: ndcg_at_100 |
|
value: 88.955 |
|
- type: ndcg_at_1000 |
|
value: 89.442 |
|
- type: ndcg_at_3 |
|
value: 85.163 |
|
- type: ndcg_at_5 |
|
value: 84.057 |
|
- type: precision_at_1 |
|
value: 89.8 |
|
- type: precision_at_10 |
|
value: 41.175 |
|
- type: precision_at_100 |
|
value: 4.744000000000001 |
|
- type: precision_at_1000 |
|
value: 0.486 |
|
- type: precision_at_3 |
|
value: 76.283 |
|
- type: precision_at_5 |
|
value: 64.41 |
|
- type: recall_at_1 |
|
value: 26.021 |
|
- type: recall_at_10 |
|
value: 87.25 |
|
- type: recall_at_100 |
|
value: 96.154 |
|
- type: recall_at_1000 |
|
value: 98.615 |
|
- type: recall_at_3 |
|
value: 56.830999999999996 |
|
- type: recall_at_5 |
|
value: 73.518 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/EcomRetrieval |
|
name: MTEB EcomRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 52.300000000000004 |
|
- type: map_at_10 |
|
value: 62.149 |
|
- type: map_at_100 |
|
value: 62.719 |
|
- type: map_at_1000 |
|
value: 62.73 |
|
- type: map_at_3 |
|
value: 59.767 |
|
- type: map_at_5 |
|
value: 61.232 |
|
- type: mrr_at_1 |
|
value: 52.300000000000004 |
|
- type: mrr_at_10 |
|
value: 62.149 |
|
- type: mrr_at_100 |
|
value: 62.719 |
|
- type: mrr_at_1000 |
|
value: 62.73 |
|
- type: mrr_at_3 |
|
value: 59.767 |
|
- type: mrr_at_5 |
|
value: 61.232 |
|
- type: ndcg_at_1 |
|
value: 52.300000000000004 |
|
- type: ndcg_at_10 |
|
value: 66.99300000000001 |
|
- type: ndcg_at_100 |
|
value: 69.672 |
|
- type: ndcg_at_1000 |
|
value: 69.95400000000001 |
|
- type: ndcg_at_3 |
|
value: 62.166 |
|
- type: ndcg_at_5 |
|
value: 64.804 |
|
- type: precision_at_1 |
|
value: 52.300000000000004 |
|
- type: precision_at_10 |
|
value: 8.219999999999999 |
|
- type: precision_at_100 |
|
value: 0.9450000000000001 |
|
- type: precision_at_1000 |
|
value: 0.097 |
|
- type: precision_at_3 |
|
value: 23.033 |
|
- type: precision_at_5 |
|
value: 15.1 |
|
- type: recall_at_1 |
|
value: 52.300000000000004 |
|
- type: recall_at_10 |
|
value: 82.19999999999999 |
|
- type: recall_at_100 |
|
value: 94.5 |
|
- type: recall_at_1000 |
|
value: 96.7 |
|
- type: recall_at_3 |
|
value: 69.1 |
|
- type: recall_at_5 |
|
value: 75.5 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/Mmarco-reranking |
|
name: MTEB MMarcoReranking |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 24.260352944026806 |
|
- type: mrr |
|
value: 22.69484126984127 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/MMarcoRetrieval |
|
name: MTEB MMarcoRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 64.888 |
|
- type: map_at_10 |
|
value: 73.921 |
|
- type: map_at_100 |
|
value: 74.28099999999999 |
|
- type: map_at_1000 |
|
value: 74.295 |
|
- type: map_at_3 |
|
value: 72.04 |
|
- type: map_at_5 |
|
value: 73.207 |
|
- type: mrr_at_1 |
|
value: 67.092 |
|
- type: mrr_at_10 |
|
value: 74.547 |
|
- type: mrr_at_100 |
|
value: 74.862 |
|
- type: mrr_at_1000 |
|
value: 74.875 |
|
- type: mrr_at_3 |
|
value: 72.908 |
|
- type: mrr_at_5 |
|
value: 73.936 |
|
- type: ndcg_at_1 |
|
value: 67.092 |
|
- type: ndcg_at_10 |
|
value: 77.687 |
|
- type: ndcg_at_100 |
|
value: 79.24600000000001 |
|
- type: ndcg_at_1000 |
|
value: 79.60000000000001 |
|
- type: ndcg_at_3 |
|
value: 74.124 |
|
- type: ndcg_at_5 |
|
value: 76.098 |
|
- type: precision_at_1 |
|
value: 67.092 |
|
- type: precision_at_10 |
|
value: 9.424000000000001 |
|
- type: precision_at_100 |
|
value: 1.019 |
|
- type: precision_at_1000 |
|
value: 0.105 |
|
- type: precision_at_3 |
|
value: 27.927000000000003 |
|
- type: precision_at_5 |
|
value: 17.797 |
|
- type: recall_at_1 |
|
value: 64.888 |
|
- type: recall_at_10 |
|
value: 88.672 |
|
- type: recall_at_100 |
|
value: 95.599 |
|
- type: recall_at_1000 |
|
value: 98.337 |
|
- type: recall_at_3 |
|
value: 79.27499999999999 |
|
- type: recall_at_5 |
|
value: 83.96000000000001 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/MedicalRetrieval |
|
name: MTEB MedicalRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 55.50000000000001 |
|
- type: map_at_10 |
|
value: 61.316 |
|
- type: map_at_100 |
|
value: 61.832 |
|
- type: map_at_1000 |
|
value: 61.867000000000004 |
|
- type: map_at_3 |
|
value: 59.9 |
|
- type: map_at_5 |
|
value: 60.685 |
|
- type: mrr_at_1 |
|
value: 55.7 |
|
- type: mrr_at_10 |
|
value: 61.416000000000004 |
|
- type: mrr_at_100 |
|
value: 61.931999999999995 |
|
- type: mrr_at_1000 |
|
value: 61.967000000000006 |
|
- type: mrr_at_3 |
|
value: 60.0 |
|
- type: mrr_at_5 |
|
value: 60.785 |
|
- type: ndcg_at_1 |
|
value: 55.50000000000001 |
|
- type: ndcg_at_10 |
|
value: 64.228 |
|
- type: ndcg_at_100 |
|
value: 67.04599999999999 |
|
- type: ndcg_at_1000 |
|
value: 68.176 |
|
- type: ndcg_at_3 |
|
value: 61.314 |
|
- type: ndcg_at_5 |
|
value: 62.743 |
|
- type: precision_at_1 |
|
value: 55.50000000000001 |
|
- type: precision_at_10 |
|
value: 7.340000000000001 |
|
- type: precision_at_100 |
|
value: 0.873 |
|
- type: precision_at_1000 |
|
value: 0.097 |
|
- type: precision_at_3 |
|
value: 21.8 |
|
- type: precision_at_5 |
|
value: 13.780000000000001 |
|
- type: recall_at_1 |
|
value: 55.50000000000001 |
|
- type: recall_at_10 |
|
value: 73.4 |
|
- type: recall_at_100 |
|
value: 87.3 |
|
- type: recall_at_1000 |
|
value: 96.6 |
|
- type: recall_at_3 |
|
value: 65.4 |
|
- type: recall_at_5 |
|
value: 68.89999999999999 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/T2Reranking |
|
name: MTEB T2Reranking |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 66.83154421352779 |
|
- type: mrr |
|
value: 76.27995669041708 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/T2Retrieval |
|
name: MTEB T2Retrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 28.303 |
|
- type: map_at_10 |
|
value: 76.943 |
|
- type: map_at_100 |
|
value: 80.585 |
|
- type: map_at_1000 |
|
value: 80.657 |
|
- type: map_at_3 |
|
value: 54.818999999999996 |
|
- type: map_at_5 |
|
value: 66.854 |
|
- type: mrr_at_1 |
|
value: 90.742 |
|
- type: mrr_at_10 |
|
value: 93.496 |
|
- type: mrr_at_100 |
|
value: 93.55799999999999 |
|
- type: mrr_at_1000 |
|
value: 93.56 |
|
- type: mrr_at_3 |
|
value: 93.083 |
|
- type: mrr_at_5 |
|
value: 93.349 |
|
- type: ndcg_at_1 |
|
value: 90.742 |
|
- type: ndcg_at_10 |
|
value: 84.94 |
|
- type: ndcg_at_100 |
|
value: 88.616 |
|
- type: ndcg_at_1000 |
|
value: 89.25 |
|
- type: ndcg_at_3 |
|
value: 86.58200000000001 |
|
- type: ndcg_at_5 |
|
value: 85.018 |
|
- type: precision_at_1 |
|
value: 90.742 |
|
- type: precision_at_10 |
|
value: 41.507 |
|
- type: precision_at_100 |
|
value: 4.984999999999999 |
|
- type: precision_at_1000 |
|
value: 0.515 |
|
- type: precision_at_3 |
|
value: 75.101 |
|
- type: precision_at_5 |
|
value: 62.543000000000006 |
|
- type: recall_at_1 |
|
value: 28.303 |
|
- type: recall_at_10 |
|
value: 83.895 |
|
- type: recall_at_100 |
|
value: 95.537 |
|
- type: recall_at_1000 |
|
value: 98.558 |
|
- type: recall_at_3 |
|
value: 56.679 |
|
- type: recall_at_5 |
|
value: 70.535 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/VideoRetrieval |
|
name: MTEB VideoRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 59.5 |
|
- type: map_at_10 |
|
value: 69.53 |
|
- type: map_at_100 |
|
value: 69.976 |
|
- type: map_at_1000 |
|
value: 69.99300000000001 |
|
- type: map_at_3 |
|
value: 67.85 |
|
- type: map_at_5 |
|
value: 68.83 |
|
- type: mrr_at_1 |
|
value: 59.5 |
|
- type: mrr_at_10 |
|
value: 69.53 |
|
- type: mrr_at_100 |
|
value: 69.976 |
|
- type: mrr_at_1000 |
|
value: 69.99300000000001 |
|
- type: mrr_at_3 |
|
value: 67.85 |
|
- type: mrr_at_5 |
|
value: 68.83 |
|
- type: ndcg_at_1 |
|
value: 59.5 |
|
- type: ndcg_at_10 |
|
value: 73.855 |
|
- type: ndcg_at_100 |
|
value: 75.831 |
|
- type: ndcg_at_1000 |
|
value: 76.227 |
|
- type: ndcg_at_3 |
|
value: 70.418 |
|
- type: ndcg_at_5 |
|
value: 72.18599999999999 |
|
- type: precision_at_1 |
|
value: 59.5 |
|
- type: precision_at_10 |
|
value: 8.72 |
|
- type: precision_at_100 |
|
value: 0.96 |
|
- type: precision_at_1000 |
|
value: 0.099 |
|
- type: precision_at_3 |
|
value: 25.933 |
|
- type: precision_at_5 |
|
value: 16.42 |
|
- type: recall_at_1 |
|
value: 59.5 |
|
- type: recall_at_10 |
|
value: 87.2 |
|
- type: recall_at_100 |
|
value: 96.0 |
|
- type: recall_at_1000 |
|
value: 99.0 |
|
- type: recall_at_3 |
|
value: 77.8 |
|
- type: recall_at_5 |
|
value: 82.1 |
|
--- |
|
|
|
| Model Name | Dimension | Sequence Length | Language | Need instruction for retrieval? | |
|
| --------------- | --------- | --------------- | -------- | ------------------------------- | |
|
| cloudy-largh-zh | 1024 | 1024 | Chinese | NO | |
