|
--- |
|
license: bigscience-bloom-rail-1.0 |
|
language: |
|
- ak |
|
- ar |
|
- as |
|
- bm |
|
- bn |
|
- ca |
|
- code |
|
- en |
|
- es |
|
- eu |
|
- fon |
|
- fr |
|
- gu |
|
- hi |
|
- id |
|
- ig |
|
- ki |
|
- kn |
|
- lg |
|
- ln |
|
- ml |
|
- mr |
|
- ne |
|
- nso |
|
- ny |
|
- or |
|
- pa |
|
- pt |
|
- rn |
|
- rw |
|
- sn |
|
- st |
|
- sw |
|
- ta |
|
- te |
|
- tn |
|
- ts |
|
- tum |
|
- tw |
|
- ur |
|
- vi |
|
- wo |
|
- xh |
|
- yo |
|
- zh |
|
- zhs |
|
- zht |
|
- zu |
|
tags: |
|
- mteb |
|
model-index: |
|
- name: udever-bloom-7b1 |
|
results: |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/AFQMC |
|
name: MTEB AFQMC |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 31.3788313486292 |
|
- type: cos_sim_spearman |
|
value: 31.87117445808444 |
|
- type: euclidean_pearson |
|
value: 30.66886666881808 |
|
- type: euclidean_spearman |
|
value: 31.28368681542041 |
|
- type: manhattan_pearson |
|
value: 30.679984531432936 |
|
- type: manhattan_spearman |
|
value: 31.22208726593753 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/ATEC |
|
name: MTEB ATEC |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 38.403248424956764 |
|
- type: cos_sim_spearman |
|
value: 38.798254852046504 |
|
- type: euclidean_pearson |
|
value: 41.154981142995084 |
|
- type: euclidean_spearman |
|
value: 38.73503172297125 |
|
- type: manhattan_pearson |
|
value: 41.20226384035751 |
|
- type: manhattan_spearman |
|
value: 38.77085234568287 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_counterfactual |
|
name: MTEB AmazonCounterfactualClassification (en) |
|
config: en |
|
split: test |
|
revision: e8379541af4e31359cca9fbcf4b00f2671dba205 |
|
metrics: |
|
- type: accuracy |
|
value: 73.11940298507463 |
|
- type: ap |
|
value: 35.692863077186466 |
|
- type: f1 |
|
value: 67.02733552778966 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_polarity |
|
name: MTEB AmazonPolarityClassification |
|
config: default |
|
split: test |
|
revision: e2d317d38cd51312af73b3d32a06d1a08b442046 |
|
metrics: |
|
- type: accuracy |
|
value: 88.885175 |
|
- type: ap |
|
value: 84.75400736514149 |
|
- type: f1 |
|
value: 88.85806225869703 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_reviews_multi |
|
name: MTEB AmazonReviewsClassification (en) |
|
config: en |
|
split: test |
|
revision: 1399c76144fd37290681b995c656ef9b2e06e26d |
|
metrics: |
|
- type: accuracy |
|
value: 43.202 |
|
- type: f1 |
|
value: 42.63847450850621 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: arguana |
|
name: MTEB ArguAna |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 25.676 |
|
- type: map_at_10 |
|
value: 42.539 |
|
- type: map_at_100 |
|
value: 43.383 |
|
- type: map_at_1000 |
|
value: 43.39 |
|
- type: map_at_3 |
|
value: 36.996 |
|
- type: map_at_5 |
|
value: 40.175 |
|
- type: mrr_at_1 |
|
value: 26.387 |
|
- type: mrr_at_10 |
|
value: 42.792 |
|
- type: mrr_at_100 |
|
value: 43.637 |
|
- type: mrr_at_1000 |
|
value: 43.644 |
|
- type: mrr_at_3 |
|
value: 37.21 |
|
- type: mrr_at_5 |
|
value: 40.407 |
|
- type: ndcg_at_1 |
|
value: 25.676 |
|
- type: ndcg_at_10 |
|
value: 52.207 |
|
- type: ndcg_at_100 |
|
value: 55.757999999999996 |
|
- type: ndcg_at_1000 |
|
value: 55.913999999999994 |
|
- type: ndcg_at_3 |
|
value: 40.853 |
|
- type: ndcg_at_5 |
|
value: 46.588 |
|
- type: precision_at_1 |
|
value: 25.676 |
|
- type: precision_at_10 |
|
value: 8.314 |
|
- type: precision_at_100 |
|
value: 0.985 |
|
- type: precision_at_1000 |
|
value: 0.1 |
|
- type: precision_at_3 |
|
value: 17.354 |
|
- type: precision_at_5 |
|
value: 13.200999999999999 |
|
- type: recall_at_1 |
|
value: 25.676 |
|
- type: recall_at_10 |
|
value: 83.14399999999999 |
|
- type: recall_at_100 |
|
value: 98.506 |
|
- type: recall_at_1000 |
|
value: 99.644 |
|
- type: recall_at_3 |
|
value: 52.063 |
|
- type: recall_at_5 |
|
value: 66.003 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/arxiv-clustering-p2p |
|
name: MTEB ArxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d |
|
metrics: |
|
- type: v_measure |
|
value: 45.66024127046263 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/arxiv-clustering-s2s |
|
name: MTEB ArxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 |
|
metrics: |
|
- type: v_measure |
|
value: 38.418361433667336 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/askubuntudupquestions-reranking |
|
name: MTEB AskUbuntuDupQuestions |
|
config: default |
|
split: test |
|
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 |
|
metrics: |
|
- type: map |
|
value: 61.60189642383972 |
|
- type: mrr |
|
value: 75.26678538451391 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/biosses-sts |
|
name: MTEB BIOSSES |
|
config: default |
|
split: test |
|
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 87.85884182572595 |
|
- type: cos_sim_spearman |
|
value: 85.5242378844044 |
|
- type: euclidean_pearson |
|
value: 85.37705073557146 |
|
- type: euclidean_spearman |
|
value: 84.65132642825964 |
|
- type: manhattan_pearson |
|
value: 85.42179213807349 |
|
- type: manhattan_spearman |
|
value: 84.6959057572829 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/BQ |
|
name: MTEB BQ |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 47.81802155652125 |
|
- type: cos_sim_spearman |
|
value: 47.66691834501235 |
|
- type: euclidean_pearson |
|
value: 47.781824357030935 |
|
- type: euclidean_spearman |
|
value: 48.03322284408188 |
|
- type: manhattan_pearson |
|
value: 47.871159981038346 |
|
- type: manhattan_spearman |
|
value: 48.18240784527666 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/bucc-bitext-mining |
|
name: MTEB BUCC (de-en) |
|
config: de-en |
|
split: test |
|
revision: d51519689f32196a32af33b075a01d0e7c51e252 |
|
metrics: |
|
- type: accuracy |
|
value: 88.29853862212944 |
|
- type: f1 |
|
value: 87.70994966904566 |
|
- type: precision |
|
value: 87.43152897902377 |
|
- type: recall |
|
value: 88.29853862212944 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/bucc-bitext-mining |
|
name: MTEB BUCC (fr-en) |
|
config: fr-en |
|
split: test |
|
revision: d51519689f32196a32af33b075a01d0e7c51e252 |
|
metrics: |
|
- type: accuracy |
|
value: 98.6022452124147 |
|
- type: f1 |
|
value: 98.40597255851495 |
|
- type: precision |
|
value: 98.30875339349916 |
|
- type: recall |
|
value: 98.6022452124147 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/bucc-bitext-mining |
|
name: MTEB BUCC (ru-en) |
|
config: ru-en |
|
split: test |
|
revision: d51519689f32196a32af33b075a01d0e7c51e252 |
|
metrics: |
|
- type: accuracy |
|
value: 79.64669206789054 |
|
- type: f1 |
|
value: 78.74831345770036 |
|
- type: precision |
|
value: 78.33899087865143 |
|
- type: recall |
|
value: 79.64669206789054 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/bucc-bitext-mining |
|
name: MTEB BUCC (zh-en) |
|
config: zh-en |
|
split: test |
|
revision: d51519689f32196a32af33b075a01d0e7c51e252 |
|
metrics: |
|
- type: accuracy |
|
value: 98.78883622959452 |
|
- type: f1 |
|
value: 98.7712831314727 |
|
- type: precision |
|
value: 98.76250658241179 |
|
- type: recall |
|
value: 98.78883622959452 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/banking77 |
|
name: MTEB Banking77Classification |
|
config: default |
|
split: test |
|
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 |
|
metrics: |
|
- type: accuracy |
|
value: 85.36363636363637 |
|
- type: f1 |
|
value: 85.33381612267455 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/biorxiv-clustering-p2p |
|
name: MTEB BiorxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 |
|
metrics: |
|
- type: v_measure |
|
value: 35.54276849354455 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/biorxiv-clustering-s2s |
|
name: MTEB BiorxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 |
|
metrics: |
|
- type: v_measure |
|
value: 32.18953191097238 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/CLSClusteringP2P |
|
name: MTEB CLSClusteringP2P |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 36.00041315364012 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/CLSClusteringS2S |
|
name: MTEB CLSClusteringS2S |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 36.35255790689628 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/CMedQAv1-reranking |
|
name: MTEB CMedQAv1 |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 70.54141681949504 |
|
- type: mrr |
|
value: 74.81400793650795 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/CMedQAv2-reranking |
|
name: MTEB CMedQAv2 |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 71.3534829537025 |
|
- type: mrr |
|
value: 75.85095238095238 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackAndroidRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 32.5 |
|
- type: map_at_10 |
|
value: 43.37 |
|
- type: map_at_100 |
|
value: 44.926 |
|
- type: map_at_1000 |
|
value: 45.047 |
|
- type: map_at_3 |
|
value: 40.083999999999996 |
|
- type: map_at_5 |
|
value: 41.71 |
|
- type: mrr_at_1 |
|
value: 40.343 |
|
- type: mrr_at_10 |
|
value: 49.706 |
|
- type: mrr_at_100 |
|
value: 50.470000000000006 |
|
- type: mrr_at_1000 |
|
value: 50.515 |
|
- type: mrr_at_3 |
|
value: 47.306 |
|
- type: mrr_at_5 |
|
value: 48.379 |
|
- type: ndcg_at_1 |
|
value: 40.343 |
|
- type: ndcg_at_10 |
|
value: 49.461 |
|
- type: ndcg_at_100 |
|
value: 55.084999999999994 |
|
- type: ndcg_at_1000 |
|
value: 56.994 |
|
- type: ndcg_at_3 |
|
value: 44.896 |
|
- type: ndcg_at_5 |
|
value: 46.437 |
|
- type: precision_at_1 |
|
value: 40.343 |
|
- type: precision_at_10 |
|
value: 9.27 |
|
- type: precision_at_100 |
|
value: 1.5190000000000001 |
|
- type: precision_at_1000 |
|
value: 0.197 |
|
- type: precision_at_3 |
|
value: 21.412 |
|
- type: precision_at_5 |
|
value: 15.021 |
|
- type: recall_at_1 |
|
value: 32.5 |
|
- type: recall_at_10 |
|
value: 60.857000000000006 |
|
- type: recall_at_100 |
|
value: 83.761 |
|
- type: recall_at_1000 |
|
value: 96.003 |
|
- type: recall_at_3 |
|
value: 46.675 |
|
- type: recall_at_5 |
|
value: 51.50900000000001 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackEnglishRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 26.931 |
|
- type: map_at_10 |
|
value: 35.769 |
|
- type: map_at_100 |
|
value: 36.8 |
|
- type: map_at_1000 |
|
value: 36.925999999999995 |
|
- type: map_at_3 |
|
value: 33.068999999999996 |
|
- type: map_at_5 |
|
value: 34.615 |
|
- type: mrr_at_1 |
|
value: 34.013 |
|
- type: mrr_at_10 |
|
value: 41.293 |
|
- type: mrr_at_100 |
|
value: 41.945 |
|
- type: mrr_at_1000 |
|
value: 42.002 |
|
- type: mrr_at_3 |
|
value: 39.204 |
|
- type: mrr_at_5 |
|
value: 40.436 |
|
- type: ndcg_at_1 |
|
value: 34.013 |
|
- type: ndcg_at_10 |
|
value: 40.935 |
|
- type: ndcg_at_100 |
|
value: 44.879999999999995 |
|
- type: ndcg_at_1000 |
|
value: 47.342 |
|
- type: ndcg_at_3 |
|
value: 37.071 |
|
- type: ndcg_at_5 |
|
value: 38.903 |
|
- type: precision_at_1 |
|
value: 34.013 |
|
- type: precision_at_10 |
|
value: 7.617999999999999 |
|
- type: precision_at_100 |
|
value: 1.185 |
|
- type: precision_at_1000 |
|
value: 0.169 |
|
- type: precision_at_3 |
|
value: 17.855999999999998 |
|
- type: precision_at_5 |
|
value: 12.65 |
|
- type: recall_at_1 |
|
value: 26.931 |
|
- type: recall_at_10 |
|
value: 50.256 |
|
- type: recall_at_100 |
|
value: 67.026 |
|
- type: recall_at_1000 |
|
value: 83.138 |
|
- type: recall_at_3 |
|
value: 38.477 |
|
- type: recall_at_5 |
|
value: 43.784 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackGamingRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 38.474000000000004 |
|
- type: map_at_10 |
|
value: 50.486 |
|
- type: map_at_100 |
|
value: 51.620999999999995 |
|
- type: map_at_1000 |
|
value: 51.675000000000004 |
|
- type: map_at_3 |
|
value: 47.64 |
|
- type: map_at_5 |
|
value: 49.187999999999995 |
|
- type: mrr_at_1 |
|
value: 43.824000000000005 |
|
- type: mrr_at_10 |
|
value: 53.910000000000004 |
|
- type: mrr_at_100 |
|
value: 54.601 |
|
- type: mrr_at_1000 |
|
value: 54.632000000000005 |
|
- type: mrr_at_3 |
|
value: 51.578 |
|
- type: mrr_at_5 |
|
value: 52.922999999999995 |
|
- type: ndcg_at_1 |
|
value: 43.824000000000005 |
|
- type: ndcg_at_10 |
|
value: 56.208000000000006 |
|
- type: ndcg_at_100 |
|
value: 60.624 |
|
- type: ndcg_at_1000 |
|
value: 61.78 |
|
- type: ndcg_at_3 |
|
value: 51.27 |
|
- type: ndcg_at_5 |
|
value: 53.578 |
|
- type: precision_at_1 |
|
value: 43.824000000000005 |
|
- type: precision_at_10 |
|
value: 8.978 |
|
- type: precision_at_100 |
|
value: 1.216 |
|
- type: precision_at_1000 |
|
value: 0.136 |
|
- type: precision_at_3 |
|
value: 22.884 |
|
- type: precision_at_5 |
|
value: 15.498000000000001 |
|
- type: recall_at_1 |
|
value: 38.474000000000004 |
|
- type: recall_at_10 |
|
value: 69.636 |
|
- type: recall_at_100 |
|
value: 88.563 |
|
- type: recall_at_1000 |
|
value: 96.86200000000001 |
|
- type: recall_at_3 |
|
value: 56.347 |
|
- type: recall_at_5 |
|
value: 61.980000000000004 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackGisRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 23.13 |
|
- type: map_at_10 |
|
value: 31.892 |
|
- type: map_at_100 |
|
value: 32.938 |
|
- type: map_at_1000 |
|
value: 33.025999999999996 |
|
- type: map_at_3 |
|
value: 29.072 |
|
- type: map_at_5 |
|
value: 30.775000000000002 |
|
- type: mrr_at_1 |
|
value: 25.197999999999997 |
|
- type: mrr_at_10 |
|
value: 34.224 |
|
- type: mrr_at_100 |
|
value: 35.149 |
|
- type: mrr_at_1000 |
|
value: 35.215999999999994 |
|
- type: mrr_at_3 |
|
value: 31.563000000000002 |
|
- type: mrr_at_5 |
|
value: 33.196 |
|
- type: ndcg_at_1 |
|
value: 25.197999999999997 |
|
- type: ndcg_at_10 |
|
value: 37.117 |
|
- type: ndcg_at_100 |
|
value: 42.244 |
|
- type: ndcg_at_1000 |
|
value: 44.432 |
|
- type: ndcg_at_3 |
|
value: 31.604 |
|
- type: ndcg_at_5 |
|
value: 34.543 |
|
- type: precision_at_1 |
|
value: 25.197999999999997 |
|
- type: precision_at_10 |
|
value: 5.876 |
|
- type: precision_at_100 |
|
value: 0.886 |
|
- type: precision_at_1000 |
|
value: 0.11100000000000002 |
|
- type: precision_at_3 |
|
value: 13.672 |
|
- type: precision_at_5 |
|
value: 9.831 |
|
- type: recall_at_1 |
|
value: 23.13 |
|
- type: recall_at_10 |
|
value: 50.980000000000004 |
|
- type: recall_at_100 |
|
value: 74.565 |
|
- type: recall_at_1000 |
|
value: 90.938 |
|
- type: recall_at_3 |
|
value: 36.038 |
|
- type: recall_at_5 |
|
value: 43.326 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackMathematicaRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 17.317 |
|
- type: map_at_10 |
|
value: 24.517 |
|
- type: map_at_100 |
|
value: 25.771 |
|
- type: map_at_1000 |
|
value: 25.915 |
|
- type: map_at_3 |
|
value: 22.332 |
|
- type: map_at_5 |
|
value: 23.526 |
|
- type: mrr_at_1 |
|
value: 21.766 |
|
- type: mrr_at_10 |
|
value: 29.096 |
|
- type: mrr_at_100 |
|
value: 30.165 |
|
- type: mrr_at_1000 |
|
value: 30.253000000000004 |
|
- type: mrr_at_3 |
|
value: 27.114 |
|
- type: mrr_at_5 |
|
value: 28.284 |
|
- type: ndcg_at_1 |
|
value: 21.766 |
|
- type: ndcg_at_10 |
|
value: 29.060999999999996 |
|
- type: ndcg_at_100 |
|
value: 35.107 |
|
- type: ndcg_at_1000 |
|
value: 38.339 |
|
- type: ndcg_at_3 |
|
value: 25.121 |
|
- type: ndcg_at_5 |
|
value: 26.953 |
|
- type: precision_at_1 |
|
value: 21.766 |
|
- type: precision_at_10 |
|
value: 5.274 |
|
- type: precision_at_100 |
|
value: 0.958 |
|
- type: precision_at_1000 |
|
value: 0.13699999999999998 |
|
- type: precision_at_3 |
|
value: 11.816 |
|
- type: precision_at_5 |
|
value: 8.433 |
|
- type: recall_at_1 |
|
value: 17.317 |
|
- type: recall_at_10 |
|
value: 38.379999999999995 |
|
- type: recall_at_100 |
|
value: 64.792 |
|
- type: recall_at_1000 |
|
value: 87.564 |
|
- type: recall_at_3 |
|
value: 27.737000000000002 |
|
- type: recall_at_5 |
|
value: 32.340999999999994 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackPhysicsRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 28.876 |
|
- type: map_at_10 |
|
value: 40.02 |
|
- type: map_at_100 |
|
value: 41.367 |
|
- type: map_at_1000 |
|
value: 41.482 |
|
- type: map_at_3 |
|
value: 36.651 |
|
- type: map_at_5 |
|
value: 38.411 |
|
- type: mrr_at_1 |
|
value: 35.804 |
|
- type: mrr_at_10 |
|
value: 45.946999999999996 |
|
- type: mrr_at_100 |
|
value: 46.696 |
|
- type: mrr_at_1000 |
|
value: 46.741 |
|
- type: mrr_at_3 |
|
value: 43.118 |
|
- type: mrr_at_5 |
|
value: 44.74 |
|
- type: ndcg_at_1 |
|
value: 35.804 |
|
- type: ndcg_at_10 |
|
value: 46.491 |
|
- type: ndcg_at_100 |
|
value: 51.803 |
|
- type: ndcg_at_1000 |
|
value: 53.845 |
|
- type: ndcg_at_3 |
|
value: 40.97 |
|
- type: ndcg_at_5 |
|
value: 43.431 |
|
- type: precision_at_1 |
|
value: 35.804 |
|
- type: precision_at_10 |
|
value: 8.595 |
|
- type: precision_at_100 |
|
value: 1.312 |
|
- type: precision_at_1000 |
|
value: 0.167 |
|
- type: precision_at_3 |
|
value: 19.634 |
|
- type: precision_at_5 |
|
value: 13.879 |
|
- type: recall_at_1 |
|
value: 28.876 |
|
- type: recall_at_10 |
|
value: 59.952000000000005 |
|
- type: recall_at_100 |
|
value: 81.978 |
|
- type: recall_at_1000 |
|
value: 95.03399999999999 |
|
- type: recall_at_3 |
|
value: 44.284 |
|
- type: recall_at_5 |
|
value: 50.885999999999996 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackProgrammersRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 24.238 |
|
- type: map_at_10 |
|
value: 34.276 |
|
- type: map_at_100 |
|
value: 35.65 |
|
- type: map_at_1000 |
|
value: 35.769 |
|
- type: map_at_3 |
|
value: 31.227 |
|
- type: map_at_5 |
|
value: 33.046 |
|
- type: mrr_at_1 |
|
value: 30.137000000000004 |
|
- type: mrr_at_10 |
|
value: 39.473 |
|
- type: mrr_at_100 |
|
value: 40.400999999999996 |
|
- type: mrr_at_1000 |
|
value: 40.455000000000005 |
|
- type: mrr_at_3 |
|
value: 36.891 |
|
- type: mrr_at_5 |
|
value: 38.391999999999996 |
|
- type: ndcg_at_1 |
|
value: 30.137000000000004 |
|
- type: ndcg_at_10 |
|
value: 40.08 |
|
- type: ndcg_at_100 |
|
value: 46.01 |
|
- type: ndcg_at_1000 |
|
value: 48.36 |
|
- type: ndcg_at_3 |
|
value: 35.163 |
|
- type: ndcg_at_5 |
|
value: 37.583 |
|
- type: precision_at_1 |
|
value: 30.137000000000004 |
|
- type: precision_at_10 |
|
value: 7.466 |
|
- type: precision_at_100 |
|
value: 1.228 |
|
- type: precision_at_1000 |
|
value: 0.16199999999999998 |
|
- type: precision_at_3 |
|
value: 17.122999999999998 |
|
- type: precision_at_5 |
|
value: 12.283 |
|
- type: recall_at_1 |
|
value: 24.238 |
|
- type: recall_at_10 |
|
value: 52.078 |
|
- type: recall_at_100 |
|
value: 77.643 |
|
- type: recall_at_1000 |
|
value: 93.49199999999999 |
|
- type: recall_at_3 |
|
value: 38.161 |
|
- type: recall_at_5 |
|
value: 44.781 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 24.915250000000004 |
|
- type: map_at_10 |
|
value: 33.98191666666666 |
|
- type: map_at_100 |
|
value: 35.19166666666667 |
|
- type: map_at_1000 |
|
value: 35.30983333333333 |
|
- type: map_at_3 |
|
value: 31.27391666666666 |
|
- type: map_at_5 |
|
value: 32.74366666666666 |
|
- type: mrr_at_1 |
|
value: 29.800749999999994 |
|
- type: mrr_at_10 |
|
value: 38.235749999999996 |
|
- type: mrr_at_100 |
|
value: 39.10616666666667 |
|
- type: mrr_at_1000 |
|
value: 39.166583333333335 |
|
- type: mrr_at_3 |
|
value: 35.91033333333334 |
|
- type: mrr_at_5 |
|
value: 37.17766666666667 |
|
- type: ndcg_at_1 |
|
value: 29.800749999999994 |
|
- type: ndcg_at_10 |
|
value: 39.287833333333325 |
|
- type: ndcg_at_100 |
|
value: 44.533833333333334 |
|
- type: ndcg_at_1000 |
|
value: 46.89608333333333 |
|
- type: ndcg_at_3 |
|
value: 34.676 |
|
- type: ndcg_at_5 |
|
value: 36.75208333333333 |
|
- type: precision_at_1 |
|
value: 29.800749999999994 |
|
- type: precision_at_10 |
|
value: 6.9134166666666665 |
|
- type: precision_at_100 |
|
value: 1.1206666666666665 |
|
- type: precision_at_1000 |
|
value: 0.15116666666666667 |
|
- type: precision_at_3 |
|
value: 16.069083333333335 |
|
- type: precision_at_5 |
|
value: 11.337916666666668 |
|
- type: recall_at_1 |
|
value: 24.915250000000004 |
|
- type: recall_at_10 |
|
value: 50.86333333333334 |
|
- type: recall_at_100 |
|
value: 73.85574999999999 |
|
- type: recall_at_1000 |
|
value: 90.24041666666666 |
|
- type: recall_at_3 |
|
value: 37.80116666666666 |
|
- type: recall_at_5 |
|
value: 43.263 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackStatsRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 22.853 |
|
- type: map_at_10 |
|
value: 30.349999999999998 |
|
- type: map_at_100 |
|
value: 31.341 |
|
- type: map_at_1000 |
|
value: 31.44 |
|
- type: map_at_3 |
|
value: 28.294999999999998 |
|
- type: map_at_5 |
|
value: 29.412 |
|
- type: mrr_at_1 |
|
value: 25.919999999999998 |
|
- type: mrr_at_10 |
|
value: 33.194 |
|
- type: mrr_at_100 |
|
value: 34.071 |
|
- type: mrr_at_1000 |
|
value: 34.136 |
|
- type: mrr_at_3 |
|
value: 31.391000000000002 |
|
- type: mrr_at_5 |
|
value: 32.311 |
|
- type: ndcg_at_1 |
|
value: 25.919999999999998 |
|
- type: ndcg_at_10 |
|
value: 34.691 |
|
- type: ndcg_at_100 |
|
value: 39.83 |
|
- type: ndcg_at_1000 |
|
value: 42.193000000000005 |
|
- type: ndcg_at_3 |
|
value: 30.91 |
|
- type: ndcg_at_5 |
|
value: 32.634 |
|
- type: precision_at_1 |
|
value: 25.919999999999998 |
|
- type: precision_at_10 |
|
value: 5.521 |
|
- type: precision_at_100 |
|
value: 0.882 |
|
- type: precision_at_1000 |
|
value: 0.117 |
|
- type: precision_at_3 |
|
value: 13.547999999999998 |
|
- type: precision_at_5 |
|
value: 9.293999999999999 |
|
- type: recall_at_1 |
|
value: 22.853 |
|
- type: recall_at_10 |
|
value: 45.145 |
|
- type: recall_at_100 |
|
value: 69.158 |
|
- type: recall_at_1000 |
|
value: 86.354 |
|
- type: recall_at_3 |
|
value: 34.466 |
|
- type: recall_at_5 |
|
value: 39.044000000000004 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackTexRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 17.151 |
|
- type: map_at_10 |
|
value: 23.674 |
|
- type: map_at_100 |
|
value: 24.738 |
|
- type: map_at_1000 |
|
value: 24.864 |
|
- type: map_at_3 |
|
value: 21.514 |
|
- type: map_at_5 |
|
value: 22.695 |
|
- type: mrr_at_1 |
|
value: 20.991 |
|
- type: mrr_at_10 |
|
value: 27.612 |
|
- type: mrr_at_100 |
|
value: 28.526 |
|
- type: mrr_at_1000 |
|
value: 28.603 |
|
- type: mrr_at_3 |
|
value: 25.618999999999996 |
|
- type: mrr_at_5 |
|
value: 26.674 |
|
- type: ndcg_at_1 |
|
value: 20.991 |
|
- type: ndcg_at_10 |
|
value: 27.983000000000004 |
|
- type: ndcg_at_100 |
|
value: 33.190999999999995 |
|
- type: ndcg_at_1000 |
|
value: 36.172 |
|
- type: ndcg_at_3 |
|
value: 24.195 |
|
- type: ndcg_at_5 |
|
value: 25.863999999999997 |
|
- type: precision_at_1 |
|
value: 20.991 |
|
- type: precision_at_10 |
|
value: 5.093 |
|
- type: precision_at_100 |
|
value: 0.8959999999999999 |
|
- type: precision_at_1000 |
|
value: 0.132 |
|
- type: precision_at_3 |
|
value: 11.402 |
|
- type: precision_at_5 |
|
value: 8.197000000000001 |
|
- type: recall_at_1 |
|
value: 17.151 |
|
- type: recall_at_10 |
|
value: 37.025000000000006 |
|
- type: recall_at_100 |
|
value: 60.787 |
|
- type: recall_at_1000 |
|
value: 82.202 |
|
- type: recall_at_3 |
|
value: 26.19 |
|
- type: recall_at_5 |
|
value: 30.657 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackUnixRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 25.463 |
|
- type: map_at_10 |
|
value: 34.372 |
|
- type: map_at_100 |
|
value: 35.475 |
|
- type: map_at_1000 |
|
value: 35.582 |
|
- type: map_at_3 |
|
value: 31.791000000000004 |
|
- type: map_at_5 |
|
value: 33.292 |
|
- type: mrr_at_1 |
|
value: 30.784 |
|
- type: mrr_at_10 |
|
value: 38.948 |
|
- type: mrr_at_100 |
|
value: 39.792 |
|
- type: mrr_at_1000 |
|
value: 39.857 |
|
- type: mrr_at_3 |
|
value: 36.614000000000004 |
|
- type: mrr_at_5 |
|
value: 37.976 |
|
- type: ndcg_at_1 |
|
value: 30.784 |
|
- type: ndcg_at_10 |
|
value: 39.631 |
|
- type: ndcg_at_100 |
|
value: 44.747 |
|
- type: ndcg_at_1000 |
|
value: 47.172 |
|
- type: ndcg_at_3 |
|
value: 34.976 |
|
- type: ndcg_at_5 |
|
value: 37.241 |
|
- type: precision_at_1 |
|
value: 30.784 |
|
- type: precision_at_10 |
|
value: 6.622999999999999 |
|
- type: precision_at_100 |
|
value: 1.04 |
|
- type: precision_at_1000 |
|
value: 0.135 |
|
- type: precision_at_3 |
|
value: 16.014 |
|
- type: precision_at_5 |
|
value: 11.286999999999999 |
|
- type: recall_at_1 |
|
value: 25.463 |
|
- type: recall_at_10 |
|
value: 51.23799999999999 |
|
- type: recall_at_100 |
|
value: 73.4 |
|
- type: recall_at_1000 |
|
value: 90.634 |
|
- type: recall_at_3 |
|
value: 38.421 |
|
- type: recall_at_5 |
|
value: 44.202999999999996 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackWebmastersRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 23.714 |
|
- type: map_at_10 |
|
value: 32.712 |
|
- type: map_at_100 |
|
value: 34.337 |
|
- type: map_at_1000 |
|
value: 34.556 |
|
- type: map_at_3 |
|
value: 29.747 |
|
- type: map_at_5 |
|
value: 31.208000000000002 |
|
- type: mrr_at_1 |
|
value: 29.051 |
|
- type: mrr_at_10 |
|
value: 37.589 |
|
- type: mrr_at_100 |
|
value: 38.638 |
|
- type: mrr_at_1000 |
|
value: 38.692 |
|
- type: mrr_at_3 |
|
value: 35.079 |
|
- type: mrr_at_5 |
|
value: 36.265 |
|
- type: ndcg_at_1 |
|
value: 29.051 |
|
- type: ndcg_at_10 |
|
value: 38.681 |
|
- type: ndcg_at_100 |
|
value: 44.775999999999996 |
|
- type: ndcg_at_1000 |
|
value: 47.354 |
|
- type: ndcg_at_3 |
|
value: 33.888 |
|
- type: ndcg_at_5 |
|
value: 35.854 |
|
- type: precision_at_1 |
|
value: 29.051 |
|
- type: precision_at_10 |
|
value: 7.489999999999999 |
|
- type: precision_at_100 |
|
value: 1.518 |
|
- type: precision_at_1000 |
|
value: 0.241 |
|
- type: precision_at_3 |
|
value: 16.008 |
|
- type: precision_at_5 |
|
value: 11.66 |
|
- type: recall_at_1 |
|
value: 23.714 |
|
- type: recall_at_10 |
|
value: 50.324000000000005 |
|
- type: recall_at_100 |
|
value: 77.16 |
|
- type: recall_at_1000 |
|
value: 93.186 |
|
- type: recall_at_3 |
|
value: 36.356 |
|
- type: recall_at_5 |
|
value: 41.457 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackWordpressRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 18.336 |
|
- type: map_at_10 |
|
value: 26.345000000000002 |
|
- type: map_at_100 |
|
value: 27.336 |
|
- type: map_at_1000 |
|
value: 27.436 |
|
- type: map_at_3 |
|
value: 23.865 |
|
- type: map_at_5 |
|
value: 25.046000000000003 |
|
- type: mrr_at_1 |
|
value: 19.778000000000002 |
|
- type: mrr_at_10 |
|
value: 27.837 |
|
- type: mrr_at_100 |
|
value: 28.82 |
|
- type: mrr_at_1000 |
|
value: 28.897000000000002 |
|
- type: mrr_at_3 |
|
value: 25.446999999999996 |
|
- type: mrr_at_5 |
|
value: 26.556 |
|
- type: ndcg_at_1 |
|
value: 19.778000000000002 |
|
- type: ndcg_at_10 |
|
value: 31.115 |
|
- type: ndcg_at_100 |
|
value: 36.109 |
|
- type: ndcg_at_1000 |
|
value: 38.769999999999996 |
|
- type: ndcg_at_3 |
|
value: 26.048 |
|
- type: ndcg_at_5 |
|
value: 28.004 |
|
- type: precision_at_1 |
|
value: 19.778000000000002 |
|
- type: precision_at_10 |
|
value: 5.157 |
|
- type: precision_at_100 |
|
value: 0.808 |
|
- type: precision_at_1000 |
|
value: 0.11 |
|
- type: precision_at_3 |
|
value: 11.459999999999999 |
|
- type: precision_at_5 |
|
value: 8.022 |
|
- type: recall_at_1 |
|
value: 18.336 |
|
- type: recall_at_10 |
|
value: 44.489000000000004 |
|
- type: recall_at_100 |
|
value: 67.43599999999999 |
|
- type: recall_at_1000 |
|
value: 87.478 |
|
- type: recall_at_3 |
|
value: 30.462 |
|
- type: recall_at_5 |
|
value: 35.188 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: climate-fever |
|
name: MTEB ClimateFEVER |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 10.747 |
|
- type: map_at_10 |
|
value: 18.625 |
|
- type: map_at_100 |
|
value: 20.465 |
|
- type: map_at_1000 |
|
value: 20.639 |
|
- type: map_at_3 |
|
value: 15.57 |
|
- type: map_at_5 |
|
value: 17.089 |
|
- type: mrr_at_1 |
|
value: 24.169 |
|
- type: mrr_at_10 |
|
value: 35.96 |
|
- type: mrr_at_100 |
|
value: 36.888 |
|
- type: mrr_at_1000 |
|
value: 36.931999999999995 |
|
- type: mrr_at_3 |
|
value: 32.443 |
|
- type: mrr_at_5 |
|
value: 34.433 |
|
- type: ndcg_at_1 |
|
value: 24.169 |
|
- type: ndcg_at_10 |
|
value: 26.791999999999998 |
|
- type: ndcg_at_100 |
|
value: 34.054 |
|
- type: ndcg_at_1000 |
|
value: 37.285000000000004 |
|
- type: ndcg_at_3 |
|
value: 21.636 |
|
- type: ndcg_at_5 |
|
value: 23.394000000000002 |
|
- type: precision_at_1 |
|
value: 24.169 |
|
- type: precision_at_10 |
|
value: 8.476 |
|
- type: precision_at_100 |
|
value: 1.6209999999999998 |
|
- type: precision_at_1000 |
|
value: 0.22200000000000003 |
|
- type: precision_at_3 |
|
value: 16.156000000000002 |
|
- type: precision_at_5 |
|
value: 12.520999999999999 |
|
- type: recall_at_1 |
|
value: 10.747 |
|
- type: recall_at_10 |
|
value: 32.969 |
|
- type: recall_at_100 |
|
value: 57.99999999999999 |
|
- type: recall_at_1000 |
|
value: 76.12299999999999 |
|
- type: recall_at_3 |
|
value: 20.315 |
|
- type: recall_at_5 |
|
value: 25.239 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/CmedqaRetrieval |
|
name: MTEB CmedqaRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 14.751 |
|
- type: map_at_10 |
|
value: 22.03 |
|
- type: map_at_100 |
|
value: 23.471 |
|
- type: map_at_1000 |
|
value: 23.644000000000002 |
|
- type: map_at_3 |
|
value: 19.559 |
|
- type: map_at_5 |
|
value: 20.863 |
|
- type: mrr_at_1 |
|
value: 23.581 |
|
- type: mrr_at_10 |
|
value: 29.863 |
|
- type: mrr_at_100 |
|
value: 30.839 |
|
- type: mrr_at_1000 |
|
value: 30.925000000000004 |
|
- type: mrr_at_3 |
|
value: 27.894000000000002 |
|
- type: mrr_at_5 |
|
value: 28.965999999999998 |
|
- type: ndcg_at_1 |
|
value: 23.581 |
|
- type: ndcg_at_10 |
|
value: 26.996 |
|
- type: ndcg_at_100 |
|
value: 33.537 |
|
- type: ndcg_at_1000 |
|
value: 37.307 |
|
- type: ndcg_at_3 |
|
value: 23.559 |
|
- type: ndcg_at_5 |
|
value: 24.839 |
|
- type: precision_at_1 |
|
value: 23.581 |
|
- type: precision_at_10 |
|
value: 6.209 |
|
- type: precision_at_100 |
|
value: 1.165 |
|
- type: precision_at_1000 |
|
value: 0.165 |
|
- type: precision_at_3 |
|
value: 13.62 |
|
- type: precision_at_5 |
|
value: 9.882 |
|
- type: recall_at_1 |
|
value: 14.751 |
|
- type: recall_at_10 |
|
value: 34.075 |
|
- type: recall_at_100 |
|
value: 61.877 |
|
- type: recall_at_1000 |
|
value: 88.212 |
|
- type: recall_at_3 |
|
value: 23.519000000000002 |
|
- type: recall_at_5 |
|
value: 27.685 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: C-MTEB/CMNLI |
|
name: MTEB Cmnli |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 76.36800962116656 |
|
- type: cos_sim_ap |
|
value: 85.14376065556142 |
|
- type: cos_sim_f1 |
|
value: 77.81474723623485 |
|
- type: cos_sim_precision |
|
value: 71.92460317460318 |
|
- type: cos_sim_recall |
|
value: 84.75566986205284 |
|
- type: dot_accuracy |
|
value: 71.94227300060132 |
|
- type: dot_ap |
|
value: 79.03676891584456 |
|
- type: dot_f1 |
|
value: 74.95833333333334 |
|
- type: dot_precision |
|
value: 67.59346233327072 |
|
- type: dot_recall |
|
value: 84.12438625204582 |
|
- type: euclidean_accuracy |
|
value: 76.043295249549 |
|
- type: euclidean_ap |
|
value: 85.28765360616536 |
|
- type: euclidean_f1 |
|
value: 78.01733248784612 |
|
- type: euclidean_precision |
|
value: 71.1861137897782 |
|
- type: euclidean_recall |
|
value: 86.29880757540333 |
|
- type: manhattan_accuracy |
|
value: 76.17558628983764 |
|
- type: manhattan_ap |
|
value: 85.52739323094916 |
|
- type: manhattan_f1 |
|
value: 78.30788804071246 |
|
- type: manhattan_precision |
|
value: 71.63918525703201 |
|
- type: manhattan_recall |
|
value: 86.34556932429273 |
|
- type: max_accuracy |
|
value: 76.36800962116656 |
|
- type: max_ap |
|
value: 85.52739323094916 |
|
- type: max_f1 |
|
value: 78.30788804071246 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/CovidRetrieval |
|
name: MTEB CovidRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 56.164 |
|
- type: map_at_10 |
|
value: 64.575 |
|
- type: map_at_100 |
|
value: 65.098 |
|
- type: map_at_1000 |
|
value: 65.118 |
|
- type: map_at_3 |
|
value: 62.329 |
|
- type: map_at_5 |
|
value: 63.535 |
|
- type: mrr_at_1 |
|
value: 56.269999999999996 |
|
- type: mrr_at_10 |
|
value: 64.63600000000001 |
|
- type: mrr_at_100 |
|
value: 65.14 |
|
- type: mrr_at_1000 |
|
value: 65.16 |
|
- type: mrr_at_3 |
|
value: 62.522 |
|
- type: mrr_at_5 |
|
value: 63.57000000000001 |
|
- type: ndcg_at_1 |
|
value: 56.269999999999996 |
|
- type: ndcg_at_10 |
|
value: 68.855 |
|
- type: ndcg_at_100 |
|
value: 71.47099999999999 |
|
- type: ndcg_at_1000 |
|
value: 72.02499999999999 |
|
- type: ndcg_at_3 |
|
value: 64.324 |
|
- type: ndcg_at_5 |
|
value: 66.417 |
|
- type: precision_at_1 |
|
value: 56.269999999999996 |
|
- type: precision_at_10 |
|
value: 8.303 |
|
- type: precision_at_100 |
|
value: 0.9570000000000001 |
|
- type: precision_at_1000 |
|
value: 0.1 |
|
- type: precision_at_3 |
|
value: 23.427999999999997 |
|
- type: precision_at_5 |
|
value: 15.09 |
|
- type: recall_at_1 |
|
value: 56.164 |
|
- type: recall_at_10 |
|
value: 82.271 |
|
- type: recall_at_100 |
|
value: 94.626 |
|
- type: recall_at_1000 |
|
value: 99.05199999999999 |
|
- type: recall_at_3 |
|
value: 69.94200000000001 |
|
- type: recall_at_5 |
|
value: 74.947 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: dbpedia-entity |
|
name: MTEB DBPedia |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 8.686 |
|
- type: map_at_10 |
|
value: 17.766000000000002 |
|
- type: map_at_100 |
|
value: 23.507 |
|
- type: map_at_1000 |
|
value: 24.757 |
|
- type: map_at_3 |
|
value: 13.238 |
|
- type: map_at_5 |
|
value: 15.161 |
|
- type: mrr_at_1 |
|
value: 65.25 |
|
- type: mrr_at_10 |
|
value: 72.88 |
|
- type: mrr_at_100 |
|
value: 73.246 |
|
- type: mrr_at_1000 |
|
value: 73.261 |
|
- type: mrr_at_3 |
|
value: 71.542 |
|
- type: mrr_at_5 |
|
value: 72.392 |
|
- type: ndcg_at_1 |
|
value: 53.75 |
|
- type: ndcg_at_10 |
|
value: 37.623 |
|
- type: ndcg_at_100 |
|
value: 40.302 |
|
- type: ndcg_at_1000 |
|
value: 47.471999999999994 |
|
- type: ndcg_at_3 |
|
value: 43.324 |
|
- type: ndcg_at_5 |
|
value: 39.887 |
|
- type: precision_at_1 |
|
value: 65.25 |
|
- type: precision_at_10 |
|
value: 28.749999999999996 |
|
- type: precision_at_100 |
|
value: 8.34 |
|
- type: precision_at_1000 |
|
value: 1.703 |
|
- type: precision_at_3 |
|
value: 46.583000000000006 |
|
- type: precision_at_5 |
|
value: 38.0 |
|
- type: recall_at_1 |
|
value: 8.686 |
|
- type: recall_at_10 |
|
value: 22.966 |
|
- type: recall_at_100 |
|
value: 44.3 |
|
- type: recall_at_1000 |
|
value: 67.77499999999999 |
|
- type: recall_at_3 |
|
value: 14.527999999999999 |
|
- type: recall_at_5 |
|
value: 17.617 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/DuRetrieval |
|
name: MTEB DuRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 22.439 |
|
- type: map_at_10 |
|
value: 68.484 |
|
- type: map_at_100 |
|
value: 71.67999999999999 |
|
- type: map_at_1000 |
|
value: 71.761 |
|
- type: map_at_3 |
|
value: 46.373999999999995 |
|
- type: map_at_5 |
|
value: 58.697 |
|
- type: mrr_at_1 |
|
value: 80.65 |
|
- type: mrr_at_10 |
|
value: 86.53 |
|
- type: mrr_at_100 |
|
value: 86.624 |
|
- type: mrr_at_1000 |
|
value: 86.631 |
|
- type: mrr_at_3 |
|
value: 85.95 |
|
- type: mrr_at_5 |
|
value: 86.297 |
|
- type: ndcg_at_1 |
|
value: 80.65 |
|
- type: ndcg_at_10 |
|
value: 78.075 |
|
- type: ndcg_at_100 |
|
value: 82.014 |
|
- type: ndcg_at_1000 |
|
value: 82.903 |
|
- type: ndcg_at_3 |
|
value: 75.785 |
|
- type: ndcg_at_5 |
|
value: 74.789 |
|
- type: precision_at_1 |
|
value: 80.65 |
|
- type: precision_at_10 |
|
value: 38.425 |
|
- type: precision_at_100 |
|
value: 4.62 |
|
- type: precision_at_1000 |
|
value: 0.483 |
|
- type: precision_at_3 |
|
value: 68.25 |
|
- type: precision_at_5 |
|
value: 57.92 |
|
- type: recall_at_1 |
|
value: 22.439 |
|
- type: recall_at_10 |
|
value: 80.396 |
|
- type: recall_at_100 |
|
value: 92.793 |
|
- type: recall_at_1000 |
|
value: 97.541 |
|
- type: recall_at_3 |
|
value: 49.611 |
|
- type: recall_at_5 |
|
value: 65.065 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/EcomRetrieval |
|
name: MTEB EcomRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 43.9 |
|
- type: map_at_10 |
|
value: 53.394 |
|
- type: map_at_100 |
|
value: 54.078 |
|
- type: map_at_1000 |
|
value: 54.105000000000004 |
|
- type: map_at_3 |
|
value: 50.583 |
|
- type: map_at_5 |
|
value: 52.443 |
|
- type: mrr_at_1 |
|
value: 43.9 |
|
- type: mrr_at_10 |
|
value: 53.394 |
|
- type: mrr_at_100 |
|
value: 54.078 |
|
- type: mrr_at_1000 |
|
value: 54.105000000000004 |
|
- type: mrr_at_3 |
|
value: 50.583 |
|
- type: mrr_at_5 |
|
value: 52.443 |
|
- type: ndcg_at_1 |
|
value: 43.9 |
|
- type: ndcg_at_10 |
|
value: 58.341 |
|
- type: ndcg_at_100 |
|
value: 61.753 |
|
- type: ndcg_at_1000 |
|
value: 62.525 |
|
- type: ndcg_at_3 |
|
value: 52.699 |
|
- type: ndcg_at_5 |
|
value: 56.042 |
|
- type: precision_at_1 |
|
value: 43.9 |
|
- type: precision_at_10 |
|
value: 7.3999999999999995 |
|
- type: precision_at_100 |
|
value: 0.901 |
|
- type: precision_at_1000 |
|
value: 0.096 |
|
- type: precision_at_3 |
|
value: 19.6 |
|
- type: precision_at_5 |
|
value: 13.38 |
|
- type: recall_at_1 |
|
value: 43.9 |
|
- type: recall_at_10 |
|
value: 74.0 |
|
- type: recall_at_100 |
|
value: 90.10000000000001 |
|
- type: recall_at_1000 |
|
value: 96.3 |
|
- type: recall_at_3 |
|
value: 58.8 |
|
- type: recall_at_5 |
|
value: 66.9 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/emotion |
|
name: MTEB EmotionClassification |
|
config: default |
|
split: test |
|
revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 |
|
metrics: |
|
- type: accuracy |
|
value: 48.765 |
|
- type: f1 |
|
value: 44.2791193129597 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: fever |
|
name: MTEB FEVER |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 56.89999999999999 |
|
- type: map_at_10 |
|
value: 68.352 |
|
- type: map_at_100 |
|
value: 68.768 |
|
- type: map_at_1000 |
|
value: 68.782 |
|
- type: map_at_3 |
|
value: 66.27300000000001 |
|
- type: map_at_5 |
|
value: 67.67699999999999 |
|
- type: mrr_at_1 |
|
value: 61.476 |
|
- type: mrr_at_10 |
|
value: 72.662 |
|
- type: mrr_at_100 |
|
value: 72.993 |
|
- type: mrr_at_1000 |
|
value: 72.99799999999999 |
|
- type: mrr_at_3 |
|
value: 70.75200000000001 |
|
- type: mrr_at_5 |
|
value: 72.056 |
|
- type: ndcg_at_1 |
|
value: 61.476 |
|
- type: ndcg_at_10 |
|
value: 73.98400000000001 |
|
- type: ndcg_at_100 |
|
value: 75.744 |
|
- type: ndcg_at_1000 |
|
value: 76.036 |
|
- type: ndcg_at_3 |
|
value: 70.162 |
|
- type: ndcg_at_5 |
|
value: 72.482 |
|
- type: precision_at_1 |
|
value: 61.476 |
|
- type: precision_at_10 |
|
value: 9.565 |
|
- type: precision_at_100 |
|
value: 1.054 |
|
- type: precision_at_1000 |
|
value: 0.109 |
|
- type: precision_at_3 |
|
value: 27.943 |
|
- type: precision_at_5 |
|
value: 18.056 |
|
- type: recall_at_1 |
|
value: 56.89999999999999 |
|
- type: recall_at_10 |
|
value: 87.122 |
|
- type: recall_at_100 |
|
value: 94.742 |
|
- type: recall_at_1000 |
|
value: 96.70100000000001 |
|
- type: recall_at_3 |
|
value: 76.911 |
|
- type: recall_at_5 |
|
value: 82.607 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: fiqa |
|
name: MTEB FiQA2018 |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 17.610999999999997 |
|
- type: map_at_10 |
|
value: 29.12 |
|
- type: map_at_100 |
|
value: 30.958000000000002 |
|
- type: map_at_1000 |
|
value: 31.151 |
|
- type: map_at_3 |
|
value: 25.369000000000003 |
|
- type: map_at_5 |
|
value: 27.445000000000004 |
|
- type: mrr_at_1 |
|
value: 35.185 |
|
- type: mrr_at_10 |
|
value: 44.533 |
|
- type: mrr_at_100 |
|
value: 45.385 |
|
- type: mrr_at_1000 |
|
value: 45.432 |
|
- type: mrr_at_3 |
|
value: 42.258 |
|
- type: mrr_at_5 |
|
value: 43.608999999999995 |
|
- type: ndcg_at_1 |
|
value: 35.185 |
|
- type: ndcg_at_10 |
|
value: 36.696 |
|
- type: ndcg_at_100 |
|
value: 43.491 |
|
- type: ndcg_at_1000 |
|
value: 46.800000000000004 |
|
- type: ndcg_at_3 |
|
value: 33.273 |
|
- type: ndcg_at_5 |
|
value: 34.336 |
|
- type: precision_at_1 |
|
value: 35.185 |
|
- type: precision_at_10 |
|
value: 10.309 |
|
- type: precision_at_100 |
|
value: 1.719 |
|
- type: precision_at_1000 |
|
value: 0.231 |
|
- type: precision_at_3 |
|
value: 22.479 |
|
- type: precision_at_5 |
|
value: 16.481 |
|
- type: recall_at_1 |
|
value: 17.610999999999997 |
|
- type: recall_at_10 |
|
value: 43.29 |
|
- type: recall_at_100 |
|
value: 68.638 |
|
- type: recall_at_1000 |
|
value: 88.444 |
|
- type: recall_at_3 |
|
value: 30.303 |
|
- type: recall_at_5 |
|
value: 35.856 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: hotpotqa |
|
name: MTEB HotpotQA |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 34.18 |
|
- type: map_at_10 |
|
value: 47.753 |
|
- type: map_at_100 |
|
value: 48.522 |
|
- type: map_at_1000 |
|
value: 48.596000000000004 |
|
- type: map_at_3 |
|
value: 45.222 |
|
- type: map_at_5 |
|
value: 46.793 |
|
- type: mrr_at_1 |
|
value: 68.35900000000001 |
|
- type: mrr_at_10 |
|
value: 74.503 |
|
- type: mrr_at_100 |
|
value: 74.811 |
|
- type: mrr_at_1000 |
|
value: 74.82799999999999 |
|
- type: mrr_at_3 |
|
value: 73.347 |
|
- type: mrr_at_5 |
|
value: 74.06700000000001 |
|
- type: ndcg_at_1 |
|
value: 68.35900000000001 |
|
- type: ndcg_at_10 |
|
value: 56.665 |
|
- type: ndcg_at_100 |
|
value: 59.629 |
|
- type: ndcg_at_1000 |
|
value: 61.222 |
|
- type: ndcg_at_3 |
|
value: 52.81400000000001 |
|
- type: ndcg_at_5 |
|
value: 54.94 |
|
- type: precision_at_1 |
|
value: 68.35900000000001 |
|
- type: precision_at_10 |
|
value: 11.535 |
|
- type: precision_at_100 |
|
value: 1.388 |
|
- type: precision_at_1000 |
|
value: 0.16 |
|
- type: precision_at_3 |
|
value: 32.784 |
|
- type: precision_at_5 |
|
value: 21.348 |
|
- type: recall_at_1 |
|
value: 34.18 |
|
- type: recall_at_10 |
|
value: 57.677 |
|
- type: recall_at_100 |
|
value: 69.379 |
|
- type: recall_at_1000 |
|
value: 80.061 |
|
- type: recall_at_3 |
|
value: 49.175999999999995 |
|
- type: recall_at_5 |
|
value: 53.369 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/IFlyTek-classification |
|
name: MTEB IFlyTek |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 46.23316660253944 |
|
- type: f1 |
|
value: 39.09397722262806 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/imdb |
|
name: MTEB ImdbClassification |
|
config: default |
|
split: test |
|
revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 |
|
metrics: |
|
- type: accuracy |
|
value: 78.46119999999999 |
|
- type: ap |
|
value: 72.53477126781094 |
|
- type: f1 |
|
value: 78.28701752379332 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/JDReview-classification |
|
name: MTEB JDReview |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 84.16510318949344 |
|
- type: ap |
|
value: 50.10324581565756 |
|
- type: f1 |
|
value: 78.34748161287605 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/LCQMC |
|
name: MTEB LCQMC |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 68.71925879533819 |
|
- type: cos_sim_spearman |
|
value: 75.33926640820977 |
|
- type: euclidean_pearson |
|
value: 74.59557932790653 |
|
- type: euclidean_spearman |
|
value: 75.76006440878783 |
|
- type: manhattan_pearson |
|
value: 74.7461963483351 |
|
- type: manhattan_spearman |
|
value: 75.87111519308131 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/MMarcoRetrieval |
|
name: MTEB MMarcoRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 66.249 |
|
- type: map_at_10 |
|
value: 75.236 |
|
- type: map_at_100 |
|
value: 75.581 |
|
- type: map_at_1000 |
|
value: 75.593 |
|
- type: map_at_3 |
|
value: 73.463 |
|
- type: map_at_5 |
|
value: 74.602 |
|
- type: mrr_at_1 |
|
value: 68.42399999999999 |
|
- type: mrr_at_10 |
|
value: 75.81099999999999 |
|
- type: mrr_at_100 |
|
value: 76.115 |
|
- type: mrr_at_1000 |
|
value: 76.126 |
|
- type: mrr_at_3 |
|
value: 74.26899999999999 |
|
- type: mrr_at_5 |
|
value: 75.24300000000001 |
|
- type: ndcg_at_1 |
|
value: 68.42399999999999 |
|
- type: ndcg_at_10 |
|
value: 78.81700000000001 |
|
- type: ndcg_at_100 |
|
value: 80.379 |
|
- type: ndcg_at_1000 |
|
value: 80.667 |
|
- type: ndcg_at_3 |
|
value: 75.476 |
|
- type: ndcg_at_5 |
|
value: 77.38199999999999 |
|
- type: precision_at_1 |
|
value: 68.42399999999999 |
|
- type: precision_at_10 |
|
value: 9.491 |
|
- type: precision_at_100 |
|
value: 1.027 |
|
- type: precision_at_1000 |
|
value: 0.105 |
|
- type: precision_at_3 |
|
value: 28.352 |
|
- type: precision_at_5 |
|
value: 18.043 |
|
- type: recall_at_1 |
|
value: 66.249 |
|
- type: recall_at_10 |
|
value: 89.238 |
|
- type: recall_at_100 |
|
value: 96.319 |
|
- type: recall_at_1000 |
|
value: 98.524 |
|
- type: recall_at_3 |
|
value: 80.438 |
|
- type: recall_at_5 |
|
value: 84.95 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: msmarco |
|
name: MTEB MSMARCO |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 23.083000000000002 |
|
- type: map_at_10 |
|
value: 35.251 |
|
- type: map_at_100 |
|
value: 36.461 |
|
- type: map_at_1000 |
|
value: 36.507 |
|
- type: map_at_3 |
|
value: 31.474999999999998 |
|
- type: map_at_5 |
|
value: 33.658 |
|
- type: mrr_at_1 |
|
value: 23.724999999999998 |
|
- type: mrr_at_10 |
|
value: 35.88 |
|
- type: mrr_at_100 |
|
value: 37.021 |
|
- type: mrr_at_1000 |
|
value: 37.062 |
|
- type: mrr_at_3 |
|
value: 32.159 |
|
- type: mrr_at_5 |
|
value: 34.325 |
|
- type: ndcg_at_1 |
|
value: 23.724999999999998 |
|
- type: ndcg_at_10 |
|
value: 42.018 |
|
- type: ndcg_at_100 |
|
value: 47.764 |
|
- type: ndcg_at_1000 |
|
value: 48.916 |
|
- type: ndcg_at_3 |
|
value: 34.369 |
|
- type: ndcg_at_5 |
|
value: 38.266 |
|
- type: precision_at_1 |
|
value: 23.724999999999998 |
|
- type: precision_at_10 |
|
value: 6.553000000000001 |
|
- type: precision_at_100 |
|
value: 0.942 |
|
- type: precision_at_1000 |
|
value: 0.104 |
|
- type: precision_at_3 |
|
value: 14.532 |
|
- type: precision_at_5 |
|
value: 10.696 |
|
- type: recall_at_1 |
|
value: 23.083000000000002 |
|
- type: recall_at_10 |
|
value: 62.739 |
|
- type: recall_at_100 |
|
value: 89.212 |
|
- type: recall_at_1000 |
|
value: 97.991 |
|
- type: recall_at_3 |
|
value: 42.064 |
|
- type: recall_at_5 |
|
value: 51.417 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_domain |
|
name: MTEB MTOPDomainClassification (en) |
|
config: en |
|
split: test |
|
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf |
|
metrics: |
|
- type: accuracy |
|
value: 93.43365253077975 |
|
- type: f1 |
|
value: 93.07455671032345 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/mtop_intent |
|
name: MTEB MTOPIntentClassification (en) |
|
config: en |
|
split: test |
|
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba |
|
metrics: |
|
- type: accuracy |
|
value: 71.72822617419061 |
|
- type: f1 |
|
value: 55.6093871673643 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_intent |
|
name: MTEB MassiveIntentClassification (en) |
|
config: en |
|
split: test |
|
revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 |
|
metrics: |
|
- type: accuracy |
|
value: 72.03765971755212 |
|
- type: f1 |
|
value: 70.88235592002572 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/amazon_massive_scenario |
|
name: MTEB MassiveScenarioClassification (en) |
|
config: en |
|
split: test |
|
revision: 7d571f92784cd94a019292a1f45445077d0ef634 |
|
metrics: |
|
- type: accuracy |
|
value: 76.86281102891728 |
|
- type: f1 |
|
value: 77.15496923811003 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/MedicalRetrieval |
|
name: MTEB MedicalRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 41.8 |
|
- type: map_at_10 |
|
value: 46.993 |
|
- type: map_at_100 |
|
value: 47.534 |
|
- type: map_at_1000 |
|
value: 47.587 |
|
- type: map_at_3 |
|
value: 45.717 |
|
- type: map_at_5 |
|
value: 46.357 |
|
- type: mrr_at_1 |
|
value: 42.0 |
|
- type: mrr_at_10 |
|
value: 47.093 |
|
- type: mrr_at_100 |
|
value: 47.634 |
|
- type: mrr_at_1000 |
|
value: 47.687000000000005 |
|
- type: mrr_at_3 |
|
value: 45.817 |
|
- type: mrr_at_5 |
|
value: 46.457 |
|
- type: ndcg_at_1 |
|
value: 41.8 |
|
- type: ndcg_at_10 |
|
value: 49.631 |
|
- type: ndcg_at_100 |
|
value: 52.53 |
|
- type: ndcg_at_1000 |
|
value: 54.238 |
|
- type: ndcg_at_3 |
|
value: 46.949000000000005 |
|
- type: ndcg_at_5 |
|
value: 48.102000000000004 |
|
- type: precision_at_1 |
|
value: 41.8 |
|
- type: precision_at_10 |
|
value: 5.800000000000001 |
|
- type: precision_at_100 |
|
value: 0.722 |
|
- type: precision_at_1000 |
|
value: 0.086 |
|
- type: precision_at_3 |
|
value: 16.833000000000002 |
|
- type: precision_at_5 |
|
value: 10.66 |
|
- type: recall_at_1 |
|
value: 41.8 |
|
- type: recall_at_10 |
|
value: 57.99999999999999 |
|
- type: recall_at_100 |
|
value: 72.2 |
|
- type: recall_at_1000 |
|
value: 86.3 |
|
- type: recall_at_3 |
|
value: 50.5 |
|
- type: recall_at_5 |
|
value: 53.300000000000004 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/medrxiv-clustering-p2p |
|
name: MTEB MedrxivClusteringP2P |
|
config: default |
|
split: test |
|
revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 |
|
metrics: |
|
- type: v_measure |
|
value: 30.949060810392886 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/medrxiv-clustering-s2s |
|
name: MTEB MedrxivClusteringS2S |
|
config: default |
|
split: test |
|
revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 |
|
metrics: |
|
- type: v_measure |
|
value: 28.87339864059011 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/mind_small |
|
name: MTEB MindSmallReranking |
|
config: default |
|
split: test |
|
revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 |
|
metrics: |
|
- type: map |
|
value: 31.217934626189926 |
|
- type: mrr |
|
value: 32.27509143911496 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/Mmarco-reranking |
|
name: MTEB MMarcoReranking |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 26.691638884089574 |
|
- type: mrr |
|
value: 25.15674603174603 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/MultilingualSentiment-classification |
|
name: MTEB MultilingualSentiment |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 68.35666666666667 |
|
- type: f1 |
|
value: 68.30294399725629 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: nfcorpus |
|
name: MTEB NFCorpus |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 5.759 |
|
- type: map_at_10 |
|
value: 13.425999999999998 |
|
- type: map_at_100 |
|
value: 16.988 |
|
- type: map_at_1000 |
|
value: 18.512 |
|
- type: map_at_3 |
|
value: 9.737 |
|
- type: map_at_5 |
|
value: 11.558 |
|
- type: mrr_at_1 |
|
value: 48.297000000000004 |
|
- type: mrr_at_10 |
|
value: 56.788000000000004 |
|
- type: mrr_at_100 |
|
value: 57.306000000000004 |
|
- type: mrr_at_1000 |
|
value: 57.349000000000004 |
|
- type: mrr_at_3 |
|
value: 54.386 |
|
- type: mrr_at_5 |
|
value: 56.135000000000005 |
|
- type: ndcg_at_1 |
|
value: 46.285 |
|
- type: ndcg_at_10 |
|
value: 36.016 |
|
- type: ndcg_at_100 |
|
value: 32.984 |
|
- type: ndcg_at_1000 |
|
value: 42.093 |
|
- type: ndcg_at_3 |
|
value: 41.743 |
|
- type: ndcg_at_5 |
|
value: 39.734 |
|
- type: precision_at_1 |
|
value: 48.297000000000004 |
|
- type: precision_at_10 |
|
value: 26.779999999999998 |
|
- type: precision_at_100 |
|
value: 8.505 |
|
- type: precision_at_1000 |
|
value: 2.1420000000000003 |
|
- type: precision_at_3 |
|
value: 39.422000000000004 |
|
- type: precision_at_5 |
|
value: 34.675 |
|
- type: recall_at_1 |
|
value: 5.759 |
|
- type: recall_at_10 |
|
value: 17.251 |
|
- type: recall_at_100 |
|
value: 33.323 |
|
- type: recall_at_1000 |
|
value: 66.759 |
|
- type: recall_at_3 |
|
value: 10.703 |
|
- type: recall_at_5 |
|
value: 13.808000000000002 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: nq |
|
name: MTEB NQ |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 31.696999999999996 |
|
- type: map_at_10 |
|
value: 46.099000000000004 |
|
- type: map_at_100 |
|
value: 47.143 |
|
- type: map_at_1000 |
|
value: 47.178 |
|
- type: map_at_3 |
|
value: 41.948 |
|
- type: map_at_5 |
|
value: 44.504 |
|
- type: mrr_at_1 |
|
value: 35.717999999999996 |
|
- type: mrr_at_10 |
|
value: 48.653 |
|
- type: mrr_at_100 |
|
value: 49.456 |
|
- type: mrr_at_1000 |
|
value: 49.479 |
|
- type: mrr_at_3 |
|
value: 45.283 |
|
- type: mrr_at_5 |
|
value: 47.422 |
|
- type: ndcg_at_1 |
|
value: 35.689 |
|
- type: ndcg_at_10 |
|
value: 53.312000000000005 |
|
- type: ndcg_at_100 |
|
value: 57.69 |
|
- type: ndcg_at_1000 |
|
value: 58.489000000000004 |
|
- type: ndcg_at_3 |
|
value: 45.678999999999995 |
|
- type: ndcg_at_5 |
|
value: 49.897000000000006 |
|
- type: precision_at_1 |
|
value: 35.689 |
|
- type: precision_at_10 |
|
value: 8.685 |
|
- type: precision_at_100 |
|
value: 1.111 |
|
- type: precision_at_1000 |
|
value: 0.11900000000000001 |
|
- type: precision_at_3 |
|
value: 20.558 |
|
- type: precision_at_5 |
|
value: 14.802999999999999 |
|
- type: recall_at_1 |
|
value: 31.696999999999996 |
|
- type: recall_at_10 |
|
value: 72.615 |
|
- type: recall_at_100 |
|
value: 91.563 |
|
- type: recall_at_1000 |
|
value: 97.52300000000001 |
|
- type: recall_at_3 |
|
value: 53.203 |
|
- type: recall_at_5 |
|
value: 62.836000000000006 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: C-MTEB/OCNLI |
|
name: MTEB Ocnli |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 67.94802382241473 |
|
- type: cos_sim_ap |
|
value: 72.1545049768353 |
|
- type: cos_sim_f1 |
|
value: 71.24658780709737 |
|
- type: cos_sim_precision |
|
value: 62.589928057553955 |
|
- type: cos_sim_recall |
|
value: 82.68215417106653 |
|
- type: dot_accuracy |
|
value: 63.56253383865729 |
|
- type: dot_ap |
|
value: 66.5298825401086 |
|
- type: dot_f1 |
|
value: 69.31953840031835 |
|
- type: dot_precision |
|
value: 55.61941251596424 |
|
- type: dot_recall |
|
value: 91.97465681098205 |
|
- type: euclidean_accuracy |
|
value: 69.46399566865186 |
|
- type: euclidean_ap |
|
value: 73.63177936887436 |
|
- type: euclidean_f1 |
|
value: 72.91028446389497 |
|
- type: euclidean_precision |
|
value: 62.25710014947683 |
|
- type: euclidean_recall |
|
value: 87.96198521647307 |
|
- type: manhattan_accuracy |
|
value: 69.89713048186248 |
|
- type: manhattan_ap |
|
value: 74.11555425121965 |
|
- type: manhattan_f1 |
|
value: 72.8923476005188 |
|
- type: manhattan_precision |
|
value: 61.71303074670571 |
|
- type: manhattan_recall |
|
value: 89.01795142555439 |
|
- type: max_accuracy |
|
value: 69.89713048186248 |
|
- type: max_ap |
|
value: 74.11555425121965 |
|
- type: max_f1 |
|
value: 72.91028446389497 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/OnlineShopping-classification |
|
name: MTEB OnlineShopping |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 90.93 |
|
- type: ap |
|
value: 88.66185083484555 |
|
- type: f1 |
|
value: 90.91685771516175 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/PAWSX |
|
name: MTEB PAWSX |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 14.385178129184318 |
|
- type: cos_sim_spearman |
|
value: 17.246549728263478 |
|
- type: euclidean_pearson |
|
value: 18.921969136664913 |
|
- type: euclidean_spearman |
|
value: 17.245713577354014 |
|
- type: manhattan_pearson |
|
value: 18.98503959815216 |
|
- type: manhattan_spearman |
|
value: 17.37740013639568 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/QBQTC |
|
name: MTEB QBQTC |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 32.04198138050403 |
|
- type: cos_sim_spearman |
|
value: 34.4844617563846 |
|
- type: euclidean_pearson |
|
value: 34.2634608256121 |
|
- type: euclidean_spearman |
|
value: 36.322207068208066 |
|
- type: manhattan_pearson |
|
value: 34.414939622012284 |
|
- type: manhattan_spearman |
|
value: 36.49437789416394 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: quora |
|
name: MTEB QuoraRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 70.858 |
|
- type: map_at_10 |
|
value: 84.516 |
|
- type: map_at_100 |
|
value: 85.138 |
|
- type: map_at_1000 |
|
value: 85.153 |
|
- type: map_at_3 |
|
value: 81.487 |
|
- type: map_at_5 |
|
value: 83.41199999999999 |
|
- type: mrr_at_1 |
|
value: 81.55 |
|
- type: mrr_at_10 |
|
value: 87.51400000000001 |
|
- type: mrr_at_100 |
|
value: 87.607 |
|
- type: mrr_at_1000 |
|
value: 87.60900000000001 |
|
- type: mrr_at_3 |
|
value: 86.49 |
|
- type: mrr_at_5 |
|
value: 87.21 |
|
- type: ndcg_at_1 |
|
value: 81.57 |
|
- type: ndcg_at_10 |
|
value: 88.276 |
|
- type: ndcg_at_100 |
|
value: 89.462 |
|
- type: ndcg_at_1000 |
|
value: 89.571 |
|
- type: ndcg_at_3 |
|
value: 85.294 |
|
- type: ndcg_at_5 |
|
value: 86.979 |
|
- type: precision_at_1 |
|
value: 81.57 |
|
- type: precision_at_10 |
|
value: 13.389999999999999 |
|
- type: precision_at_100 |
|
value: 1.532 |
|
- type: precision_at_1000 |
|
value: 0.157 |
|
- type: precision_at_3 |
|
value: 37.2 |
|
- type: precision_at_5 |
|
value: 24.544 |
|
- type: recall_at_1 |
|
value: 70.858 |
|
- type: recall_at_10 |
|
value: 95.428 |
|
- type: recall_at_100 |
|
value: 99.46000000000001 |
|
- type: recall_at_1000 |
|
value: 99.98 |
|
- type: recall_at_3 |
|
value: 86.896 |
|
- type: recall_at_5 |
|
value: 91.617 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/reddit-clustering |
|
name: MTEB RedditClustering |
|
config: default |
|
split: test |
|
revision: 24640382cdbf8abc73003fb0fa6d111a705499eb |
|
metrics: |
|
- type: v_measure |
|
value: 47.90089115942085 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/reddit-clustering-p2p |
|
name: MTEB RedditClusteringP2P |
|
config: default |
|
split: test |
|
revision: 282350215ef01743dc01b456c7f5241fa8937f16 |
|
metrics: |
|
- type: v_measure |
|
value: 55.948584594903515 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: scidocs |
|
name: MTEB SCIDOCS |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 4.513 |
|
- type: map_at_10 |
|
value: 11.189 |
|
- type: map_at_100 |
|
value: 13.034 |
|
- type: map_at_1000 |
|
value: 13.312 |
|
- type: map_at_3 |
|
value: 8.124 |
|
- type: map_at_5 |
|
value: 9.719999999999999 |
|
- type: mrr_at_1 |
|
value: 22.1 |
|
- type: mrr_at_10 |
|
value: 32.879999999999995 |
|
- type: mrr_at_100 |
|
value: 33.916000000000004 |
|
- type: mrr_at_1000 |
|
value: 33.982 |
|
- type: mrr_at_3 |
|
value: 29.633 |
|
- type: mrr_at_5 |
|
value: 31.663000000000004 |
|
- type: ndcg_at_1 |
|
value: 22.1 |
|
- type: ndcg_at_10 |
|
value: 18.944 |
|
- type: ndcg_at_100 |
|
value: 26.240000000000002 |
|
- type: ndcg_at_1000 |
|
value: 31.282 |
|
- type: ndcg_at_3 |
|
value: 18.17 |
|
- type: ndcg_at_5 |
|
value: 15.976 |
|
- type: precision_at_1 |
|
value: 22.1 |
|
- type: precision_at_10 |
|
value: 9.700000000000001 |
|
- type: precision_at_100 |
|
value: 2.025 |
|
- type: precision_at_1000 |
|
value: 0.32299999999999995 |
|
- type: precision_at_3 |
|
value: 16.933 |
|
- type: precision_at_5 |
|
value: 14.02 |
|
- type: recall_at_1 |
|
value: 4.513 |
|
- type: recall_at_10 |
|
value: 19.723 |
|
- type: recall_at_100 |
|
value: 41.117 |
|
- type: recall_at_1000 |
|
value: 65.718 |
|
- type: recall_at_3 |
|
value: 10.333 |
|
- type: recall_at_5 |
|
value: 14.252 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sickr-sts |
|
name: MTEB SICK-R |
|
config: default |
|
split: test |
|
revision: a6ea5a8cab320b040a23452cc28066d9beae2cee |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 85.93526522406187 |
|
- type: cos_sim_spearman |
|
value: 81.4067321748142 |
|
- type: euclidean_pearson |
|
value: 82.23783344725466 |
|
- type: euclidean_spearman |
|
value: 80.88990344685583 |
|
- type: manhattan_pearson |
|
value: 82.3367264631989 |
|
- type: manhattan_spearman |
|
value: 80.9278067738814 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts12-sts |
|
name: MTEB STS12 |
|
config: default |
|
split: test |
|
revision: a0d554a64d88156834ff5ae9920b964011b16384 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 85.23458296088118 |
|
- type: cos_sim_spearman |
|
value: 77.47310329678291 |
|
- type: euclidean_pearson |
|
value: 83.73584591194671 |
|
- type: euclidean_spearman |
|
value: 80.15616176452284 |
|
- type: manhattan_pearson |
|
value: 84.03063128849925 |
|
- type: manhattan_spearman |
|
value: 80.36472448270416 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts13-sts |
|
name: MTEB STS13 |
|
config: default |
|
split: test |
|
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 86.11807249122802 |
|
- type: cos_sim_spearman |
|
value: 86.37854318479079 |
|
- type: euclidean_pearson |
|
value: 86.65850909046301 |
|
- type: euclidean_spearman |
|
value: 87.85344963531178 |
|
- type: manhattan_pearson |
|
value: 86.77920459868837 |
|
- type: manhattan_spearman |
|
value: 87.97331161741792 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts14-sts |
|
name: MTEB STS14 |
|
config: default |
|
split: test |
|
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 84.4649953305265 |
|
- type: cos_sim_spearman |
|
value: 81.17166984686445 |
|
- type: euclidean_pearson |
|
value: 82.36880883967271 |
|
- type: euclidean_spearman |
|
value: 81.28206358558401 |
|
- type: manhattan_pearson |
|
value: 82.56994704487155 |
|
- type: manhattan_spearman |
|
value: 81.52094918949243 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts15-sts |
|
name: MTEB STS15 |
|
config: default |
|
split: test |
|
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 87.5328930220188 |
|
- type: cos_sim_spearman |
|
value: 88.23398394823562 |
|
- type: euclidean_pearson |
|
value: 88.0817998861656 |
|
- type: euclidean_spearman |
|
value: 88.68995789914679 |
|
- type: manhattan_pearson |
|
value: 88.11885742601258 |
|
- type: manhattan_spearman |
|
value: 88.7318106493293 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts16-sts |
|
name: MTEB STS16 |
|
config: default |
|
split: test |
|
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 84.81883368511858 |
|
- type: cos_sim_spearman |
|
value: 86.28679308000675 |
|
- type: euclidean_pearson |
|
value: 84.33705182713047 |
|
- type: euclidean_spearman |
|
value: 84.83018555455023 |
|
- type: manhattan_pearson |
|
value: 84.3271850394614 |
|
- type: manhattan_spearman |
|
value: 84.77974015415639 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (en-en) |
|
config: en-en |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 90.71845282522295 |
|
- type: cos_sim_spearman |
|
value: 90.6215253553308 |
|
- type: euclidean_pearson |
|
value: 89.486847313806 |
|
- type: euclidean_spearman |
|
value: 89.11692037511729 |
|
- type: manhattan_pearson |
|
value: 89.53911733450684 |
|
- type: manhattan_spearman |
|
value: 89.2507288145461 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (en) |
|
config: en |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 65.81961557635002 |
|
- type: cos_sim_spearman |
|
value: 65.01437718770094 |
|
- type: euclidean_pearson |
|
value: 66.53720271639384 |
|
- type: euclidean_spearman |
|
value: 65.66538718470727 |
|
- type: manhattan_pearson |
|
value: 66.85160833477023 |
|
- type: manhattan_spearman |
|
value: 65.86253623736344 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: C-MTEB/STSB |
|
name: MTEB STSB |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 81.74904608584143 |
|
- type: cos_sim_spearman |
|
value: 82.02672847550606 |
|
- type: euclidean_pearson |
|
value: 81.47843718306068 |
|
- type: euclidean_spearman |
|
value: 81.7259314292303 |
|
- type: manhattan_pearson |
|
value: 81.70320276859634 |
|
- type: manhattan_spearman |
|
value: 81.94903024173293 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/stsbenchmark-sts |
|
name: MTEB STSBenchmark |
|
config: default |
|
split: test |
|
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 87.37129233774877 |
|
- type: cos_sim_spearman |
|
value: 88.02311088852667 |
|
- type: euclidean_pearson |
|
value: 85.864664021262 |
|
- type: euclidean_spearman |
|
value: 86.24775921494894 |
|
- type: manhattan_pearson |
|
value: 85.85401868812795 |
|
- type: manhattan_spearman |
|
value: 86.22999105137849 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/scidocs-reranking |
|
name: MTEB SciDocsRR |
|
config: default |
|
split: test |
|
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab |
|
metrics: |
|
- type: map |
|
value: 80.2684105571225 |
|
- type: mrr |
|
value: 94.3528194753685 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: scifact |
|
name: MTEB SciFact |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 55.161 |
|
- type: map_at_10 |
|
value: 64.794 |
|
- type: map_at_100 |
|
value: 65.66499999999999 |
|
- type: map_at_1000 |
|
value: 65.684 |
|
- type: map_at_3 |
|
value: 62.326 |
|
- type: map_at_5 |
|
value: 63.863 |
|
- type: mrr_at_1 |
|
value: 58.333 |
|
- type: mrr_at_10 |
|
value: 66.396 |
|
- type: mrr_at_100 |
|
value: 67.07300000000001 |
|
- type: mrr_at_1000 |
|
value: 67.092 |
|
- type: mrr_at_3 |
|
value: 64.61099999999999 |
|
- type: mrr_at_5 |
|
value: 65.744 |
|
- type: ndcg_at_1 |
|
value: 58.333 |
|
- type: ndcg_at_10 |
|
value: 69.294 |
|
- type: ndcg_at_100 |
|
value: 72.612 |
|
- type: ndcg_at_1000 |
|
value: 73.083 |
|
- type: ndcg_at_3 |
|
value: 65.226 |
|
- type: ndcg_at_5 |
|
value: 67.44 |
|
- type: precision_at_1 |
|
value: 58.333 |
|
- type: precision_at_10 |
|
value: 9.2 |
|
- type: precision_at_100 |
|
value: 1.083 |
|
- type: precision_at_1000 |
|
value: 0.11199999999999999 |
|
- type: precision_at_3 |
|
value: 25.667 |
|
- type: precision_at_5 |
|
value: 16.866999999999997 |
|
- type: recall_at_1 |
|
value: 55.161 |
|
- type: recall_at_10 |
|
value: 81.289 |
|
- type: recall_at_100 |
|
value: 95.333 |
|
- type: recall_at_1000 |
|
value: 99.0 |
|
- type: recall_at_3 |
|
value: 70.45 |
|
- type: recall_at_5 |
|
value: 76.128 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/sprintduplicatequestions-pairclassification |
|
name: MTEB SprintDuplicateQuestions |
|
config: default |
|
split: test |
|
revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 99.81980198019802 |
|
- type: cos_sim_ap |
|
value: 95.61939598272275 |
|
- type: cos_sim_f1 |
|
value: 91.00684261974584 |
|
- type: cos_sim_precision |
|
value: 89.0057361376673 |
|
- type: cos_sim_recall |
|
value: 93.10000000000001 |
|
- type: dot_accuracy |
|
value: 99.78910891089109 |
|
- type: dot_ap |
|
value: 94.52852299178002 |
|
- type: dot_f1 |
|
value: 89.2586989409985 |
|
- type: dot_precision |
|
value: 90.03051881993896 |
|
- type: dot_recall |
|
value: 88.5 |
|
- type: euclidean_accuracy |
|
value: 99.81782178217821 |
|
- type: euclidean_ap |
|
value: 95.41313424258671 |
|
- type: euclidean_f1 |
|
value: 90.91806515301086 |
|
- type: euclidean_precision |
|
value: 89.76608187134502 |
|
- type: euclidean_recall |
|
value: 92.10000000000001 |
|
- type: manhattan_accuracy |
|
value: 99.81584158415842 |
|
- type: manhattan_ap |
|
value: 95.52722650384223 |
|
- type: manhattan_f1 |
|
value: 90.86444007858546 |
|
- type: manhattan_precision |
|
value: 89.28571428571429 |
|
- type: manhattan_recall |
|
value: 92.5 |
|
- type: max_accuracy |
|
value: 99.81980198019802 |
|
- type: max_ap |
|
value: 95.61939598272275 |
|
- type: max_f1 |
|
value: 91.00684261974584 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/stackexchange-clustering |
|
name: MTEB StackExchangeClustering |
|
config: default |
|
split: test |
|
revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 |
|
metrics: |
|
- type: v_measure |
|
value: 60.2736951820551 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/stackexchange-clustering-p2p |
|
name: MTEB StackExchangeClusteringP2P |
|
config: default |
|
split: test |
|
revision: 815ca46b2622cec33ccafc3735d572c266efdb44 |
|
metrics: |
|
- type: v_measure |
|
value: 32.34316824844043 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/stackoverflowdupquestions-reranking |
|
name: MTEB StackOverflowDupQuestions |
|
config: default |
|
split: test |
|
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 |
|
metrics: |
|
- type: map |
|
value: 50.55034024386463 |
|
- type: mrr |
|
value: 51.468598803157626 |
|
- task: |
|
type: Summarization |
|
dataset: |
|
type: mteb/summeval |
|
name: MTEB SummEval |
|
config: default |
|
split: test |
|
revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 31.20772719310616 |
|
- type: cos_sim_spearman |
|
value: 30.966269993937523 |
|
- type: dot_pearson |
|
value: 30.866563682880965 |
|
- type: dot_spearman |
|
value: 29.906699130890875 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: C-MTEB/T2Reranking |
|
name: MTEB T2Reranking |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map |
|
value: 67.87990805824984 |
|
- type: mrr |
|
value: 78.16078682657897 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/T2Retrieval |
|
name: MTEB T2Retrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 26.009 |
|
- type: map_at_10 |
|
value: 71.319 |
|
- type: map_at_100 |
|
value: 74.895 |
|
- type: map_at_1000 |
|
value: 74.995 |
|
- type: map_at_3 |
|
value: 50.778 |
|
- type: map_at_5 |
|
value: 62.00599999999999 |
|
- type: mrr_at_1 |
|
value: 87.41 |
|
- type: mrr_at_10 |
|
value: 90.18599999999999 |
|
- type: mrr_at_100 |
|
value: 90.29700000000001 |
|
- type: mrr_at_1000 |
|
value: 90.302 |
|
- type: mrr_at_3 |
|
value: 89.701 |
|
- type: mrr_at_5 |
|
value: 89.992 |
|
- type: ndcg_at_1 |
|
value: 87.41 |
|
- type: ndcg_at_10 |
|
value: 79.822 |
|
- type: ndcg_at_100 |
|
value: 83.877 |
|
- type: ndcg_at_1000 |
|
value: 84.882 |
|
- type: ndcg_at_3 |
|
value: 82.391 |
|
- type: ndcg_at_5 |
|
value: 80.339 |
|
- type: precision_at_1 |
|
value: 87.41 |
|
- type: precision_at_10 |
|
value: 39.546 |
|
- type: precision_at_100 |
|
value: 4.824 |
|
- type: precision_at_1000 |
|
value: 0.507 |
|
- type: precision_at_3 |
|
value: 72.129 |
|
- type: precision_at_5 |
|
value: 59.915 |
|
- type: recall_at_1 |
|
value: 26.009 |
|
- type: recall_at_10 |
|
value: 78.144 |
|
- type: recall_at_100 |
|
value: 91.375 |
|
- type: recall_at_1000 |
|
value: 96.42399999999999 |
|
- type: recall_at_3 |
|
value: 52.529 |
|
- type: recall_at_5 |
|
value: 65.46 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/TNews-classification |
|
name: MTEB TNews |
|
config: default |
|
split: validation |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 47.803000000000004 |
|
- type: f1 |
|
value: 46.298520969605775 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: trec-covid |
|
name: MTEB TRECCOVID |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 0.252 |
|
- type: map_at_10 |
|
value: 2.181 |
|
- type: map_at_100 |
|
value: 12.82 |
|
- type: map_at_1000 |
|
value: 30.307000000000002 |
|
- type: map_at_3 |
|
value: 0.716 |
|
- type: map_at_5 |
|
value: 1.133 |
|
- type: mrr_at_1 |
|
value: 96.0 |
|
- type: mrr_at_10 |
|
value: 98.0 |
|
- type: mrr_at_100 |
|
value: 98.0 |
|
- type: mrr_at_1000 |
|
value: 98.0 |
|
- type: mrr_at_3 |
|
value: 98.0 |
|
- type: mrr_at_5 |
|
value: 98.0 |
|
- type: ndcg_at_1 |
|
value: 92.0 |
|
- type: ndcg_at_10 |
|
value: 83.818 |
|
- type: ndcg_at_100 |
|
value: 63.327999999999996 |
|
- type: ndcg_at_1000 |
|
value: 55.883 |
|
- type: ndcg_at_3 |
|
value: 87.16199999999999 |
|
- type: ndcg_at_5 |
|
value: 85.03 |
|
- type: precision_at_1 |
|
value: 96.0 |
|
- type: precision_at_10 |
|
value: 88.0 |
|
- type: precision_at_100 |
|
value: 64.94 |
|
- type: precision_at_1000 |
|
value: 24.688 |
|
- type: precision_at_3 |
|
value: 91.333 |
|
- type: precision_at_5 |
|
value: 88.8 |
|
- type: recall_at_1 |
|
value: 0.252 |
|
- type: recall_at_10 |
|
value: 2.326 |
|
- type: recall_at_100 |
|
value: 15.665000000000001 |
|
- type: recall_at_1000 |
|
value: 52.559999999999995 |
|
- type: recall_at_3 |
|
value: 0.735 |
|
- type: recall_at_5 |
|
value: 1.175 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (sqi-eng) |
|
config: sqi-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 19.0 |
|
- type: f1 |
|
value: 15.331629955575188 |
|
- type: precision |
|
value: 14.38509724403208 |
|
- type: recall |
|
value: 19.0 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (fry-eng) |
|
config: fry-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 39.884393063583815 |
|
- type: f1 |
|
value: 32.369942196531795 |
|
- type: precision |
|
value: 30.036929993577395 |
|
- type: recall |
|
value: 39.884393063583815 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (kur-eng) |
|
config: kur-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 15.365853658536585 |
|
- type: f1 |
|
value: 12.49755078527547 |
|
- type: precision |
|
value: 11.840415442997939 |
|
- type: recall |
|
value: 15.365853658536585 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tur-eng) |
|
config: tur-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 11.1 |
|
- type: f1 |
|
value: 8.955359175928436 |
|
- type: precision |
|
value: 8.324461412770235 |
|
- type: recall |
|
value: 11.1 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (deu-eng) |
|
config: deu-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 87.7 |
|
- type: f1 |
|
value: 85.06214285714286 |
|
- type: precision |
|
value: 83.98761904761905 |
|
- type: recall |
|
value: 87.7 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (nld-eng) |
|
config: nld-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 56.00000000000001 |
|
- type: f1 |
|
value: 49.8456850459482 |
|
- type: precision |
|
value: 47.80084415584415 |
|
- type: recall |
|
value: 56.00000000000001 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ron-eng) |
|
config: ron-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 38.1 |
|
- type: f1 |
|
value: 33.85465329991646 |
|
- type: precision |
|
value: 32.37519841269841 |
|
- type: recall |
|
value: 38.1 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ang-eng) |
|
config: ang-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 42.53731343283582 |
|
- type: f1 |
|
value: 34.67903986560703 |
|
- type: precision |
|
value: 32.17128642501776 |
|
- type: recall |
|
value: 42.53731343283582 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ido-eng) |
|
config: ido-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 53.900000000000006 |
|
- type: f1 |
|
value: 47.83909812409812 |
|
- type: precision |
|
value: 45.67887667887668 |
|
- type: recall |
|
value: 53.900000000000006 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (jav-eng) |
|
config: jav-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 26.34146341463415 |
|
- type: f1 |
|
value: 22.264125162260022 |
|
- type: precision |
|
value: 21.384015912351636 |
|
- type: recall |
|
value: 26.34146341463415 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (isl-eng) |
|
config: isl-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 10.2 |
|
- type: f1 |
|
value: 8.001233870597419 |
|
- type: precision |
|
value: 7.383838204560821 |
|
- type: recall |
|
value: 10.2 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (slv-eng) |
|
config: slv-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 17.253948967193196 |
|
- type: f1 |
|
value: 13.055189087650387 |
|
- type: precision |
|
value: 12.105642744272275 |
|
- type: recall |
|
value: 17.253948967193196 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (cym-eng) |
|
config: cym-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 10.26086956521739 |
|
- type: f1 |
|
value: 8.31837824011737 |
|
- type: precision |
|
value: 7.879315672736052 |
|
- type: recall |
|
value: 10.26086956521739 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (kaz-eng) |
|
config: kaz-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 11.826086956521738 |
|
- type: f1 |
|
value: 9.663030581871162 |
|
- type: precision |
|
value: 9.152605557273077 |
|
- type: recall |
|
value: 11.826086956521738 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (est-eng) |
|
config: est-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 6.800000000000001 |
|
- type: f1 |
|
value: 5.608697757594542 |
|
- type: precision |
|
value: 5.333727335466467 |
|
- type: recall |
|
value: 6.800000000000001 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (heb-eng) |
|
config: heb-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 11.3 |
|
- type: f1 |
|
value: 7.4866384899217335 |
|
- type: precision |
|
value: 6.580321536442861 |
|
- type: recall |
|
value: 11.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (gla-eng) |
|
config: gla-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 4.101326899879373 |
|
- type: f1 |
|
value: 3.0988364784130122 |
|
- type: precision |
|
value: 2.925923150618102 |
|
- type: recall |
|
value: 4.101326899879373 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (mar-eng) |
|
config: mar-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 76.3 |
|
- type: f1 |
|
value: 71.55912698412699 |
|
- type: precision |
|
value: 69.55511904761904 |
|
- type: recall |
|
value: 76.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (lat-eng) |
|
config: lat-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 53.6 |
|
- type: f1 |
|
value: 46.74811085685228 |
|
- type: precision |
|
value: 44.41049616018656 |
|
- type: recall |
|
value: 53.6 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (bel-eng) |
|
config: bel-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 23.400000000000002 |
|
- type: f1 |
|
value: 18.485309948823105 |
|
- type: precision |
|
value: 17.12104734130107 |
|
- type: recall |
|
value: 23.400000000000002 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (pms-eng) |
|
config: pms-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 41.523809523809526 |
|
- type: f1 |
|
value: 36.577269291555005 |
|
- type: precision |
|
value: 35.00219198790627 |
|
- type: recall |
|
value: 41.523809523809526 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (gle-eng) |
|
config: gle-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 4.9 |
|
- type: f1 |
|
value: 3.909842412258181 |
|
- type: precision |
|
value: 3.7099694121032796 |
|
- type: recall |
|
value: 4.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (pes-eng) |
|
config: pes-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 26.900000000000002 |
|
- type: f1 |
|
value: 21.587309161426806 |
|
- type: precision |
|
value: 19.877234126984124 |
|
- type: recall |
|
value: 26.900000000000002 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (nob-eng) |
|
config: nob-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 37.3 |
|
- type: f1 |
|
value: 31.940531675926408 |
|
- type: precision |
|
value: 30.414405457464277 |
|
- type: recall |
|
value: 37.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (bul-eng) |
|
config: bul-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 34.4 |
|
- type: f1 |
|
value: 28.460500740394355 |
|
- type: precision |
|
value: 26.630818170746558 |
|
- type: recall |
|
value: 34.4 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (cbk-eng) |
|
config: cbk-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 67.5 |
|
- type: f1 |
|
value: 61.492367158984806 |
|
- type: precision |
|
value: 59.23266755904913 |
|
- type: recall |
|
value: 67.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (hun-eng) |
|
config: hun-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 7.9 |
|
- type: f1 |
|
value: 6.652063929922994 |
|
- type: precision |
|
value: 6.392931096681097 |
|
- type: recall |
|
value: 7.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (uig-eng) |
|
config: uig-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 2.6 |
|
- type: f1 |
|
value: 2.0216271963330783 |
|
- type: precision |
|
value: 1.9467343791901313 |
|
- type: recall |
|
value: 2.6 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (rus-eng) |
|
config: rus-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 76.6 |
|
- type: f1 |
|
value: 71.23357142857142 |
|
- type: precision |
|
value: 69.03261904761905 |
|
- type: recall |
|
value: 76.6 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (spa-eng) |
|
config: spa-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 98.6 |
|
- type: f1 |
|
value: 98.13333333333333 |
|
- type: precision |
|
value: 97.89999999999999 |
|
- type: recall |
|
value: 98.6 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (hye-eng) |
|
config: hye-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 1.8867924528301887 |
|
- type: f1 |
|
value: 0.9184016421339141 |
|
- type: precision |
|
value: 0.8343646123610833 |
|
- type: recall |
|
value: 1.8867924528301887 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tel-eng) |
|
config: tel-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 84.1880341880342 |
|
- type: f1 |
|
value: 80.56369556369557 |
|
- type: precision |
|
value: 79.02421652421653 |
|
- type: recall |
|
value: 84.1880341880342 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (afr-eng) |
|
config: afr-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 27.200000000000003 |
|
- type: f1 |
|
value: 22.55873107448107 |
|
- type: precision |
|
value: 21.13610950874723 |
|
- type: recall |
|
value: 27.200000000000003 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (mon-eng) |
|
config: mon-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 9.090909090909092 |
|
- type: f1 |
|
value: 7.37323521273764 |
|
- type: precision |
|
value: 7.01229523252768 |
|
- type: recall |
|
value: 9.090909090909092 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (arz-eng) |
|
config: arz-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 79.24528301886792 |
|
- type: f1 |
|
value: 74.80483178596387 |
|
- type: precision |
|
value: 72.8336827393431 |
|
- type: recall |
|
value: 79.24528301886792 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (hrv-eng) |
|
config: hrv-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 21.3 |
|
- type: f1 |
|
value: 17.754399705471684 |
|
- type: precision |
|
value: 16.81516621898026 |
|
- type: recall |
|
value: 21.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (nov-eng) |
|
config: nov-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 59.92217898832685 |
|
- type: f1 |
|
value: 54.92807451951421 |
|
- type: precision |
|
value: 53.071150639244024 |
|
- type: recall |
|
value: 59.92217898832685 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (gsw-eng) |
|
config: gsw-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 30.76923076923077 |
|
- type: f1 |
|
value: 23.70099036765703 |
|
- type: precision |
|
value: 21.666666666666664 |
|
- type: recall |
|
value: 30.76923076923077 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (nds-eng) |
|
config: nds-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 35.6 |
|
- type: f1 |
|
value: 29.87713276919159 |
|
- type: precision |
|
value: 28.07062211509473 |
|
- type: recall |
|
value: 35.6 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ukr-eng) |
|
config: ukr-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 38.1 |
|
- type: f1 |
|
value: 31.123585858585855 |
|
- type: precision |
|
value: 28.995893769823304 |
|
- type: recall |
|
value: 38.1 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (uzb-eng) |
|
config: uzb-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 10.74766355140187 |
|
- type: f1 |
|
value: 8.280338473537247 |
|
- type: precision |
|
value: 7.806134675293554 |
|
- type: recall |
|
value: 10.74766355140187 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (lit-eng) |
|
config: lit-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 7.6 |
|
- type: f1 |
|
value: 5.872095040470223 |
|
- type: precision |
|
value: 5.557777361527362 |
|
- type: recall |
|
value: 7.6 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ina-eng) |
|
config: ina-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 86.8 |
|
- type: f1 |
|
value: 83.72833333333332 |
|
- type: precision |
|
value: 82.4259523809524 |
|
- type: recall |
|
value: 86.8 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (lfn-eng) |
|
config: lfn-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 52.0 |
|
- type: f1 |
|
value: 46.48058132211534 |
|
- type: precision |
|
value: 44.52753032676945 |
|
- type: recall |
|
value: 52.0 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (zsm-eng) |
|
config: zsm-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 90.4 |
|
- type: f1 |
|
value: 88.10999999999999 |
|
- type: precision |
|
value: 87.10333333333334 |
|
- type: recall |
|
value: 90.4 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ita-eng) |
|
config: ita-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 79.5 |
|
- type: f1 |
|
value: 74.95746031746032 |
|
- type: precision |
|
value: 73.03249999999998 |
|
- type: recall |
|
value: 79.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (cmn-eng) |
|
config: cmn-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 96.7 |
|
- type: f1 |
|
value: 95.7 |
|
- type: precision |
|
value: 95.21666666666667 |
|
- type: recall |
|
value: 96.7 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (lvs-eng) |
|
config: lvs-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 10.7 |
|
- type: f1 |
|
value: 8.576412755390276 |
|
- type: precision |
|
value: 8.046714349557488 |
|
- type: recall |
|
value: 10.7 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (glg-eng) |
|
config: glg-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 86.0 |
|
- type: f1 |
|
value: 82.54523809523809 |
|
- type: precision |
|
value: 81.06166666666665 |
|
- type: recall |
|
value: 86.0 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ceb-eng) |
|
config: ceb-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 11.0 |
|
- type: f1 |
|
value: 9.080509354193564 |
|
- type: precision |
|
value: 8.57587968815845 |
|
- type: recall |
|
value: 11.0 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (bre-eng) |
|
config: bre-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 9.6 |
|
- type: f1 |
|
value: 7.409451659451658 |
|
- type: precision |
|
value: 6.8121069441897415 |
|
- type: recall |
|
value: 9.6 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ben-eng) |
|
config: ben-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 87.1 |
|
- type: f1 |
|
value: 83.88999999999999 |
|
- type: precision |
|
value: 82.395 |
|
- type: recall |
|
value: 87.1 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (swg-eng) |
|
config: swg-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 34.82142857142857 |
|
- type: f1 |
|
value: 29.175170068027214 |
|
- type: precision |
|
value: 27.40499084249084 |
|
- type: recall |
|
value: 34.82142857142857 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (arq-eng) |
|
config: arq-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 42.9198682766191 |
|
- type: f1 |
|
value: 37.21120707205811 |
|
- type: precision |
|
value: 35.23526784229309 |
|
- type: recall |
|
value: 42.9198682766191 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (kab-eng) |
|
config: kab-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 2.3 |
|
- type: f1 |
|
value: 1.5401826425879608 |
|
- type: precision |
|
value: 1.424235527544351 |
|
- type: recall |
|
value: 2.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (fra-eng) |
|
config: fra-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 95.6 |
|
- type: f1 |
|
value: 94.32333333333334 |
|
- type: precision |
|
value: 93.72500000000001 |
|
- type: recall |
|
value: 95.6 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (por-eng) |
|
config: por-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 95.5 |
|
- type: f1 |
|
value: 94.43333333333334 |
|
- type: precision |
|
value: 93.89999999999999 |
|
- type: recall |
|
value: 95.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tat-eng) |
|
config: tat-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 6.7 |
|
- type: f1 |
|
value: 4.9622522983552395 |
|
- type: precision |
|
value: 4.528962761017515 |
|
- type: recall |
|
value: 6.7 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (oci-eng) |
|
config: oci-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 50.8 |
|
- type: f1 |
|
value: 45.736438587556236 |
|
- type: precision |
|
value: 44.010822829131655 |
|
- type: recall |
|
value: 50.8 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (pol-eng) |
|
config: pol-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 23.9 |
|
- type: f1 |
|
value: 20.267261904761906 |
|
- type: precision |
|
value: 19.16142408316321 |
|
- type: recall |
|
value: 23.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (war-eng) |
|
config: war-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 13.4 |
|
- type: f1 |
|
value: 11.232209832252995 |
|
- type: precision |
|
value: 10.714445160103056 |
|
- type: recall |
|
value: 13.4 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (aze-eng) |
|
config: aze-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 10.299999999999999 |
|
- type: f1 |
|
value: 8.161916387744503 |
|
- type: precision |
|
value: 7.678631905405786 |
|
- type: recall |
|
value: 10.299999999999999 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (vie-eng) |
|
config: vie-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 96.7 |
|
- type: f1 |
|
value: 95.83333333333334 |
|
- type: precision |
|
value: 95.41666666666667 |
|
- type: recall |
|
value: 96.7 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (nno-eng) |
|
config: nno-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 24.9 |
|
- type: f1 |
|
value: 20.794749162495066 |
|
- type: precision |
|
value: 19.575997295469914 |
|
- type: recall |
|
value: 24.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (cha-eng) |
|
config: cha-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 32.11678832116788 |
|
- type: f1 |
|
value: 26.960375391032326 |
|
- type: precision |
|
value: 25.498078211502524 |
|
- type: recall |
|
value: 32.11678832116788 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (mhr-eng) |
|
config: mhr-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 4.9 |
|
- type: f1 |
|
value: 3.251889552842259 |
|
- type: precision |
|
value: 2.9281137342615295 |
|
- type: recall |
|
value: 4.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (dan-eng) |
|
config: dan-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 38.9 |
|
- type: f1 |
|
value: 33.59595154442981 |
|
- type: precision |
|
value: 31.906759791342587 |
|
- type: recall |
|
value: 38.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ell-eng) |
|
config: ell-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 16.900000000000002 |
|
- type: f1 |
|
value: 13.082818919542666 |
|
- type: precision |
|
value: 12.125554724968518 |
|
- type: recall |
|
value: 16.900000000000002 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (amh-eng) |
|
config: amh-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 0.5952380952380952 |
|
- type: f1 |
|
value: 0.09920634920634923 |
|
- type: precision |
|
value: 0.05411255411255411 |
|
- type: recall |
|
value: 0.5952380952380952 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (pam-eng) |
|
config: pam-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 8.1 |
|
- type: f1 |
|
value: 7.033911671727207 |
|
- type: precision |
|
value: 6.759952905986985 |
|
- type: recall |
|
value: 8.1 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (hsb-eng) |
|
config: hsb-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 16.149068322981368 |
|
- type: f1 |
|
value: 13.314287609382625 |
|
- type: precision |
|
value: 12.588291889534126 |
|
- type: recall |
|
value: 16.149068322981368 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (srp-eng) |
|
config: srp-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 22.3 |
|
- type: f1 |
|
value: 18.754672526177103 |
|
- type: precision |
|
value: 17.77463320976479 |
|
- type: recall |
|
value: 22.3 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (epo-eng) |
|
config: epo-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 39.5 |
|
- type: f1 |
|
value: 33.91659439373835 |
|
- type: precision |
|
value: 32.244738455988454 |
|
- type: recall |
|
value: 39.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (kzj-eng) |
|
config: kzj-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 7.5 |
|
- type: f1 |
|
value: 6.300929449087343 |
|
- type: precision |
|
value: 6.05555758176835 |
|
- type: recall |
|
value: 7.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (awa-eng) |
|
config: awa-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 57.14285714285714 |
|
- type: f1 |
|
value: 53.011353725639445 |
|
- type: precision |
|
value: 51.78829107400536 |
|
- type: recall |
|
value: 57.14285714285714 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (fao-eng) |
|
config: fao-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 16.030534351145036 |
|
- type: f1 |
|
value: 14.424487352192786 |
|
- type: precision |
|
value: 13.98739301411057 |
|
- type: recall |
|
value: 16.030534351145036 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (mal-eng) |
|
config: mal-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 96.21542940320232 |
|
- type: f1 |
|
value: 95.0509461426492 |
|
- type: precision |
|
value: 94.46870451237264 |
|
- type: recall |
|
value: 96.21542940320232 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ile-eng) |
|
config: ile-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 69.1 |
|
- type: f1 |
|
value: 63.649573934837086 |
|
- type: precision |
|
value: 61.44357142857143 |
|
- type: recall |
|
value: 69.1 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (bos-eng) |
|
config: bos-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 23.728813559322035 |
|
- type: f1 |
|
value: 19.281200536513545 |
|
- type: precision |
|
value: 18.11042731593579 |
|
- type: recall |
|
value: 23.728813559322035 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (cor-eng) |
|
config: cor-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 4.9 |
|
- type: f1 |
|
value: 3.8602777777777777 |
|
- type: precision |
|
value: 3.553962393468025 |
|
- type: recall |
|
value: 4.9 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (cat-eng) |
|
config: cat-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 92.10000000000001 |
|
- type: f1 |
|
value: 90.24190476190476 |
|
- type: precision |
|
value: 89.41666666666667 |
|
- type: recall |
|
value: 92.10000000000001 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (eus-eng) |
|
config: eus-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 78.8 |
|
- type: f1 |
|
value: 74.53390756302521 |
|
- type: precision |
|
value: 72.79386904761904 |
|
- type: recall |
|
value: 78.8 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (yue-eng) |
|
config: yue-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 91.60000000000001 |
|
- type: f1 |
|
value: 89.39 |
|
- type: precision |
|
value: 88.375 |
|
- type: recall |
|
value: 91.60000000000001 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (swe-eng) |
|
config: swe-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 32.4 |
|
- type: f1 |
|
value: 27.824399979105863 |
|
- type: precision |
|
value: 26.434715247715246 |
|
- type: recall |
|
value: 32.4 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (dtp-eng) |
|
config: dtp-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 6.800000000000001 |
|
- type: f1 |
|
value: 5.258204523374802 |
|
- type: precision |
|
value: 4.940595825661615 |
|
- type: recall |
|
value: 6.800000000000001 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (kat-eng) |
|
config: kat-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 2.0107238605898123 |
|
- type: f1 |
|
value: 1.4770600435024532 |
|
- type: precision |
|
value: 1.4215975441361408 |
|
- type: recall |
|
value: 2.0107238605898123 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (jpn-eng) |
|
config: jpn-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 87.2 |
|
- type: f1 |
|
value: 83.88333333333333 |
|
- type: precision |
|
value: 82.44166666666668 |
|
- type: recall |
|
value: 87.2 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (csb-eng) |
|
config: csb-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 14.229249011857709 |
|
- type: f1 |
|
value: 11.043453048700425 |
|
- type: precision |
|
value: 10.285902503293807 |
|
- type: recall |
|
value: 14.229249011857709 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (xho-eng) |
|
config: xho-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 9.859154929577464 |
|
- type: f1 |
|
value: 7.960154086914651 |
|
- type: precision |
|
value: 7.679678785726838 |
|
- type: recall |
|
value: 9.859154929577464 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (orv-eng) |
|
config: orv-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 12.574850299401197 |
|
- type: f1 |
|
value: 8.435162337247867 |
|
- type: precision |
|
value: 7.5408084342568324 |
|
- type: recall |
|
value: 12.574850299401197 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ind-eng) |
|
config: ind-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 93.5 |
|
- type: f1 |
|
value: 91.90666666666665 |
|
- type: precision |
|
value: 91.14166666666668 |
|
- type: recall |
|
value: 93.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tuk-eng) |
|
config: tuk-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 8.866995073891626 |
|
- type: f1 |
|
value: 6.8479221927497775 |
|
- type: precision |
|
value: 6.431102386508143 |
|
- type: recall |
|
value: 8.866995073891626 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (max-eng) |
|
config: max-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 46.12676056338028 |
|
- type: f1 |
|
value: 41.447273383893105 |
|
- type: precision |
|
value: 39.80374351371386 |
|
- type: recall |
|
value: 46.12676056338028 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (swh-eng) |
|
config: swh-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 35.38461538461539 |
|
- type: f1 |
|
value: 27.80912253371418 |
|
- type: precision |
|
value: 25.588007434161277 |
|
- type: recall |
|
value: 35.38461538461539 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (hin-eng) |
|
config: hin-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 96.1 |
|
- type: f1 |
|
value: 94.88333333333333 |
|
- type: precision |
|
value: 94.3 |
|
- type: recall |
|
value: 96.1 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (dsb-eng) |
|
config: dsb-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 18.16283924843424 |
|
- type: f1 |
|
value: 15.00273275898725 |
|
- type: precision |
|
value: 14.135773519036146 |
|
- type: recall |
|
value: 18.16283924843424 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ber-eng) |
|
config: ber-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 6.4 |
|
- type: f1 |
|
value: 5.169780886652615 |
|
- type: precision |
|
value: 4.901094815916798 |
|
- type: recall |
|
value: 6.4 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tam-eng) |
|
config: tam-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 85.66775244299674 |
|
- type: f1 |
|
value: 81.86753528773072 |
|
- type: precision |
|
value: 80.13029315960912 |
|
- type: recall |
|
value: 85.66775244299674 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (slk-eng) |
|
config: slk-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 14.7 |
|
- type: f1 |
|
value: 12.296409553542203 |
|
- type: precision |
|
value: 11.643939628482972 |
|
- type: recall |
|
value: 14.7 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tgl-eng) |
|
config: tgl-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 14.000000000000002 |
|
- type: f1 |
|
value: 11.188658083109301 |
|
- type: precision |
|
value: 10.439068547503426 |
|
- type: recall |
|
value: 14.000000000000002 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ast-eng) |
|
config: ast-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 74.01574803149606 |
|
- type: f1 |
|
value: 68.3727034120735 |
|
- type: precision |
|
value: 66.06299212598424 |
|
- type: recall |
|
value: 74.01574803149606 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (mkd-eng) |
|
config: mkd-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 20.200000000000003 |
|
- type: f1 |
|
value: 15.584321026350167 |
|
- type: precision |
|
value: 14.220359087863855 |
|
- type: recall |
|
value: 20.200000000000003 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (khm-eng) |
|
config: khm-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 1.2465373961218837 |
|
- type: f1 |
|
value: 0.7009849184364421 |
|
- type: precision |
|
value: 0.6369121979354991 |
|
- type: recall |
|
value: 1.2465373961218837 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ces-eng) |
|
config: ces-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 15.5 |
|
- type: f1 |
|
value: 12.992671904350203 |
|
- type: precision |
|
value: 12.323623108157992 |
|
- type: recall |
|
value: 15.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tzl-eng) |
|
config: tzl-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 37.5 |
|
- type: f1 |
|
value: 32.70299145299145 |
|
- type: precision |
|
value: 31.066176470588236 |
|
- type: recall |
|
value: 37.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (urd-eng) |
|
config: urd-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 86.2 |
|
- type: f1 |
|
value: 82.87166666666667 |
|
- type: precision |
|
value: 81.44261904761906 |
|
- type: recall |
|
value: 86.2 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (ara-eng) |
|
config: ara-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 92.5 |
|
- type: f1 |
|
value: 90.61666666666667 |
|
- type: precision |
|
value: 89.71666666666668 |
|
- type: recall |
|
value: 92.5 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (kor-eng) |
|
config: kor-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 51.7 |
|
- type: f1 |
|
value: 44.78806599832916 |
|
- type: precision |
|
value: 42.26749389499389 |
|
- type: recall |
|
value: 51.7 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (yid-eng) |
|
config: yid-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 0.9433962264150944 |
|
- type: f1 |
|
value: 0.48704516529471925 |
|
- type: precision |
|
value: 0.41179094097726165 |
|
- type: recall |
|
value: 0.9433962264150944 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (fin-eng) |
|
config: fin-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 6.800000000000001 |
|
- type: f1 |
|
value: 5.480668860234897 |
|
- type: precision |
|
value: 5.195067371791852 |
|
- type: recall |
|
value: 6.800000000000001 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (tha-eng) |
|
config: tha-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 10.036496350364963 |
|
- type: f1 |
|
value: 6.784271238735886 |
|
- type: precision |
|
value: 6.159462364744479 |
|
- type: recall |
|
value: 10.036496350364963 |
|
- task: |
|
type: BitextMining |
|
dataset: |
|
type: mteb/tatoeba-bitext-mining |
|
name: MTEB Tatoeba (wuu-eng) |
|
config: wuu-eng |
|
split: test |
|
revision: 9080400076fbadbb4c4dcb136ff4eddc40b42553 |
|
metrics: |
|
- type: accuracy |
|
value: 90.3 |
|
- type: f1 |
|
value: 87.91499999999999 |
|
- type: precision |
|
value: 86.82595238095237 |
|
- type: recall |
|
value: 90.3 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/ThuNewsClusteringP2P |
|
name: MTEB ThuNewsClusteringP2P |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 49.19154423543331 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: C-MTEB/ThuNewsClusteringS2S |
|
name: MTEB ThuNewsClusteringS2S |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: v_measure |
|
value: 47.76345036893387 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: webis-touche2020 |
|
name: MTEB Touche2020 |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 2.396 |
|
- type: map_at_10 |
|
value: 9.994 |
|
- type: map_at_100 |
|
value: 16.067999999999998 |
|
- type: map_at_1000 |
|
value: 17.59 |
|
- type: map_at_3 |
|
value: 4.733 |
|
- type: map_at_5 |
|
value: 6.7589999999999995 |
|
- type: mrr_at_1 |
|
value: 28.571 |
|
- type: mrr_at_10 |
|
value: 47.678 |
|
- type: mrr_at_100 |
|
value: 48.311 |
|
- type: mrr_at_1000 |
|
value: 48.317 |
|
- type: mrr_at_3 |
|
value: 43.878 |
|
- type: mrr_at_5 |
|
value: 46.224 |
|
- type: ndcg_at_1 |
|
value: 25.509999999999998 |
|
- type: ndcg_at_10 |
|
value: 25.189 |
|
- type: ndcg_at_100 |
|
value: 36.179 |
|
- type: ndcg_at_1000 |
|
value: 47.562 |
|
- type: ndcg_at_3 |
|
value: 26.858999999999998 |
|
- type: ndcg_at_5 |
|
value: 26.825 |
|
- type: precision_at_1 |
|
value: 28.571 |
|
- type: precision_at_10 |
|
value: 23.469 |
|
- type: precision_at_100 |
|
value: 7.550999999999999 |
|
- type: precision_at_1000 |
|
value: 1.51 |
|
- type: precision_at_3 |
|
value: 29.252 |
|
- type: precision_at_5 |
|
value: 28.571 |
|
- type: recall_at_1 |
|
value: 2.396 |
|
- type: recall_at_10 |
|
value: 16.551 |
|
- type: recall_at_100 |
|
value: 46.438 |
|
- type: recall_at_1000 |
|
value: 81.04 |
|
- type: recall_at_3 |
|
value: 6.145 |
|
- type: recall_at_5 |
|
value: 9.728 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/toxic_conversations_50k |
|
name: MTEB ToxicConversationsClassification |
|
config: default |
|
split: test |
|
revision: d7c0de2777da35d6aae2200a62c6e0e5af397c4c |
|
metrics: |
|
- type: accuracy |
|
value: 71.5842 |
|
- type: ap |
|
value: 14.770823761227014 |
|
- type: f1 |
|
value: 55.22772349179383 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: mteb/tweet_sentiment_extraction |
|
name: MTEB TweetSentimentExtractionClassification |
|
config: default |
|
split: test |
|
revision: d604517c81ca91fe16a244d1248fc021f9ecee7a |
|
metrics: |
|
- type: accuracy |
|
value: 62.13921901528015 |
|
- type: f1 |
|
value: 62.450042974251694 |
|
- task: |
|
type: Clustering |
|
dataset: |
|
type: mteb/twentynewsgroups-clustering |
|
name: MTEB TwentyNewsgroupsClustering |
|
config: default |
|
split: test |
|
revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 |
|
metrics: |
|
- type: v_measure |
|
value: 40.81463922932671 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/twittersemeval2015-pairclassification |
|
name: MTEB TwitterSemEval2015 |
|
config: default |
|
split: test |
|
revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 85.86755677415509 |
|
- type: cos_sim_ap |
|
value: 73.8131664470889 |
|
- type: cos_sim_f1 |
|
value: 68.03196803196803 |
|
- type: cos_sim_precision |
|
value: 64.58036984352773 |
|
- type: cos_sim_recall |
|
value: 71.87335092348285 |
|
- type: dot_accuracy |
|
value: 84.58604041246946 |
|
- type: dot_ap |
|
value: 69.43165607336826 |
|
- type: dot_f1 |
|
value: 65.84285381207741 |
|
- type: dot_precision |
|
value: 58.980785296574766 |
|
- type: dot_recall |
|
value: 74.51187335092348 |
|
- type: euclidean_accuracy |
|
value: 85.60529296060082 |
|
- type: euclidean_ap |
|
value: 72.48939155702391 |
|
- type: euclidean_f1 |
|
value: 66.84775898259045 |
|
- type: euclidean_precision |
|
value: 62.822000464144814 |
|
- type: euclidean_recall |
|
value: 71.42480211081794 |
|
- type: manhattan_accuracy |
|
value: 85.5456875484294 |
|
- type: manhattan_ap |
|
value: 72.37178636434892 |
|
- type: manhattan_f1 |
|
value: 66.6751398068124 |
|
- type: manhattan_precision |
|
value: 64.32074546346249 |
|
- type: manhattan_recall |
|
value: 69.2084432717678 |
|
- type: max_accuracy |
|
value: 85.86755677415509 |
|
- type: max_ap |
|
value: 73.8131664470889 |
|
- type: max_f1 |
|
value: 68.03196803196803 |
|
- task: |
|
type: PairClassification |
|
dataset: |
|
type: mteb/twitterurlcorpus-pairclassification |
|
name: MTEB TwitterURLCorpus |
|
config: default |
|
split: test |
|
revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf |
|
metrics: |
|
- type: cos_sim_accuracy |
|
value: 89.39341017580627 |
|
- type: cos_sim_ap |
|
value: 86.7769866448429 |
|
- type: cos_sim_f1 |
|
value: 79.26586570354536 |
|
- type: cos_sim_precision |
|
value: 76.02149017390076 |
|
- type: cos_sim_recall |
|
value: 82.79950723744996 |
|
- type: dot_accuracy |
|
value: 89.15861373074087 |
|
- type: dot_ap |
|
value: 85.15235322715995 |
|
- type: dot_f1 |
|
value: 78.97118887294403 |
|
- type: dot_precision |
|
value: 75.6290083867785 |
|
- type: dot_recall |
|
value: 82.62242069602709 |
|
- type: euclidean_accuracy |
|
value: 89.0266620095471 |
|
- type: euclidean_ap |
|
value: 86.18904940615533 |
|
- type: euclidean_f1 |
|
value: 78.37750135208222 |
|
- type: euclidean_precision |
|
value: 73.70312605953754 |
|
- type: euclidean_recall |
|
value: 83.68493994456422 |
|
- type: manhattan_accuracy |
|
value: 88.98397174680794 |
|
- type: manhattan_ap |
|
value: 86.18302538523727 |
|
- type: manhattan_f1 |
|
value: 78.42197035745423 |
|
- type: manhattan_precision |
|
value: 74.23658872077029 |
|
- type: manhattan_recall |
|
value: 83.10748383122882 |
|
- type: max_accuracy |
|
value: 89.39341017580627 |
|
- type: max_ap |
|
value: 86.7769866448429 |
|
- type: max_f1 |
|
value: 79.26586570354536 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: C-MTEB/VideoRetrieval |
|
name: MTEB VideoRetrieval |
|
config: default |
|
split: dev |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 46.9 |
|
- type: map_at_10 |
|
value: 57.399 |
|
- type: map_at_100 |
|
value: 57.976000000000006 |
|
- type: map_at_1000 |
|
value: 58.00300000000001 |
|
- type: map_at_3 |
|
value: 54.967 |
|
- type: map_at_5 |
|
value: 56.562 |
|
- type: mrr_at_1 |
|
value: 46.800000000000004 |
|
- type: mrr_at_10 |
|
value: 57.349000000000004 |
|
- type: mrr_at_100 |
|
value: 57.926 |
|
- type: mrr_at_1000 |
|
value: 57.952999999999996 |
|
- type: mrr_at_3 |
|
value: 54.917 |
|
- type: mrr_at_5 |
|
value: 56.51199999999999 |
|
- type: ndcg_at_1 |
|
value: 46.9 |
|
- type: ndcg_at_10 |
|
value: 62.437 |
|
- type: ndcg_at_100 |
|
value: 65.273 |
|
- type: ndcg_at_1000 |
|
value: 65.999 |
|
- type: ndcg_at_3 |
|
value: 57.524 |
|
- type: ndcg_at_5 |
|
value: 60.402 |
|
- type: precision_at_1 |
|
value: 46.9 |
|
- type: precision_at_10 |
|
value: 7.82 |
|
- type: precision_at_100 |
|
value: 0.915 |
|
- type: precision_at_1000 |
|
value: 0.097 |
|
- type: precision_at_3 |
|
value: 21.633 |
|
- type: precision_at_5 |
|
value: 14.38 |
|
- type: recall_at_1 |
|
value: 46.9 |
|
- type: recall_at_10 |
|
value: 78.2 |
|
- type: recall_at_100 |
|
value: 91.5 |
|
- type: recall_at_1000 |
|
value: 97.2 |
|
- type: recall_at_3 |
|
value: 64.9 |
|
- type: recall_at_5 |
|
value: 71.89999999999999 |
|
- task: |
|
type: Classification |
|
dataset: |
|
type: C-MTEB/waimai-classification |
|
name: MTEB Waimai |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: accuracy |
|
value: 84.68 |
|
- type: ap |
|
value: 66.4749730574293 |
|
- type: f1 |
|
value: 82.93606561551698 |
|
--- |
|
# Model Card for udever-bloom |
|
|
|
<!-- Provide a quick summary of what the model is/does. --> |
|
|
|
`udever-bloom-7b1` is finetuned from [bigscience/bloom-7b1](https://huggingface.co/bigscience/bloom-7b1) via [BitFit](https://aclanthology.org/2022.acl-short.1/) on MS MARCO Passage Ranking, SNLI and MultiNLI data. |
|
It is a universal embedding model across tasks, natural and programming languages. |
|
(From the technical view, `udever` is merely with some minor improvements to `sgpt-bloom`) |
|
|
|
<img width="338" height="259" src="https://user-images.githubusercontent.com/26690193/277643721-cdb7f227-cae5-40e1-b6e1-a201bde00339.png" /> |
|
|
|
|
|
## Model Details |
|
|
|
### Model Description |
|
|
|
- **Developed by:** Alibaba Group |
|
- **Model type:** Transformer-based Language Model (decoder-only) |
|
- **Language(s) (NLP):** Multiple; see [bloom training data](https://huggingface.co/bigscience/bloom-7b1#training-data) |
|
- **Finetuned from model :** [bigscience/bloom-7b1](https://huggingface.co/bigscience/bloom-7b1) |
|
|
|
### Model Sources |
|
|
|
<!-- Provide the basic links for the model. --> |
|
|
|
- **Repository:** [github.com/izhx/uni-rep](https://github.com/izhx/uni-rep) |
|
- **Paper :** [Language Models are Universal Embedders](https://arxiv.org/pdf/2310.08232.pdf) |
|
- **Training Date :** 2023-06 |
|
|
|
|
|
### Checkpoints |
|
- [udever-bloom-560m](https://huggingface.co/izhx/udever-bloom-560m) |
|
- [udever-bloom-1b1](https://huggingface.co/izhx/udever-bloom-1b1) |
|
- [udever-bloom-3b](https://huggingface.co/izhx/udever-bloom-3b) |
|
- [udever-bloom-7b1](https://huggingface.co/izhx/udever-bloom-7b1) |
|
|
|
On ModelScope / 魔搭社区: [udever-bloom-560m](https://modelscope.cn/models/damo/udever-bloom-560m), [udever-bloom-1b1](https://modelscope.cn/models/damo/udever-bloom-1b1), [udever-bloom-3b](https://modelscope.cn/models/damo/udever-bloom-3b), [udever-bloom-7b1](https://modelscope.cn/models/damo/udever-bloom-7b1) |
|
|
|
|
|
## How to Get Started with the Model |
|
|
|
Use the code below to get started with the model. |
|
|
|
```python |
|
import torch |
|
from transformers import AutoTokenizer, BloomModel |
|
|
|
tokenizer = AutoTokenizer.from_pretrained('izhx/udever-bloom-7b1') |
|
model = BloomModel.from_pretrained('izhx/udever-bloom-7b1') |
|
|
|
boq, eoq, bod, eod = '[BOQ]', '[EOQ]', '[BOD]', '[EOD]' |
|
eoq_id, eod_id = tokenizer.convert_tokens_to_ids([eoq, eod]) |
|
|
|
if tokenizer.padding_side != 'left': |
|
print('!!!', tokenizer.padding_side) |
|
tokenizer.padding_side = 'left' |
|
|
|
|
|
def encode(texts: list, is_query: bool = True, max_length=300): |
|
bos = boq if is_query else bod |
|
eos_id = eoq_id if is_query else eod_id |
|
texts = [bos + t for t in texts] |
|
encoding = tokenizer( |
|
texts, truncation=True, max_length=max_length - 1, padding=True |
|
) |
|
for ids, mask in zip(encoding['input_ids'], encoding['attention_mask']): |
|
ids.append(eos_id) |
|
mask.append(1) |
|
inputs = tokenizer.pad(encoding, return_tensors='pt') |
|
with torch.inference_mode(): |
|
outputs = model(**inputs) |
|
embeds = outputs.last_hidden_state[:, -1] |
|
return embeds |
|
|
|
encode(['I am Bert', 'You are Elmo']) |
|
|
|
``` |
|
|
|
## Training Details |
|
|
|
### Training Data |
|
|
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. --> |
|
|
|
- MS MARCO Passage Ranking, retrieved by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86) |
|
- SNLI and MultiNLI (https://sbert.net/datasets/AllNLI.tsv.gz) |
|
|
|
|
|
### Training Procedure |
|
|
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. --> |
|
|
|
#### Preprocessing |
|
|
|
MS MARCO hard negatives provided by (https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_bi-encoder_mnrl.py#L86). |
|
Negatives for SNLI and MultiNLI are randomly sampled. |
|
|
|
|
|
#### Training Hyperparameters |
|
|
|
- **Training regime:** tf32, BitFit |
|
- **Batch size:** 1024 |
|
- **Epochs:** 3 |
|
- **Optimizer:** AdamW |
|
- **Learning rate:** 1e-4 |
|
- **Scheduler:** constant with warmup. |
|
- **Warmup:** 0.25 epoch |
|
|
|
|
|
## Evaluation |
|
|
|
### Table 1: Massive Text Embedding Benchmark [MTEB](https://huggingface.co/spaces/mteb/leaderboard) |
|
|
|
| MTEB | Avg. | Class. | Clust. | PairClass. | Rerank. | Retr. | STS | Summ. | |
|
|-----------------------------|--------------|--------------|--------------|--------------|--------------|--------------|--------------|--------| |
|
| #Datasets ➡️ | 56 | 12 | 11 | 3 | 4 | 15 | 10 | 1 | |
|
|| |
|
| bge-large-en-v1.5 | **64.23** | **75.97** | 46.08| **87.12** | **60.03** | **54.29** | 83.11| 31.61 | |
|
| bge-base-en-v1.5 | 63.55| 75.53| 45.77| 86.55| 58.86| 53.25| 82.4| 31.07 | |
|
| gte-large | 63.13| 73.33| **46.84** | 85| 59.13| 52.22| **83.35** | 31.66 | |
|
| gte-base | 62.39| 73.01| 46.2| 84.57| 58.61| 51.14| 82.3| 31.17 | |
|
| e5-large-v2 | 62.25| 75.24| 44.49| 86.03| 56.61| 50.56| 82.05| 30.19 | |
|
| instructor-xl | 61.79| 73.12| 44.74| 86.62| 57.29| 49.26| 83.06| 32.32 | |
|
| instructor-large | 61.59| 73.86| 45.29| 85.89| 57.54| 47.57| 83.15| 31.84 | |
|
| e5-base-v2 | 61.5 | 73.84| 43.8| 85.73| 55.91| 50.29| 81.05| 30.28 | |
|
| e5-large | 61.42| 73.14| 43.33| 85.94| 56.53| 49.99| 82.06| 30.97 | |
|
| text-embedding-ada-002 (OpenAI API) | 60.99| 70.93| 45.9 | 84.89| 56.32| 49.25| 80.97| 30.8 | |
|
| e5-base | 60.44| 72.63| 42.11| 85.09| 55.7 | 48.75| 80.96| 31.01 | |
|
| SGPT-5.8B-msmarco | 58.93| 68.13| 40.34| 82 | 56.56| 50.25| 78.1 | 31.46 | |
|
| sgpt-bloom-7b1-msmarco | 57.59| 66.19| 38.93| 81.9 | 55.65| 48.22| 77.74| **33.6** | |
|
|| |
|
| Udever-bloom-560m | 55.80| 68.04| 36.89| 81.05| 52.60| 41.19| 79.93| 32.06 | |
|
| Udever-bloom-1b1 | 58.28| 70.18| 39.11| 83.11| 54.28| 45.27| 81.52| 31.10 | |
|
| Udever-bloom-3b | 59.86| 71.91| 40.74| 84.06| 54.90| 47.67| 82.37| 30.62 | |
|
| Udever-bloom-7b1 | 60.63 | 72.13| 40.81| 85.40| 55.91| 49.34| 83.01| 30.97 | |
|
|
|
|
|
### Table 2: [CodeSearchNet](https://github.com/github/CodeSearchNet) |
|
|
|
| CodeSearchNet | Go | Ruby | Python | Java | JS | PHP | Avg. | |
|
|-|-|-|-|-|-|-|-| |
|
| CodeBERT | 69.3 | 70.6 | 84.0 | 86.8 | 74.8 | 70.6 | 76.0 | |
|
| GraphCodeBERT | 84.1 | 73.2 | 87.9 | 75.7 | 71.1 | 72.5 | 77.4 | |
|
| cpt-code S | **97.7** | **86.3** | 99.8 | 94.0 | 86.0 | 96.7 | 93.4 | |
|
| cpt-code M | 97.5 | 85.5 | **99.9** | **94.4** | **86.5** | **97.2** | **93.5** | |
|
| sgpt-bloom-7b1-msmarco | 76.79 | 69.25 | 95.68 | 77.93 | 70.35 | 73.45 | 77.24 | |
|
|| |
|
| Udever-bloom-560m | 75.38 | 66.67 | 96.23 | 78.99 | 69.39 | 73.69 | 76.73 | |
|
| Udever-bloom-1b1 | 78.76 | 72.85 | 97.67 | 82.77 | 74.38 | 78.97 | 80.90 | |
|
| Udever-bloom-3b | 80.63 | 75.40 | 98.02 | 83.88 | 76.18 | 79.67 | 82.29 | |
|
| Udever-bloom-7b1 | 79.37 | 76.59 | 98.38 | 84.68 | 77.49 | 80.03 | 82.76 | |
|
|
|
|
|
### Table 3: Chinese multi-domain retrieval [Multi-cpr](https://dl.acm.org/doi/10.1145/3477495.3531736) |
|
|
|
| | | |E-commerce | | Entertainment video | | Medical | | |
|
|--|--|--|--|--|--|--|--|--| |
|
| Model | Train | Backbone | MRR@10 | Recall@1k | MRR@10 | Recall@1k | MRR@10 | Recall@1k | |
|
|| |
|
| BM25 | - | - | 0.225 | 0.815 | 0.225 | 0.780 | 0.187 | 0.482 | |
|
| Doc2Query | - | - | 0.239 | 0.826 | 0.238 | 0.794 | 0.210 | 0.505 | |
|
| DPR-1 | In-Domain | BERT | 0.270 | 0.921 | 0.254 | 0.934 | 0.327 | 0.747 | |
|
| DPR-2 | In-Domain | BERT-CT | 0.289 | **0.926** | 0.263 | **0.935** | 0.339 | **0.769** | |
|
| text-embedding-ada-002 | General | GPT | 0.183 | 0.825 | 0.159 | 0.786 | 0.245 | 0.593 | |
|
| sgpt-bloom-7b1-msmarco | General | BLOOM | 0.242 | 0.840 | 0.227 | 0.829 | 0.311 | 0.675 | |
|
|| |
|
| Udever-bloom-560m | General | BLOOM | 0.156 | 0.802 | 0.149 | 0.749 | 0.245 | 0.571 | |
|
| Udever-bloom-1b1 | General | BLOOM | 0.244 | 0.863 | 0.208 | 0.815 | 0.241 | 0.557 | |
|
| Udever-bloom-3b | General | BLOOM | 0.267 | 0.871 | 0.228 | 0.836 | 0.288 | 0.619 | |
|
| Udever-bloom-7b1 | General | BLOOM | **0.296** | 0.889 | **0.267** | 0.907 | **0.343** | 0.705 | |
|
|
|
#### More results refer to [paper](https://arxiv.org/pdf/2310.08232.pdf) section 3. |
|
|
|
|
|
|
|
## Technical Specifications |
|
|
|
### Model Architecture and Objective |
|
|
|
- Model: [bigscience/bloom-7b1](https://huggingface.co/bigscience/bloom-7b1). |
|
- Objective: Constrastive loss with hard negatives (refer to [paper](https://arxiv.org/pdf/2310.08232.pdf) section 2.2). |
|
|
|
|
|
### Compute Infrastructure |
|
|
|
- Nvidia A100 SXM4 80GB. |
|
- torch 2.0.0, transformers 4.29.2. |
|
|
|
|
|
## Citation |
|
|
|
**BibTeX:** |
|
|
|
```BibTeX |
|
@article{zhang2023language, |
|
title={Language Models are Universal Embedders}, |
|
author={Zhang, Xin and Li, Zehan and Zhang, Yanzhao and Long, Dingkun and Xie, Pengjun and Zhang, Meishan and Zhang, Min}, |
|
journal={arXiv preprint arXiv:2310.08232}, |
|
year={2023} |
|
} |
|
``` |
|
|
