|
--- |
|
license: apache-2.0 |
|
pipeline_tag: sentence-similarity |
|
tags: |
|
- sentence-transformers |
|
- feature-extraction |
|
- sentence-similarity |
|
- transformers |
|
- mteb |
|
model-index: |
|
- name: cai-lunaris-text-embeddings |
|
results: |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: arguana |
|
name: MTEB ArguAna |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 17.07 |
|
- type: map_at_10 |
|
value: 29.372999999999998 |
|
- type: map_at_100 |
|
value: 30.79 |
|
- type: map_at_1000 |
|
value: 30.819999999999997 |
|
- type: map_at_3 |
|
value: 24.395 |
|
- type: map_at_5 |
|
value: 27.137 |
|
- type: mrr_at_1 |
|
value: 17.923000000000002 |
|
- type: mrr_at_10 |
|
value: 29.695 |
|
- type: mrr_at_100 |
|
value: 31.098 |
|
- type: mrr_at_1000 |
|
value: 31.128 |
|
- type: mrr_at_3 |
|
value: 24.704 |
|
- type: mrr_at_5 |
|
value: 27.449 |
|
- type: ndcg_at_1 |
|
value: 17.07 |
|
- type: ndcg_at_10 |
|
value: 37.269000000000005 |
|
- type: ndcg_at_100 |
|
value: 43.716 |
|
- type: ndcg_at_1000 |
|
value: 44.531 |
|
- type: ndcg_at_3 |
|
value: 26.839000000000002 |
|
- type: ndcg_at_5 |
|
value: 31.845000000000002 |
|
- type: precision_at_1 |
|
value: 17.07 |
|
- type: precision_at_10 |
|
value: 6.3020000000000005 |
|
- type: precision_at_100 |
|
value: 0.922 |
|
- type: precision_at_1000 |
|
value: 0.099 |
|
- type: precision_at_3 |
|
value: 11.309 |
|
- type: precision_at_5 |
|
value: 9.246 |
|
- type: recall_at_1 |
|
value: 17.07 |
|
- type: recall_at_10 |
|
value: 63.016000000000005 |
|
- type: recall_at_100 |
|
value: 92.24799999999999 |
|
- type: recall_at_1000 |
|
value: 98.72 |
|
- type: recall_at_3 |
|
value: 33.926 |
|
- type: recall_at_5 |
|
value: 46.23 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/askubuntudupquestions-reranking |
|
name: MTEB AskUbuntuDupQuestions |
|
config: default |
|
split: test |
|
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 |
|
metrics: |
|
- type: map |
|
value: 53.44266265900711 |
|
- type: mrr |
|
value: 66.54695950402322 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/biosses-sts |
|
name: MTEB BIOSSES |
|
config: default |
|
split: test |
|
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 75.9652953730204 |
|
- type: cos_sim_spearman |
|
value: 73.96554077670989 |
|
- type: euclidean_pearson |
|
value: 75.68477255792381 |
|
- type: euclidean_spearman |
|
value: 74.59447076995703 |
|
- type: manhattan_pearson |
|
value: 75.94984623881341 |
|
- type: manhattan_spearman |
|
value: 74.72218452337502 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackAndroidRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 14.119000000000002 |
|
- type: map_at_10 |
|
value: 19.661 |
|
- type: map_at_100 |
|
value: 20.706 |
|
- type: map_at_1000 |
|
value: 20.848 |
|
- type: map_at_3 |
|
value: 17.759 |
|
- type: map_at_5 |
|
value: 18.645 |
|
- type: mrr_at_1 |
|
value: 17.166999999999998 |
|
- type: mrr_at_10 |
|
value: 23.313 |
|
- type: mrr_at_100 |
|
value: 24.263 |
|
- type: mrr_at_1000 |
|
value: 24.352999999999998 |
|
- type: mrr_at_3 |
|
value: 21.412 |
|
- type: mrr_at_5 |
|
value: 22.313 |
|
- type: ndcg_at_1 |
|
value: 17.166999999999998 |
|
- type: ndcg_at_10 |
|
value: 23.631 |
|
- type: ndcg_at_100 |
|
value: 28.427000000000003 |
|
- type: ndcg_at_1000 |
|
value: 31.862000000000002 |
|
- type: ndcg_at_3 |
|
value: 20.175 |
|
- type: ndcg_at_5 |
|
value: 21.397 |
|
- type: precision_at_1 |
|
value: 17.166999999999998 |
|
- type: precision_at_10 |
|
value: 4.549 |
|
- type: precision_at_100 |
|
value: 0.8370000000000001 |
|
- type: precision_at_1000 |
|
value: 0.136 |
|
- type: precision_at_3 |
|
value: 9.68 |
|
- type: precision_at_5 |
|
value: 6.981 |
|
- type: recall_at_1 |
|
value: 14.119000000000002 |
|
- type: recall_at_10 |
|
value: 32.147999999999996 |
|
- type: recall_at_100 |
|
value: 52.739999999999995 |
|
- type: recall_at_1000 |
|
value: 76.67 |
|
- type: recall_at_3 |
|
value: 22.019 |
|
- type: recall_at_5 |
|
value: 25.361 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackEnglishRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 16.576 |
|
- type: map_at_10 |
|
value: 22.281000000000002 |
|
- type: map_at_100 |
|
value: 23.066 |
|
- type: map_at_1000 |
|
value: 23.166 |
|
- type: map_at_3 |
|
value: 20.385 |
|
- type: map_at_5 |
|
value: 21.557000000000002 |
|
- type: mrr_at_1 |
|
value: 20.892 |
|
- type: mrr_at_10 |
|
value: 26.605 |
|
- type: mrr_at_100 |
|
value: 27.229 |
|
- type: mrr_at_1000 |
|
value: 27.296 |
|
- type: mrr_at_3 |
|
value: 24.809 |
|
- type: mrr_at_5 |
|
value: 25.927 |
|
- type: ndcg_at_1 |
|
value: 20.892 |
|
- type: ndcg_at_10 |
|
value: 26.092 |
|
- type: ndcg_at_100 |
|
value: 29.398999999999997 |
|
- type: ndcg_at_1000 |
|
value: 31.884 |
|
- type: ndcg_at_3 |
|
value: 23.032 |
|
- type: ndcg_at_5 |
|
value: 24.634 |
|
- type: precision_at_1 |
|
value: 20.892 |
|
- type: precision_at_10 |
|
value: 4.885 |
|
- type: precision_at_100 |
|
value: 0.818 |
|
- type: precision_at_1000 |
|
value: 0.126 |
|
- type: precision_at_3 |
|
value: 10.977 |
|
- type: precision_at_5 |
|
value: 8.013 |
|
- type: recall_at_1 |
|
value: 16.576 |
|
- type: recall_at_10 |
|
value: 32.945 |
|
- type: recall_at_100 |
|
value: 47.337 |
|
- type: recall_at_1000 |
|
value: 64.592 |
|
- type: recall_at_3 |
|
value: 24.053 |
|
- type: recall_at_5 |
|
value: 28.465 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackGamingRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 20.604 |
|
- type: map_at_10 |
|
value: 28.754999999999995 |
|
- type: map_at_100 |
|
value: 29.767 |
|
- type: map_at_1000 |
|
value: 29.852 |
|
- type: map_at_3 |
|
value: 26.268 |
|
- type: map_at_5 |
|
value: 27.559 |
|
- type: mrr_at_1 |
|
value: 24.326 |
|
- type: mrr_at_10 |
|
value: 31.602000000000004 |
|
- type: mrr_at_100 |
|
value: 32.46 |
|
- type: mrr_at_1000 |
|
value: 32.521 |
|
- type: mrr_at_3 |
|
value: 29.415000000000003 |
|
- type: mrr_at_5 |
|
value: 30.581000000000003 |
|
- type: ndcg_at_1 |
|
value: 24.326 |
|
- type: ndcg_at_10 |
|
value: 33.335 |
|
- type: ndcg_at_100 |
|
value: 38.086 |
|
- type: ndcg_at_1000 |
|
value: 40.319 |
|
- type: ndcg_at_3 |
|
value: 28.796 |
|
- type: ndcg_at_5 |
|
value: 30.758999999999997 |
|
- type: precision_at_1 |
|
value: 24.326 |
|
- type: precision_at_10 |
|
value: 5.712 |
|
- type: precision_at_100 |
|
value: 0.893 |
|
- type: precision_at_1000 |
|
value: 0.11499999999999999 |
|
- type: precision_at_3 |
|
value: 13.208 |
|
- type: precision_at_5 |
|
value: 9.329 |
|
- type: recall_at_1 |
|
value: 20.604 |
|
- type: recall_at_10 |
|
value: 44.505 |
|
- type: recall_at_100 |
|
value: 65.866 |
|
- type: recall_at_1000 |
|
value: 82.61800000000001 |
|
- type: recall_at_3 |
|
value: 31.794 |
|
- type: recall_at_5 |
|
value: 36.831 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackGisRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 8.280999999999999 |
|
- type: map_at_10 |
|
value: 11.636000000000001 |
|
- type: map_at_100 |
|
value: 12.363 |
|
- type: map_at_1000 |
|
value: 12.469 |
|
- type: map_at_3 |
|
value: 10.415000000000001 |
|
- type: map_at_5 |
|
value: 11.144 |
|
- type: mrr_at_1 |
|
value: 9.266 |
|
- type: mrr_at_10 |
|
value: 12.838 |
|
- type: mrr_at_100 |
|
value: 13.608999999999998 |
|
- type: mrr_at_1000 |
|
value: 13.700999999999999 |
|
- type: mrr_at_3 |
|
value: 11.507000000000001 |
|
- type: mrr_at_5 |
|
value: 12.343 |
|
- type: ndcg_at_1 |
|
value: 9.266 |
|
- type: ndcg_at_10 |
|
value: 13.877 |
|
- type: ndcg_at_100 |
|
value: 18.119 |
|
- type: ndcg_at_1000 |
|
value: 21.247 |
|
- type: ndcg_at_3 |
|
value: 11.376999999999999 |
|
- type: ndcg_at_5 |
|
value: 12.675 |
|
- type: precision_at_1 |
|
value: 9.266 |
|
- type: precision_at_10 |
|
value: 2.226 |
|
- type: precision_at_100 |
|
value: 0.47200000000000003 |
|
- type: precision_at_1000 |
|
value: 0.077 |
|
- type: precision_at_3 |
|
value: 4.859 |
|
- type: precision_at_5 |
|
value: 3.6380000000000003 |
|
- type: recall_at_1 |
|
value: 8.280999999999999 |
|
- type: recall_at_10 |
|
value: 19.872999999999998 |
|
- type: recall_at_100 |
|
value: 40.585 |
|
- type: recall_at_1000 |
|
value: 65.225 |
|
- type: recall_at_3 |
|
value: 13.014000000000001 |
|
- type: recall_at_5 |
|
value: 16.147 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackMathematicaRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 4.1209999999999996 |
|
- type: map_at_10 |
|
value: 7.272 |
|
- type: map_at_100 |
|
value: 8.079 |
|
- type: map_at_1000 |
|
value: 8.199 |
|
- type: map_at_3 |
|
value: 6.212 |
|
- type: map_at_5 |
|
value: 6.736000000000001 |
|
- type: mrr_at_1 |
|
value: 5.721 |
|
- type: mrr_at_10 |
|
value: 9.418 |
|
- type: mrr_at_100 |
|
value: 10.281 |
|
- type: mrr_at_1000 |
|
value: 10.385 |
|
- type: mrr_at_3 |
|
value: 8.126 |
|
- type: mrr_at_5 |
|
value: 8.779 |
|
- type: ndcg_at_1 |
|
value: 5.721 |
|
- type: ndcg_at_10 |
|
value: 9.673 |
|
- type: ndcg_at_100 |
|
value: 13.852999999999998 |
|
- type: ndcg_at_1000 |
|
value: 17.546999999999997 |
|
- type: ndcg_at_3 |
|
value: 7.509 |
|
- type: ndcg_at_5 |
|
value: 8.373 |
|
- type: precision_at_1 |
|
value: 5.721 |
|
- type: precision_at_10 |
|
value: 2.04 |
|
- type: precision_at_100 |
|
value: 0.48 |
|
- type: precision_at_1000 |
|
value: 0.093 |
|
- type: precision_at_3 |
|
value: 4.022 |
|
- type: precision_at_5 |
|
value: 3.06 |
|
- type: recall_at_1 |
|
value: 4.1209999999999996 |
|
- type: recall_at_10 |
|
value: 15.201 |
|
- type: recall_at_100 |
|
value: 33.922999999999995 |
|
- type: recall_at_1000 |
|
value: 61.529999999999994 |
|
- type: recall_at_3 |
|
value: 8.869 |
|
- type: recall_at_5 |
|
value: 11.257 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackPhysicsRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 14.09 |
|
- type: map_at_10 |
|
value: 19.573999999999998 |
|
- type: map_at_100 |
|
value: 20.580000000000002 |
|
- type: map_at_1000 |
|
value: 20.704 |
|
- type: map_at_3 |
|
value: 17.68 |
|
- type: map_at_5 |
|
value: 18.64 |
|
- type: mrr_at_1 |
|
value: 17.227999999999998 |
|
- type: mrr_at_10 |
|
value: 23.152 |
|
- type: mrr_at_100 |
|
value: 24.056 |
|
- type: mrr_at_1000 |
|
value: 24.141000000000002 |
|
- type: mrr_at_3 |
|
value: 21.142 |
|
- type: mrr_at_5 |
|
value: 22.201 |
|
- type: ndcg_at_1 |
|
value: 17.227999999999998 |
|
- type: ndcg_at_10 |
|
value: 23.39 |
|
- type: ndcg_at_100 |
|
value: 28.483999999999998 |
|
- type: ndcg_at_1000 |
|
value: 31.709 |
|
- type: ndcg_at_3 |
|
value: 19.883 |
|
- type: ndcg_at_5 |
|
value: 21.34 |
|
- type: precision_at_1 |
|
value: 17.227999999999998 |
|
- type: precision_at_10 |
|
value: 4.3790000000000004 |
|
- type: precision_at_100 |
|
value: 0.826 |
|
- type: precision_at_1000 |
|
value: 0.128 |
|
- type: precision_at_3 |
|
value: 9.496 |
|
- type: precision_at_5 |
|
value: 6.872 |
|
- type: recall_at_1 |
|
value: 14.09 |
|
- type: recall_at_10 |
|
value: 31.580000000000002 |
|
- type: recall_at_100 |
|
value: 54.074 |
|
- type: recall_at_1000 |
|
value: 77.092 |
|
- type: recall_at_3 |
|
value: 21.601 |
|
- type: recall_at_5 |
|
value: 25.333 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackProgrammersRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 10.538 |
|
- type: map_at_10 |
|
value: 15.75 |
|
- type: map_at_100 |
|
value: 16.71 |
|
- type: map_at_1000 |
|
value: 16.838 |
|
- type: map_at_3 |
|
value: 13.488 |
|
- type: map_at_5 |
|
value: 14.712 |
|
- type: mrr_at_1 |
|
value: 13.813 |
|
- type: mrr_at_10 |
|
value: 19.08 |
|
- type: mrr_at_100 |
|
value: 19.946 |
|
- type: mrr_at_1000 |
|
value: 20.044 |
|
- type: mrr_at_3 |
|
value: 16.838 |
|
- type: mrr_at_5 |
|
value: 17.951 |
|
- type: ndcg_at_1 |
|
value: 13.813 |
|
- type: ndcg_at_10 |
|
value: 19.669 |
|
- type: ndcg_at_100 |
|
value: 24.488 |
|
- type: ndcg_at_1000 |
|
value: 27.87 |
|
- type: ndcg_at_3 |
|
value: 15.479000000000001 |
|
- type: ndcg_at_5 |
|
value: 17.229 |
|
- type: precision_at_1 |
|
value: 13.813 |
|
- type: precision_at_10 |
|
value: 3.916 |
|
- type: precision_at_100 |
|
value: 0.743 |
|
- type: precision_at_1000 |
|
value: 0.122 |
|
- type: precision_at_3 |
|
value: 7.534000000000001 |
|
- type: precision_at_5 |
|
value: 5.822 |
|
- type: recall_at_1 |
|
value: 10.538 |
|
- type: recall_at_10 |
|
value: 28.693 |
|
- type: recall_at_100 |
|
value: 50.308 |
|
- type: recall_at_1000 |
|
value: 74.44 |
|
- type: recall_at_3 |
|
value: 16.866999999999997 |
|
- type: recall_at_5 |
|
value: 21.404999999999998 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 11.044583333333332 |
|
- type: map_at_10 |
|
value: 15.682833333333335 |
|
- type: map_at_100 |
|
value: 16.506500000000003 |
|
- type: map_at_1000 |
|
value: 16.623833333333334 |
|
- type: map_at_3 |
|
value: 14.130833333333333 |
|
- type: map_at_5 |
|
value: 14.963583333333332 |
|
- type: mrr_at_1 |
|
value: 13.482833333333332 |
|
- type: mrr_at_10 |
|
value: 18.328500000000002 |
|
- type: mrr_at_100 |
|
value: 19.095416666666665 |
|
- type: mrr_at_1000 |
|
value: 19.18241666666666 |
|
- type: mrr_at_3 |
|
value: 16.754749999999998 |
|
- type: mrr_at_5 |
|
value: 17.614749999999997 |
|
- type: ndcg_at_1 |
|
value: 13.482833333333332 |
|
- type: ndcg_at_10 |
|
value: 18.81491666666667 |
|
- type: ndcg_at_100 |
|
value: 22.946833333333334 |
|
- type: ndcg_at_1000 |
|
value: 26.061083333333336 |
|
- type: ndcg_at_3 |
|
value: 15.949333333333332 |
|
- type: ndcg_at_5 |
|
value: 17.218333333333334 |
|
- type: precision_at_1 |
|
value: 13.482833333333332 |
|
- type: precision_at_10 |
|
value: 3.456583333333333 |
|
- type: precision_at_100 |
|
value: 0.6599166666666666 |
|
- type: precision_at_1000 |
|
value: 0.109 |
|
- type: precision_at_3 |
|
value: 7.498833333333332 |
|
- type: precision_at_5 |
|
value: 5.477166666666667 |
|
- type: recall_at_1 |
|
value: 11.044583333333332 |
|
- type: recall_at_10 |
|
value: 25.737750000000005 |
|
- type: recall_at_100 |
|
value: 44.617916666666666 |
|
- type: recall_at_1000 |
|
value: 67.56524999999999 |
|
- type: recall_at_3 |
|
value: 17.598249999999997 |
|
- type: recall_at_5 |
|
value: 20.9035 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackStatsRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 9.362 |
|
- type: map_at_10 |
|
value: 13.414000000000001 |
|
- type: map_at_100 |
|
value: 14.083000000000002 |
|
- type: map_at_1000 |
|
value: 14.168 |
|
- type: map_at_3 |
|
value: 12.098 |
|
- type: map_at_5 |
|
value: 12.803999999999998 |
|
- type: mrr_at_1 |
|
value: 11.043 |
|
- type: mrr_at_10 |
|
value: 15.158 |
|
- type: mrr_at_100 |
|
value: 15.845999999999998 |
|
- type: mrr_at_1000 |
|
value: 15.916 |
|
- type: mrr_at_3 |
|
value: 13.88 |
|
- type: mrr_at_5 |
|
value: 14.601 |
|
- type: ndcg_at_1 |
|
value: 11.043 |
|
- type: ndcg_at_10 |
|
value: 16.034000000000002 |
|
- type: ndcg_at_100 |
|
value: 19.686 |
|
- type: ndcg_at_1000 |
|
value: 22.188 |
|
- type: ndcg_at_3 |
|
value: 13.530000000000001 |
|
- type: ndcg_at_5 |
|
value: 14.704 |
|
- type: precision_at_1 |
|
value: 11.043 |
|
- type: precision_at_10 |
|
value: 2.791 |
|
- type: precision_at_100 |
|
value: 0.5 |
|
- type: precision_at_1000 |
|
value: 0.077 |
|
- type: precision_at_3 |
|
value: 6.237 |
|
- type: precision_at_5 |
|
value: 4.5089999999999995 |
|
- type: recall_at_1 |
|
value: 9.362 |
|
- type: recall_at_10 |
|
value: 22.396 |
|
- type: recall_at_100 |
|
value: 39.528999999999996 |
|
- type: recall_at_1000 |
|
value: 58.809 |
|
- type: recall_at_3 |
|
value: 15.553 |
|
- type: recall_at_5 |
|
value: 18.512 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackTexRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 5.657 |
|
- type: map_at_10 |
|
value: 8.273 |
|
- type: map_at_100 |
|
value: 8.875 |
|
- type: map_at_1000 |
|
value: 8.977 |
|
- type: map_at_3 |
|
value: 7.32 |
|
- type: map_at_5 |
|
value: 7.792000000000001 |
|
- type: mrr_at_1 |
|
value: 7.02 |
|
- type: mrr_at_10 |
|
value: 9.966999999999999 |
|
- type: mrr_at_100 |
|
value: 10.636 |
|
- type: mrr_at_1000 |
|
value: 10.724 |
|
- type: mrr_at_3 |
|
value: 8.872 |
|
- type: mrr_at_5 |
|
value: 9.461 |
|
- type: ndcg_at_1 |
|
value: 7.02 |
|
- type: ndcg_at_10 |
|
value: 10.199 |
|
- type: ndcg_at_100 |
|
value: 13.642000000000001 |
|
- type: ndcg_at_1000 |
|
value: 16.643 |
|
- type: ndcg_at_3 |
|
value: 8.333 |
|
- type: ndcg_at_5 |
|
value: 9.103 |
|
- type: precision_at_1 |
|
value: 7.02 |
|
- type: precision_at_10 |
|
value: 1.8929999999999998 |
|
- type: precision_at_100 |
|
value: 0.43 |
|
- type: precision_at_1000 |
|
value: 0.08099999999999999 |
|
- type: precision_at_3 |
|
value: 3.843 |
|
- type: precision_at_5 |
|
value: 2.884 |
|
- type: recall_at_1 |
|
value: 5.657 |
|
- type: recall_at_10 |
|
value: 14.563 |
|
- type: recall_at_100 |
|
value: 30.807000000000002 |
|
- type: recall_at_1000 |
|
value: 53.251000000000005 |
|
- type: recall_at_3 |
|
value: 9.272 |
|
- type: recall_at_5 |
|
value: 11.202 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackUnixRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 10.671999999999999 |
|
- type: map_at_10 |
|
value: 14.651 |
|
- type: map_at_100 |
|
value: 15.406 |
|
- type: map_at_1000 |
|
value: 15.525 |
|
- type: map_at_3 |
|
value: 13.461 |
|
- type: map_at_5 |
|
value: 14.163 |
|
- type: mrr_at_1 |
|
value: 12.407 |
|
- type: mrr_at_10 |
|
value: 16.782 |
|
- type: mrr_at_100 |
|
value: 17.562 |
|
- type: mrr_at_1000 |
|
value: 17.653 |
|
- type: mrr_at_3 |
|
value: 15.47 |
|
- type: mrr_at_5 |
|
value: 16.262 |
|
- type: ndcg_at_1 |
|
value: 12.407 |
|
- type: ndcg_at_10 |
|
value: 17.251 |
|
- type: ndcg_at_100 |
|
value: 21.378 |
|
- type: ndcg_at_1000 |
|
value: 24.689 |
|
- type: ndcg_at_3 |
|
value: 14.915000000000001 |
|
- type: ndcg_at_5 |
|
value: 16.1 |
|
- type: precision_at_1 |
|
value: 12.407 |
|
- type: precision_at_10 |
|
value: 2.91 |
|
- type: precision_at_100 |
|
value: 0.573 |
|
- type: precision_at_1000 |
|
value: 0.096 |
|
- type: precision_at_3 |
|
value: 6.779 |
|
- type: precision_at_5 |
|
value: 4.888 |
|
- type: recall_at_1 |
|
value: 10.671999999999999 |
|
- type: recall_at_10 |
|
value: 23.099 |
|
- type: recall_at_100 |
|
value: 41.937999999999995 |
|
- type: recall_at_1000 |
|
value: 66.495 |
|
- type: recall_at_3 |
|
value: 16.901 |
|
- type: recall_at_5 |
|
value: 19.807 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackWebmastersRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 13.364 |
|
- type: map_at_10 |
|
value: 17.772 |
|
- type: map_at_100 |
|
value: 18.659 |
|
- type: map_at_1000 |
|
value: 18.861 |
|
- type: map_at_3 |
|
value: 16.659 |
|
- type: map_at_5 |
|
value: 17.174 |
|
- type: mrr_at_1 |
|
value: 16.996 |
|
- type: mrr_at_10 |
|
value: 21.687 |
|
- type: mrr_at_100 |
|
value: 22.313 |
|
- type: mrr_at_1000 |
|
value: 22.422 |
|
- type: mrr_at_3 |
|
value: 20.652 |
|
- type: mrr_at_5 |
|
value: 21.146 |
|
- type: ndcg_at_1 |
|
value: 16.996 |
|
- type: ndcg_at_10 |
|
value: 21.067 |
|
- type: ndcg_at_100 |
|
value: 24.829 |
|
- type: ndcg_at_1000 |
|
value: 28.866999999999997 |
|
- type: ndcg_at_3 |
|
value: 19.466 |
|
- type: ndcg_at_5 |
|
value: 19.993 |
|
- type: precision_at_1 |
|
value: 16.996 |
|
- type: precision_at_10 |
|
value: 4.071000000000001 |
|
- type: precision_at_100 |
|
value: 0.9329999999999999 |
|
- type: precision_at_1000 |
|
value: 0.183 |
|
- type: precision_at_3 |
|
value: 9.223 |
|
- type: precision_at_5 |
|
value: 6.4030000000000005 |
|
- type: recall_at_1 |
|
value: 13.364 |
|
- type: recall_at_10 |
|
value: 25.976 |
|
- type: recall_at_100 |
|
value: 44.134 |
|
- type: recall_at_1000 |
|
value: 73.181 |
|
- type: recall_at_3 |
|
value: 20.503 |
|
- type: recall_at_5 |
|
value: 22.409000000000002 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: BeIR/cqadupstack |
|
name: MTEB CQADupstackWordpressRetrieval |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 5.151 |
|
- type: map_at_10 |
|
value: 9.155000000000001 |
|
- type: map_at_100 |
|
value: 9.783999999999999 |
|
- type: map_at_1000 |
|
value: 9.879 |
|
- type: map_at_3 |
|
value: 7.825 |
|
- type: map_at_5 |
|
value: 8.637 |
|
- type: mrr_at_1 |
|
value: 5.915 |
|
- type: mrr_at_10 |
|
value: 10.34 |
|
- type: mrr_at_100 |
|
value: 10.943999999999999 |
|
- type: mrr_at_1000 |
|
value: 11.033 |
|
- type: mrr_at_3 |
|
value: 8.934000000000001 |
|
- type: mrr_at_5 |
|
value: 9.812 |
|
- type: ndcg_at_1 |
|
value: 5.915 |
|
- type: ndcg_at_10 |
|
value: 11.561 |
|
- type: ndcg_at_100 |
|
value: 14.971 |
|
- type: ndcg_at_1000 |
|
value: 17.907999999999998 |
|
- type: ndcg_at_3 |
|
value: 8.896999999999998 |
|
- type: ndcg_at_5 |
|
value: 10.313 |
|
- type: precision_at_1 |
|
value: 5.915 |
|
- type: precision_at_10 |
|
value: 2.1069999999999998 |
|
- type: precision_at_100 |
|
value: 0.414 |
|
- type: precision_at_1000 |
|
value: 0.074 |
|
- type: precision_at_3 |
|
value: 4.128 |
|
- type: precision_at_5 |
|
value: 3.327 |
|
- type: recall_at_1 |
|
value: 5.151 |
|
- type: recall_at_10 |
|
value: 17.874000000000002 |
|
- type: recall_at_100 |
|
value: 34.174 |
|
- type: recall_at_1000 |
|
value: 56.879999999999995 |
|
- type: recall_at_3 |
|
value: 10.732999999999999 |
|
- type: recall_at_5 |
|
value: 14.113000000000001 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: climate-fever |
|
name: MTEB ClimateFEVER |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 3.101 |
|
- type: map_at_10 |
|
value: 5.434 |
|
- type: map_at_100 |
|
value: 6.267 |
|
- type: map_at_1000 |
|
value: 6.418 |
|
- type: map_at_3 |
|
value: 4.377000000000001 |
|
- type: map_at_5 |
|
value: 4.841 |
|
- type: mrr_at_1 |
|
value: 7.166 |
|
- type: mrr_at_10 |
|
value: 12.012 |
|
- type: mrr_at_100 |
|
value: 13.144 |
|
- type: mrr_at_1000 |
|
value: 13.229 |
|
- type: mrr_at_3 |
|
value: 9.826 |
|
- type: mrr_at_5 |
|
value: 10.921 |
|
- type: ndcg_at_1 |
|
value: 7.166 |
|
- type: ndcg_at_10 |
|
value: 8.687000000000001 |
|
- type: ndcg_at_100 |
|
value: 13.345 |
|
- type: ndcg_at_1000 |
|
value: 16.915 |
|
- type: ndcg_at_3 |
|
value: 6.276 |
|
- type: ndcg_at_5 |
|
value: 7.013 |
|
- type: precision_at_1 |
|
value: 7.166 |
|
- type: precision_at_10 |
|
value: 2.9250000000000003 |
|
- type: precision_at_100 |
|
value: 0.771 |
|
- type: precision_at_1000 |
|
value: 0.13999999999999999 |
|
- type: precision_at_3 |
|
value: 4.734 |
|
- type: precision_at_5 |
|
value: 3.8830000000000005 |
|
- type: recall_at_1 |
|
value: 3.101 |
|
- type: recall_at_10 |
|
value: 11.774999999999999 |
|
- type: recall_at_100 |
|
value: 28.819 |
|
- type: recall_at_1000 |
|
value: 49.886 |
|
- type: recall_at_3 |
|
value: 5.783 |
|
- type: recall_at_5 |
|
value: 7.692 |
|
- task: |
|
type: Retrieval |
|
dataset: |
|
type: dbpedia-entity |
|
name: MTEB DBPedia |
|
config: default |
|
split: test |
|
revision: None |
|
metrics: |
|
- type: map_at_1 |
|
value: 2.758 |
|
- type: map_at_10 |
|
value: 5.507 |
|
- type: map_at_100 |
|
value: 7.1819999999999995 |
|
- type: map_at_1000 |
|
value: 7.652 |
|
- type: map_at_3 |
|
value: 4.131 |
|
- type: map_at_5 |
|
value: 4.702 |
|
- type: mrr_at_1 |
|
value: 28.499999999999996 |
|
- type: mrr_at_10 |
|
value: 37.693 |
|
- type: mrr_at_100 |
|
value: 38.657000000000004 |
|
- type: mrr_at_1000 |
|
value: 38.704 |
|
- type: mrr_at_3 |
|
value: 34.792 |
|
- type: mrr_at_5 |
|
value: 36.417 |
|
- type: ndcg_at_1 |
|
value: 20.625 |
|
- type: ndcg_at_10 |
|
value: 14.771999999999998 |
|
- type: ndcg_at_100 |
|
value: 16.821 |
|
- type: ndcg_at_1000 |
|
value: 21.546000000000003 |
|
- type: ndcg_at_3 |
|
value: 16.528000000000002 |
|
- type: ndcg_at_5 |
|
value: 15.573 |
|
- type: precision_at_1 |
|
value: 28.499999999999996 |
|
- type: precision_at_10 |
|
value: 12.25 |
|
- type: precision_at_100 |
|
value: 3.7600000000000002 |
|
- type: precision_at_1000 |
|
value: 0.86 |
|
- type: precision_at_3 |
|
value: 19.167 |
|
- type: precision_at_5 |
|
value: 16.25 |
|
- type: recall_at_1 |
|
value: 2.758 |
|
- type: recall_at_10 |
|
value: 9.164 |
|
- type: recall_at_100 |
|
value: 21.022 |
|
- type: recall_at_1000 |
|
value: 37.053999999999995 |
|
- type: recall_at_3 |
|
value: 5.112 |
|
- type: recall_at_5 |
|
value: 6.413 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/mind_small |
|
name: MTEB MindSmallReranking |
|
config: default |
|
split: test |
|
revision: 3bdac13927fdc888b903db93b2ffdbd90b295a69 |
|
metrics: |
|
- type: map |
|
value: 28.53554681148413 |
|
- type: mrr |
|
value: 29.290078704990325 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sickr-sts |
|
name: MTEB SICK-R |
|
config: default |
|
split: test |
|
revision: a6ea5a8cab320b040a23452cc28066d9beae2cee |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 76.52926207453477 |
|
- type: cos_sim_spearman |
|
value: 68.98528351149498 |
|
- type: euclidean_pearson |
|
value: 73.7744559091218 |
|
- type: euclidean_spearman |
|
value: 69.03481995814735 |
|
- type: manhattan_pearson |
|
value: 73.72818267270651 |
|
- type: manhattan_spearman |
|
value: 69.00576442086793 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts12-sts |
|
name: MTEB STS12 |
|
config: default |
|
split: test |
|
revision: a0d554a64d88156834ff5ae9920b964011b16384 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 61.71540153163407 |
|
- type: cos_sim_spearman |
|
value: 58.502746406116614 |
|
- type: euclidean_pearson |
|
value: 60.82817999438477 |
|
- type: euclidean_spearman |
|
value: 58.988494433752756 |
|
- type: manhattan_pearson |
|
value: 60.87147859170236 |
|
- type: manhattan_spearman |
|
value: 59.03527382025516 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts13-sts |
|
name: MTEB STS13 |
|
config: default |
|
split: test |
|
revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 72.89990498692094 |
|
- type: cos_sim_spearman |
|
value: 74.03028513377879 |
|
- type: euclidean_pearson |
|
value: 73.8252088833803 |
|
- type: euclidean_spearman |
|
value: 74.15554246478399 |
|
- type: manhattan_pearson |
|
value: 73.80947397334666 |
|
- type: manhattan_spearman |
|
value: 74.13117958176566 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts14-sts |
|
name: MTEB STS14 |
|
config: default |
|
split: test |
|
revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 70.67974206005906 |
|
- type: cos_sim_spearman |
|
value: 66.18263558486296 |
|
- type: euclidean_pearson |
|
value: 69.5048876024341 |
|
- type: euclidean_spearman |
|
value: 66.36380457878391 |
|
- type: manhattan_pearson |
|
value: 69.4895372451589 |
|
- type: manhattan_spearman |
|
value: 66.36941569935124 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts15-sts |
|
name: MTEB STS15 |
|
config: default |
|
split: test |
|
revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 73.99856913569187 |
|
- type: cos_sim_spearman |
|
value: 75.54712054246464 |
|
- type: euclidean_pearson |
|
value: 74.55692573876115 |
|
- type: euclidean_spearman |
|
value: 75.34499056740096 |
|
- type: manhattan_pearson |
|
value: 74.59342318869683 |
|
- type: manhattan_spearman |
|
value: 75.35708317926819 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts16-sts |
|
name: MTEB STS16 |
|
config: default |
|
split: test |
|
revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 72.3343670787494 |
|
- type: cos_sim_spearman |
|
value: 73.7136650302399 |
|
- type: euclidean_pearson |
|
value: 73.86004257913046 |
|
- type: euclidean_spearman |
|
value: 73.9557418048638 |
|
- type: manhattan_pearson |
|
value: 73.78919091538661 |
|
- type: manhattan_spearman |
|
value: 73.86316425954108 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts17-crosslingual-sts |
|
name: MTEB STS17 (en-en) |
|
config: en-en |
|
split: test |
|
revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 79.08159601556619 |
|
- type: cos_sim_spearman |
|
value: 80.13910828685532 |
|
- type: euclidean_pearson |
|
value: 79.39197806617453 |
|
- type: euclidean_spearman |
|
value: 79.85692277871196 |
|
- type: manhattan_pearson |
|
value: 79.32452246324705 |
|
- type: manhattan_spearman |
|
value: 79.70120373587193 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/sts22-crosslingual-sts |
|
name: MTEB STS22 (en) |
|
config: en |
|
split: test |
|
revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 62.29720207747786 |
|
- type: cos_sim_spearman |
|
value: 65.65260681394685 |
|
- type: euclidean_pearson |
|
value: 64.49002165983158 |
|
- type: euclidean_spearman |
|
value: 65.25917651158736 |
|
- type: manhattan_pearson |
|
value: 64.49981108236335 |
|
- type: manhattan_spearman |
|
value: 65.20426825202405 |
|
- task: |
|
type: STS |
|
dataset: |
|
type: mteb/stsbenchmark-sts |
|
name: MTEB STSBenchmark |
|
config: default |
|
split: test |
|
revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 71.1871068550574 |
|
- type: cos_sim_spearman |
|
value: 71.40167034949341 |
|
- type: euclidean_pearson |
|
value: 72.2373684855404 |
|
- type: euclidean_spearman |
|
value: 71.90255429812984 |
|
- type: manhattan_pearson |
|
value: 72.23173532049509 |
|
- type: manhattan_spearman |
|
value: 71.87843489689064 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/scidocs-reranking |
|
name: MTEB SciDocsRR |
|
config: default |
|
split: test |
|
revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab |
|
metrics: |
|
- type: map |
|
value: 68.65000574464773 |
|
- type: mrr |
|
value: 88.29363084265044 |
|
- task: |
|
type: Reranking |
|
dataset: |
|
type: mteb/stackoverflowdupquestions-reranking |
|
name: MTEB StackOverflowDupQuestions |
|
config: default |
|
split: test |
|
revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 |
|
metrics: |
|
- type: map |
|
value: 40.76107749144358 |
|
- type: mrr |
|
value: 41.03689202953908 |
|
- task: |
|
type: Summarization |
|
dataset: |
|
type: mteb/summeval |
|
name: MTEB SummEval |
|
config: default |
|
split: test |
|
revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c |
|
metrics: |
|
- type: cos_sim_pearson |
|
value: 28.68520527813894 |
|
- type: cos_sim_spearman |
|
value: 29.017620841627433 |
|
- type: dot_pearson |
|
value: 29.25380949876322 |
|
- type: dot_spearman |
|
value: 29.33885250837327 |
|
--- |
|
|
|
# {MODEL_NAME} |
|
|
|
This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 1024 dimensional dense vector space and can be used for tasks like clustering or semantic search. |
|
|
|
<!--- Describe your model here --> |
|
|
|
## Usage (Sentence-Transformers) |
|
|
|
Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed: |
|
|
|
``` |
|
pip install -U sentence-transformers |
|
``` |
|
|
|
Then you can use the model like this: |
|
|
|
```python |
|
from sentence_transformers import SentenceTransformer |
|
sentences = ["This is an example sentence", "Each sentence is converted"] |
|
|
|
model = SentenceTransformer('{MODEL_NAME}') |
|
embeddings = model.encode(sentences) |
|
print(embeddings) |
|
``` |
|
|
|
|
|
|
|
## Usage (HuggingFace Transformers) |
|
Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings. |
|
|
|
```python |
|
from transformers import AutoTokenizer, AutoModel |
|
import torch |
|
|
|
|
|
#Mean Pooling - Take attention mask into account for correct averaging |
|
def mean_pooling(model_output, attention_mask): |
|
token_embeddings = model_output[0] #First element of model_output contains all token embeddings |
|
input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() |
|
return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) |
|
|
|
|
|
# Sentences we want sentence embeddings for |
|
sentences = ['This is an example sentence', 'Each sentence is converted'] |
|
|
|
# Load model from HuggingFace Hub |
|
tokenizer = AutoTokenizer.from_pretrained('{MODEL_NAME}') |
|
model = AutoModel.from_pretrained('{MODEL_NAME}') |
|
|
|
# Tokenize sentences |
|
encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt') |
|
|
|
# Compute token embeddings |
|
with torch.no_grad(): |
|
model_output = model(**encoded_input) |
|
|
|
# Perform pooling. In this case, mean pooling. |
|
sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask']) |
|
|
|
print("Sentence embeddings:") |
|
print(sentence_embeddings) |
|
``` |
|
|
