Model: AdrienB134/llm2vec-croissant-mntp
Author: AdrienB134
Library: transformers
Tags: transformers, safetensors, llama, generated_from_trainer, mteb, fr, dataset:asi/wikitext_fr, base_model:croissantllm/CroissantCool-v0.2, base_model:finetune:croissantllm/CroissantCool-v0.2, license:mit, model-index, text-generation-inference, endpoints_compatible, region:us
Base models (finetune): id: croissantllm/CroissantCool-v0.2
Card metadata: license: mit
base_model: croissantllm/CroissantCool-v0.2
datasets: asi/wikitext_fr
tags: generated_from_trainer, mteb
metrics: accuracy
model-index: name: final
results: task: type: Clustering
dataset: type: lyon-nlp/alloprof
name: MTEB AlloProfClusteringP2P (fra-Latn)
config: fra-Latn
split: test
revision: 392ba3f5bcc8c51f578786c1fc3dae648662cb9b
metrics: type: v_measure
value: 62.345943052433995
verified: no, task: type: Clustering
dataset: type: lyon-nlp/alloprof
name: MTEB AlloProfClusteringS2S (fra-Latn)
config: fra-Latn
split: test
revision: 392ba3f5bcc8c51f578786c1fc3dae648662cb9b
metrics: type: v_measure
value: 25.729454984521148
verified: no, task: type: Reranking
dataset: type: lyon-nlp/mteb-fr-reranking-alloprof-s2p
name: MTEB AlloprofReranking (fra-Latn)
config: fra-Latn
split: test
revision: 65393d0d7a08a10b4e348135e824f385d420b0fd
metrics: type: map
value: 26.596323297349183
verified: no, type: mrr
value: 26.091629657044162
verified: no, task: type: Retrieval
dataset: type: lyon-nlp/alloprof
name: MTEB AlloprofRetrieval (fra-Latn)
config: fra-Latn
split: test
revision: fcf295ea64c750f41fadbaa37b9b861558e1bfbd
metrics: type: map_at_1
value: 0.345
verified: no, type: map_at_10
value: 0.9339999999999999
verified: no, type: map_at_100
value: 1.191
verified: no, type: map_at_1000
value: 1.3419999999999999
verified: no, type: map_at_20
value: 1.02
verified: no, type: map_at_3
value: 0.6689999999999999
verified: no, type: map_at_5
value: 0.753
verified: no, type: mrr_at_1
value: 0.345
verified: no, type: mrr_at_10
value: 0.9339999999999999
verified: no, type: mrr_at_100
value: 1.191
verified: no, type: mrr_at_1000
value: 1.3419999999999999
verified: no, type: mrr_at_20
value: 1.02
verified: no, type: mrr_at_3
value: 0.6689999999999999
verified: no, type: mrr_at_5
value: 0.753
verified: no, type: ndcg_at_1
value: 0.345
verified: no, type: ndcg_at_10
value: 1.384
verified: no, type: ndcg_at_100
value: 3.1510000000000002
verified: no, type: ndcg_at_1000
value: 9.014
verified: no, type: ndcg_at_20
value: 1.6920000000000002
verified: no, type: ndcg_at_3
value: 0.7849999999999999
verified: no, type: ndcg_at_5
value: 0.941
verified: no, type: precision_at_1
value: 0.345
verified: no, type: precision_at_10
value: 0.28900000000000003
verified: no, type: precision_at_100
value: 0.124
verified: no, type: precision_at_1000
value: 0.063
verified: no, type: precision_at_20
value: 0.20500000000000002
verified: no, type: precision_at_3
value: 0.374
verified: no, type: precision_at_5
value: 0.302
verified: no, type: recall_at_1
value: 0.345
verified: no, type: recall_at_10
value: 2.8930000000000002
verified: no, type: recall_at_100
value: 12.435
verified: no, type: recall_at_1000
value: 62.867
verified: no, type: recall_at_20
value: 4.102
verified: no, type: recall_at_3
value: 1.123
verified: no, type: recall_at_5
value: 1.5110000000000001
verified: no, task: type: Classification
dataset: type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (fra-Latn)
config: fra-Latn
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics: type: accuracy
value: 32.662
verified: no, type: f1
value: 32.443152253731846
verified: no, task: type: Retrieval
dataset: type: maastrichtlawtech/bsard
name: MTEB BSARDRetrieval (fra-Latn)
config: fra-Latn
split: test
revision: 5effa1b9b5fa3b0f9e12523e6e43e5f86a6e6d59
metrics: type: map_at_1
value: 0
verified: no, type: map_at_10
value: 0
verified: no, type: map_at_100
value: 0.062
verified: no, type: map_at_1000
value: 0.077
verified: no, type: map_at_20
value: 0
verified: no, type: map_at_3
value: 0
verified: no, type: map_at_5
value: 0
verified: no, type: mrr_at_1
value: 0
verified: no, type: mrr_at_10
value: 0
verified: no, type: mrr_at_100
value: 0.062
verified: no, type: mrr_at_1000
value: 0.077
verified: no, type: mrr_at_20
value: 0
verified: no, type: mrr_at_3
value: 0
verified: no, type: mrr_at_5
value: 0
verified: no, type: ndcg_at_1
value: 0
verified: no, type: ndcg_at_10
value: 0
verified: no, type: ndcg_at_100
value: 0.484
verified: no, type: ndcg_at_1000
value: 1.054
verified: no, type: ndcg_at_20
value: 0
verified: no, type: ndcg_at_3
value: 0
verified: no, type: ndcg_at_5
value: 0
verified: no, type: precision_at_1
value: 0
verified: no, type: precision_at_10
value: 0
verified: no, type: precision_at_100
value: 0.027
verified: no, type: precision_at_1000
value: 0.008
verified: no, type: precision_at_20
value: 0
verified: no, type: precision_at_3
value: 0
verified: no, type: precision_at_5
value: 0
verified: no, type: recall_at_1
value: 0
verified: no, type: recall_at_10
value: 0
verified: no, type: recall_at_100
value: 2.703
verified: no, type: recall_at_1000
value: 7.6579999999999995
verified: no, type: recall_at_20
value: 0
verified: no, type: recall_at_3
value: 0
verified: no, type: recall_at_5
value: 0
verified: no, task: type: Clustering
dataset: type: lyon-nlp/clustering-hal-s2s
name: MTEB HALClusteringS2S (fra-Latn)
config: fra-Latn
split: test
revision: e06ebbbb123f8144bef1a5d18796f3dec9ae2915
metrics: type: v_measure
value: 13.77084465510841
verified: no, task: type: Clustering
dataset: type: mlsum
name: MTEB MLSUMClusteringP2P (fra-Latn)
config: fra-Latn
split: test
revision: b5d54f8f3b61ae17845046286940f03c6bc79bc7
metrics: type: v_measure
value: 45.43375637260015
verified: no, task: type: Clustering
dataset: type: mlsum
name: MTEB MLSUMClusteringS2S (fra-Latn)
config: fra-Latn
split: test
revision: b5d54f8f3b61ae17845046286940f03c6bc79bc7
metrics: type: v_measure
value: 45.20564648796975
verified: no, task: type: Classification
dataset: type: mteb/mtop_domain
name: MTEB MTOPDomainClassification (fra-Latn)
config: fra-Latn
split: test
revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
metrics: type: accuracy
value: 73.42937676166615
verified: no, type: f1
value: 72.65861284500563
verified: no, task: type: Classification
dataset: type: mteb/mtop_intent
name: MTEB MTOPIntentClassification (fra-Latn)
config: fra-Latn
split: test
revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
metrics: type: accuracy
value: 58.54368932038836
verified: no, type: f1
value: 37.51985447597095
verified: no, task: type: Classification
dataset: type: mteb/masakhanews
name: MTEB MasakhaNEWSClassification (fra-Latn)
config: fra-Latn
split: test
revision: 18193f187b92da67168c655c9973a165ed9593dd
metrics: type: accuracy
value: 75.56872037914692
verified: no, type: f1
value: 71.99185345982795
verified: no, task: type: Clustering
dataset: type: masakhane/masakhanews
name: MTEB MasakhaNEWSClusteringP2P (fra-Latn)
config: fra-Latn
split: test
revision: 8ccc72e69e65f40c70e117d8b3c08306bb788b60
metrics: type: v_measure
value: 38.20382948117535
verified: no, task: type: Clustering
dataset: type: masakhane/masakhanews
name: MTEB MasakhaNEWSClusteringS2S (fra-Latn)
config: fra-Latn
split: test
revision: 8ccc72e69e65f40c70e117d8b3c08306bb788b60
metrics: type: v_measure
value: 26.943825642352117
verified: no, task: type: Classification
dataset: type: mteb/amazon_massive_intent
name: MTEB MassiveIntentClassification (fra-Latn)
config: fra-Latn
split: test
revision: 4672e20407010da34463acc759c162ca9734bca6
metrics: type: accuracy
value: 50.20847343644924
verified: no, type: f1
value: 47.32281768380685
verified: no, task: type: Classification
dataset: type: mteb/amazon_massive_scenario
name: MTEB MassiveScenarioClassification (fra-Latn)
config: fra-Latn
split: test
revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8
metrics: type: accuracy
value: 52.57565568258238
verified: no, type: f1
value: 50.95953249242336
verified: no, task: type: Retrieval
dataset: type: jinaai/mintakaqa
name: MTEB MintakaRetrieval (fra-Latn)
config: fra-Latn
split: test
revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e
metrics: type: map_at_1
value: 0.164
verified: no, type: map_at_10
value: 0.584
verified: no, type: map_at_100
value: 0.8240000000000001
verified: no, type: map_at_1000
value: 0.9769999999999999
verified: no, type: map_at_20
value: 0.6669999999999999
verified: no, type: map_at_3
value: 0.40299999999999997
verified: no, type: map_at_5
value: 0.47600000000000003
verified: no, type: mrr_at_1
value: 0.164
verified: no, type: mrr_at_10
value: 0.584
verified: no, type: mrr_at_100
value: 0.8240000000000001
verified: no, type: mrr_at_1000
value: 0.9769999999999999
verified: no, type: mrr_at_20
value: 0.6669999999999999
verified: no, type: mrr_at_3
value: 0.40299999999999997
verified: no, type: mrr_at_5
value: 0.47600000000000003
verified: no, type: ndcg_at_1
value: 0.164
verified: no, type: ndcg_at_10
value: 0.8670000000000001
verified: no, type: ndcg_at_100
value: 2.443
verified: no, type: ndcg_at_1000
value: 8.671
verified: no, type: ndcg_at_20
value: 1.176
verified: no, type: ndcg_at_3
value: 0.47800000000000004
verified: no, type: ndcg_at_5
value: 0.612
verified: no, type: precision_at_1
value: 0.164
verified: no, type: precision_at_10
value: 0.18
verified: no, type: precision_at_100
value: 0.10200000000000001
verified: no, type: precision_at_1000
value: 0.064
verified: no, type: precision_at_20
value: 0.152
verified: no, type: precision_at_3
value: 0.232
verified: no, type: precision_at_5
value: 0.20500000000000002
verified: no, type: recall_at_1
value: 0.164
verified: no, type: recall_at_10
value: 1.802
verified: no, type: recall_at_100
value: 10.156
verified: no, type: recall_at_1000
value: 64.21
verified: no, type: recall_at_20
value: 3.0300000000000002
verified: no, type: recall_at_3
value: 0.696
verified: no, type: recall_at_5
value: 1.024
verified: no, task: type: PairClassification
dataset: type: GEM/opusparcus
name: MTEB OpusparcusPC (fra-Latn)
config: fra-Latn
split: test
revision: 9e9b1f8ef51616073f47f306f7f47dd91663f86a
metrics: type: cos_sim_accuracy
value: 73.433242506812
verified: no, type: cos_sim_ap
value: 86.03577758642086
verified: no, type: cos_sim_f1
value: 82.1602478972997
verified: no, type: cos_sim_precision
value: 74.12140575079871
verified: no, type: cos_sim_recall
value: 92.15491559086395
verified: no, type: dot_accuracy
value: 68.8692098092643
verified: no, type: dot_ap
value: 75.51070462676913
verified: no, type: dot_f1
value: 81.47547628698824
verified: no, type: dot_precision
value: 68.83561643835617
verified: no, type: dot_recall
value: 99.80139026812313
verified: no, type: euclidean_accuracy
value: 73.84196185286103
verified: no, type: euclidean_ap
value: 86.27910998502644
verified: no, type: euclidean_f1
value: 82.5531914893617
verified: no, type: euclidean_precision
value: 72.22635889798957
verified: no, type: euclidean_recall
value: 96.32571996027805
verified: no, type: manhattan_accuracy
value: 73.9100817438692
verified: no, type: manhattan_ap
value: 86.43527306280204
verified: no, type: manhattan_f1
value: 82.57349808265872
verified: no, type: manhattan_precision
value: 72.31343283582089
verified: no, type: manhattan_recall
value: 96.22641509433963
verified: no, type: max_accuracy
value: 73.9100817438692
verified: no, type: max_ap
value: 86.43527306280204
verified: no, type: max_f1
value: 82.57349808265872
verified: no, task: type: PairClassification
dataset: type: paws-x
name: MTEB PawsX (fra-Latn)
config: fra-Latn
split: test
revision: 8a04d940a42cd40658986fdd8e3da561533a3646
metrics: type: cos_sim_accuracy
value: 61.550000000000004
verified: no, type: cos_sim_ap
value: 60.30864957174996
verified: no, type: cos_sim_f1
value: 62.891311994372145
verified: no, type: cos_sim_precision
value: 46.08247422680412
verified: no, type: cos_sim_recall
value: 99.00332225913621
verified: no, type: dot_accuracy
value: 55.35
verified: no, type: dot_ap
value: 47.540176633815165
verified: no, type: dot_f1
value: 62.20227821884707
verified: no, type: dot_precision
value: 45.18555667001003
verified: no, type: dot_recall
value: 99.77851605758582
verified: no, type: euclidean_accuracy
value: 61.95
verified: no, type: euclidean_ap
value: 60.44070441806914
verified: no, type: euclidean_f1
value: 62.89978678038379
verified: no, type: euclidean_precision
value: 46.31083202511774
verified: no, type: euclidean_recall
value: 98.00664451827242
verified: no, type: manhattan_accuracy
value: 61.9
verified: no, type: manhattan_ap
value: 60.52939878134297
verified: no, type: manhattan_f1
value: 63.034188034188034
verified: no, type: manhattan_precision
value: 46.45669291338583
verified: no, type: manhattan_recall
value: 98.00664451827242
verified: no, type: max_accuracy
value: 61.95
verified: no, type: max_ap
value: 60.52939878134297
verified: no, type: max_f1
value: 63.034188034188034
verified: no, task: type: STS
dataset: type: Lajavaness/SICK-fr
name: MTEB SICKFr (fra-Latn)
config: fra-Latn
split: test
revision: e077ab4cf4774a1e36d86d593b150422fafd8e8a
metrics: type: cos_sim_pearson
value: 55.697943925847646
verified: no, type: cos_sim_spearman
value: 53.33151992866752
verified: no, type: euclidean_pearson
value: 54.32882764397367
verified: no, type: euclidean_spearman
value: 53.54968438609837
verified: no, type: manhattan_pearson
value: 54.56634524641888
verified: no, type: manhattan_spearman
value: 53.81344727168701
verified: no, task: type: STS
dataset: type: mteb/sts22-crosslingual-sts
name: MTEB STS22 (fra-Latn)
config: fra-Latn
split: test
revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3
metrics: type: cos_sim_pearson
value: 22.771197036286605
verified: no, type: cos_sim_spearman
value: 60.29016180301653
verified: no, type: euclidean_pearson
value: 35.31319988418939
verified: no, type: euclidean_spearman
value: 59.61398871828641
verified: no, type: manhattan_pearson
value: 36.10315029818106
verified: no, type: manhattan_spearman
value: 60.5122301133988
verified: no, task: type: STS
dataset: type: mteb/stsb_multi_mt
name: MTEB STSBenchmarkMultilingualSTS (fra-Latn)
config: fra-Latn
split: test
revision: 29afa2569dcedaaa2fe6a3dcfebab33d28b82e8c
metrics: type: cos_sim_pearson
value: 47.730796921644384
verified: no, type: cos_sim_spearman
value: 49.54059034135741
verified: no, type: euclidean_pearson
value: 49.48474815018905
verified: no, type: euclidean_spearman
value: 50.71533884079761
verified: no, type: manhattan_pearson
value: 50.10488858533032
verified: no, type: manhattan_spearman
value: 51.1375710610132
verified: no, task: type: Summarization
dataset: type: lyon-nlp/summarization-summeval-fr-p2p
name: MTEB SummEvalFr (fra-Latn)
config: fra-Latn
split: test
revision: b385812de6a9577b6f4d0f88c6a6e35395a94054
metrics: type: cos_sim_pearson
value: 29.102661066592816
verified: no, type: cos_sim_spearman
value: 29.615000554218955
verified: no, type: dot_pearson
value: 19.77690299595119
verified: no, type: dot_spearman
value: 19.112834848310158
verified: no, task: type: Reranking
dataset: type: lyon-nlp/mteb-fr-reranking-syntec-s2p
name: MTEB SyntecReranking (fra-Latn)
config: fra-Latn
split: test
revision: daf0863838cd9e3ba50544cdce3ac2b338a1b0ad
metrics: type: map
value: 37.372655122655125
verified: no, type: mrr
value: 37.28174603174604
verified: no, task: type: Retrieval
dataset: type: lyon-nlp/mteb-fr-retrieval-syntec-s2p
name: MTEB SyntecRetrieval (fra-Latn)
config: fra-Latn
split: test
revision: 19661ccdca4dfc2d15122d776b61685f48c68ca9
metrics: type: map_at_1
value: 2
verified: no, type: map_at_10
value: 6.816999999999999
verified: no, type: map_at_100
value: 9.522
verified: no, type: map_at_1000
value: 9.522
verified: no, type: map_at_20
value: 8.402
verified: no, type: map_at_3
value: 4.167
verified: no, type: map_at_5
value: 4.867
verified: no, type: mrr_at_1
value: 2
verified: no, type: mrr_at_10
value: 6.816999999999999
verified: no, type: mrr_at_100
value: 9.522
verified: no, type: mrr_at_1000
value: 9.522
verified: no, type: mrr_at_20
value: 8.402
verified: no, type: mrr_at_3
value: 4.167
verified: no, type: mrr_at_5
value: 4.867
verified: no, type: ndcg_at_1
value: 2
verified: no, type: ndcg_at_10
value: 10.940999999999999
verified: no, type: ndcg_at_100
value: 25.96
verified: no, type: ndcg_at_1000
value: 25.96
verified: no, type: ndcg_at_20
value: 16.742
verified: no, type: ndcg_at_3
value: 4.893
verified: no, type: ndcg_at_5
value: 6.141
verified: no, type: precision_at_1
value: 2
verified: no, type: precision_at_10
value: 2.5
verified: no, type: precision_at_100
value: 1
verified: no, type: precision_at_1000
value: 0.1
verified: no, type: precision_at_20
value: 2.4
verified: no, type: precision_at_3
value: 2.333
verified: no, type: precision_at_5
value: 2
verified: no, type: recall_at_1
value: 2
verified: no, type: recall_at_10
value: 25
verified: no, type: recall_at_100
value: 100
verified: no, type: recall_at_1000
value: 100
verified: no, type: recall_at_20
value: 48
verified: no, type: recall_at_3
value: 7.000000000000001
verified: no, type: recall_at_5
value: 10
verified: no, task: type: Retrieval
dataset: type: jinaai/xpqa
name: MTEB XPQARetrieval (fra-Latn)
config: fra-Latn
split: test
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
metrics: type: map_at_1
value: 9.437
verified: no, type: map_at_10
value: 13.574
verified: no, type: map_at_100
value: 14.265
verified: no, type: map_at_1000
value: 14.527999999999999
verified: no, type: map_at_20
value: 13.834
verified: no, type: map_at_3
value: 12.277000000000001
verified: no, type: map_at_5
value: 12.936
verified: no, type: mrr_at_1
value: 14.285999999999998
verified: no, type: mrr_at_10
value: 18.269
verified: no, type: mrr_at_100
value: 18.991
verified: no, type: mrr_at_1000
value: 19.15
verified: no, type: mrr_at_20
value: 18.598
verified: no, type: mrr_at_3
value: 17
verified: no, type: mrr_at_5
value: 17.681
verified: no, type: ndcg_at_1
value: 14.285999999999998
verified: no, type: ndcg_at_10
value: 16.447
verified: no, type: ndcg_at_100
value: 20.617
verified: no, type: ndcg_at_1000
value: 27.589000000000002
verified: no, type: ndcg_at_20
value: 17.455000000000002
verified: no, type: ndcg_at_3
value: 14.540000000000001
verified: no, type: ndcg_at_5
value: 15.084
verified: no, type: precision_at_1
value: 14.285999999999998
verified: no, type: precision_at_10
value: 3.698
verified: no, type: precision_at_100
value: 0.734
verified: no, type: precision_at_1000
value: 0.18
verified: no, type: precision_at_20
value: 2.163
verified: no, type: precision_at_3
value: 8.366999999999999
verified: no, type: precision_at_5
value: 5.928
verified: no, type: recall_at_1
value: 9.437
verified: no, type: recall_at_10
value: 20.16
verified: no, type: recall_at_100
value: 38.527
verified: no, type: recall_at_1000
value: 85.102
verified: no, type: recall_at_20
value: 23.632
verified: no, type: recall_at_3
value: 14.562
verified: no, type: recall_at_5
value: 16.8
verified: no
language: fr
Likes: 0 | All-time downloads: 11487
Created: 2024-05-14 09:25:42