Updated Leaderboard. Added multi-dataset models, added dataset-AUGMENTED models, also performed some reproductions

add_new_model/mteb_metadata.yaml

@@ -1,114 +0,0 @@
----
-tags:
-- mteb
-model-index:
-- name: multilingual-e5-large-STSAUGMENTED-b16-e10
-  results:
-  - task:
-      type: Classification
-    dataset:
-      type: mteb/amazon_reviews_multi
-      name: MTEB AmazonReviewsClassification (es)
-      config: es
-      split: test
-      revision: 1399c76144fd37290681b995c656ef9b2e06e26d
-    metrics:
-    - type: accuracy
-      value: 42.114
-    - type: f1
-      value: 40.962817058318876
-  - task:
-      type: Classification
-    dataset:
-      type: mteb/mtop_domain
-      name: MTEB MTOPDomainClassification (es)
-      config: es
-      split: test
-      revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf
-    metrics:
-    - type: accuracy
-      value: 80.92394929953302
-    - type: f1
-      value: 80.87019762034018
-  - task:
-      type: Classification
-    dataset:
-      type: mteb/mtop_intent
-      name: MTEB MTOPIntentClassification (es)
-      config: es
-      split: test
-      revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba
-    metrics:
-    - type: accuracy
-      value: 53.32555036691128
-    - type: f1
-      value: 37.93993994854238
-  - task:
-      type: Classification
-    dataset:
-      type: mteb/amazon_massive_intent
-      name: MTEB MassiveIntentClassification (es)
-      config: es
-      split: test
-      revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7
-    metrics:
-    - type: accuracy
-      value: 60.178211163416286
-    - type: f1
-      value: 60.03972849482204
-  - task:
-      type: Classification
-    dataset:
-      type: mteb/amazon_massive_scenario
-      name: MTEB MassiveScenarioClassification (es)
-      config: es
-      split: test
-      revision: 7d571f92784cd94a019292a1f45445077d0ef634
-    metrics:
-    - type: accuracy
-      value: 64.1930060524546
-    - type: f1
-      value: 64.02551940330545
-  - task:
-      type: STS
-    dataset:
-      type: mteb/sts17-crosslingual-sts
-      name: MTEB STS17 (es-es)
-      config: es-es
-      split: test
-      revision: af5e6fb845001ecf41f4c1e033ce921939a2a68d
-    metrics:
-    - type: cos_sim_pearson
-      value: 88.27146953394649
-    - type: cos_sim_spearman
-      value: 88.57657349986717
-    - type: euclidean_pearson
-      value: 89.07302053356283
-    - type: euclidean_spearman
-      value: 88.57431075359631
-    - type: manhattan_pearson
-      value: 89.11306099359507
-    - type: manhattan_spearman
-      value: 88.61591595032135
-  - task:
-      type: STS
-    dataset:
-      type: mteb/sts22-crosslingual-sts
-      name: MTEB STS22 (es)
-      config: es
-      split: test
-      revision: 6d1ba47164174a496b7fa5d3569dae26a6813b80
-    metrics:
-    - type: cos_sim_pearson
-      value: 65.57570700756301
-    - type: cos_sim_spearman
-      value: 70.33230514473709
-    - type: euclidean_pearson
-      value: 66.56411956377407
-    - type: euclidean_spearman
-      value: 70.33230514473709
-    - type: manhattan_pearson
-      value: 66.32203605146594
-    - type: manhattan_spearman
-      value: 70.34809182527435
----

data/classification.csv

@@ -12,8 +12,6 @@ paraphrase-multilingual-mpnet-BASE-v2-ft-stsb_multi_mt-embeddings,64.0,37.25,86.
 mstsb-paraphrase-multilingual-mpnet-BASE-v2,64.47,38.29,86.04,67.06,63.47,67.53
 multilingual-e5-BASE-b16-e10,65.09,43.4,89.02,61.7,63.06,68.25
 multilingual-e5-LARGE-stsb-tuned-b32-e10,66.19,43.31,89.3,64.04,64.62,69.69
-multilingual-e5-LARGE-stsb-tuned-b16-e10,67.1,43.72,90.29,65.51,65.13,70.84
-multilingual-e5-LARGE-stsb-tuned-b16-e5,66.23,43.62,89.33,62.93,65.11,70.16
 multilingual-e5-LARGE-stsb-tuned-b64-e10,64.58,43.71,88.84,60.2,62.74,67.4
 LaBSE,61.97,39.39,84.07,64.44,58.32,63.61
 multilingual-e5-BASE-stsb-tuned-b16-e10-all-languages,64.39,43.04,88.81,59.25,63.04,67.8
@@ -21,5 +19,12 @@ bge-BASE-tuned-b16-e10,50.83,31.34,74.1,45.63,48.72,54.36
 multilingual-e5-LARGE-stsb-tuned-b16-e10-all-languages,64.93,42.98,87.96,60.91,63.78,69.02
 bge-LARGE-tuned-b16-e10,51.67,31.72,76.04,48.01,48.9,53.66
 multilingual-e5-LARGE-stsb-tuned-b16-e15,67.56,43.35,90.98,67.27,65.35,70.87
-multilingual-e5-LARGE-stsb-tuned-b64-e10-all-languages,64.85,43.91,87.2,62.25,63.09,67.83
-multilingual-e5-LARGE-STSAUGMENTED-b16-e5,63.11,44.38,85.51,57.27,62.15,66.25
+text2vec-base-multilingual,58.64,34.82,79.24,58.69,58.28,62.2
+paraphrase-multilingual-MiniLM-L12-v2,61.1,37.49,83.04,60.28,59.66,65.04
+text2vec-stsb-tuned-b16-e5,57.19,34.64,77.72,56.46,56.94,60.22
+paraphrase-multilingual-MiniLM-L12-v2-stsb-tuned-b16-e5,60.41,37.28,82.83,60.45,58.56,62.96
+multilingual-e5-LARGE-tuned-double-dataset-b16-e5,62.65,42.33,85.21,57.01,62.09,66.61
+multilingual-e5-LARGE-stsb-tuned-b16-e5,66.33,43.02,89.71,64.4,64.8,69.72
+multilingual-e5-LARGE-stsb-tuned-b16-e10,66.77,43.86,89.45,64.87,65.13,70.51
+multilingual-e5-LARGE-STSAUGMENTED-b16-e5,64.23,44.25,86.83,59.76,63.08,67.21
+multilingual-e5-LARGE-double-finetuning-b16-e5,63.55,44.23,86.21,57.96,62.58,66.75

data/general.csv

@@ -12,8 +12,6 @@ paraphrase-multilingual-mpnet-BASE-v2-ft-stsb_multi_mt-embeddings,,,68.62,64.0,,
 mstsb-paraphrase-multilingual-mpnet-BASE-v2,,,69.39,64.48,,74.29,
 multilingual-e5-BASE-b16-e10,,,71.97,65.09,,78.86,
 multilingual-e5-LARGE-stsb-tuned-b32-e10,,,72.73,66.19,,79.27,
-multilingual-e5-LARGE-stsb-tuned-b16-e10,,,73.07,67.1,,79.05,
-multilingual-e5-LARGE-stsb-tuned-b16-e5,,,72.84,66.23,,79.46,
 multilingual-e5-LARGE-stsb-tuned-b64-e10,,,71.83,64.58,,79.08,
 LaBSE,,,66.99,61.97,,72.01,
 multilingual-e5-BASE-stsb-tuned-b16-e10-all-languages,,,71.88,64.39,,79.38,
@@ -21,5 +19,12 @@ bge-BASE-tuned-b16-e10,,,59.69,50.83,,68.55,
 multilingual-e5-LARGE-stsb-tuned-b16-e10-all-languages,,,72.22,64.93,,79.5,
 bge-LARGE-tuned-b16-e10,,,61.5,51.67,,71.34,
 multilingual-e5-LARGE-stsb-tuned-b16-e15,,,73.38,67.56,,79.19,
-multilingual-e5-LARGE-stsb-tuned-b64-e10-all-languages,,,71.68,64.85,,78.52,
-multilingual-e5-LARGE-STSAUGMENTED-b16-e5,,,71.28,63.11,,79.44,
+text2vec-base-multilingual,,,66.04,58.64,,73.43,
+paraphrase-multilingual-MiniLM-L12-v2,,,66.08,61.1,,71.06,
+text2vec-stsb-tuned-b16-e5,,,66.0,57.19,,74.81,
+paraphrase-multilingual-MiniLM-L12-v2-stsb-tuned-b16-e5,,,68.21,60.41,,76.01,
+multilingual-e5-LARGE-tuned-double-dataset-b16-e5,,,71.19,62.65,,79.74,
+multilingual-e5-LARGE-stsb-tuned-b16-e5,,,72.9,66.33,,79.47,
+multilingual-e5-LARGE-stsb-tuned-b16-e10,,,73.03,66.77,,79.29,
+multilingual-e5-LARGE-STSAUGMENTED-b16-e5,,,71.84,64.23,,79.44,
+multilingual-e5-LARGE-double-finetuning-b16-e5,,,71.53,63.55,,79.5,

data/sts.csv

@@ -12,8 +12,6 @@ paraphrase-multilingual-mpnet-BASE-v2-ft-stsb_multi_mt-embeddings,73.24,86.89,59
 mstsb-paraphrase-multilingual-mpnet-BASE-v2,74.28,88.22,60.36
 multilingual-e5-BASE-b16-e10,78.86,87.51,70.21
 multilingual-e5-LARGE-stsb-tuned-b32-e10,79.27,88.1,70.44
-multilingual-e5-LARGE-stsb-tuned-b16-e10,79.05,88.53,69.58
-multilingual-e5-LARGE-stsb-tuned-b16-e5,79.46,88.44,70.48
 multilingual-e5-LARGE-stsb-tuned-b64-e10,79.08,88.03,70.12
 LaBSE,72.01,80.83,63.18
 multilingual-e5-BASE-stsb-tuned-b16-e10-all-languages,79.38,87.77,70.99
@@ -21,5 +19,12 @@ bge-BASE-tuned-b16-e10,68.55,82.01,55.08
 multilingual-e5-LARGE-stsb-tuned-b16-e10-all-languages,79.5,88.66,70.35
 bge-LARGE-tuned-b16-e10,71.34,81.57,61.12
 multilingual-e5-LARGE-stsb-tuned-b16-e15,79.19,89.04,69.34
-multilingual-e5-LARGE-stsb-tuned-b64-e10-all-languages,78.52,88.37,68.67
-multilingual-e5-LARGE-STSAUGMENTED-b16-e5,79.44,88.46,70.41
+text2vec-base-multilingual,73.43,87.91,58.94
+paraphrase-multilingual-MiniLM-L12-v2,71.06,85.56,56.56
+text2vec-stsb-tuned-b16-e5,74.81,88.28,61.34
+paraphrase-multilingual-MiniLM-L12-v2-stsb-tuned-b16-e5,76.01,88.03,63.99
+multilingual-e5-LARGE-tuned-double-dataset-b16-e5,79.74,88.98,70.5
+multilingual-e5-LARGE-stsb-tuned-b16-e5,79.47,88.72,70.21
+multilingual-e5-LARGE-stsb-tuned-b16-e10,79.29,89.1,69.48
+multilingual-e5-LARGE-STSAUGMENTED-b16-e5,79.44,88.78,70.11
+multilingual-e5-LARGE-double-finetuning-b16-e5,79.5,88.63,70.37