Muennighoff's picture
Update README.md
2dbba11
---
pipeline_tag: sentence-similarity
tags:
- sentence-transformers
- feature-extraction
- sentence-similarity
- mteb
model-index:
- name: SGPT-5.8B-weightedmean-msmarco-specb-bitfit
results:
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (en)
config: en
split: test
revision: 2d8a100785abf0ae21420d2a55b0c56e3e1ea996
metrics:
- type: accuracy
value: 69.22388059701493
- type: ap
value: 32.04724673950256
- type: f1
value: 63.25719825770428
- task:
type: Classification
dataset:
type: mteb/amazon_polarity
name: MTEB AmazonPolarityClassification
config: default
split: test
revision: 80714f8dcf8cefc218ef4f8c5a966dd83f75a0e1
metrics:
- type: accuracy
value: 71.26109999999998
- type: ap
value: 66.16336378255403
- type: f1
value: 70.89719145825303
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (en)
config: en
split: test
revision: c379a6705fec24a2493fa68e011692605f44e119
metrics:
- type: accuracy
value: 39.19199999999999
- type: f1
value: 38.580766731113826
- task:
type: Retrieval
dataset:
type: arguana
name: MTEB ArguAna
config: default
split: test
revision: 5b3e3697907184a9b77a3c99ee9ea1a9cbb1e4e3
metrics:
- type: map_at_1
value: 27.311999999999998
- type: map_at_10
value: 42.620000000000005
- type: map_at_100
value: 43.707
- type: map_at_1000
value: 43.714999999999996
- type: map_at_3
value: 37.624
- type: map_at_5
value: 40.498
- type: mrr_at_1
value: 27.667
- type: mrr_at_10
value: 42.737
- type: mrr_at_100
value: 43.823
- type: mrr_at_1000
value: 43.830999999999996
- type: mrr_at_3
value: 37.743
- type: mrr_at_5
value: 40.616
- type: ndcg_at_1
value: 27.311999999999998
- type: ndcg_at_10
value: 51.37500000000001
- type: ndcg_at_100
value: 55.778000000000006
- type: ndcg_at_1000
value: 55.96600000000001
- type: ndcg_at_3
value: 41.087
- type: ndcg_at_5
value: 46.269
- type: precision_at_1
value: 27.311999999999998
- type: precision_at_10
value: 7.945
- type: precision_at_100
value: 0.9820000000000001
- type: precision_at_1000
value: 0.1
- type: precision_at_3
value: 17.046
- type: precision_at_5
value: 12.745000000000001
- type: recall_at_1
value: 27.311999999999998
- type: recall_at_10
value: 79.445
- type: recall_at_100
value: 98.151
- type: recall_at_1000
value: 99.57300000000001
- type: recall_at_3
value: 51.13799999999999
- type: recall_at_5
value: 63.727000000000004
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-p2p
name: MTEB ArxivClusteringP2P
config: default
split: test
revision: 0bbdb47bcbe3a90093699aefeed338a0f28a7ee8
metrics:
- type: v_measure
value: 45.59037428592033
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-s2s
name: MTEB ArxivClusteringS2S
config: default
split: test
revision: b73bd54100e5abfa6e3a23dcafb46fe4d2438dc3
metrics:
- type: v_measure
value: 38.86371701986363
- task:
type: Reranking
dataset:
type: mteb/askubuntudupquestions-reranking
name: MTEB AskUbuntuDupQuestions
config: default
split: test
revision: 4d853f94cd57d85ec13805aeeac3ae3e5eb4c49c
metrics:
- type: map
value: 61.625568691427766
- type: mrr
value: 75.83256386580486
- task:
type: STS
dataset:
type: mteb/biosses-sts
name: MTEB BIOSSES
config: default
split: test
revision: 9ee918f184421b6bd48b78f6c714d86546106103
metrics:
- type: cos_sim_pearson
value: 89.96074355094802
- type: cos_sim_spearman
value: 86.2501580394454
- type: euclidean_pearson
value: 82.18427440380462
- type: euclidean_spearman
value: 80.14760935017947
- type: manhattan_pearson
value: 82.24621578156392
- type: manhattan_spearman
value: 80.00363016590163
- task:
type: Classification
dataset:
type: mteb/banking77
name: MTEB Banking77Classification
config: default
split: test
revision: 44fa15921b4c889113cc5df03dd4901b49161ab7
metrics:
- type: accuracy
value: 84.49350649350649
- type: f1
value: 84.4249343233736
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-p2p
name: MTEB BiorxivClusteringP2P
config: default
split: test
revision: 11d0121201d1f1f280e8cc8f3d98fb9c4d9f9c55
metrics:
- type: v_measure
value: 36.551459722989385
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-s2s
name: MTEB BiorxivClusteringS2S
config: default
split: test
revision: c0fab014e1bcb8d3a5e31b2088972a1e01547dc1
metrics:
- type: v_measure
value: 33.69901851846774
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackAndroidRetrieval
config: default
split: test
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db
metrics:
- type: map_at_1
value: 30.499
- type: map_at_10
value: 41.208
- type: map_at_100
value: 42.638
- type: map_at_1000
value: 42.754
- type: map_at_3
value: 37.506
- type: map_at_5
value: 39.422000000000004
- type: mrr_at_1
value: 37.339
- type: mrr_at_10
value: 47.051
- type: mrr_at_100
value: 47.745
- type: mrr_at_1000
value: 47.786
- type: mrr_at_3
value: 44.086999999999996
- type: mrr_at_5
value: 45.711
- type: ndcg_at_1
value: 37.339
- type: ndcg_at_10
value: 47.666
- type: ndcg_at_100
value: 52.994
- type: ndcg_at_1000
value: 54.928999999999995
- type: ndcg_at_3
value: 41.982
- type: ndcg_at_5
value: 44.42
- type: precision_at_1
value: 37.339
- type: precision_at_10
value: 9.127
- type: precision_at_100
value: 1.4749999999999999
- type: precision_at_1000
value: 0.194
- type: precision_at_3
value: 20.076
- type: precision_at_5
value: 14.449000000000002
- type: recall_at_1
value: 30.499
- type: recall_at_10
value: 60.328
- type: recall_at_100
value: 82.57900000000001
- type: recall_at_1000
value: 95.074
- type: recall_at_3
value: 44.17
- type: recall_at_5
value: 50.94
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackEnglishRetrieval
config: default
split: test
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db
metrics:
- type: map_at_1
value: 30.613
- type: map_at_10
value: 40.781
- type: map_at_100
value: 42.018
- type: map_at_1000
value: 42.132999999999996
- type: map_at_3
value: 37.816
- type: map_at_5
value: 39.389
- type: mrr_at_1
value: 38.408
- type: mrr_at_10
value: 46.631
- type: mrr_at_100
value: 47.332
- type: mrr_at_1000
value: 47.368
- type: mrr_at_3
value: 44.384
- type: mrr_at_5
value: 45.661
- type: ndcg_at_1
value: 38.408
- type: ndcg_at_10
value: 46.379999999999995
- type: ndcg_at_100
value: 50.81
- type: ndcg_at_1000
value: 52.663000000000004
- type: ndcg_at_3
value: 42.18
- type: ndcg_at_5
value: 43.974000000000004
- type: precision_at_1
value: 38.408
- type: precision_at_10
value: 8.656
- type: precision_at_100
value: 1.3860000000000001
- type: precision_at_1000
value: 0.184
- type: precision_at_3
value: 20.276
- type: precision_at_5
value: 14.241999999999999
- type: recall_at_1
value: 30.613
- type: recall_at_10
value: 56.44
- type: recall_at_100
value: 75.044
- type: recall_at_1000
value: 86.426
- type: recall_at_3
value: 43.766
- type: recall_at_5
value: 48.998000000000005
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGamingRetrieval
config: default
split: test
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db
metrics:
- type: map_at_1
value: 37.370999999999995
- type: map_at_10
value: 49.718
- type: map_at_100
value: 50.737
- type: map_at_1000
value: 50.79
- type: map_at_3
value: 46.231
- type: map_at_5
value: 48.329
- type: mrr_at_1
value: 42.884
- type: mrr_at_10
value: 53.176
- type: mrr_at_100
value: 53.81700000000001
- type: mrr_at_1000
value: 53.845
- type: mrr_at_3
value: 50.199000000000005
- type: mrr_at_5
value: 52.129999999999995
- type: ndcg_at_1
value: 42.884
- type: ndcg_at_10
value: 55.826
- type: ndcg_at_100
value: 59.93000000000001
- type: ndcg_at_1000
value: 61.013
- type: ndcg_at_3
value: 49.764
- type: ndcg_at_5
value: 53.025999999999996
- type: precision_at_1
value: 42.884
- type: precision_at_10
value: 9.046999999999999
- type: precision_at_100
value: 1.212
- type: precision_at_1000
value: 0.135
- type: precision_at_3
value: 22.131999999999998
- type: precision_at_5
value: 15.524
- type: recall_at_1
value: 37.370999999999995
- type: recall_at_10
value: 70.482
- type: recall_at_100
value: 88.425
- type: recall_at_1000
value: 96.03399999999999
- type: recall_at_3
value: 54.43
- type: recall_at_5
value: 62.327999999999996
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGisRetrieval
config: default
split: test
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db
metrics:
- type: map_at_1
value: 22.875999999999998
- type: map_at_10
value: 31.715
- type: map_at_100
value: 32.847
- type: map_at_1000
value: 32.922000000000004
- type: map_at_3
value: 29.049999999999997
- type: map_at_5
value: 30.396
- type: mrr_at_1
value: 24.52
- type: mrr_at_10
value: 33.497
- type: mrr_at_100
value: 34.455000000000005
- type: mrr_at_1000
value: 34.510000000000005
- type: mrr_at_3
value: 30.791
- type: mrr_at_5
value: 32.175
- type: ndcg_at_1
value: 24.52
- type: ndcg_at_10
value: 36.95
- type: ndcg_at_100
value: 42.238
- type: ndcg_at_1000
value: 44.147999999999996
- type: ndcg_at_3
value: 31.435000000000002
- type: ndcg_at_5
value: 33.839000000000006
- type: precision_at_1
value: 24.52
- type: precision_at_10
value: 5.9319999999999995
- type: precision_at_100
value: 0.901
- type: precision_at_1000
value: 0.11
- type: precision_at_3
value: 13.446
- type: precision_at_5
value: 9.469
- type: recall_at_1
value: 22.875999999999998
- type: recall_at_10
value: 51.38
- type: recall_at_100
value: 75.31099999999999
- type: recall_at_1000
value: 89.718
- type: recall_at_3
value: 36.26
- type: recall_at_5
value: 42.248999999999995
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackMathematicaRetrieval
config: default
split: test
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db
metrics:
- type: map_at_1
value: 14.984
- type: map_at_10
value: 23.457
- type: map_at_100
value: 24.723
- type: map_at_1000
value: 24.846
- type: map_at_3
value: 20.873
- type: map_at_5
value: 22.357
- type: mrr_at_1
value: 18.159
- type: mrr_at_10
value: 27.431
- type: mrr_at_100
value: 28.449
- type: mrr_at_1000
value: 28.52
- type: mrr_at_3
value: 24.979000000000003
- type: mrr_at_5
value: 26.447
- type: ndcg_at_1
value: 18.159
- type: ndcg_at_10
value: 28.627999999999997
- type: ndcg_at_100
value: 34.741
- type: ndcg_at_1000
value: 37.516
- type: ndcg_at_3
value: 23.902
- type: ndcg_at_5
value: 26.294
- type: precision_at_1
value: 18.159
- type: precision_at_10
value: 5.485
- type: precision_at_100
value: 0.985
- type: precision_at_1000
value: 0.136
- type: precision_at_3
value: 11.774
- type: precision_at_5
value: 8.731
- type: recall_at_1
value: 14.984
- type: recall_at_10
value: 40.198
- type: recall_at_100
value: 67.11500000000001
- type: recall_at_1000
value: 86.497
- type: recall_at_3
value: 27.639000000000003
- type: recall_at_5
value: 33.595000000000006
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackPhysicsRetrieval
config: default
split: test
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db
metrics:
- type: map_at_1
value: 29.067
- type: map_at_10
value: 39.457
- type: map_at_100
value: 40.83
- type: map_at_1000
value: 40.94
- type: map_at_3
value: 35.995
- type: map_at_5
value: 38.159
- type: mrr_at_1
value: 34.937000000000005
- type: mrr_at_10
value: 44.755
- type: mrr_at_100
value: 45.549
- type: mrr_at_1000
value: 45.589
- type: mrr_at_3
value: 41.947
- type: mrr_at_5
value: 43.733
- type: ndcg_at_1
value: 34.937000000000005
- type: ndcg_at_10
value: 45.573
- type: ndcg_at_100
value: 51.266999999999996
- type: ndcg_at_1000
value: 53.184
- type: ndcg_at_3
value: 39.961999999999996
- type: ndcg_at_5
value: 43.02
- type: precision_at_1
value: 34.937000000000005
- type: precision_at_10
value: 8.296000000000001
- type: precision_at_100
value: 1.32
- type: precision_at_1000
value: 0.167
- type: precision_at_3
value: 18.8
- type: precision_at_5
value: 13.763
- type: recall_at_1
value: 29.067
- type: recall_at_10
value: 58.298
- type: recall_at_100
value: 82.25099999999999
- type: recall_at_1000
value: 94.476
- type: recall_at_3
value: 42.984
- type: recall_at_5
value: 50.658
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackProgrammersRetrieval
config: default
split: test
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db
metrics:
- type: map_at_1
value: 25.985999999999997
- type: map_at_10
value: 35.746
- type: map_at_100
value: 37.067
- type: map_at_1000
value: 37.191
- type: map_at_3
value: 32.599000000000004
- type: map_at_5
value: 34.239000000000004
- type: mrr_at_1
value: 31.735000000000003
- type: mrr_at_10
value: 40.515
- type: mrr_at_100
value: 41.459
- type: mrr_at_1000
value: 41.516
- type: mrr_at_3
value: 37.938
- type: mrr_at_5
value: 39.25
- type: ndcg_at_1
value: 31.735000000000003
- type: ndcg_at_10
value: 41.484
- type: ndcg_at_100
value: 47.047
- type: ndcg_at_1000
value: 49.427
- type: ndcg_at_3
value: 36.254999999999995
- type: ndcg_at_5
value: 38.375
- type: precision_at_1
value: 31.735000000000003
- type: precision_at_10
value: 7.66
- type: precision_at_100
value: 1.234
- type: precision_at_1000
value: 0.16
- type: precision_at_3
value: 17.427999999999997
- type: precision_at_5
value: 12.328999999999999
- type: recall_at_1
value: 25.985999999999997
- type: recall_at_10
value: 53.761
- type: recall_at_100
value: 77.149
- type: recall_at_1000
value: 93.342
- type: recall_at_3
value: 39.068000000000005
- type: recall_at_5
value: 44.693
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackRetrieval
config: default
split: test
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db
metrics:
- type: map_at_1
value: 24.949749999999998
- type: map_at_10
value: 34.04991666666667
- type: map_at_100
value: 35.26825
- type: map_at_1000
value: 35.38316666666667
- type: map_at_3
value: 31.181333333333335
- type: map_at_5
value: 32.77391666666667
- type: mrr_at_1
value: 29.402833333333334
- type: mrr_at_10
value: 38.01633333333333
- type: mrr_at_100
value: 38.88033333333334
- type: mrr_at_1000
value: 38.938500000000005
- type: mrr_at_3
value: 35.5175
- type: mrr_at_5
value: 36.93808333333333
- type: ndcg_at_1
value: 29.402833333333334
- type: ndcg_at_10
value: 39.403166666666664
- type: ndcg_at_100
value: 44.66408333333333
- type: ndcg_at_1000
value: 46.96283333333333
- type: ndcg_at_3
value: 34.46633333333334
- type: ndcg_at_5
value: 36.78441666666667
- type: precision_at_1
value: 29.402833333333334
- type: precision_at_10
value: 6.965833333333333
- type: precision_at_100
value: 1.1330833333333334
- type: precision_at_1000
value: 0.15158333333333335
- type: precision_at_3
value: 15.886666666666665
- type: precision_at_5
value: 11.360416666666667
- type: recall_at_1
value: 24.949749999999998
- type: recall_at_10
value: 51.29325
- type: recall_at_100
value: 74.3695
- type: recall_at_1000
value: 90.31299999999999
- type: recall_at_3
value: 37.580083333333334
- type: recall_at_5
value: 43.529666666666664
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackStatsRetrieval
config: default
split: test
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db
metrics:
- type: map_at_1
value: 22.081999999999997
- type: map_at_10
value: 29.215999999999998
- type: map_at_100
value: 30.163
- type: map_at_1000
value: 30.269000000000002
- type: map_at_3
value: 26.942
- type: map_at_5
value: 28.236
- type: mrr_at_1
value: 24.847
- type: mrr_at_10
value: 31.918999999999997
- type: mrr_at_100
value: 32.817
- type: mrr_at_1000
value: 32.897
- type: mrr_at_3
value: 29.831000000000003
- type: mrr_at_5
value: 31.019999999999996
- type: ndcg_at_1
value: 24.847
- type: ndcg_at_10
value: 33.4
- type: ndcg_at_100
value: 38.354
- type: ndcg_at_1000
value: 41.045
- type: ndcg_at_3
value: 29.236
- type: ndcg_at_5
value: 31.258000000000003
- type: precision_at_1
value: 24.847
- type: precision_at_10
value: 5.353
- type: precision_at_100
value: 0.853
- type: precision_at_1000
value: 0.116
- type: precision_at_3
value: 12.679000000000002
- type: precision_at_5
value: 8.988
- type: recall_at_1
value: 22.081999999999997
- type: recall_at_10
value: 43.505
- type: recall_at_100
value: 66.45400000000001
- type: recall_at_1000
value: 86.378
- type: recall_at_3
value: 32.163000000000004
- type: recall_at_5
value: 37.059999999999995
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackTexRetrieval
config: default
split: test
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db
metrics:
- type: map_at_1
value: 15.540000000000001
- type: map_at_10
value: 22.362000000000002
- type: map_at_100
value: 23.435
- type: map_at_1000
value: 23.564
- type: map_at_3
value: 20.143
- type: map_at_5
value: 21.324
- type: mrr_at_1
value: 18.892
- type: mrr_at_10
value: 25.942999999999998
- type: mrr_at_100
value: 26.883000000000003
- type: mrr_at_1000
value: 26.968999999999998
- type: mrr_at_3
value: 23.727
- type: mrr_at_5
value: 24.923000000000002
- type: ndcg_at_1
value: 18.892
- type: ndcg_at_10
value: 26.811
- type: ndcg_at_100
value: 32.066
- type: ndcg_at_1000
value: 35.166
- type: ndcg_at_3
value: 22.706
- type: ndcg_at_5
value: 24.508
- type: precision_at_1
value: 18.892
- type: precision_at_10
value: 4.942
- type: precision_at_100
value: 0.878
- type: precision_at_1000
value: 0.131
- type: precision_at_3
value: 10.748000000000001
- type: precision_at_5
value: 7.784000000000001
- type: recall_at_1
value: 15.540000000000001
- type: recall_at_10
value: 36.742999999999995
- type: recall_at_100
value: 60.525
- type: recall_at_1000
value: 82.57600000000001
- type: recall_at_3
value: 25.252000000000002
- type: recall_at_5
value: 29.872
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackUnixRetrieval
config: default
split: test
revision: 2b9f5791698b5be7bc5e10535c8690f20043c3db
metrics:
- type: map_at_1
value: 24.453
- type: map_at_10
value: 33.363
- type: map_at_100
value: 34.579
- type: map_at_1000
value: 34.686
- type: map_at_3
value: 30.583
- type: map_at_5
value: 32.118
- type: mrr_at_1
value: 28.918
- type: mrr_at_10
value: 37.675
- type: mrr_at_100
value: 38.567
- type: mrr_at_1000
value: 38.632
- type: mrr_at_3
value: 35.260999999999996
- type: mrr_at_5
value: 36.576
- type: ndcg_at_1
value: 28.918
- type: ndcg_at_10
value: 38.736
- type: ndcg_at_100
value: 44.261
- type: ndcg_at_1000
value: 46.72
- type: ndcg_at_3
value: 33.81
- type: ndcg_at_5
value: 36.009
- type: precision_at_1
value: 28.918
- type: precision_at_10
value: 6.586
- type: precision_at_100
value: 1.047
- type: precision_at_1000
value: 0.13699999999999998
- type: precision_at_3
value: 15.360999999999999
- type: precision_at_5
value: 10.857999999999999
- type: recall_at_1
value: 24.453
- type: recall_at_10
value: 50.885999999999996
- type: recall_at_100
value: 75.03
- type: recall_at_1000
value: 92.123
- type: recall_at_3
value: 37.138
- type: recall_at_5
value: 42.864999999999995
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWebmastersRetrieval
config: default
split: test