instructor-xl / README.md
Eyalyoli's picture
fix readme
5279e3d
<
---
pipeline_tag: sentence-similarity
tags:
- text-embedding
- embeddings
- information-retrieval
- beir
- text-classification
- language-model
- text-clustering
- text-semantic-similarity
- text-evaluation
- prompt-retrieval
- text-reranking
- sentence-transformers
- feature-extraction
- sentence-similarity
- transformers
- t5
- English
- Sentence Similarity
- natural_questions
- ms_marco
- fever
- hotpot_qa
- mteb
language: en
inference: false
license: apache-2.0
model-index:
- name: final_xl_results
results:
- task:
type: Classification
dataset:
type: mteb/amazon_counterfactual
name: MTEB AmazonCounterfactualClassification (en)
config: en
split: test
revision: e8379541af4e31359cca9fbcf4b00f2671dba205
metrics:
- type: accuracy
value: 85.08955223880596
- type: ap
value: 52.66066378722476
- type: f1
value: 79.63340218960269
- task:
type: Classification
dataset:
type: mteb/amazon_polarity
name: MTEB AmazonPolarityClassification
config: default
split: test
revision: e2d317d38cd51312af73b3d32a06d1a08b442046
metrics:
- type: accuracy
value: 86.542
- type: ap
value: 81.92695193008987
- type: f1
value: 86.51466132573681
- task:
type: Classification
dataset:
type: mteb/amazon_reviews_multi
name: MTEB AmazonReviewsClassification (en)
config: en
split: test
revision: 1399c76144fd37290681b995c656ef9b2e06e26d
metrics:
- type: accuracy
value: 42.964
- type: f1
value: 41.43146249774862
- task:
type: Retrieval
dataset:
type: arguana
name: MTEB ArguAna
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 29.872
- type: map_at_10
value: 46.342
- type: map_at_100
value: 47.152
- type: map_at_1000
value: 47.154
- type: map_at_3
value: 41.216
- type: map_at_5
value: 44.035999999999994
- type: mrr_at_1
value: 30.939
- type: mrr_at_10
value: 46.756
- type: mrr_at_100
value: 47.573
- type: mrr_at_1000
value: 47.575
- type: mrr_at_3
value: 41.548
- type: mrr_at_5
value: 44.425
- type: ndcg_at_1
value: 29.872
- type: ndcg_at_10
value: 55.65
- type: ndcg_at_100
value: 58.88099999999999
- type: ndcg_at_1000
value: 58.951
- type: ndcg_at_3
value: 45.0
- type: ndcg_at_5
value: 50.09
- type: precision_at_1
value: 29.872
- type: precision_at_10
value: 8.549
- type: precision_at_100
value: 0.991
- type: precision_at_1000
value: 0.1
- type: precision_at_3
value: 18.658
- type: precision_at_5
value: 13.669999999999998
- type: recall_at_1
value: 29.872
- type: recall_at_10
value: 85.491
- type: recall_at_100
value: 99.075
- type: recall_at_1000
value: 99.644
- type: recall_at_3
value: 55.974000000000004
- type: recall_at_5
value: 68.35
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-p2p
name: MTEB ArxivClusteringP2P
config: default
split: test
revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d
metrics:
- type: v_measure
value: 42.452729850641276
- task:
type: Clustering
dataset:
type: mteb/arxiv-clustering-s2s
name: MTEB ArxivClusteringS2S
config: default
split: test
revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53
metrics:
- type: v_measure
value: 32.21141846480423
- task:
type: Reranking
dataset:
type: mteb/askubuntudupquestions-reranking
name: MTEB AskUbuntuDupQuestions
config: default
split: test
revision: 2000358ca161889fa9c082cb41daa8dcfb161a54
metrics:
- type: map
value: 65.34710928952622
- type: mrr
value: 77.61124301983028
- task:
type: STS
dataset:
type: mteb/biosses-sts
name: MTEB BIOSSES
config: default
split: test
revision: d3fb88f8f02e40887cd149695127462bbcf29b4a
metrics:
- type: cos_sim_spearman
value: 84.15312230525639
- task:
type: Classification
dataset:
type: mteb/banking77
name: MTEB Banking77Classification
config: default
split: test
revision: 0fd18e25b25c072e09e0d92ab615fda904d66300
metrics:
- type: accuracy
value: 82.66233766233766
- type: f1
value: 82.04175284777669
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-p2p
name: MTEB BiorxivClusteringP2P
config: default
split: test
revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40
metrics:
- type: v_measure
value: 37.36697339826455
- task:
type: Clustering
dataset:
type: mteb/biorxiv-clustering-s2s
name: MTEB BiorxivClusteringS2S
config: default
split: test
revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908
metrics:
- type: v_measure
value: 30.551241447593092
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackAndroidRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 36.797000000000004
- type: map_at_10
value: 48.46
- type: map_at_100
value: 49.968
- type: map_at_1000
value: 50.080000000000005
- type: map_at_3
value: 44.71
- type: map_at_5
value: 46.592
- type: mrr_at_1
value: 45.494
- type: mrr_at_10
value: 54.747
- type: mrr_at_100
value: 55.43599999999999
- type: mrr_at_1000
value: 55.464999999999996
- type: mrr_at_3
value: 52.361000000000004
- type: mrr_at_5
value: 53.727000000000004
- type: ndcg_at_1
value: 45.494
- type: ndcg_at_10
value: 54.989
- type: ndcg_at_100
value: 60.096000000000004
- type: ndcg_at_1000
value: 61.58
- type: ndcg_at_3
value: 49.977
- type: ndcg_at_5
value: 51.964999999999996
- type: precision_at_1
value: 45.494
- type: precision_at_10
value: 10.558
- type: precision_at_100
value: 1.6049999999999998
- type: precision_at_1000
value: 0.203
- type: precision_at_3
value: 23.796
- type: precision_at_5
value: 16.881
- type: recall_at_1
value: 36.797000000000004
- type: recall_at_10
value: 66.83
- type: recall_at_100
value: 88.34100000000001
- type: recall_at_1000
value: 97.202
- type: recall_at_3
value: 51.961999999999996
- type: recall_at_5
value: 57.940000000000005
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackEnglishRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 32.597
- type: map_at_10
value: 43.424
- type: map_at_100
value: 44.78
- type: map_at_1000
value: 44.913
- type: map_at_3
value: 40.315
- type: map_at_5
value: 41.987
- type: mrr_at_1
value: 40.382
- type: mrr_at_10
value: 49.219
- type: mrr_at_100
value: 49.895
- type: mrr_at_1000
value: 49.936
- type: mrr_at_3
value: 46.996
- type: mrr_at_5
value: 48.231
- type: ndcg_at_1
value: 40.382
- type: ndcg_at_10
value: 49.318
- type: ndcg_at_100
value: 53.839999999999996
- type: ndcg_at_1000
value: 55.82899999999999
- type: ndcg_at_3
value: 44.914
- type: ndcg_at_5
value: 46.798
- type: precision_at_1
value: 40.382
- type: precision_at_10
value: 9.274000000000001
- type: precision_at_100
value: 1.497
- type: precision_at_1000
value: 0.198
- type: precision_at_3
value: 21.592
- type: precision_at_5
value: 15.159
- type: recall_at_1
value: 32.597
- type: recall_at_10
value: 59.882000000000005
- type: recall_at_100
value: 78.446
- type: recall_at_1000
value: 90.88000000000001
- type: recall_at_3
value: 46.9
- type: recall_at_5
value: 52.222
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGamingRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 43.8
- type: map_at_10
value: 57.293000000000006
- type: map_at_100
value: 58.321
- type: map_at_1000
value: 58.361
- type: map_at_3
value: 53.839999999999996
- type: map_at_5
value: 55.838
- type: mrr_at_1
value: 49.592000000000006
- type: mrr_at_10
value: 60.643
- type: mrr_at_100
value: 61.23499999999999
- type: mrr_at_1000
value: 61.251999999999995
- type: mrr_at_3
value: 58.265
- type: mrr_at_5
value: 59.717
- type: ndcg_at_1
value: 49.592000000000006
- type: ndcg_at_10
value: 63.364
- type: ndcg_at_100
value: 67.167
- type: ndcg_at_1000
value: 67.867
- type: ndcg_at_3
value: 57.912
- type: ndcg_at_5
value: 60.697
- type: precision_at_1
value: 49.592000000000006
- type: precision_at_10
value: 10.088
- type: precision_at_100
value: 1.2930000000000001
- type: precision_at_1000
value: 0.13899999999999998
- type: precision_at_3
value: 25.789
- type: precision_at_5
value: 17.541999999999998
- type: recall_at_1
value: 43.8
- type: recall_at_10
value: 77.635
- type: recall_at_100
value: 93.748
- type: recall_at_1000
value: 98.468
- type: recall_at_3
value: 63.223
- type: recall_at_5
value: 70.122
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGisRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 27.721
- type: map_at_10
value: 35.626999999999995
- type: map_at_100
value: 36.719
- type: map_at_1000
value: 36.8
- type: map_at_3
value: 32.781
- type: map_at_5
value: 34.333999999999996
- type: mrr_at_1
value: 29.604999999999997
- type: mrr_at_10
value: 37.564
- type: mrr_at_100
value: 38.505
- type: mrr_at_1000
value: 38.565
- type: mrr_at_3
value: 34.727000000000004
- type: mrr_at_5
value: 36.207
- type: ndcg_at_1
value: 29.604999999999997
- type: ndcg_at_10
value: 40.575
- type: ndcg_at_100
value: 45.613
- type: ndcg_at_1000
value: 47.676
- type: ndcg_at_3
value: 34.811
- type: ndcg_at_5
value: 37.491
- type: precision_at_1
value: 29.604999999999997
- type: precision_at_10
value: 6.1690000000000005
- type: precision_at_100
value: 0.906
- type: precision_at_1000
value: 0.11199999999999999
- type: precision_at_3
value: 14.237
- type: precision_at_5
value: 10.056
- type: recall_at_1
value: 27.721
- type: recall_at_10
value: 54.041
- type: recall_at_100
value: 76.62299999999999
- type: recall_at_1000
value: 92.134
- type: recall_at_3
value: 38.582
- type: recall_at_5
value: 44.989000000000004
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackMathematicaRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 16.553
- type: map_at_10
value: 25.384
- type: map_at_100
value: 26.655
- type: map_at_1000
value: 26.778000000000002
- type: map_at_3
value: 22.733
- type: map_at_5
value: 24.119
- type: mrr_at_1
value: 20.149
- type: mrr_at_10
value: 29.705
- type: mrr_at_100
value: 30.672
- type: mrr_at_1000
value: 30.737
- type: mrr_at_3
value: 27.032
- type: mrr_at_5
value: 28.369
- type: ndcg_at_1
value: 20.149
- type: ndcg_at_10
value: 30.843999999999998
- type: ndcg_at_100
value: 36.716
- type: ndcg_at_1000
value: 39.495000000000005
- type: ndcg_at_3
value: 25.918999999999997
- type: ndcg_at_5
value: 27.992
- type: precision_at_1
value: 20.149
- type: precision_at_10
value: 5.858
- type: precision_at_100
value: 1.009
- type: precision_at_1000
value: 0.13799999999999998
- type: precision_at_3
value: 12.645000000000001
- type: precision_at_5
value: 9.179
- type: recall_at_1
value: 16.553
- type: recall_at_10
value: 43.136
- type: recall_at_100
value: 68.562
- type: recall_at_1000
value: 88.208
- type: recall_at_3
value: 29.493000000000002
- type: recall_at_5
value: 34.751
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackPhysicsRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 28.000999999999998
- type: map_at_10
value: 39.004
- type: map_at_100
value: 40.461999999999996
- type: map_at_1000
value: 40.566
- type: map_at_3
value: 35.805
- type: map_at_5
value: 37.672
- type: mrr_at_1
value: 33.782000000000004
- type: mrr_at_10
value: 44.702
- type: mrr_at_100
value: 45.528
- type: mrr_at_1000
value: 45.576
- type: mrr_at_3
value: 42.14
- type: mrr_at_5
value: 43.651
- type: ndcg_at_1
value: 33.782000000000004
- type: ndcg_at_10
value: 45.275999999999996
- type: ndcg_at_100
value: 50.888
- type: ndcg_at_1000
value: 52.879
- type: ndcg_at_3
value: 40.191
- type: ndcg_at_5
value: 42.731
- type: precision_at_1
value: 33.782000000000004
- type: precision_at_10
value: 8.200000000000001
- type: precision_at_100
value: 1.287
- type: precision_at_1000
value: 0.16199999999999998
- type: precision_at_3
value: 19.185
- type: precision_at_5
value: 13.667000000000002
- type: recall_at_1
value: 28.000999999999998
- type: recall_at_10
value: 58.131
- type: recall_at_100
value: 80.869
- type: recall_at_1000
value: 93.931
- type: recall_at_3
value: 44.161
- type: recall_at_5
value: 50.592000000000006
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackProgrammersRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 28.047
- type: map_at_10
value: 38.596000000000004
- type: map_at_100
value: 40.116
- type: map_at_1000
value: 40.232
- type: map_at_3
value: 35.205
- type: map_at_5
value: 37.076
- type: mrr_at_1
value: 34.932
- type: mrr_at_10
value: 44.496
- type: mrr_at_100
value: 45.47
- type: mrr_at_1000
value: 45.519999999999996
- type: mrr_at_3
value: 41.743
- type: mrr_at_5
value: 43.352000000000004
- type: ndcg_at_1
value: 34.932
- type: ndcg_at_10
value: 44.901
- type: ndcg_at_100
value: 50.788999999999994
- type: ndcg_at_1000
value: 52.867
- type: ndcg_at_3
value: 39.449
- type: ndcg_at_5
value: 41.929
- type: precision_at_1
value: 34.932
- type: precision_at_10
value: 8.311
- type: precision_at_100
value: 1.3050000000000002
- type: precision_at_1000
value: 0.166
- type: precision_at_3
value: 18.836
- type: precision_at_5
value: 13.447000000000001
- type: recall_at_1
value: 28.047
- type: recall_at_10
value: 57.717
- type: recall_at_100
value: 82.182
- type: recall_at_1000
value: 95.82000000000001
- type: recall_at_3
value: 42.448
- type: recall_at_5
value: 49.071
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 27.861250000000005
- type: map_at_10
value: 37.529583333333335
- type: map_at_100
value: 38.7915
- type: map_at_1000
value: 38.90558333333335
- type: map_at_3
value: 34.57333333333333
- type: map_at_5
value: 36.187166666666656
- type: mrr_at_1
value: 32.88291666666666
- type: mrr_at_10
value: 41.79750000000001
- type: mrr_at_100
value: 42.63183333333333
- type: mrr_at_1000
value: 42.68483333333333
- type: mrr_at_3
value: 39.313750000000006
- type: mrr_at_5
value: 40.70483333333333
- type: ndcg_at_1
value: 32.88291666666666
- type: ndcg_at_10
value: 43.09408333333333
- type: ndcg_at_100
value: 48.22158333333333
- type: ndcg_at_1000
value: 50.358000000000004
- type: ndcg_at_3
value: 38.129583333333336
- type: ndcg_at_5
value: 40.39266666666666
- type: precision_at_1
value: 32.88291666666666
- type: precision_at_10
value: 7.5584999999999996
- type: precision_at_100
value: 1.1903333333333332
- type: precision_at_1000
value: 0.15658333333333332
- type: precision_at_3
value: 17.495916666666666
- type: precision_at_5
value: 12.373833333333332
- type: recall_at_1
value: 27.861250000000005
- type: recall_at_10
value: 55.215916666666665
- type: recall_at_100
value: 77.392
- type: recall_at_1000
value: 92.04908333333334
- type: recall_at_3
value: 41.37475
- type: recall_at_5
value: 47.22908333333333
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackStatsRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 25.064999999999998
- type: map_at_10
value: 31.635999999999996
- type: map_at_100
value: 32.596000000000004
- type: map_at_1000
value: 32.695
- type: map_at_3
value: 29.612
- type: map_at_5
value: 30.768
- type: mrr_at_1
value: 28.528
- type: mrr_at_10
value: 34.717
- type: mrr_at_100
value: 35.558
- type: mrr_at_1000
value: 35.626000000000005
- type: mrr_at_3
value: 32.745000000000005
- type: mrr_at_5
value: 33.819
- type: ndcg_at_1
value: 28.528
- type: ndcg_at_10
value: 35.647
- type: ndcg_at_100
value: 40.207
- type: ndcg_at_1000
value: 42.695
- type: ndcg_at_3
value: 31.878
- type: ndcg_at_5
value: 33.634
- type: precision_at_1
value: 28.528
- type: precision_at_10
value: 5.46
- type: precision_at_100
value: 0.84
- type: precision_at_1000
value: 0.11399999999999999
- type: precision_at_3
value: 13.547999999999998
- type: precision_at_5
value: 9.325
- type: recall_at_1
value: 25.064999999999998
- type: recall_at_10
value: 45.096000000000004
- type: recall_at_100
value: 65.658
- type: recall_at_1000
value: 84.128
- type: recall_at_3
value: 34.337
- type: recall_at_5
value: 38.849000000000004
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackTexRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 17.276
- type: map_at_10
value: 24.535
- type: map_at_100
value: 25.655
- type: map_at_1000
value: 25.782
- type: map_at_3
value: 22.228
- type: map_at_5
value: 23.612
- type: mrr_at_1
value: 21.266
- type: mrr_at_10
value: 28.474
- type: mrr_at_100
value: 29.398000000000003
- type: mrr_at_1000
value: 29.482000000000003
- type: mrr_at_3
value: 26.245
- type: mrr_at_5
value: 27.624
- type: ndcg_at_1
value: 21.266
- type: ndcg_at_10
value: 29.087000000000003
- type: ndcg_at_100
value: 34.374
- type: ndcg_at_1000
value: 37.433
- type: ndcg_at_3
value: 25.040000000000003
- type: ndcg_at_5
value: 27.116
- type: precision_at_1
value: 21.266
- type: precision_at_10
value: 5.258
- type: precision_at_100
value: 0.9299999999999999
- type: precision_at_1000
value: 0.13699999999999998
- type: precision_at_3
value: 11.849
- type: precision_at_5
value: 8.699
- type: recall_at_1
value: 17.276
- type: recall_at_10
value: 38.928000000000004
- type: recall_at_100
value: 62.529
- type: recall_at_1000
value: 84.44800000000001
- type: recall_at_3
value: 27.554000000000002
- type: recall_at_5
value: 32.915
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackUnixRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 27.297
- type: map_at_10
value: 36.957
- type: map_at_100
value: 38.252
- type: map_at_1000
value: 38.356
- type: map_at_3
value: 34.121
- type: map_at_5
value: 35.782000000000004
- type: mrr_at_1