metadata
tags:
- feature-extraction
- mteb
pipeline_tag: feature-extraction
model-index:
- name: dragon-plus
results:
- task:
type: Retrieval
dataset:
type: arguana
name: MTEB ArguAna
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 22.973
- type: map_at_10
value: 38.242
- type: map_at_100
value: 39.326
- type: map_at_1000
value: 39.342
- type: map_at_3
value: 33.144
- type: map_at_5
value: 35.818
- type: mrr_at_1
value: 23.115
- type: mrr_at_10
value: 38.31
- type: mrr_at_100
value: 39.387
- type: mrr_at_1000
value: 39.403
- type: mrr_at_3
value: 33.167
- type: mrr_at_5
value: 35.856
- type: ndcg_at_1
value: 22.973
- type: ndcg_at_10
value: 47.251
- type: ndcg_at_100
value: 51.937
- type: ndcg_at_1000
value: 52.288000000000004
- type: ndcg_at_3
value: 36.569
- type: ndcg_at_5
value: 41.396
- type: precision_at_1
value: 22.973
- type: precision_at_10
value: 7.632
- type: precision_at_100
value: 0.9690000000000001
- type: precision_at_1000
value: 0.1
- type: precision_at_3
value: 15.504999999999999
- type: precision_at_5
value: 11.65
- type: recall_at_1
value: 22.973
- type: recall_at_10
value: 76.31599999999999
- type: recall_at_100
value: 96.942
- type: recall_at_1000
value: 99.57300000000001
- type: recall_at_3
value: 46.515
- type: recall_at_5
value: 58.25
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackAndroidRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 28.793000000000003
- type: map_at_10
value: 38.686
- type: map_at_100
value: 39.848
- type: map_at_1000
value: 39.989999999999995
- type: map_at_3
value: 35.437000000000005
- type: map_at_5
value: 37.067
- type: mrr_at_1
value: 35.05
- type: mrr_at_10
value: 43.903999999999996
- type: mrr_at_100
value: 44.612
- type: mrr_at_1000
value: 44.669
- type: mrr_at_3
value: 41.321000000000005
- type: mrr_at_5
value: 42.573
- type: ndcg_at_1
value: 35.05
- type: ndcg_at_10
value: 44.564
- type: ndcg_at_100
value: 49.252
- type: ndcg_at_1000
value: 51.791
- type: ndcg_at_3
value: 39.576
- type: ndcg_at_5
value: 41.426
- type: precision_at_1
value: 35.05
- type: precision_at_10
value: 8.455
- type: precision_at_100
value: 1.3299999999999998
- type: precision_at_1000
value: 0.187
- type: precision_at_3
value: 18.645999999999997
- type: precision_at_5
value: 13.247
- type: recall_at_1
value: 28.793000000000003
- type: recall_at_10
value: 56.351
- type: recall_at_100
value: 76.542
- type: recall_at_1000
value: 93.14099999999999
- type: recall_at_3
value: 41.581
- type: recall_at_5
value: 47.066
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackEnglishRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 29.828
- type: map_at_10
value: 39.312999999999995
- type: map_at_100
value: 40.487
- type: map_at_1000
value: 40.607
- type: map_at_3
value: 36.525
- type: map_at_5
value: 38.121
- type: mrr_at_1
value: 37.197
- type: mrr_at_10
value: 45.091
- type: mrr_at_100
value: 45.726
- type: mrr_at_1000
value: 45.769999999999996
- type: mrr_at_3
value: 42.856
- type: mrr_at_5
value: 44.056
- type: ndcg_at_1
value: 37.197
- type: ndcg_at_10
value: 44.737
- type: ndcg_at_100
value: 49.02
- type: ndcg_at_1000
value: 51.052
- type: ndcg_at_3
value: 40.685
- type: ndcg_at_5
value: 42.519
- type: precision_at_1
value: 37.197
- type: precision_at_10
value: 8.363
- type: precision_at_100
value: 1.329
- type: precision_at_1000
value: 0.179
- type: precision_at_3
value: 19.533
- type: precision_at_5
value: 13.732
- type: recall_at_1
value: 29.828
- type: recall_at_10
value: 54.339000000000006
- type: recall_at_100
value: 72.217
- type: recall_at_1000
value: 85.185
- type: recall_at_3
value: 42.331
- type: recall_at_5
value: 47.612
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGamingRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 37.919000000000004
- type: map_at_10
value: 49.225
- type: map_at_100
value: 50.306
- type: map_at_1000
value: 50.364
- type: map_at_3
value: 46.459
- type: map_at_5
value: 48.173
- type: mrr_at_1
value: 43.072
- type: mrr_at_10
value: 52.437
- type: mrr_at_100
value: 53.2
- type: mrr_at_1000
value: 53.233
- type: mrr_at_3
value: 50.219
- type: mrr_at_5
value: 51.629999999999995
- type: ndcg_at_1
value: 43.072
- type: ndcg_at_10
value: 54.468
- type: ndcg_at_100
value: 58.912
- type: ndcg_at_1000
value: 60.179
- type: ndcg_at_3
value: 49.836999999999996
- type: ndcg_at_5
value: 52.371
- type: precision_at_1
value: 43.072
- type: precision_at_10
value: 8.52
- type: precision_at_100
value: 1.168
- type: precision_at_1000
value: 0.133
- type: precision_at_3
value: 21.923000000000002
- type: precision_at_5
value: 14.997
- type: recall_at_1
value: 37.919000000000004
- type: recall_at_10
value: 66.682
- type: recall_at_100
value: 85.81
- type: recall_at_1000
value: 94.812
- type: recall_at_3
value: 54.515
- type: recall_at_5
value: 60.684000000000005
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackGisRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 21.04
- type: map_at_10
value: 27.665
- type: map_at_100
value: 28.716
- type: map_at_1000
value: 28.794999999999998
- type: map_at_3
value: 25.338
- type: map_at_5
value: 26.815
- type: mrr_at_1
value: 22.712
- type: mrr_at_10
value: 29.447000000000003
- type: mrr_at_100
value: 30.457
- type: mrr_at_1000
value: 30.522
- type: mrr_at_3
value: 27.119
- type: mrr_at_5
value: 28.582
- type: ndcg_at_1
value: 22.712
- type: ndcg_at_10
value: 31.77
- type: ndcg_at_100
value: 37.104
- type: ndcg_at_1000
value: 39.371
- type: ndcg_at_3
value: 27.171
- type: ndcg_at_5
value: 29.698999999999998
- type: precision_at_1
value: 22.712
- type: precision_at_10
value: 4.859
- type: precision_at_100
value: 0.7929999999999999
- type: precision_at_1000
value: 0.10300000000000001
- type: precision_at_3
value: 11.299
- type: precision_at_5
value: 8.203000000000001
- type: recall_at_1
value: 21.04
- type: recall_at_10
value: 42.848000000000006
- type: recall_at_100
value: 67.694
- type: recall_at_1000
value: 85.179
- type: recall_at_3
value: 30.54
- type: recall_at_5
value: 36.555
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackMathematicaRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 13.403
- type: map_at_10
value: 19.663
- type: map_at_100
value: 20.799
- type: map_at_1000
value: 20.915
- type: map_at_3
value: 17.465
- type: map_at_5
value: 18.665000000000003
- type: mrr_at_1
value: 16.418
- type: mrr_at_10
value: 23.394000000000002
- type: mrr_at_100
value: 24.363
- type: mrr_at_1000
value: 24.44
- type: mrr_at_3
value: 20.916
- type: mrr_at_5
value: 22.241
- type: ndcg_at_1
value: 16.418
- type: ndcg_at_10
value: 24.013
- type: ndcg_at_100
value: 29.62
- type: ndcg_at_1000
value: 32.518
- type: ndcg_at_3
value: 19.747
- type: ndcg_at_5
value: 21.689
- type: precision_at_1
value: 16.418
- type: precision_at_10
value: 4.515000000000001
- type: precision_at_100
value: 0.8410000000000001
- type: precision_at_1000
value: 0.123
- type: precision_at_3
value: 9.411
- type: precision_at_5
value: 6.965000000000001
- type: recall_at_1
value: 13.403
- type: recall_at_10
value: 33.731
- type: recall_at_100
value: 58.743
- type: recall_at_1000
value: 79.343
- type: recall_at_3
value: 22.148
- type: recall_at_5
value: 26.998
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackPhysicsRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 25.782
- type: map_at_10
value: 34.891
- type: map_at_100
value: 36.186
- type: map_at_1000
value: 36.303999999999995
- type: map_at_3
value: 32.099
- type: map_at_5
value: 33.777
- type: mrr_at_1
value: 30.895
- type: mrr_at_10
value: 40.049
- type: mrr_at_100
value: 40.953
- type: mrr_at_1000
value: 41
- type: mrr_at_3
value: 37.424
- type: mrr_at_5
value: 39.07
- type: ndcg_at_1
value: 30.895
- type: ndcg_at_10
value: 40.436
- type: ndcg_at_100
value: 46.046
- type: ndcg_at_1000
value: 48.324
- type: ndcg_at_3
value: 35.66
- type: ndcg_at_5
value: 38.167
- type: precision_at_1
value: 30.895
- type: precision_at_10
value: 7.151000000000001
- type: precision_at_100
value: 1.171
- type: precision_at_1000
value: 0.155
- type: precision_at_3
value: 16.619
- type: precision_at_5
value: 11.935
- type: recall_at_1
value: 25.782
- type: recall_at_10
value: 52.013
- type: recall_at_100
value: 75.736
- type: recall_at_1000
value: 90.823
- type: recall_at_3
value: 38.763
- type: recall_at_5
value: 45.023
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackProgrammersRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 22.491
- type: map_at_10
value: 30.434
- type: map_at_100
value: 31.611
- type: map_at_1000
value: 31.732
- type: map_at_3
value: 27.776
- type: map_at_5
value: 29.271
- type: mrr_at_1
value: 27.74
- type: mrr_at_10
value: 34.964
- type: mrr_at_100
value: 35.943000000000005
- type: mrr_at_1000
value: 36.012
- type: mrr_at_3
value: 32.667
- type: mrr_at_5
value: 33.975
- type: ndcg_at_1
value: 27.74
- type: ndcg_at_10
value: 35.32
- type: ndcg_at_100
value: 40.812
- type: ndcg_at_1000
value: 43.49
- type: ndcg_at_3
value: 30.843999999999998
- type: ndcg_at_5
value: 32.838
- type: precision_at_1
value: 27.74
- type: precision_at_10
value: 6.358
- type: precision_at_100
value: 1.078
- type: precision_at_1000
value: 0.147
- type: precision_at_3
value: 14.421999999999999
- type: precision_at_5
value: 10.32
- type: recall_at_1
value: 22.491
- type: recall_at_10
value: 45.659
- type: recall_at_100
value: 69.303
- type: recall_at_1000
value: 87.849
- type: recall_at_3
value: 33.155
- type: recall_at_5
value: 38.369
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 22.955500000000008
- type: map_at_10
value: 30.754000000000005
- type: map_at_100
value: 31.85208333333333
- type: map_at_1000
value: 31.968416666666666
- type: map_at_3
value: 28.35166666666667
- type: map_at_5
value: 29.717333333333336
- type: mrr_at_1
value: 27.0815
- type: mrr_at_10
value: 34.50116666666666
- type: mrr_at_100
value: 35.361583333333336
- type: mrr_at_1000
value: 35.42583333333334
- type: mrr_at_3
value: 32.30499999999999
- type: mrr_at_5
value: 33.56175
- type: ndcg_at_1
value: 27.0815
- type: ndcg_at_10
value: 35.40033333333333
- type: ndcg_at_100
value: 40.3485
- type: ndcg_at_1000
value: 42.86816666666667
- type: ndcg_at_3
value: 31.24325
- type: ndcg_at_5
value: 33.21525
- type: precision_at_1
value: 27.0815
- type: precision_at_10
value: 6.118666666666667
- type: precision_at_100
value: 1.0085833333333334
- type: precision_at_1000
value: 0.14150000000000001
- type: precision_at_3
value: 14.19175
- type: precision_at_5
value: 10.064583333333331
- type: recall_at_1
value: 22.955500000000008
- type: recall_at_10
value: 45.51058333333333
- type: recall_at_100
value: 67.49925
- type: recall_at_1000
value: 85.24766666666666
- type: recall_at_3
value: 33.885
- type: recall_at_5
value: 38.99608333333334
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackStatsRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 21.371000000000002
- type: map_at_10
value: 27.532
- type: map_at_100
value: 28.443
- type: map_at_1000
value: 28.525
- type: map_at_3
value: 25.689
- type: map_at_5
value: 26.677
- type: mrr_at_1
value: 24.08
- type: mrr_at_10
value: 30.128
- type: mrr_at_100
value: 30.953999999999997
- type: mrr_at_1000
value: 31.022
- type: mrr_at_3
value: 28.298000000000002
- type: mrr_at_5
value: 29.317
- type: ndcg_at_1
value: 24.08
- type: ndcg_at_10
value: 31.212
- type: ndcg_at_100
value: 35.72
- type: ndcg_at_1000
value: 38.061
- type: ndcg_at_3
value: 27.705000000000002
- type: ndcg_at_5
value: 29.26
- type: precision_at_1
value: 24.08
- type: precision_at_10
value: 4.8469999999999995
- type: precision_at_100
value: 0.753
- type: precision_at_1000
value: 0.104
- type: precision_at_3
value: 11.759
- type: precision_at_5
value: 8.097999999999999
- type: recall_at_1
value: 21.371000000000002
- type: recall_at_10
value: 40.089000000000006
- type: recall_at_100
value: 60.879000000000005
- type: recall_at_1000
value: 78.325
- type: recall_at_3
value: 30.175
- type: recall_at_5
value: 34.168
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackTexRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 15.043999999999999
- type: map_at_10
value: 20.794
- type: map_at_100
value: 21.636
- type: map_at_1000
value: 21.753
- type: map_at_3
value: 19.006
- type: map_at_5
value: 19.994999999999997
- type: mrr_at_1
value: 18.066
- type: mrr_at_10
value: 24.157999999999998
- type: mrr_at_100
value: 24.936
- type: mrr_at_1000
value: 25.018
- type: mrr_at_3
value: 22.345000000000002
- type: mrr_at_5
value: 23.396
- type: ndcg_at_1
value: 18.066
- type: ndcg_at_10
value: 24.584
- type: ndcg_at_100
value: 28.869
- type: ndcg_at_1000
value: 31.94
- type: ndcg_at_3
value: 21.295
- type: ndcg_at_5
value: 22.820999999999998
- type: precision_at_1
value: 18.066
- type: precision_at_10
value: 4.381
- type: precision_at_100
value: 0.754
- type: precision_at_1000
value: 0.117
- type: precision_at_3
value: 9.956
- type: precision_at_5
value: 7.123
- type: recall_at_1
value: 15.043999999999999
- type: recall_at_10
value: 32.665
- type: recall_at_100
value: 52.342
- type: recall_at_1000
value: 74.896
- type: recall_at_3
value: 23.402
- type: recall_at_5
value: 27.397
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackUnixRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 22.712
- type: map_at_10
value: 28.963
- type: map_at_100
value: 29.934
- type: map_at_1000
value: 30.049
- type: map_at_3
value: 27.086
- type: map_at_5
value: 28.163
- type: mrr_at_1
value: 26.586
- type: mrr_at_10
value: 32.792
- type: mrr_at_100
value: 33.692
- type: mrr_at_1000
value: 33.767
- type: mrr_at_3
value: 30.939
- type: mrr_at_5
value: 32.012
- type: ndcg_at_1
value: 26.586
- type: ndcg_at_10
value: 32.92
- type: ndcg_at_100
value: 37.891000000000005
- type: ndcg_at_1000
value: 40.647
- type: ndcg_at_3
value: 29.465000000000003
- type: ndcg_at_5
value: 31.106
- type: precision_at_1
value: 26.586
- type: precision_at_10
value: 5.177
- type: precision_at_100
value: 0.8540000000000001
- type: precision_at_1000
value: 0.121
- type: precision_at_3
value: 12.903999999999998
- type: precision_at_5
value: 8.881
- type: recall_at_1
value: 22.712
- type: recall_at_10
value: 41.382000000000005
- type: recall_at_100
value: 63.866
- type: recall_at_1000
value: 83.29299999999999
- type: recall_at_3
value: 31.739
- type: recall_at_5
value: 35.988
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWebmastersRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 19.64
- type: map_at_10
value: 28.432000000000002
- type: map_at_100
value: 29.848999999999997
- type: map_at_1000
value: 30.072
- type: map_at_3
value: 25.862000000000002
- type: map_at_5
value: 27.339000000000002
- type: mrr_at_1
value: 24.308
- type: mrr_at_10
value: 32.475
- type: mrr_at_100
value: 33.404
- type: mrr_at_1000
value: 33.477000000000004
- type: mrr_at_3
value: 30.203999999999997
- type: mrr_at_5
value: 31.558000000000003
- type: ndcg_at_1
value: 24.308
- type: ndcg_at_10
value: 33.79
- type: ndcg_at_100
value: 39.113
- type: ndcg_at_1000
value: 42.388
- type: ndcg_at_3
value: 29.738999999999997
- type: ndcg_at_5
value: 31.734
- type: precision_at_1
value: 24.308
- type: precision_at_10
value: 6.621
- type: precision_at_100
value: 1.322
- type: precision_at_1000
value: 0.22499999999999998
- type: precision_at_3
value: 14.032
- type: precision_at_5
value: 10.435
- type: recall_at_1
value: 19.64
- type: recall_at_10
value: 44.147999999999996
- type: recall_at_100
value: 68.31099999999999
- type: recall_at_1000
value: 90.022
- type: recall_at_3
value: 32.275999999999996
- type: recall_at_5
value: 37.717
- task:
type: Retrieval
dataset:
type: BeIR/cqadupstack
name: MTEB CQADupstackWordpressRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 17.443
- type: map_at_10
value: 23.45
- type: map_at_100
value: 24.41
- type: map_at_1000
value: 24.515
- type: map_at_3
value: 21.478
- type: map_at_5
value: 22.545
- type: mrr_at_1
value: 18.854000000000003
- type: mrr_at_10
value: 25.174999999999997
- type: mrr_at_100
value: 26.099
- type: mrr_at_1000
value: 26.179999999999996
- type: mrr_at_3
value: 23.352
- type: mrr_at_5
value: 24.331
- type: ndcg_at_1
value: 18.854000000000003
- type: ndcg_at_10
value: 26.99
- type: ndcg_at_100
value: 31.823
- type: ndcg_at_1000
value: 34.657
- type: ndcg_at_3
value: 23.195
- type: ndcg_at_5
value: 24.953
- type: precision_at_1
value: 18.854000000000003
- type: precision_at_10
value: 4.1770000000000005
- type: precision_at_100
value: 0.7100000000000001
- type: precision_at_1000
value: 0.104
- type: precision_at_3
value: 9.797
- type: precision_at_5
value: 6.839
- type: recall_at_1
value: 17.443
- type: recall_at_10
value: 36.22
- type: recall_at_100
value: 58.548
- type: recall_at_1000
value: 80.104
- type: recall_at_3
value: 25.995
- type: recall_at_5
value: 30.375999999999998
- task:
type: Retrieval
dataset:
type: climate-fever
name: MTEB ClimateFEVER
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 10.283000000000001
- type: map_at_10
value: 16.121
- type: map_at_100
value: 17.818
- type: map_at_1000
value: 18.015
- type: map_at_3
value: 13.655000000000001
- type: map_at_5
value: 14.854999999999999
- type: mrr_at_1
value: 22.15
- type: mrr_at_10
value: 31.139
- type: mrr_at_100
value: 32.336999999999996
- type: mrr_at_1000
value: 32.39
- type: mrr_at_3
value: 27.861000000000004
- type: mrr_at_5
value: 29.754
- type: ndcg_at_1
value: 22.15
- type: ndcg_at_10
value: 22.852
- type: ndcg_at_100
value: 30.233999999999998
- type: ndcg_at_1000
value: 34.02
- type: ndcg_at_3
value: 18.394
- type: ndcg_at_5
value: 19.973
- type: precision_at_1
value: 22.15
- type: precision_at_10
value: 6.912
- type: precision_at_100
value: 1.4829999999999999
- type: precision_at_1000
value: 0.218
- type: precision_at_3
value: 12.899
- type: precision_at_5
value: 10.111
- type: recall_at_1
value: 10.283000000000001
- type: recall_at_10
value: 27.587
- type: recall_at_100
value: 53.273
- type: recall_at_1000
value: 74.74499999999999
- type: recall_at_3
value: 16.897000000000002
- type: recall_at_5
value: 21.084
- task:
type: Retrieval
dataset:
type: dbpedia-entity
name: MTEB DBPedia
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 9.038
- type: map_at_10
value: 20.153
- type: map_at_100
value: 28.610999999999997
- type: map_at_1000
value: 30.285
- type: map_at_3
value: 14.249
- type: map_at_5
value: 16.715
- type: mrr_at_1
value: 66.75
- type: mrr_at_10
value: 74.477
- type: mrr_at_100
value: 74.678
- type: mrr_at_1000
value: 74.695
- type: mrr_at_3
value: 72.625
- type: mrr_at_5
value: 73.8
- type: ndcg_at_1
value: 55.125
- type: ndcg_at_10
value: 41.837999999999994
- type: ndcg_at_100
value: 46.182
- type: ndcg_at_1000
value: 53.144000000000005
- type: ndcg_at_3
value: 46.084
- type: ndcg_at_5
value: 43.751
- type: precision_at_1
value: 66.75
- type: precision_at_10
value: 33.775
- type: precision_at_100
value: 10.803
- type: precision_at_1000
value: 2.191
- type: precision_at_3
value: 49.5
- type: precision_at_5
value: 42.4
- type: recall_at_1
value: 9.038
- type: recall_at_10
value: 25.988
- type: recall_at_100
value: 52.158
- type: recall_at_1000
value: 74.617
- type: recall_at_3
value: 15.675
- type: recall_at_5
value: 19.570999999999998
- task:
type: Retrieval
dataset:
type: fever
name: MTEB FEVER
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 62.551
- type: map_at_10
value: 73.124
- type: map_at_100
value: 73.432
- type: map_at_1000
value: 73.447
- type: map_at_3
value: 71.297
- type: map_at_5
value: 72.489
- type: mrr_at_1
value: 67.23700000000001
- type: mrr_at_10
value: 77.438
- type: mrr_at_100
value: 77.645
- type: mrr_at_1000
value: 77.64999999999999
- type: mrr_at_3
value: 75.788
- type: mrr_at_5
value: 76.886
- type: ndcg_at_1
value: 67.23700000000001
- type: ndcg_at_10
value: 78.306
- type: ndcg_at_100
value: 79.526
- type: ndcg_at_1000
value: 79.825
- type: ndcg_at_3
value: 74.961
- type: ndcg_at_5
value: 76.91900000000001
- type: precision_at_1
value: 67.23700000000001
- type: precision_at_10
value: 9.875
- type: precision_at_100
value: 1.065
- type: precision_at_1000
value: 0.11
- type: precision_at_3
value: 29.353
- type: precision_at_5
value: 18.749
- type: recall_at_1
value: 62.551
- type: recall_at_10
value: 90.011
- type: recall_at_100
value: 95.06
- type: recall_at_1000
value: 97.033
- type: recall_at_3
value: 81.081
- type: recall_at_5
value: 85.87599999999999
- task:
type: Retrieval
dataset:
type: fiqa
name: MTEB FiQA2018
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 17.636
- type: map_at_10
value: 28.627000000000002
- type: map_at_100
value: 30.262
- type: map_at_1000
value: 30.442000000000004
- type: map_at_3
value: 25.091
- type: map_at_5
value: 27.12
- type: mrr_at_1
value: 34.259
- type: mrr_at_10
value: 42.733
- type: mrr_at_100
value: 43.613
- type: mrr_at_1000
value: 43.663000000000004
- type: mrr_at_3
value: 40.406
- type: mrr_at_5
value: 41.687000000000005
- type: ndcg_at_1
value: 34.259
- type: ndcg_at_10
value: 35.613
- type: ndcg_at_100
value: 42.027
- type: ndcg_at_1000
value: 45.336999999999996
- type: ndcg_at_3
value: 32.435
- type: ndcg_at_5
value: 33.482
- type: precision_at_1
value: 34.259
- type: precision_at_10
value: 9.66
- type: precision_at_100
value: 1.6219999999999999
- type: precision_at_1000
value: 0.22300000000000003
- type: precision_at_3
value: 21.399
- type: precision_at_5
value: 15.741
- type: recall_at_1
value: 17.636
- type: recall_at_10
value: 41.955999999999996
- type: recall_at_100
value: 66.17
- type: recall_at_1000
value: 85.79599999999999
- type: recall_at_3
value: 29.853
- type: recall_at_5
value: 35.18
- task:
type: Retrieval
dataset:
type: hotpotqa
name: MTEB HotpotQA
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 39.487
- type: map_at_10
value: 56.765
- type: map_at_100
value: 57.616
- type: map_at_1000
value: 57.679
- type: map_at_3
value: 53.616
- type: map_at_5
value: 55.623999999999995
- type: mrr_at_1
value: 78.974
- type: mrr_at_10
value: 84.622
- type: mrr_at_100
value: 84.776
- type: mrr_at_1000
value: 84.783
- type: mrr_at_3
value: 83.747
- type: mrr_at_5
value: 84.27900000000001
- type: ndcg_at_1
value: 78.974
- type: ndcg_at_10
value: 66.164
- type: ndcg_at_100
value: 69.03099999999999
- type: ndcg_at_1000
value: 70.261
- type: ndcg_at_3
value: 61.712
- type: ndcg_at_5
value: 64.22
- type: precision_at_1
value: 78.974
- type: precision_at_10
value: 13.520999999999999
- type: precision_at_100
value: 1.575
- type: precision_at_1000
value: 0.174
- type: precision_at_3
value: 38.501000000000005
- type: precision_at_5
value: 25.083
- type: recall_at_1
value: 39.487
- type: recall_at_10
value: 67.60300000000001
- type: recall_at_100
value: 78.744
- type: recall_at_1000
value: 86.914
- type: recall_at_3
value: 57.752
- type: recall_at_5
value: 62.708
- task:
type: Retrieval
dataset:
type: msmarco
name: MTEB MSMARCO
config: default
split: dev
revision: None
metrics:
- type: map_at_1
value: 24.224999999999998
- type: map_at_10
value: 37.791000000000004
- type: map_at_100
value: 38.899
- type: map_at_1000
value: 38.937
- type: map_at_3
value: 33.584
- type: map_at_5
value: 36.142
- type: mrr_at_1
value: 24.871
- type: mrr_at_10
value: 38.361000000000004
- type: mrr_at_100
value: 39.394
- type: mrr_at_1000
value: 39.427
- type: mrr_at_3
value: 34.224
- type: mrr_at_5
value: 36.767
- type: ndcg_at_1
value: 24.871
- type: ndcg_at_10
value: 45.231
- type: ndcg_at_100
value: 50.42100000000001
- type: ndcg_at_1000
value: 51.329
- type: ndcg_at_3
value: 36.77
- type: ndcg_at_5
value: 41.33
- type: precision_at_1
value: 24.871
- type: precision_at_10
value: 7.124999999999999
- type: precision_at_100
value: 0.971
- type: precision_at_1000
value: 0.105
- type: precision_at_3
value: 15.659
- type: precision_at_5
value: 11.708
- type: recall_at_1
value: 24.224999999999998
- type: recall_at_10
value: 68.081
- type: recall_at_100
value: 91.818
- type: recall_at_1000
value: 98.65
- type: recall_at_3
value: 45.355000000000004
- type: recall_at_5
value: 56.26
- task:
type: Retrieval
dataset:
type: nfcorpus
name: MTEB NFCorpus
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 5.904
- type: map_at_10
value: 12.784
- type: map_at_100
value: 15.628
- type: map_at_1000
value: 17.006
- type: map_at_3
value: 9.695
- type: map_at_5
value: 10.961
- type: mrr_at_1
value: 46.44
- type: mrr_at_10
value: 54.106
- type: mrr_at_100
value: 54.81700000000001
- type: mrr_at_1000
value: 54.858
- type: mrr_at_3
value: 52.837999999999994
- type: mrr_at_5
value: 53.627
- type: ndcg_at_1
value: 44.737
- type: ndcg_at_10
value: 33.967999999999996
- type: ndcg_at_100
value: 30.451
- type: ndcg_at_1000
value: 39.151
- type: ndcg_at_3
value: 39.871
- type: ndcg_at_5
value: 37.138
- type: precision_at_1
value: 46.44
- type: precision_at_10
value: 24.582
- type: precision_at_100
value: 7.715
- type: precision_at_1000
value: 2.0500000000000003
- type: precision_at_3
value: 37.461
- type: precision_at_5
value: 31.517
- type: recall_at_1
value: 5.904
- type: recall_at_10
value: 16.522000000000002
- type: recall_at_100
value: 29.413
- type: recall_at_1000
value: 61.611000000000004
- type: recall_at_3
value: 10.649000000000001
- type: recall_at_5
value: 12.642999999999999
- task:
type: Retrieval
dataset:
type: nq
name: MTEB NQ
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 31.561
- type: map_at_10
value: 46.406
- type: map_at_100
value: 47.499
- type: map_at_1000
value: 47.526
- type: map_at_3
value: 42.26
- type: map_at_5
value: 44.724000000000004
- type: mrr_at_1
value: 35.168
- type: mrr_at_10
value: 48.914
- type: mrr_at_100
value: 49.727
- type: mrr_at_1000
value: 49.744
- type: mrr_at_3
value: 45.418
- type: mrr_at_5
value: 47.53
- type: ndcg_at_1
value: 35.138999999999996
- type: ndcg_at_10
value: 53.943
- type: ndcg_at_100
value: 58.50300000000001
- type: ndcg_at_1000
value: 59.144
- type: ndcg_at_3
value: 46.135999999999996
- type: ndcg_at_5
value: 50.227999999999994
- type: precision_at_1
value: 35.138999999999996
- type: precision_at_10
value: 8.812000000000001
- type: precision_at_100
value: 1.138
- type: precision_at_1000
value: 0.12
- type: precision_at_3
value: 20.867
- type: precision_at_5
value: 14.878
- type: recall_at_1
value: 31.561
- type: recall_at_10
value: 74.343
- type: recall_at_100
value: 93.975
- type: recall_at_1000
value: 98.75699999999999
- type: recall_at_3
value: 54.169
- type: recall_at_5
value: 63.56
- task:
type: Retrieval
dataset:
type: quora
name: MTEB QuoraRetrieval
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 69.753
- type: map_at_10
value: 83.56400000000001
- type: map_at_100
value: 84.19200000000001
- type: map_at_1000
value: 84.211
- type: map_at_3
value: 80.568
- type: map_at_5
value: 82.44500000000001
- type: mrr_at_1
value: 79.99000000000001
- type: mrr_at_10
value: 86.542
- type: mrr_at_100
value: 86.655
- type: mrr_at_1000
value: 86.656
- type: mrr_at_3
value: 85.505
- type: mrr_at_5
value: 86.21
- type: ndcg_at_1
value: 79.99000000000001
- type: ndcg_at_10
value: 87.449
- type: ndcg_at_100
value: 88.739
- type: ndcg_at_1000
value: 88.87
- type: ndcg_at_3
value: 84.418
- type: ndcg_at_5
value: 86.09599999999999
- type: precision_at_1
value: 79.99000000000001
- type: precision_at_10
value: 13.236999999999998
- type: precision_at_100
value: 1.516
- type: precision_at_1000
value: 0.156
- type: precision_at_3
value: 36.736999999999995
- type: precision_at_5
value: 24.227999999999998
- type: recall_at_1
value: 69.753
- type: recall_at_10
value: 94.967
- type: recall_at_100
value: 99.378
- type: recall_at_1000
value: 99.953
- type: recall_at_3
value: 86.408
- type: recall_at_5
value: 91.03
- task:
type: Retrieval
dataset:
type: scidocs
name: MTEB SCIDOCS
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 3.8080000000000003
- type: map_at_10
value: 9.222
- type: map_at_100
value: 10.779
- type: map_at_1000
value: 11.027000000000001
- type: map_at_3
value: 6.729
- type: map_at_5
value: 7.872999999999999
- type: mrr_at_1
value: 18.7
- type: mrr_at_10
value: 28.084999999999997
- type: mrr_at_100
value: 29.134999999999998
- type: mrr_at_1000
value: 29.214000000000002
- type: mrr_at_3
value: 24.917
- type: mrr_at_5
value: 26.651999999999997
- type: ndcg_at_1
value: 18.7
- type: ndcg_at_10
value: 15.969
- type: ndcg_at_100
value: 22.535
- type: ndcg_at_1000
value: 27.337
- type: ndcg_at_3
value: 15.112
- type: ndcg_at_5
value: 13.089
- type: precision_at_1
value: 18.7
- type: precision_at_10
value: 8.32
- type: precision_at_100
value: 1.786
- type: precision_at_1000
value: 0.293
- type: precision_at_3
value: 14.099999999999998
- type: precision_at_5
value: 11.42
- type: recall_at_1
value: 3.8080000000000003
- type: recall_at_10
value: 16.872
- type: recall_at_100
value: 36.235
- type: recall_at_1000
value: 59.587
- type: recall_at_3
value: 8.583
- type: recall_at_5
value: 11.562999999999999
- task:
type: Retrieval
dataset:
type: scifact
name: MTEB SciFact
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 53.994
- type: map_at_10
value: 63.56
- type: map_at_100
value: 64.247
- type: map_at_1000
value: 64.275
- type: map_at_3
value: 61.23499999999999
- type: map_at_5
value: 62.638000000000005
- type: mrr_at_1
value: 57.333
- type: mrr_at_10
value: 65.23299999999999
- type: mrr_at_100
value: 65.762
- type: mrr_at_1000
value: 65.78699999999999
- type: mrr_at_3
value: 63.556000000000004
- type: mrr_at_5
value: 64.572
- type: ndcg_at_1
value: 57.333
- type: ndcg_at_10
value: 67.88300000000001
- type: ndcg_at_100
value: 70.99
- type: ndcg_at_1000
value: 71.66
- type: ndcg_at_3
value: 64.16
- type: ndcg_at_5
value: 66.042
- type: precision_at_1
value: 57.333
- type: precision_at_10
value: 8.967
- type: precision_at_100
value: 1.06
- type: precision_at_1000
value: 0.11199999999999999
- type: precision_at_3
value: 25.222
- type: precision_at_5
value: 16.467000000000002
- type: recall_at_1
value: 53.994
- type: recall_at_10
value: 79.289
- type: recall_at_100
value: 93.533
- type: recall_at_1000
value: 98.667
- type: recall_at_3
value: 69.267
- type: recall_at_5
value: 74.128
- task:
type: Retrieval
dataset:
type: trec-covid
name: MTEB TRECCOVID
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 0.212
- type: map_at_10
value: 1.925
- type: map_at_100
value: 9.235
- type: map_at_1000
value: 22.111
- type: map_at_3
value: 0.626
- type: map_at_5
value: 1.031
- type: mrr_at_1
value: 82
- type: mrr_at_10
value: 90.5
- type: mrr_at_100
value: 90.5
- type: mrr_at_1000
value: 90.5
- type: mrr_at_3
value: 90
- type: mrr_at_5
value: 90.5
- type: ndcg_at_1
value: 75
- type: ndcg_at_10
value: 75.851
- type: ndcg_at_100
value: 53.190000000000005
- type: ndcg_at_1000
value: 45.507999999999996
- type: ndcg_at_3
value: 80.19500000000001
- type: ndcg_at_5
value: 78.448
- type: precision_at_1
value: 82
- type: precision_at_10
value: 82.6
- type: precision_at_100
value: 54.48
- type: precision_at_1000
value: 20.785999999999998
- type: precision_at_3
value: 86.667
- type: precision_at_5
value: 85.2
- type: recall_at_1
value: 0.212
- type: recall_at_10
value: 2.13
- type: recall_at_100
value: 12.152000000000001
- type: recall_at_1000
value: 42.403
- type: recall_at_3
value: 0.6689999999999999
- type: recall_at_5
value: 1.121
- task:
type: Retrieval
dataset:
type: webis-touche2020
name: MTEB Touche2020
config: default
split: test
revision: None
metrics:
- type: map_at_1
value: 2.701
- type: map_at_10
value: 10.488999999999999
- type: map_at_100
value: 17.258000000000003
- type: map_at_1000
value: 18.797
- type: map_at_3
value: 5.563
- type: map_at_5
value: 7.268
- type: mrr_at_1
value: 30.612000000000002
- type: mrr_at_10
value: 48.197
- type: mrr_at_100
value: 48.762
- type: mrr_at_1000
value: 48.762
- type: mrr_at_3
value: 44.218
- type: mrr_at_5
value: 46.666999999999994
- type: ndcg_at_1
value: 28.571
- type: ndcg_at_10
value: 26.512
- type: ndcg_at_100
value: 38.356
- type: ndcg_at_1000
value: 49.57
- type: ndcg_at_3
value: 27.704
- type: ndcg_at_5
value: 27.342
- type: precision_at_1
value: 30.612000000000002
- type: precision_at_10
value: 24.285999999999998
- type: precision_at_100
value: 8
- type: precision_at_1000
value: 1.541
- type: precision_at_3
value: 29.252
- type: precision_at_5
value: 27.346999999999998
- type: recall_at_1
value: 2.701
- type: recall_at_10
value: 17.197000000000003
- type: recall_at_100
value: 49.061
- type: recall_at_1000
value: 82.82300000000001
- type: recall_at_3
value: 6.687
- type: recall_at_5
value: 9.868
DRAGON+ is a BERT-base sized dense retriever initialized from RetroMAE and further trained on the data augmented from MS MARCO corpus, following the approach described in How to Train Your DRAGON: Diverse Augmentation Towards Generalizable Dense Retrieval.
The associated GitHub repository is available here https://github.com/facebookresearch/dpr-scale/tree/main/dragon. We use asymmetric dual encoder, with two distinctly parameterized encoders. The following models are also available:
| Model | Initialization | MARCO Dev | BEIR | Query Encoder Path | Context Encoder Path |
|---|---|---|---|---|---|
| DRAGON+ | Shitao/RetroMAE | 39.0 | 47.4 | facebook/dragon-plus-query-encoder | facebook/dragon-plus-context-encoder |
| DRAGON-RoBERTa | RoBERTa-base | 39.4 | 47.2 | facebook/dragon-roberta-query-encoder | facebook/dragon-roberta-context-encoder |
Usage (HuggingFace Transformers)
Using the model directly available in HuggingFace transformers .
import torch
from transformers import AutoTokenizer, AutoModel
tokenizer = AutoTokenizer.from_pretrained('facebook/dragon-plus-query-encoder')
query_encoder = AutoModel.from_pretrained('facebook/dragon-plus-query-encoder')
context_encoder = AutoModel.from_pretrained('facebook/dragon-plus-context-encoder')
# We use msmarco query and passages as an example
query = "Where was Marie Curie born?"
contexts = [
"Maria Sklodowska, later known as Marie Curie, was born on November 7, 1867.",
"Born in Paris on 15 May 1859, Pierre Curie was the son of Eugène Curie, a doctor of French Catholic origin from Alsace."
]
# Apply tokenizer
query_input = tokenizer(query, return_tensors='pt')
ctx_input = tokenizer(contexts, padding=True, truncation=True, return_tensors='pt')
# Compute embeddings: take the last-layer hidden state of the [CLS] token
query_emb = query_encoder(**query_input).last_hidden_state[:, 0, :]
ctx_emb = context_encoder(**ctx_input).last_hidden_state[:, 0, :]
# Compute similarity scores using dot product
score1 = query_emb @ ctx_emb[0] # 396.5625
score2 = query_emb @ ctx_emb[1] # 393.8340