Hugging Face's logo

bwang0911
/
jev2-legal

Sentence Similarity
sentence-transformers
Safetensors
English
mpnet
feature-extraction
Generated from Trainer
dataset_size:53224
loss:MultipleNegativesRankingLoss
Eval Results
Inference Endpoints
Model card Files Community
bwang0911 commited on
Commit
04d5fd0
•
1 Parent(s): 64a9a01

Add new SentenceTransformer model

Browse files
Files changed (2) hide show
  1. README.md +106 -106
  2. model.safetensors +1 -1
README.md CHANGED
@@ -233,46 +233,46 @@ model-index:
233
  value: 0.24
234
  name: Cosine Accuracy@1
235
  - type: cosine_accuracy@3
236
- value: 0.34
237
  name: Cosine Accuracy@3
238
  - type: cosine_accuracy@5
239
- value: 0.4
240
  name: Cosine Accuracy@5
241
  - type: cosine_accuracy@10
242
- value: 0.52
243
  name: Cosine Accuracy@10
244
  - type: cosine_precision@1
245
  value: 0.24
246
  name: Cosine Precision@1
247
  - type: cosine_precision@3
248
- value: 0.16666666666666663
249
  name: Cosine Precision@3
250
  - type: cosine_precision@5
251
- value: 0.136
252
  name: Cosine Precision@5
253
  - type: cosine_precision@10
254
- value: 0.094
255
  name: Cosine Precision@10
256
  - type: cosine_recall@1
257
- value: 0.06678088578088578
258
  name: Cosine Recall@1
259
  - type: cosine_recall@3
260
- value: 0.1388193473193473
261
  name: Cosine Recall@3
262
  - type: cosine_recall@5
263
- value: 0.18372843822843823
264
  name: Cosine Recall@5
265
  - type: cosine_recall@10
266
- value: 0.2667284382284382
267
  name: Cosine Recall@10
268
  - type: cosine_ndcg@10
269
- value: 0.22218705752805715
270
  name: Cosine Ndcg@10
271
  - type: cosine_mrr@10
272
- value: 0.3134126984126984
273
  name: Cosine Mrr@10
274
  - type: cosine_map@100
275
- value: 0.18539536890113958
276
  name: Cosine Map@100
277
  - task:
278
  type: information-retrieval
@@ -282,49 +282,49 @@ model-index:
282
  type: mteb/AILA_statutes
283
  metrics:
284
  - type: cosine_accuracy@1
285
- value: 0.28
286
  name: Cosine Accuracy@1
287
  - type: cosine_accuracy@3
288
- value: 0.58
289
  name: Cosine Accuracy@3
290
  - type: cosine_accuracy@5
291
- value: 0.8
292
  name: Cosine Accuracy@5
293
  - type: cosine_accuracy@10
294
- value: 0.9
295
  name: Cosine Accuracy@10
296
  - type: cosine_precision@1
297
- value: 0.28
298
  name: Cosine Precision@1
299
  - type: cosine_precision@3
300
- value: 0.22666666666666668
301
  name: Cosine Precision@3
302
  - type: cosine_precision@5
303
- value: 0.22399999999999998
304
  name: Cosine Precision@5
305
  - type: cosine_precision@10
306
- value: 0.15799999999999997
307
  name: Cosine Precision@10
308
  - type: cosine_recall@1
309
- value: 0.073
310
  name: Cosine Recall@1
311
  - type: cosine_recall@3
312
- value: 0.17266666666666666
313
  name: Cosine Recall@3
314
  - type: cosine_recall@5
315
- value: 0.2763333333333334
316
  name: Cosine Recall@5
317
  - type: cosine_recall@10
318
- value: 0.3773333333333333
319
  name: Cosine Recall@10
320
  - type: cosine_ndcg@10
321
- value: 0.32396168684748544
322
  name: Cosine Ndcg@10
323
  - type: cosine_mrr@10
324
- value: 0.48524603174603165
325
  name: Cosine Mrr@10
326
  - type: cosine_map@100
327
- value: 0.26147750527977026
328
  name: Cosine Map@100
329
  - task:
330
  type: information-retrieval
@@ -334,49 +334,49 @@ model-index:
334
  type: mteb/legalbench_consumer_contracts_qa
335
  metrics:
336
  - type: cosine_accuracy@1
337
- value: 0.4292929292929293
338
  name: Cosine Accuracy@1
339
  - type: cosine_accuracy@3
340
- value: 0.6363636363636364
341
  name: Cosine Accuracy@3
342
  - type: cosine_accuracy@5
343
- value: 0.7095959595959596
344
  name: Cosine Accuracy@5
345
  - type: cosine_accuracy@10
346
- value: 0.8156565656565656
347
  name: Cosine Accuracy@10
348
  - type: cosine_precision@1
349
- value: 0.4292929292929293
350
  name: Cosine Precision@1
351
  - type: cosine_precision@3
352
- value: 0.21212121212121207
353
  name: Cosine Precision@3
354
  - type: cosine_precision@5
355
- value: 0.1419191919191919
356
  name: Cosine Precision@5
357
  - type: cosine_precision@10
358
- value: 0.08156565656565656
359
  name: Cosine Precision@10
360
  - type: cosine_recall@1
361
- value: 0.4292929292929293
362
  name: Cosine Recall@1
363
  - type: cosine_recall@3
364
- value: 0.6363636363636364
365
  name: Cosine Recall@3
366
  - type: cosine_recall@5
367
- value: 0.7095959595959596
368
  name: Cosine Recall@5
369
  - type: cosine_recall@10
370
- value: 0.8156565656565656
371
  name: Cosine Recall@10
372
  - type: cosine_ndcg@10
373
- value: 0.6114603730669577
374
  name: Cosine Ndcg@10
375
  - type: cosine_mrr@10
376
- value: 0.5472532868366202
377
  name: Cosine Mrr@10
378
  - type: cosine_map@100
379
- value: 0.555387361338846
380
  name: Cosine Map@100
381
  - task:
382
  type: information-retrieval
@@ -386,49 +386,49 @@ model-index:
386
  type: mteb/legalbench_corporate_lobbying
387
  metrics:
388
  - type: cosine_accuracy@1
389
- value: 0.6441176470588236
390
  name: Cosine Accuracy@1
391
  - type: cosine_accuracy@3
392
- value: 0.8558823529411764
393
  name: Cosine Accuracy@3
394
  - type: cosine_accuracy@5
395
- value: 0.8823529411764706
396
  name: Cosine Accuracy@5
397
  - type: cosine_accuracy@10
398
- value: 0.9147058823529411
399
  name: Cosine Accuracy@10
400
  - type: cosine_precision@1
401
- value: 0.6441176470588236
402
  name: Cosine Precision@1
403
  - type: cosine_precision@3
404
- value: 0.2852941176470588
405
  name: Cosine Precision@3
406
  - type: cosine_precision@5
407
- value: 0.17647058823529413
408
  name: Cosine Precision@5
409
  - type: cosine_precision@10
410
- value: 0.09147058823529411
411
  name: Cosine Precision@10
412
  - type: cosine_recall@1
413
- value: 0.6441176470588236
414
  name: Cosine Recall@1
415
  - type: cosine_recall@3
416
- value: 0.8558823529411764
417
  name: Cosine Recall@3
418
  - type: cosine_recall@5
419
- value: 0.8823529411764706
420
  name: Cosine Recall@5
421
  - type: cosine_recall@10
422
- value: 0.9147058823529411
423
  name: Cosine Recall@10
424
  - type: cosine_ndcg@10
425
- value: 0.7924078571703878
426
  name: Cosine Ndcg@10
427
  - type: cosine_mrr@10
428
- value: 0.751936274509804
429
  name: Cosine Mrr@10
430
  - type: cosine_map@100
431
- value: 0.754712212674935
432
  name: Cosine Map@100
433
  - task:
434
  type: information-retrieval
@@ -438,49 +438,49 @@ model-index:
438
  type: mteb/legal_summarization
439
  metrics:
440
  - type: cosine_accuracy@1
441
- value: 0.41901408450704225
442
  name: Cosine Accuracy@1
443
  - type: cosine_accuracy@3
444
- value: 0.5563380281690141
445
  name: Cosine Accuracy@3
446
  - type: cosine_accuracy@5
447
- value: 0.6338028169014085
448
  name: Cosine Accuracy@5
449
  - type: cosine_accuracy@10
450
- value: 0.7183098591549296
451
  name: Cosine Accuracy@10
452
  - type: cosine_precision@1
453
- value: 0.41901408450704225
454
  name: Cosine Precision@1
455
  - type: cosine_precision@3
456
- value: 0.20070422535211266
457
  name: Cosine Precision@3
458
  - type: cosine_precision@5
459
- value: 0.14295774647887324
460
  name: Cosine Precision@5
461
  - type: cosine_precision@10
462
- value: 0.08838028169014084
463
  name: Cosine Precision@10
464
  - type: cosine_recall@1
465
- value: 0.35939538747637334
466
  name: Cosine Recall@1
467
  - type: cosine_recall@3
468
- value: 0.4814835985610633
469
  name: Cosine Recall@3
470
  - type: cosine_recall@5
471
- value: 0.5483042192549235
472
  name: Cosine Recall@5
473
  - type: cosine_recall@10
474
- value: 0.6505441741357234
475
  name: Cosine Recall@10
476
  - type: cosine_ndcg@10
477
- value: 0.5155518221457815
478
  name: Cosine Ndcg@10
479
  - type: cosine_mrr@10
480
- value: 0.5074348871003801
481
  name: Cosine Mrr@10
482
  - type: cosine_map@100
483
- value: 0.46706462134757426
484
  name: Cosine Map@100
485
  ---
486
 
@@ -591,21 +591,21 @@ You can finetune this model on your own dataset.
591
 
592
  | Metric | mteb/AILA_casedocs | mteb/AILA_statutes | mteb/legalbench_consumer_contracts_qa | mteb/legalbench_corporate_lobbying | mteb/legal_summarization |
593
  |:--------------------|:-------------------|:-------------------|:--------------------------------------|:-----------------------------------|:-------------------------|
594
- | cosine_accuracy@1 | 0.24 | 0.28 | 0.4293 | 0.6441 | 0.419 |
595
- | cosine_accuracy@3 | 0.34 | 0.58 | 0.6364 | 0.8559 | 0.5563 |
596
- | cosine_accuracy@5 | 0.4 | 0.8 | 0.7096 | 0.8824 | 0.6338 |
597
- | cosine_accuracy@10 | 0.52 | 0.9 | 0.8157 | 0.9147 | 0.7183 |
598
- | cosine_precision@1 | 0.24 | 0.28 | 0.4293 | 0.6441 | 0.419 |
599
- | cosine_precision@3 | 0.1667 | 0.2267 | 0.2121 | 0.2853 | 0.2007 |
600
- | cosine_precision@5 | 0.136 | 0.224 | 0.1419 | 0.1765 | 0.143 |
601
- | cosine_precision@10 | 0.094 | 0.158 | 0.0816 | 0.0915 | 0.0884 |
602
- | cosine_recall@1 | 0.0668 | 0.073 | 0.4293 | 0.6441 | 0.3594 |
603
- | cosine_recall@3 | 0.1388 | 0.1727 | 0.6364 | 0.8559 | 0.4815 |
604
- | cosine_recall@5 | 0.1837 | 0.2763 | 0.7096 | 0.8824 | 0.5483 |
605
- | cosine_recall@10 | 0.2667 | 0.3773 | 0.8157 | 0.9147 | 0.6505 |
606
- | **cosine_ndcg@10** | **0.2222** | **0.324** | **0.6115** | **0.7924** | **0.5156** |
607
- | cosine_mrr@10 | 0.3134 | 0.4852 | 0.5473 | 0.7519 | 0.5074 |
608
- | cosine_map@100 | 0.1854 | 0.2615 | 0.5554 | 0.7547 | 0.4671 |
609
 
610
  <!--
611
  ## Bias, Risks and Limitations
@@ -820,7 +820,7 @@ You can finetune this model on your own dataset.
820
 
821
  - `eval_strategy`: steps
822
  - `per_device_train_batch_size`: 64
823
- - `learning_rate`: 5e-06
824
  - `num_train_epochs`: 2
825
  - `warmup_ratio`: 0.1
826
  - `fp16`: True
@@ -840,7 +840,7 @@ You can finetune this model on your own dataset.
840
  - `gradient_accumulation_steps`: 1
841
  - `eval_accumulation_steps`: None
842
  - `torch_empty_cache_steps`: None
843
- - `learning_rate`: 5e-06
844
  - `weight_decay`: 0.0
845
  - `adam_beta1`: 0.9
846
  - `adam_beta2`: 0.999
@@ -949,22 +949,22 @@ You can finetune this model on your own dataset.
949
  | Epoch | Step | Training Loss | mteb/AILA_casedocs_cosine_ndcg@10 | mteb/AILA_statutes_cosine_ndcg@10 | mteb/legalbench_consumer_contracts_qa_cosine_ndcg@10 | mteb/legalbench_corporate_lobbying_cosine_ndcg@10 | mteb/legal_summarization_cosine_ndcg@10 |
950
  |:------:|:----:|:-------------:|:---------------------------------:|:---------------------------------:|:----------------------------------------------------:|:-------------------------------------------------:|:---------------------------------------:|
951
  | 0 | 0 | - | 0.1704 | 0.2351 | 0.6781 | 0.8793 | 0.5766 |
952
- | 0.1196 | 100 | - | 0.2192 | 0.2808 | 0.6816 | 0.8857 | 0.6033 |
953
- | 0.2392 | 200 | - | 0.2285 | 0.2958 | 0.6637 | 0.8878 | 0.6141 |
954
- | 0.3589 | 300 | - | 0.2384 | 0.3174 | 0.6504 | 0.8820 | 0.6103 |
955
- | 0.4785 | 400 | - | 0.2349 | 0.3105 | 0.6379 | 0.8626 | 0.5871 |
956
- | 0.5981 | 500 | 1.9344 | 0.2223 | 0.3026 | 0.6288 | 0.8476 | 0.5743 |
957
- | 0.7177 | 600 | - | 0.2155 | 0.3078 | 0.6247 | 0.8277 | 0.5571 |
958
- | 0.8373 | 700 | - | 0.2179 | 0.3183 | 0.6244 | 0.8389 | 0.5469 |
959
- | 0.9569 | 800 | - | 0.2145 | 0.3207 | 0.6230 | 0.8368 | 0.5374 |
960
- | 1.0766 | 900 | - | 0.2045 | 0.3241 | 0.6257 | 0.8331 | 0.5360 |
961
- | 1.1962 | 1000 | 0.9429 | 0.2162 | 0.3450 | 0.6145 | 0.8216 | 0.5296 |
962
- | 1.3158 | 1100 | - | 0.2175 | 0.3369 | 0.6149 | 0.8160 | 0.5308 |
963
- | 1.4354 | 1200 | - | 0.2274 | 0.3246 | 0.6095 | 0.8020 | 0.5262 |
964
- | 1.5550 | 1300 | - | 0.2217 | 0.3273 | 0.6182 | 0.8030 | 0.5244 |
965
- | 1.6746 | 1400 | - | 0.2186 | 0.3226 | 0.6145 | 0.7935 | 0.5196 |
966
- | 1.7943 | 1500 | 0.9098 | 0.2222 | 0.3203 | 0.6129 | 0.7898 | 0.5178 |
967
- | 1.9139 | 1600 | - | 0.2222 | 0.3240 | 0.6115 | 0.7924 | 0.5156 |
968
 
969
 
970
  ### Framework Versions
 
233
  value: 0.24
234
  name: Cosine Accuracy@1
235
  - type: cosine_accuracy@3
236
+ value: 0.4
237
  name: Cosine Accuracy@3
238
  - type: cosine_accuracy@5
239
+ value: 0.44
240
  name: Cosine Accuracy@5
241
  - type: cosine_accuracy@10
242
+ value: 0.5
243
  name: Cosine Accuracy@10
244
  - type: cosine_precision@1
245
  value: 0.24
246
  name: Cosine Precision@1
247
  - type: cosine_precision@3
248
+ value: 0.2
249
  name: Cosine Precision@3
250
  - type: cosine_precision@5
251
+ value: 0.14400000000000002
252
  name: Cosine Precision@5
253
  - type: cosine_precision@10
254
+ value: 0.096
255
  name: Cosine Precision@10
256
  - type: cosine_recall@1
257
+ value: 0.06261421911421912
258
  name: Cosine Recall@1
259
  - type: cosine_recall@3
260
+ value: 0.1773951048951049
261
  name: Cosine Recall@3
262
  - type: cosine_recall@5
263
+ value: 0.21672843822843824
264
  name: Cosine Recall@5
265
  - type: cosine_recall@10
266
+ value: 0.28030419580419585
267
  name: Cosine Recall@10
268
  - type: cosine_ndcg@10
269
+ value: 0.23571318760075094
270
  name: Cosine Ndcg@10
271
  - type: cosine_mrr@10
272
+ value: 0.32385714285714284
273
  name: Cosine Mrr@10
274
  - type: cosine_map@100
275
+ value: 0.19099315576955767
276
  name: Cosine Map@100
277
  - task:
278
  type: information-retrieval
 
282
  type: mteb/AILA_statutes
283
  metrics:
284
  - type: cosine_accuracy@1
285
+ value: 0.24
286
  name: Cosine Accuracy@1
287
  - type: cosine_accuracy@3
288
+ value: 0.52
289
  name: Cosine Accuracy@3
290
  - type: cosine_accuracy@5
291
+ value: 0.72
292
  name: Cosine Accuracy@5
293
  - type: cosine_accuracy@10
294
+ value: 0.8
295
  name: Cosine Accuracy@10
296
  - type: cosine_precision@1
297
+ value: 0.24
298
  name: Cosine Precision@1
299
  - type: cosine_precision@3
300
+ value: 0.20666666666666667
301
  name: Cosine Precision@3
302
  - type: cosine_precision@5
303
+ value: 0.19999999999999996
304
  name: Cosine Precision@5
305
  - type: cosine_precision@10
306
+ value: 0.144
307
  name: Cosine Precision@10
308
  - type: cosine_recall@1
309
+ value: 0.068
310
  name: Cosine Recall@1
311
  - type: cosine_recall@3
312
+ value: 0.16066666666666665
313
  name: Cosine Recall@3
314
  - type: cosine_recall@5
315
+ value: 0.25033333333333335
316
  name: Cosine Recall@5
317
  - type: cosine_recall@10
318
+ value: 0.35100000000000003
319
  name: Cosine Recall@10
320
  - type: cosine_ndcg@10
321
+ value: 0.2945290400206784
322
  name: Cosine Ndcg@10
323
  - type: cosine_mrr@10
324
+ value: 0.4145238095238095
325
  name: Cosine Mrr@10
326
  - type: cosine_map@100
327
+ value: 0.23863257355862635
328
  name: Cosine Map@100
329
  - task:
330
  type: information-retrieval
 
334
  type: mteb/legalbench_consumer_contracts_qa
335
  metrics:
336
  - type: cosine_accuracy@1
337
+ value: 0.48737373737373735
338
  name: Cosine Accuracy@1
339
  - type: cosine_accuracy@3
340
+ value: 0.6515151515151515
341
  name: Cosine Accuracy@3
342
  - type: cosine_accuracy@5
343
+ value: 0.73989898989899
344
  name: Cosine Accuracy@5
345
  - type: cosine_accuracy@10
346
+ value: 0.8560606060606061
347
  name: Cosine Accuracy@10
348
  - type: cosine_precision@1
349
+ value: 0.48737373737373735
350
  name: Cosine Precision@1
351
  - type: cosine_precision@3
352
+ value: 0.21717171717171713
353
  name: Cosine Precision@3
354
  - type: cosine_precision@5
355
+ value: 0.14797979797979796
356
  name: Cosine Precision@5
357
  - type: cosine_precision@10
358
+ value: 0.0856060606060606
359
  name: Cosine Precision@10
360
  - type: cosine_recall@1
361
+ value: 0.48737373737373735
362
  name: Cosine Recall@1
363
  - type: cosine_recall@3
364
+ value: 0.6515151515151515
365
  name: Cosine Recall@3
366
  - type: cosine_recall@5
367
+ value: 0.73989898989899
368
  name: Cosine Recall@5
369
  - type: cosine_recall@10
370
+ value: 0.8560606060606061
371
  name: Cosine Recall@10
372
  - type: cosine_ndcg@10
373
+ value: 0.6575720798646046
374
  name: Cosine Ndcg@10
375
  - type: cosine_mrr@10
376
+ value: 0.5956780102613435
377
  name: Cosine Mrr@10
378
  - type: cosine_map@100
379
+ value: 0.6021553873830202
380
  name: Cosine Map@100
381
  - task:
382
  type: information-retrieval
 
386
  type: mteb/legalbench_corporate_lobbying
387
  metrics:
388
  - type: cosine_accuracy@1
389
+ value: 0.788235294117647
390
  name: Cosine Accuracy@1
391
  - type: cosine_accuracy@3
392
+ value: 0.9205882352941176
393
  name: Cosine Accuracy@3
394
  - type: cosine_accuracy@5
395
+ value: 0.9382352941176471
396
  name: Cosine Accuracy@5
397
  - type: cosine_accuracy@10
398
+ value: 0.9588235294117647
399
  name: Cosine Accuracy@10
400
  - type: cosine_precision@1
401
+ value: 0.788235294117647
402
  name: Cosine Precision@1
403
  - type: cosine_precision@3
404
+ value: 0.3068627450980392
405
  name: Cosine Precision@3
406
  - type: cosine_precision@5
407
+ value: 0.1876470588235294
408
  name: Cosine Precision@5
409
  - type: cosine_precision@10
410
+ value: 0.09588235294117646
411
  name: Cosine Precision@10
412
  - type: cosine_recall@1
413
+ value: 0.788235294117647
414
  name: Cosine Recall@1
415
  - type: cosine_recall@3
416
+ value: 0.9205882352941176
417
  name: Cosine Recall@3
418
  - type: cosine_recall@5
419
+ value: 0.9382352941176471
420
  name: Cosine Recall@5
421
  - type: cosine_recall@10
422
+ value: 0.9588235294117647
423
  name: Cosine Recall@10
424
  - type: cosine_ndcg@10
425
+ value: 0.8823720261303867
426
  name: Cosine Ndcg@10
427
  - type: cosine_mrr@10
428
+ value: 0.8569596171802053
429
  name: Cosine Mrr@10
430
  - type: cosine_map@100
431
+ value: 0.8589677781368958
432
  name: Cosine Map@100
433
  - task:
434
  type: information-retrieval
 
438
  type: mteb/legal_summarization
439
  metrics:
440
  - type: cosine_accuracy@1
441
+ value: 0.4788732394366197
442
  name: Cosine Accuracy@1
443
  - type: cosine_accuracy@3
444
+ value: 0.6373239436619719
445
  name: Cosine Accuracy@3
446
  - type: cosine_accuracy@5
447
+ value: 0.721830985915493
448
  name: Cosine Accuracy@5
449
  - type: cosine_accuracy@10
450
+ value: 0.8204225352112676
451
  name: Cosine Accuracy@10
452
  - type: cosine_precision@1
453
+ value: 0.4788732394366197
454
  name: Cosine Precision@1
455
  - type: cosine_precision@3
456
+ value: 0.23474178403755866
457
  name: Cosine Precision@3
458
  - type: cosine_precision@5
459
+ value: 0.16830985915492958
460
  name: Cosine Precision@5
461
  - type: cosine_precision@10
462
+ value: 0.1028169014084507
463
  name: Cosine Precision@10
464
  - type: cosine_recall@1
465
+ value: 0.4233891988293397
466
  name: Cosine Recall@1
467
  - type: cosine_recall@3
468
+ value: 0.5632004146088653
469
  name: Cosine Recall@3
470
  - type: cosine_recall@5
471
+ value: 0.6415233827205657
472
  name: Cosine Recall@5
473
  - type: cosine_recall@10
474
+ value: 0.7539452624839948
475
  name: Cosine Recall@10
476
  - type: cosine_ndcg@10
477
+ value: 0.602922176130265
478
  name: Cosine Ndcg@10
479
  - type: cosine_mrr@10
480
+ value: 0.5816705790297337
481
  name: Cosine Mrr@10
482
  - type: cosine_map@100
483
+ value: 0.5513678334926079
484
  name: Cosine Map@100
485
  ---
486
 
 
591
 
592
  | Metric | mteb/AILA_casedocs | mteb/AILA_statutes | mteb/legalbench_consumer_contracts_qa | mteb/legalbench_corporate_lobbying | mteb/legal_summarization |
593
  |:--------------------|:-------------------|:-------------------|:--------------------------------------|:-----------------------------------|:-------------------------|
594
+ | cosine_accuracy@1 | 0.24 | 0.24 | 0.4874 | 0.7882 | 0.4789 |
595
+ | cosine_accuracy@3 | 0.4 | 0.52 | 0.6515 | 0.9206 | 0.6373 |
596
+ | cosine_accuracy@5 | 0.44 | 0.72 | 0.7399 | 0.9382 | 0.7218 |
597
+ | cosine_accuracy@10 | 0.5 | 0.8 | 0.8561 | 0.9588 | 0.8204 |
598
+ | cosine_precision@1 | 0.24 | 0.24 | 0.4874 | 0.7882 | 0.4789 |
599
+ | cosine_precision@3 | 0.2 | 0.2067 | 0.2172 | 0.3069 | 0.2347 |
600
+ | cosine_precision@5 | 0.144 | 0.2 | 0.148 | 0.1876 | 0.1683 |
601
+ | cosine_precision@10 | 0.096 | 0.144 | 0.0856 | 0.0959 | 0.1028 |
602
+ | cosine_recall@1 | 0.0626 | 0.068 | 0.4874 | 0.7882 | 0.4234 |
603
+ | cosine_recall@3 | 0.1774 | 0.1607 | 0.6515 | 0.9206 | 0.5632 |
604
+ | cosine_recall@5 | 0.2167 | 0.2503 | 0.7399 | 0.9382 | 0.6415 |
605
+ | cosine_recall@10 | 0.2803 | 0.351 | 0.8561 | 0.9588 | 0.7539 |
606
+ | **cosine_ndcg@10** | **0.2357** | **0.2945** | **0.6576** | **0.8824** | **0.6029** |
607
+ | cosine_mrr@10 | 0.3239 | 0.4145 | 0.5957 | 0.857 | 0.5817 |
608
+ | cosine_map@100 | 0.191 | 0.2386 | 0.6022 | 0.859 | 0.5514 |
609
 
610
  <!--
611
  ## Bias, Risks and Limitations
 
820
 
821
  - `eval_strategy`: steps
822
  - `per_device_train_batch_size`: 64
823
+ - `learning_rate`: 1e-06
824
  - `num_train_epochs`: 2
825
  - `warmup_ratio`: 0.1
826
  - `fp16`: True
 
840
  - `gradient_accumulation_steps`: 1
841
  - `eval_accumulation_steps`: None
842
  - `torch_empty_cache_steps`: None
843
+ - `learning_rate`: 1e-06
844
  - `weight_decay`: 0.0
845
  - `adam_beta1`: 0.9
846
  - `adam_beta2`: 0.999
 
949
  | Epoch | Step | Training Loss | mteb/AILA_casedocs_cosine_ndcg@10 | mteb/AILA_statutes_cosine_ndcg@10 | mteb/legalbench_consumer_contracts_qa_cosine_ndcg@10 | mteb/legalbench_corporate_lobbying_cosine_ndcg@10 | mteb/legal_summarization_cosine_ndcg@10 |
950
  |:------:|:----:|:-------------:|:---------------------------------:|:---------------------------------:|:----------------------------------------------------:|:-------------------------------------------------:|:---------------------------------------:|
951
  | 0 | 0 | - | 0.1704 | 0.2351 | 0.6781 | 0.8793 | 0.5766 |
952
+ | 0.1196 | 100 | - | 0.1709 | 0.2434 | 0.6791 | 0.8834 | 0.5820 |
953
+ | 0.2392 | 200 | - | 0.2164 | 0.2702 | 0.6808 | 0.8832 | 0.6015 |
954
+ | 0.3589 | 300 | - | 0.2221 | 0.2707 | 0.6739 | 0.8855 | 0.6089 |
955
+ | 0.4785 | 400 | - | 0.2170 | 0.2705 | 0.6681 | 0.8857 | 0.6149 |
956
+ | 0.5981 | 500 | 2.757 | 0.2138 | 0.2644 | 0.6711 | 0.8830 | 0.6116 |
957
+ | 0.7177 | 600 | - | 0.2124 | 0.2725 | 0.6671 | 0.8861 | 0.6142 |
958
+ | 0.8373 | 700 | - | 0.2235 | 0.2919 | 0.6656 | 0.8856 | 0.6112 |
959
+ | 0.9569 | 800 | - | 0.2258 | 0.2902 | 0.6632 | 0.8848 | 0.6128 |
960
+ | 1.0766 | 900 | - | 0.2220 | 0.2999 | 0.6597 | 0.8865 | 0.6120 |
961
+ | 1.1962 | 1000 | 1.6406 | 0.2264 | 0.3015 | 0.6582 | 0.8870 | 0.6106 |
962
+ | 1.3158 | 1100 | - | 0.2266 | 0.2996 | 0.6576 | 0.8859 | 0.6097 |
963
+ | 1.4354 | 1200 | - | 0.2337 | 0.2944 | 0.6581 | 0.8863 | 0.6066 |
964
+ | 1.5550 | 1300 | - | 0.2343 | 0.2928 | 0.6572 | 0.8829 | 0.6064 |
965
+ | 1.6746 | 1400 | - | 0.2342 | 0.2920 | 0.6566 | 0.8822 | 0.6041 |
966
+ | 1.7943 | 1500 | 1.6345 | 0.2358 | 0.2947 | 0.6575 | 0.8824 | 0.6026 |
967
+ | 1.9139 | 1600 | - | 0.2357 | 0.2945 | 0.6576 | 0.8824 | 0.6029 |
968
 
969
 
970
  ### Framework Versions
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b6d829c685c2a71831ec9bcba3b2fb9e107b46825a36f9eed6f4a5d0c91fb174
3
  size 437967672
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5cd1bca50fc6e4f2852fc538a94425f95c95644fa3cce35c32d2e47f18b07a58
3
  size 437967672