Update README.md
Browse files
README.md
CHANGED
|
@@ -27,3 +27,52 @@ DPOpenHermes was trained on a single H100 80GB hosted on RunPod for ~10h for 0.6
|
|
| 27 |
|
| 28 |
https://wandb.ai/oaaic/openhermes-dpo/reports/DPOpenHermes--Vmlldzo2MTQ3NDg2
|
| 29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
https://wandb.ai/oaaic/openhermes-dpo/reports/DPOpenHermes--Vmlldzo2MTQ3NDg2
|
| 29 |
|
| 30 |
+
# Benchmarks
|
| 31 |
+
|
| 32 |
+
## AGIEval
|
| 33 |
+
|
| 34 |
+
```
|
| 35 |
+
| Task |Version| Metric |Value | |Stderr|
|
| 36 |
+
|------------------------------|------:|--------|-----:|---|-----:|
|
| 37 |
+
|agieval_aqua_rat | 0|acc |0.2480|_ |0.0272|
|
| 38 |
+
| | |acc_norm|0.2520|_ |0.0273|
|
| 39 |
+
|agieval_logiqa_en | 0|acc |0.3810|_ |0.0190|
|
| 40 |
+
| | |acc_norm|0.3856|_ |0.0191|
|
| 41 |
+
|agieval_lsat_ar | 0|acc |0.2348|_ |0.0280|
|
| 42 |
+
| | |acc_norm|0.2304|_ |0.0278|
|
| 43 |
+
|agieval_lsat_lr | 0|acc |0.5118|_ |0.0222|
|
| 44 |
+
| | |acc_norm|0.5196|_ |0.0221|
|
| 45 |
+
|agieval_lsat_rc | 0|acc |0.5948|_ |0.0300|
|
| 46 |
+
| | |acc_norm|0.5688|_ |0.0303|
|
| 47 |
+
|agieval_sat_en | 0|acc |0.7427|_ |0.0305|
|
| 48 |
+
| | |acc_norm|0.7427|_ |0.0305|
|
| 49 |
+
|agieval_sat_en_without_passage| 0|acc |0.4563|_ |0.0348|
|
| 50 |
+
| | |acc_norm|0.4515|_ |0.0348|
|
| 51 |
+
|agieval_sat_math | 0|acc |0.3818|_ |0.0328|
|
| 52 |
+
| | |acc_norm|0.3682|_ |0.0326|
|
| 53 |
+
```
|
| 54 |
+
|
| 55 |
+
Average: 0.4399
|
| 56 |
+
|
| 57 |
+
## GPT4All
|
| 58 |
+
|
| 59 |
+
```
|
| 60 |
+
| Task |Version| Metric |Value | |Stderr|
|
| 61 |
+
|-------------|------:|--------|-----:|---|-----:|
|
| 62 |
+
|arc_challenge| 0|acc |0.5930|_ |0.0144|
|
| 63 |
+
| | |acc_norm|0.6323|_ |0.0141|
|
| 64 |
+
|arc_easy | 0|acc |0.8443|_ |0.0074|
|
| 65 |
+
| | |acc_norm|0.8295|_ |0.0077|
|
| 66 |
+
|boolq | 1|acc |0.8599|_ |0.0061|
|
| 67 |
+
|hellaswag | 0|acc |0.6548|_ |0.0047|
|
| 68 |
+
| | |acc_norm|0.8365|_ |0.0037|
|
| 69 |
+
|openbookqa | 0|acc |0.3520|_ |0.0214|
|
| 70 |
+
| | |acc_norm|0.4640|_ |0.0223|
|
| 71 |
+
|piqa | 0|acc |0.8210|_ |0.0089|
|
| 72 |
+
| | |acc_norm|0.8335|_ |0.0087|
|
| 73 |
+
|winogrande | 0|acc |0.7466|_ |0.0122|
|
| 74 |
+
```
|
| 75 |
+
|
| 76 |
+
Average: 0.7431
|
| 77 |
+
|
| 78 |
+
|