Update README.md
Browse files
README.md
CHANGED
|
@@ -1,39 +1 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
BF16:
|
| 4 |
-
```
|
| 5 |
-
I1005 10:12:54.337877 967332 eval_ppl.py:66] wikitext2 perplexity: 6.49992036819458
|
| 6 |
-
I1005 10:17:19.718732 967332 eval_ppl.py:66] c4 perplexity: 8.022844314575195
|
| 7 |
-
```
|
| 8 |
-
3 Bit QTIP (this model):
|
| 9 |
-
```
|
| 10 |
-
I1005 10:13:30.440096 967334 eval_ppl.py:66] wikitext2 perplexity: 6.804006099700928
|
| 11 |
-
I1005 10:19:49.873933 967334 eval_ppl.py:66] c4 perplexity: 8.314777374267578
|
| 12 |
-
```
|
| 13 |
-
|
| 14 |
-
## Zeroshot Results with lm_eval 0.3.0
|
| 15 |
-
|
| 16 |
-
BF16:
|
| 17 |
-
```
|
| 18 |
-
|-------------|------:|--------|-----:|---|-----:|
|
| 19 |
-
|arc_challenge| 0|acc |0.5196|± |0.0146|
|
| 20 |
-
| | |acc_norm|0.5512|± |0.0145|
|
| 21 |
-
|arc_easy | 0|acc |0.8182|± |0.0079|
|
| 22 |
-
| | |acc_norm|0.7980|± |0.0082|
|
| 23 |
-
|boolq | 1|acc |0.8410|± |0.0064|
|
| 24 |
-
|piqa | 0|acc |0.8003|± |0.0093|
|
| 25 |
-
| | |acc_norm|0.8090|± |0.0092|
|
| 26 |
-
|winogrande | 0|acc |0.7380|± |0.0124|
|
| 27 |
-
```
|
| 28 |
-
3 Bit QTIP (this model):
|
| 29 |
-
```
|
| 30 |
-
|-------------|------:|--------|-----:|---|-----:|
|
| 31 |
-
|arc_challenge| 0|acc |0.5077|± |0.0146|
|
| 32 |
-
| | |acc_norm|0.5469|± |0.0145|
|
| 33 |
-
|arc_easy | 0|acc |0.8093|± |0.0081|
|
| 34 |
-
| | |acc_norm|0.7917|± |0.0083|
|
| 35 |
-
|boolq | 1|acc |0.8419|± |0.0064|
|
| 36 |
-
|piqa | 0|acc |0.7922|± |0.0095|
|
| 37 |
-
| | |acc_norm|0.8025|± |0.0093|
|
| 38 |
-
|winogrande | 0|acc |0.7198|± |0.0126|
|
| 39 |
-
```
|
|
|
|
| 1 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|