Update README.md
Browse files
README.md
CHANGED
|
@@ -26,6 +26,13 @@ ctx_len = 1024
|
|
| 26 |
n_layer = 24
|
| 27 |
n_embd = 2048
|
| 28 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
Preview checkpoint: RWKV-4-Pile-1B5-20220814-4526.pth : Trained on the Pile for 187B tokens.
|
| 30 |
* Pile loss 2.0635
|
| 31 |
* LAMBADA ppl 7.34, acc 55.64%
|
|
|
|
| 26 |
n_layer = 24
|
| 27 |
n_embd = 2048
|
| 28 |
|
| 29 |
+
Preview checkpoint: RWKV-4-Pile-1B5-20220822-5809.pth : Trained on the Pile for 240B tokens.
|
| 30 |
+
* Pile loss 2.0518
|
| 31 |
+
* LAMBADA ppl 7.14, acc 56.36%
|
| 32 |
+
* PIQA acc 71.71%
|
| 33 |
+
* SC2016 acc 68.15%
|
| 34 |
+
* Hellaswag acc_norm 52.04%
|
| 35 |
+
|
| 36 |
Preview checkpoint: RWKV-4-Pile-1B5-20220814-4526.pth : Trained on the Pile for 187B tokens.
|
| 37 |
* Pile loss 2.0635
|
| 38 |
* LAMBADA ppl 7.34, acc 55.64%
|