Update README.md
Browse files
README.md
CHANGED
@@ -25,8 +25,12 @@ n_layer = 12
|
|
25 |
n_embd = 768
|
26 |
|
27 |
Final checkpoint:
|
28 |
-
RWKV-3-Pile-20220720-10704.pth : Trained on the Pile for 328B tokens.
|
29 |
-
|
|
|
|
|
|
|
|
|
30 |
|
31 |
Preview checkpoint:
|
32 |
20220703-1652.pth : Trained on the Pile for 50B tokens. Pile loss 2.6375, LAMBADA ppl 33.30, acc 31.24%.
|
|
|
25 |
n_embd = 768
|
26 |
|
27 |
Final checkpoint:
|
28 |
+
RWKV-3-Pile-20220720-10704.pth : Trained on the Pile for 328B tokens.
|
29 |
+
* Pile loss 2.5596
|
30 |
+
* LAMBADA ppl 28.82, acc 32.33%
|
31 |
+
* PIQA acc 64.15%
|
32 |
+
* SC2016 acc 57.88%
|
33 |
+
* Hellaswag acc_norm 32.45%
|
34 |
|
35 |
Preview checkpoint:
|
36 |
20220703-1652.pth : Trained on the Pile for 50B tokens. Pile loss 2.6375, LAMBADA ppl 33.30, acc 31.24%.
|