Hugging Face's logo

code1phoenix
/
NanoChat

Model card Files
xet
 Community
code1phoenix commited on
Commit
efcc0d2
·
verified ·
1 Parent(s): 03ed2a0

Upload folder using huggingface_hub

Browse files
Files changed (10) hide show
  1. meta_021400.json +50 -0
  2. model_021400.pt +3 -0
  3. optim_021400_rank0.pt +3 -0
  4. optim_021400_rank1.pt +3 -0
  5. optim_021400_rank2.pt +3 -0
  6. optim_021400_rank3.pt +3 -0
  7. optim_021400_rank4.pt +3 -0
  8. optim_021400_rank5.pt +3 -0
  9. optim_021400_rank6.pt +3 -0
  10. optim_021400_rank7.pt +3 -0
meta_021400.json ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "step": 21400,
3
+ "val_bpb": 0.811933320366963,
4
+ "model_config": {
5
+ "sequence_len": 2048,
6
+ "vocab_size": 65536,
7
+ "n_layer": 20,
8
+ "n_head": 10,
9
+ "n_kv_head": 10,
10
+ "n_embd": 1280
11
+ },
12
+ "user_config": {
13
+ "run": "speedrun_fixed",
14
+ "device_type": "",
15
+ "depth": 20,
16
+ "max_seq_len": 2048,
17
+ "num_iterations": -1,
18
+ "target_flops": -1.0,
19
+ "target_param_data_ratio": 20,
20
+ "device_batch_size": 32,
21
+ "total_batch_size": 524288,
22
+ "embedding_lr": 0.2,
23
+ "unembedding_lr": 0.004,
24
+ "weight_decay": 0.0,
25
+ "matrix_lr": 0.02,
26
+ "grad_clip": 1.0,
27
+ "warmup_ratio": 0.0,
28
+ "warmdown_ratio": 0.2,
29
+ "final_lr_frac": 0.0,
30
+ "resume_from_step": -1,
31
+ "eval_every": 250,
32
+ "eval_tokens": 10485760,
33
+ "core_metric_every": -1,
34
+ "core_metric_max_per_task": 500,
35
+ "sample_every": 2000,
36
+ "save_every": 500,
37
+ "model_tag": ""
38
+ },
39
+ "device_batch_size": 32,
40
+ "max_seq_len": 2048,
41
+ "dataloader_state_dict": {
42
+ "pq_idx": 198,
43
+ "rg_idx": 48
44
+ },
45
+ "loop_state": {
46
+ "min_val_bpb": 0.811933320366963,
47
+ "smooth_train_loss": 2.715633973883271,
48
+ "total_training_time": 10385.098405361176
49
+ }
50
+ }
model_021400.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:702f6352c011848fa3d71d6cf05e888ea10137ed87780334007907808b6b1c96
3
+ size 2076230219
optim_021400_rank0.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b38bb5ed042e987c11d16ee4a53c3d522e6916bddf0764e8a15b5bf631d877d
3
+ size 407643053
optim_021400_rank1.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b8e9d0bc89e5c201e87e5ffa94b00690e58384bf5caf1b7276c9dda273c1aac9
3
+ size 407643053
optim_021400_rank2.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e9c107e657bc2e8c173cbdd1e984ce52da24460886826d060e7dce936f9a3017
3
+ size 407643053
optim_021400_rank3.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fa6ec89738adc64fe5cebb31eba044d3db220f5fd8a5cc1325461bbcf6a72b25
3
+ size 407643053
optim_021400_rank4.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:427f33306c8aa25cb07c5c5d989649a406e4504a51fd8aaeb633cf092dfeeea6
3
+ size 355213661
optim_021400_rank5.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ec803553e4e9ed258e70aa30a179fa9b37c80643ce67cfeacce2bd3a40cafde9
3
+ size 355213661
optim_021400_rank6.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:56c58d2cae52f894b3fb077edcd6f46ddfdbf29cf6a7388fd0cd9f4fe2b53a59
3
+ size 355213661
optim_021400_rank7.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79e0ed3e2060654c2df8aacce99ad1809b285e62199d8fed09b9667367aa59c5
3
+ size 355213661