Hugging Face's logo

ThomasTheMaker
/
pico-decoder-tiny-experiments

Safetensors
Model card Files
xet
Community
ThomasTheMaker commited on
Commit
f2524ca
·
verified ·
1 Parent(s): 00f2284

Upload folder using huggingface_hub

Browse files
Files changed (9) hide show
  1. pico-decoder-tiny-dolma29k/checkpoints/step_1000/fabric_state/checkpoint.pt +1 -1
  2. pico-decoder-tiny-dolma29k/checkpoints/step_1000/learning_dynamics/train_activations.pt +3 -0
  3. pico-decoder-tiny-dolma29k/checkpoints/step_1000/learning_dynamics/train_data/data-00000-of-00001.arrow +3 -0
  4. pico-decoder-tiny-dolma29k/checkpoints/step_1000/learning_dynamics/train_data/dataset_info.json +19 -0
  5. pico-decoder-tiny-dolma29k/checkpoints/step_1000/learning_dynamics/train_data/state.json +13 -0
  6. pico-decoder-tiny-dolma29k/checkpoints/step_1000/learning_dynamics/train_gradients.pt +3 -0
  7. pico-decoder-tiny-dolma29k/checkpoints/step_1000/learning_dynamics/train_weights.pt +3 -0
  8. pico-decoder-tiny-dolma29k/eval_results/step_1000.json +1 -0
  9. pico-decoder-tiny-dolma29k/logs/log_20250828_225300.log +128 -0
pico-decoder-tiny-dolma29k/checkpoints/step_1000/fabric_state/checkpoint.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5439f07f8fadc43c74e673d43dbc67f5e8fe885ca452e0d09676da3aa54610d1
3
  size 135543171
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:17ff5a63c62f790e672d56533953307ebf889aaf949cb56ac6808c3c7ad3b766
3
  size 135543171
pico-decoder-tiny-dolma29k/checkpoints/step_1000/learning_dynamics/train_activations.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:86f6894546ad09ade1abd8844ed94f3c6d1eb34b6f3278911777c42c50ddd512
3
+ size 33819
pico-decoder-tiny-dolma29k/checkpoints/step_1000/learning_dynamics/train_data/data-00000-of-00001.arrow ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:908a522ebc31350e7397330dd99c9ad2253e2efdc1bee1ff76f9babef6078d28
3
+ size 66408
pico-decoder-tiny-dolma29k/checkpoints/step_1000/learning_dynamics/train_data/dataset_info.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "citation": "",
3
+ "description": "",
4
+ "features": {
5
+ "input_ids": {
6
+ "feature": {
7
+ "dtype": "int32",
8
+ "_type": "Value"
9
+ },
10
+ "_type": "Sequence"
11
+ },
12
+ "text": {
13
+ "dtype": "string",
14
+ "_type": "Value"
15
+ }
16
+ },
17
+ "homepage": "",
18
+ "license": ""
19
+ }
pico-decoder-tiny-dolma29k/checkpoints/step_1000/learning_dynamics/train_data/state.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_data_files": [
3
+ {
4
+ "filename": "data-00000-of-00001.arrow"
5
+ }
6
+ ],
7
+ "_fingerprint": "86e249409514e027",
8
+ "_format_columns": null,
9
+ "_format_kwargs": {},
10
+ "_format_type": null,
11
+ "_output_all_columns": false,
12
+ "_split": null
13
+ }
pico-decoder-tiny-dolma29k/checkpoints/step_1000/learning_dynamics/train_gradients.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:49877366118749ef2726caa90137d614bcca1690852e7dcf1d992f0fe93a9189
3
+ size 2371527
pico-decoder-tiny-dolma29k/checkpoints/step_1000/learning_dynamics/train_weights.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ed0f3b897d6c11ea78de275cfe4e3f7e56b3bc07ed14d9b64ced24f1bce4fda5
3
+ size 2371443
pico-decoder-tiny-dolma29k/eval_results/step_1000.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"paloma": 2.5468931158531133e+19}
pico-decoder-tiny-dolma29k/logs/log_20250828_225300.log ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2025-08-28 22:55:45 - pico-train - INFO - Step 1000 -- 📊 Evaluation Results
2
+ 2025-08-28 22:55:45 - pico-train - INFO - └── paloma: 2.5468931158531133e+19
3
+ 2025-08-28 22:55:47 - pico-train - INFO - ==================================================
4
+ 2025-08-28 22:55:47 - pico-train - INFO - ✨ Training Configuration
5
+ 2025-08-28 22:55:47 - pico-train - INFO - ==================================================
6
+ 2025-08-28 22:55:47 - pico-train - INFO - ╭─────────────────────────────────────────────────────╮
7
+ 2025-08-28 22:55:47 - pico-train - INFO - │ checkpointing: │
8
+ 2025-08-28 22:55:47 - pico-train - INFO - │ checkpoints_dir: checkpoints │
9
+ 2025-08-28 22:55:47 - pico-train - INFO - │ evaluation: │
10
+ 2025-08-28 22:55:47 - pico-train - INFO - │ eval_results_dir: eval_results │
11
+ 2025-08-28 22:55:47 - pico-train - INFO - │ fabric_checkpoint_dir: fabric_state │
12
+ 2025-08-28 22:55:47 - pico-train - INFO - │ fabric_checkpoint_filename: checkpoint.pt │
13
+ 2025-08-28 22:55:47 - pico-train - INFO - │ hf_checkpoint: │
14
+ 2025-08-28 22:55:47 - pico-train - INFO - │ collection_slug: null │
15
+ 2025-08-28 22:55:47 - pico-train - INFO - │ repo_id: ThomasTheMaker/pico-decoder-tiny │
16
+ 2025-08-28 22:55:47 - pico-train - INFO - │ learning_dynamics: │
17
+ 2025-08-28 22:55:47 - pico-train - INFO - │ batch_size: 1 │
18
+ 2025-08-28 22:55:47 - pico-train - INFO - │ eval_data: null │
19
+ 2025-08-28 22:55:47 - pico-train - INFO - │ layer_suffixes: │
20
+ 2025-08-28 22:55:47 - pico-train - INFO - │ - attention.v_proj │
21
+ 2025-08-28 22:55:47 - pico-train - INFO - │ - attention.o_proj │
22
+ 2025-08-28 22:55:47 - pico-train - INFO - │ - swiglu.w_2 │
23
+ 2025-08-28 22:55:47 - pico-train - INFO - │ sequence_idx: -1 │
24
+ 2025-08-28 22:55:47 - pico-train - INFO - │ learning_dynamics_dir: learning_dynamics │
25
+ 2025-08-28 22:55:47 - pico-train - INFO - │ logs_dir: logs │
26
+ 2025-08-28 22:55:47 - pico-train - INFO - │ run_name: pico-decoder-tiny-dolma29k │
27
+ 2025-08-28 22:55:47 - pico-train - INFO - │ runs_dir: runs │
28
+ 2025-08-28 22:55:47 - pico-train - INFO - │ save_every_n_steps: 1000 │
29
+ 2025-08-28 22:55:47 - pico-train - INFO - │ save_to_hf: true │
30
+ 2025-08-28 22:55:47 - pico-train - INFO - │ training: │
31
+ 2025-08-28 22:55:47 - pico-train - INFO - │ auto_resume: true │
32
+ 2025-08-28 22:55:47 - pico-train - INFO - │ data: │
33
+ 2025-08-28 22:55:47 - pico-train - INFO - │ dataloader: │
34
+ 2025-08-28 22:55:47 - pico-train - INFO - │ batch_size: 4 │
35
+ 2025-08-28 22:55:47 - pico-train - INFO - │ dataset: │
36
+ 2025-08-28 22:55:47 - pico-train - INFO - │ name: pico-lm/pretokenized-dolma │
37
+ 2025-08-28 22:55:47 - pico-train - INFO - │ tokenizer: │
38
+ 2025-08-28 22:55:47 - pico-train - INFO - │ name: allenai/OLMo-7B-0724-hf │
39
+ 2025-08-28 22:55:47 - pico-train - INFO - │ vocab_size: 50304 │
40
+ 2025-08-28 22:55:47 - pico-train - INFO - │ evaluation: │
41
+ 2025-08-28 22:55:47 - pico-train - INFO - │ metrics: │
42
+ 2025-08-28 22:55:47 - pico-train - INFO - │ - paloma │
43
+ 2025-08-28 22:55:47 - pico-train - INFO - │ paloma: │
44
+ 2025-08-28 22:55:47 - pico-train - INFO - │ batch_size: 1 │
45
+ 2025-08-28 22:55:47 - pico-train - INFO - │ dataset_name: pico-lm/pretokenized-paloma-tinsy │
46
+ 2025-08-28 22:55:47 - pico-train - INFO - │ dataset_split: val │
47
+ 2025-08-28 22:55:47 - pico-train - INFO - │ max_length: 2048 │
48
+ 2025-08-28 22:55:47 - pico-train - INFO - │ model: │
49
+ 2025-08-28 22:55:47 - pico-train - INFO - │ activation_hidden_dim: 384 │
50
+ 2025-08-28 22:55:47 - pico-train - INFO - │ attention_n_heads: 12 │
51
+ 2025-08-28 22:55:47 - pico-train - INFO - │ attention_n_kv_heads: 4 │
52
+ 2025-08-28 22:55:47 - pico-train - INFO - │ batch_size: 1024 │
53
+ 2025-08-28 22:55:47 - pico-train - INFO - │ d_model: 96 │
54
+ 2025-08-28 22:55:47 - pico-train - INFO - │ max_seq_len: 2048 │
55
+ 2025-08-28 22:55:47 - pico-train - INFO - │ model_type: pico_decoder │
56
+ 2025-08-28 22:55:47 - pico-train - INFO - │ n_layers: 12 │
57
+ 2025-08-28 22:55:47 - pico-train - INFO - │ norm_eps: 1.0e-06 │
58
+ 2025-08-28 22:55:47 - pico-train - INFO - │ position_emb_theta: 10000.0 │
59
+ 2025-08-28 22:55:47 - pico-train - INFO - │ vocab_size: 50304 │
60
+ 2025-08-28 22:55:47 - pico-train - INFO - │ monitoring: │
61
+ 2025-08-28 22:55:47 - pico-train - INFO - │ logging: │
62
+ 2025-08-28 22:55:47 - pico-train - INFO - │ log_every_n_steps: 100 │
63
+ 2025-08-28 22:55:47 - pico-train - INFO - │ log_level: INFO │
64
+ 2025-08-28 22:55:47 - pico-train - INFO - │ save_to_wandb: false │
65
+ 2025-08-28 22:55:47 - pico-train - INFO - │ wandb: │
66
+ 2025-08-28 22:55:47 - pico-train - INFO - │ entity: boymyc │
67
+ 2025-08-28 22:55:47 - pico-train - INFO - │ project: pico-decoder-tiny │
68
+ 2025-08-28 22:55:47 - pico-train - INFO - │ training: │
69
+ 2025-08-28 22:55:47 - pico-train - INFO - │ fabric: │
70
+ 2025-08-28 22:55:47 - pico-train - INFO - │ accelerator: cuda │
71
+ 2025-08-28 22:55:47 - pico-train - INFO - │ num_devices: 1 │
72
+ 2025-08-28 22:55:47 - pico-train - INFO - │ num_nodes: 1 │
73
+ 2025-08-28 22:55:47 - pico-train - INFO - │ precision: bf16-mixed │
74
+ 2025-08-28 22:55:47 - pico-train - INFO - │ max_steps: 200000 │
75
+ 2025-08-28 22:55:47 - pico-train - INFO - │ optimization: │
76
+ 2025-08-28 22:55:47 - pico-train - INFO - │ gradient_accumulation_steps: 4 │
77
+ 2025-08-28 22:55:47 - pico-train - INFO - │ lr: 0.0003 │
78
+ 2025-08-28 22:55:47 - pico-train - INFO - │ lr_scheduler: linear_with_warmup │
79
+ 2025-08-28 22:55:47 - pico-train - INFO - │ lr_warmup_steps: 2500 │
80
+ 2025-08-28 22:55:47 - pico-train - INFO - │ optimizer: adamw │
81
+ 2025-08-28 22:55:47 - pico-train - INFO - │ │
82
+ 2025-08-28 22:55:47 - pico-train - INFO - ╰─────────────────────────────────────────────────────╯
83
+ 2025-08-28 22:55:47 - pico-train - INFO - ==================================================
84
+ 2025-08-28 22:55:47 - pico-train - INFO - ⛭ Runtime Summary:
85
+ 2025-08-28 22:55:47 - pico-train - INFO - ==================================================
86
+ 2025-08-28 22:55:47 - pico-train - INFO - Starting from step: 1000
87
+ 2025-08-28 22:55:47 - pico-train - INFO - Model Setup:
88
+ 2025-08-28 22:55:47 - pico-train - INFO - └─ Total Parameters: 11,282,784
89
+ 2025-08-28 22:55:47 - pico-train - INFO - └─ Trainable Parameters: 11,282,784
90
+ 2025-08-28 22:55:47 - pico-train - INFO - Distributed Setup:
91
+ 2025-08-28 22:55:47 - pico-train - INFO - └─ Number of Devices: 1
92
+ 2025-08-28 22:55:47 - pico-train - INFO - └─ Device Type: NVIDIA GeForce RTX 5090
93
+ 2025-08-28 22:55:47 - pico-train - INFO - └─ Available Memory: 33.68 GB
94
+ 2025-08-28 22:55:47 - pico-train - INFO - Software Setup:
95
+ 2025-08-28 22:55:47 - pico-train - INFO - └─ Python Version: 3.10.12
96
+ 2025-08-28 22:55:47 - pico-train - INFO - └─ PyTorch Version: 2.8.0+cu128
97
+ 2025-08-28 22:55:47 - pico-train - INFO - └─ CUDA Version: 12.8
98
+ 2025-08-28 22:55:47 - pico-train - INFO - └─ Operating System: Linux 6.8.0-63-generic
99
+ 2025-08-28 22:55:47 - pico-train - INFO - Batch Size Configuration:
100
+ 2025-08-28 22:55:47 - pico-train - INFO - └─ Global Batch Size: 4
101
+ 2025-08-28 22:55:47 - pico-train - INFO - └─ Per Device Batch Size: 1
102
+ 2025-08-28 22:55:47 - pico-train - INFO - └─ Gradient Accumulation Steps: 4
103
+ 2025-08-28 22:55:47 - pico-train - INFO - ==================================================
104
+ 2025-08-28 22:55:49 - pico-train - INFO - Step 1000 -- 🔄 Training Metrics
105
+ 2025-08-28 22:55:49 - pico-train - INFO - ├── Loss: 7.7657
106
+ 2025-08-28 22:55:49 - pico-train - INFO - ├── Learning Rate: 1.20e-04
107
+ 2025-08-28 22:55:49 - pico-train - INFO - └── Inf/NaN count: 0
108
+ 2025-08-28 22:55:49 - pico-train - INFO - Step 1000 -- 📈 Saving Learning Dynamics
109
+ 2025-08-28 22:56:43 - pico-train - INFO - Step 1100 -- 🔄 Training Metrics
110
+ 2025-08-28 22:56:43 - pico-train - INFO - ├── Loss: 7.6733
111
+ 2025-08-28 22:56:43 - pico-train - INFO - ├── Learning Rate: 1.32e-04
112
+ 2025-08-28 22:56:43 - pico-train - INFO - └── Inf/NaN count: 0
113
+ 2025-08-28 22:57:34 - pico-train - INFO - Step 1200 -- 🔄 Training Metrics
114
+ 2025-08-28 22:57:34 - pico-train - INFO - ├── Loss: 7.5969
115
+ 2025-08-28 22:57:34 - pico-train - INFO - ├── Learning Rate: 1.44e-04
116
+ 2025-08-28 22:57:34 - pico-train - INFO - └── Inf/NaN count: 0
117
+ 2025-08-28 22:58:25 - pico-train - INFO - Step 1300 -- 🔄 Training Metrics
118
+ 2025-08-28 22:58:25 - pico-train - INFO - ├── Loss: 7.4765
119
+ 2025-08-28 22:58:25 - pico-train - INFO - ├── Learning Rate: 1.56e-04
120
+ 2025-08-28 22:58:25 - pico-train - INFO - └── Inf/NaN count: 0
121
+ 2025-08-28 22:59:16 - pico-train - INFO - Step 1400 -- 🔄 Training Metrics
122
+ 2025-08-28 22:59:16 - pico-train - INFO - ├── Loss: 7.3686
123
+ 2025-08-28 22:59:16 - pico-train - INFO - ├── Learning Rate: 1.68e-04
124
+ 2025-08-28 22:59:16 - pico-train - INFO - └── Inf/NaN count: 0
125
+ 2025-08-28 23:00:07 - pico-train - INFO - Step 1500 -- 🔄 Training Metrics
126
+ 2025-08-28 23:00:07 - pico-train - INFO - ├── Loss: 7.3251
127
+ 2025-08-28 23:00:07 - pico-train - INFO - ├── Learning Rate: 1.80e-04
128
+ 2025-08-28 23:00:07 - pico-train - INFO - └── Inf/NaN count: 0