| Overriding config with config/finetune_shakespeare.py: | |
| import time | |
| out_dir = 'out-shakespeare' | |
| eval_interval = 5 | |
| eval_iters = 40 | |
| wandb_log = False # feel free to turn on | |
| wandb_project = 'shakespeare' | |
| wandb_run_name = 'ft-' + str(time.time()) | |
| dataset = 'shakespeare' | |
| init_from = 'gpt2' # this is the largest GPT-2 model | |
| # only save checkpoints if the validation loss improves | |
| always_save_checkpoint = False | |
| # the number of examples per iter: | |
| # 1 batch_size * 32 grad_accum * 1024 tokens = 32,768 tokens/iter | |
| # shakespeare has 301,966 tokens, so 1 epoch ~= 9.2 iters | |
| batch_size = 1 | |
| gradient_accumulation_steps = 32 | |
| max_iters = 120 | |
| # finetune at constant LR | |
| learning_rate = 3e-5 | |
| decay_lr = False | |
| Initializing from OpenAI GPT-2 weights: gpt2 | |
| loading weights from pretrained gpt: gpt2 | |
| forcing vocab_size=50257, block_size=1024, bias=True | |
| overriding dropout rate to 0.0 | |
| number of parameters: 123.65M | |
| Downloading (…)lve/main/config.json: 100% 665/665 [00:00<00:00, 98.3kB/s] | |
| Downloading pytorch_model.bin: 100% 548M/548M [00:05<00:00, 92.8MB/s] | |
| Downloading (…)neration_config.json: 100% 124/124 [00:00<00:00, 19.2kB/s] | |
| using fused AdamW: True | |
| compiling the model... (takes a ~minute) | |
| [2023-03-21 14:22:50,795] torch._inductor.utils: [WARNING] make_fallback(aten.addmv): a decomposition exists, we should switch to it | |
| step 0: train loss 3.4423, val loss 3.0369 | |
| iter 0: loss 3.2863, time 77202.23ms, mfu -100.00% | |
| iter 1: loss 2.7469, time 22529.17ms, mfu -100.00% | |
| iter 2: loss 3.7087, time 23101.21ms, mfu -100.00% | |
| iter 3: loss 3.6040, time 23363.38ms, mfu -100.00% | |
| iter 4: loss 2.6769, time 23118.49ms, mfu -100.00% | |
| step 5: train loss 3.4339, val loss 2.9363 | |
| saving checkpoint to out-shakespeare | |
| iter 5: loss 3.1141, time 30621.41ms, mfu 2.35% | |
| iter 6: loss 3.3365, time 23426.49ms, mfu 2.42% | |
| iter 7: loss 3.8965, time 23144.13ms, mfu 2.49% | |
| iter 8: loss 3.4058, time 23061.69ms, mfu 2.55% | |
| iter 9: loss 3.2569, time 23230.68ms, mfu 2.60% | |
| step 10: train loss 3.2385, val loss 2.9982 | |
| iter 10: loss 3.1935, time 25160.57ms, mfu 2.63% | |
| iter 11: loss 3.9526, time 23125.77ms, mfu 2.68% | |
| iter 12: loss 2.4570, time 23136.22ms, mfu 2.72% | |
| iter 13: loss 3.5092, time 23120.81ms, mfu 2.76% | |
| iter 14: loss 3.4771, time 23226.29ms, mfu 2.79% | |
| step 15: train loss 2.9026, val loss 2.8705 | |
| saving checkpoint to out-shakespeare | |
| iter 15: loss 3.4825, time 30931.56ms, mfu 2.75% | |
| iter 16: loss 3.3583, time 23307.64ms, mfu 2.78% | |
| iter 17: loss 2.2991, time 23143.53ms, mfu 2.81% | |
| iter 18: loss 3.2513, time 23131.39ms, mfu 2.84% | |
| iter 19: loss 2.9859, time 23160.12ms, mfu 2.87% | |
| step 20: train loss 2.9491, val loss 2.7808 | |
| saving checkpoint to out-shakespeare | |
| iter 20: loss 3.0525, time 30909.27ms, mfu 2.81% | |
| iter 21: loss 2.9295, time 23294.73ms, mfu 2.84% | |
| iter 22: loss 2.2879, time 23094.34ms, mfu 2.87% | |
| iter 23: loss 1.8019, time 23103.56ms, mfu 2.89% | |
| iter 24: loss 3.4942, time 23172.01ms, mfu 2.91% | |
| step 25: train loss 2.8004, val loss 2.8107 | |
| iter 25: loss 2.2264, time 25127.64ms, mfu 2.91% | |
| iter 26: loss 3.4194, time 23174.40ms, mfu 2.93% | |
| iter 27: loss 2.8144, time 23152.02ms, mfu 2.94% | |
| iter 28: loss 3.0488, time 23133.18ms, mfu 2.96% | |
| iter 29: loss 3.1027, time 23085.89ms, mfu 2.98% | |
| step 30: train loss 2.6644, val loss 2.6210 | |
| saving checkpoint to out-shakespeare | |
| iter 30: loss 2.4424, time 31309.61ms, mfu 2.91% | |
| iter 31: loss 3.0193, time 23415.64ms, mfu 2.92% | |
| iter 32: loss 2.8735, time 23054.64ms, mfu 2.94% | |
| iter 33: loss 2.9842, time 23053.71ms, mfu 2.96% | |
| iter 34: loss 2.8148, time 23136.92ms, mfu 2.97% | |
| step 35: train loss 2.8676, val loss 2.5965 | |
| saving checkpoint to out-shakespeare | |
| iter 35: loss 2.8556, time 31228.61ms, mfu 2.91% | |
| iter 36: loss 2.1186, time 23332.51ms, mfu 2.92% | |
| iter 37: loss 2.4768, time 23039.16ms, mfu 2.94% | |
| iter 38: loss 2.7992, time 23035.59ms, mfu 2.96% | |
| iter 39: loss 2.7109, time 23218.08ms, mfu 2.97% | |
| step 40: train loss 2.5840, val loss 2.6467 | |
| iter 40: loss 3.0349, time 25092.98ms, mfu 2.96% | |
| iter 41: loss 2.8766, time 23084.39ms, mfu 2.98% | |
| iter 42: loss 2.5366, time 23099.15ms, mfu 2.99% | |
| iter 43: loss 2.7461, time 23183.70ms, mfu 3.00% | |
| iter 44: loss 1.4962, time 23190.74ms, mfu 3.01% | |
| step 45: train loss 2.6357, val loss 2.6529 | |
| iter 45: loss 2.1228, time 25011.92ms, mfu 3.00% | |
| iter 46: loss 1.9382, time 23127.95ms, mfu 3.01% | |
| iter 47: loss 1.7129, time 23168.21ms, mfu 3.02% | |
| iter 48: loss 2.4555, time 23162.14ms, mfu 3.03% | |
| iter 49: loss 1.3368, time 23152.22ms, mfu 3.03% | |
| step 50: train loss 2.3167, val loss 2.6496 | |
| iter 50: loss 2.3815, time 24969.84ms, mfu 3.02% | |
| iter 51: loss 1.5433, time 23013.56ms, mfu 3.03% | |
| iter 52: loss 2.5276, time 22951.87ms, mfu 3.04% | |
| iter 53: loss 2.0912, time 22989.47ms, mfu 3.05% | |
| iter 54: loss 1.6236, time 23016.77ms, mfu 3.06% | |
| step 55: train loss 2.2718, val loss 2.6701 | |
| iter 55: loss 0.9116, time 24910.16ms, mfu 3.04% |