Overriding config with config/finetune_shakespeare.py:
import time

out_dir = 'out-shakespeare'
eval_interval = 5
eval_iters = 40
wandb_log = False # feel free to turn on
wandb_project = 'shakespeare'
wandb_run_name = 'ft-' + str(time.time())

dataset = 'shakespeare'
init_from = 'gpt2' # this is the largest GPT-2 model

# only save checkpoints if the validation loss improves
always_save_checkpoint = False

# the number of examples per iter:
# 1 batch_size * 32 grad_accum * 1024 tokens = 32,768 tokens/iter
# shakespeare has 301,966 tokens, so 1 epoch ~= 9.2 iters
batch_size = 1
gradient_accumulation_steps = 32
max_iters = 1000

# finetune at constant LR
learning_rate = 3e-5
decay_lr = False

Initializing from OpenAI GPT-2 weights: gpt2
loading weights from pretrained gpt: gpt2
forcing vocab_size=50257, block_size=1024, bias=True
overriding dropout rate to 0.0
number of parameters: 123.65M
using fused AdamW: True
compiling the model... (takes a ~minute)
[2023-03-20 21:31:13,957] torch._inductor.utils: [WARNING] make_fallback(aten.addmv): a decomposition exists, we should switch to it
step 0: train loss 4.1871, val loss 4.0326
iter 0: loss 4.8126, time 53610.16ms, mfu -100.00%
iter 1: loss 3.8469, time 22853.81ms, mfu -100.00%
iter 2: loss 4.1342, time 23058.41ms, mfu -100.00%
iter 3: loss 4.2060, time 23164.17ms, mfu -100.00%
iter 4: loss 4.6711, time 23070.16ms, mfu -100.00%
step 5: train loss 4.3096, val loss 3.9636
saving checkpoint to out-shakespeare
iter 5: loss 3.4577, time 30970.06ms, mfu 2.32%
iter 6: loss 2.9587, time 23298.83ms, mfu 2.40%
iter 7: loss 3.2116, time 23132.08ms, mfu 2.47%
iter 8: loss 3.4900, time 23106.50ms, mfu 2.53%
iter 9: loss 3.8003, time 23125.60ms, mfu 2.59%
step 10: train loss 3.6215, val loss 3.4816
saving checkpoint to out-shakespeare
iter 10: loss 3.6364, time 30978.89ms, mfu 2.56%
iter 11: loss 3.4725, time 23263.91ms, mfu 2.61%
iter 12: loss 3.4080, time 23053.16ms, mfu 2.67%
iter 13: loss 3.9510, time 23091.76ms, mfu 2.71%
iter 14: loss 3.6421, time 23142.46ms, mfu 2.75%
step 15: train loss 3.5292, val loss 3.2960
saving checkpoint to out-shakespeare
iter 15: loss 3.2916, time 31036.47ms, mfu 2.71%
iter 16: loss 3.8844, time 23232.40ms, mfu 2.74%
iter 17: loss 3.2954, time 23076.36ms, mfu 2.78%
iter 18: loss 2.9807, time 23073.19ms, mfu 2.81%
iter 19: loss 3.4524, time 23090.94ms, mfu 2.84%
step 20: train loss 3.4621, val loss 3.3625
iter 20: loss 3.3737, time 25115.53ms, mfu 2.85%
iter 21: loss 3.6565, time 23165.72ms, mfu 2.87%
iter 22: loss 3.3047, time 23174.77ms, mfu 2.89%
iter 23: loss 3.8091, time 23135.82ms, mfu 2.92%
iter 24: loss 3.1955, time 23097.90ms, mfu 2.94%
step 25: train loss 3.5139, val loss 3.2854
saving checkpoint to out-shakespeare
iter 25: loss 3.8481, time 30838.74ms, mfu 2.87%
iter 26: loss 3.2716, time 23304.59ms, mfu 2.90%
iter 27: loss 3.3729, time 23056.31ms, mfu 2.92%
iter 28: loss 3.3545, time 23107.46ms, mfu 2.94%
iter 29: loss 2.7101, time 23209.45ms, mfu 2.95%
step 30: train loss 3.3706, val loss 3.2958
iter 30: loss 3.0968, time 25123.31ms, mfu 2.94%
iter 31: loss 2.9495, time 23116.72ms, mfu 2.96%
iter 32: loss 3.0179, time 23101.19ms, mfu 2.97%
iter 33: loss 2.9648, time 23117.17ms, mfu 2.99%
iter 34: loss 3.6522, time 23132.76ms, mfu 3.00%
step 35: train loss 3.3923, val loss 3.2125
saving checkpoint to out-shakespeare
iter 35: loss 3.2469, time 31079.08ms, mfu 2.93%
iter 36: loss 3.1450, time 23273.02ms, mfu 2.95%
iter 37: loss 3.4624, time 23046.04ms, mfu 2.96%
iter 38: loss 3.4371, time 23102.73ms, mfu 2.98%
iter 39: loss 3.3130, time 23178.65ms, mfu 2.99%
step 40: train loss 3.3233, val loss 3.2543
iter 40: loss 3.0743, time 25069.68ms, mfu 2.98%
iter 41: loss 3.1269, time 23084.39ms, mfu 2.99%
iter 42: loss 3.6785, time 23076.30ms, mfu 3.00%
iter 43: loss 3.3787, time 23075.87ms, mfu 3.01%
iter 44: loss 3.2637, time 23098.68ms, mfu 3.02%
step 45: train loss 3.1971, val loss 3.2642
iter 45: loss 3.1861, time 25003.67ms, mfu 3.01%
iter 46: loss 3.4037, time 23106.62ms, mfu 3.02%
iter 47: loss 3.4947, time 23109.37ms, mfu 3.03%
iter 48: loss 3.3276, time 23098.50ms, mfu 3.04%
iter 49: loss 2.9062, time 23171.38ms, mfu 3.04%
step 50: train loss 3.2188, val loss 3.2460
iter 50: loss 3.5280, time 25111.46ms, mfu 3.02%
iter 51: loss 3.5470, time 23143.40ms, mfu 3.03%
iter 52: loss 3.1881, time 23109.22ms, mfu 3.04%
iter 53: loss 3.4332, time 23083.68ms, mfu 3.05%
iter 54: loss 3.1956, time 23117.10ms, mfu 3.05%
step 55: train loss 3.2902, val loss 3.1846
saving checkpoint to out-shakespeare
iter 55: loss 3.4816, time 31132.51ms, mfu 2.98%
iter 56: loss 3.2971, time 23207.94ms, mfu 2.99%
iter 57: loss 2.9543, time 23064.74ms, mfu 3.00%
iter 58: loss 2.8729, time 23093.16ms, mfu 3.01%
iter 59: loss 3.0883, time 23129.34ms, mfu 3.02%
step 60: train loss 3.1288, val loss 3.1545
saving checkpoint to out-shakespeare
iter 60: loss 3.7098, time 31022.27ms, mfu 2.95%
iter 61: loss 3.4157, time 23229.02ms, mfu 2.97%
iter 62: loss 3.0020, time 23059.02ms, mfu 2.98%
iter 63: loss 3.0751, time 23063.51ms, mfu 2.99%
iter 64: loss 2.9081, time 23134.60ms, mfu 3.01%
step 65: train loss 3.2254, val loss 3.1772
iter 65: loss 3.3802, time 25114.58ms, mfu 2.99%
iter 66: loss 3.1073, time 23118.96ms, mfu 3.00%
iter 67: loss 3.1010, time 23081.32ms, mfu 3.01%
iter 68: loss 3.2594, time 23058.54ms, mfu 3.02%
iter 69: loss 3.4402, time 23062.45ms, mfu 3.03%
step 70: train loss 3.1511, val loss 3.2315
iter 70: loss 3.4094, time 24967.39ms, mfu 3.02%
iter 71: loss 3.0997, time 23070.28ms, mfu 3.03%
iter 72: loss 2.1573, time 23072.48ms, mfu 3.04%
iter 73: loss 3.3926, time 23060.80ms, mfu 3.04%
iter 74: loss 3.2284, time 23080.48ms, mfu 3.05%
step 75: train loss 3.1102, val loss 3.1017
saving checkpoint to out-shakespeare
iter 75: loss 3.3760, time 31003.52ms, mfu 2.98%
iter 76: loss 3.3387, time 23207.33ms, mfu 2.99%
iter 77: loss 2.9299, time 23040.87ms, mfu 3.00%
iter 78: loss 2.9623, time 23069.43ms, mfu 3.01%
iter 79: loss 3.0674, time 23111.04ms, mfu 3.02%
step 80: train loss 3.0574, val loss 3.2178
iter 80: loss 2.6808, time 25072.69ms, mfu 3.01%
iter 81: loss 2.7986, time 23144.88ms, mfu 3.02%
iter 82: loss 2.9121, time 23094.25ms, mfu 3.03%
iter 83: loss 2.7153, time 23114.27ms, mfu 3.03%
iter 84: loss 2.8444, time 23089.41ms, mfu 3.04%
step 85: train loss 2.9855, val loss 3.2298
iter 85: loss 3.0517, time 25033.77ms, mfu 3.03%
iter 86: loss 2.5920, time 23088.89ms, mfu 3.03%
iter 87: loss 3.1241, time 23084.88ms, mfu 3.04%
iter 88: loss 2.5355, time 23070.40ms, mfu 3.05%
iter 89: loss 3.4543, time 23060.05ms, mfu 3.06%
step 90: train loss 3.0426, val loss 3.2664
iter 90: loss 3.3099, time 24997.54ms, mfu 3.04%
iter 91: loss 2.8099, time 23108.94ms, mfu 3.04%
iter 92: loss 3.2419, time 23103.54ms, mfu 3.05%
iter 93: loss 3.4718, time 23089.71ms, mfu 3.06%
iter 94: loss 3.0708, time 23137.11ms, mfu 3.06%
step 95: train loss 3.0225, val loss 3.2529
iter 95: loss 2.8545, time 25072.26ms, mfu 3.04%
iter 96: loss 3.3059, time 23120.57ms, mfu 3.05%
iter 97: loss 2.7528, time 23111.60ms, mfu 3.06%
iter 98: loss 3.1788, time 23106.26ms, mfu 3.06%
iter 99: loss 2.9023, time 23103.06ms, mfu 3.07%
step 100: train loss 2.9153, val loss 3.2140
iter 100: loss 3.0090, time 24968.37ms, mfu 3.05%
iter 101: loss 3.0753, time 23093.87ms, mfu 3.05%
iter 102: loss 3.1295, time 23108.81ms, mfu 3.06%
iter 103: loss 2.9033, time 23136.51ms, mfu 3.06%
iter 104: loss 3.1117, time 23127.17ms, mfu 3.07%
step 105: train loss 2.9402, val loss 3.2071
iter 105: loss 2.8862, time 25050.88ms, mfu 3.05%
iter 106: loss 2.6040, time 23141.23ms, mfu 3.05%
iter 107: loss 3.1831, time 23146.47ms, mfu 3.06%
iter 108: loss 3.1619, time 23078.47ms, mfu 3.06%
iter 109: loss 3.0995, time 23098.26ms, mfu 3.07%
step 110: train loss 2.7568, val loss 3.2857
iter 110: loss 3.0392, time 24959.72ms, mfu 3.05%
iter 111: loss 3.1982, time 23121.36ms, mfu 3.06%
iter 112: loss 3.1794, time 23124.92ms, mfu 3.06%
iter 113: loss 2.8230, time 23138.96ms, mfu 3.07%
iter 114: loss 2.2634, time 23121.12ms, mfu 3.07%
step 115: train loss 2.8576, val loss 3.2603
iter 115: loss 3.0414, time 24960.16ms, mfu 3.05%
iter 116: loss 2.2827, time 23077.89ms, mfu 3.06%
iter 117: loss 2.5435, time 23054.11ms, mfu 3.06%