totally-not-an-llm's picture
Upload folder using huggingface_hub
ebe1b41 verified
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 4.0,
"eval_steps": 8,
"global_step": 128,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.03,
"grad_norm": 0.5859375,
"learning_rate": 2e-05,
"loss": 3.9295,
"step": 1
},
{
"epoch": 0.03,
"eval_loss": 3.907318115234375,
"eval_runtime": 1.2713,
"eval_samples_per_second": 11.012,
"eval_steps_per_second": 5.506,
"step": 1
},
{
"epoch": 0.06,
"grad_norm": 0.5078125,
"learning_rate": 4e-05,
"loss": 3.805,
"step": 2
},
{
"epoch": 0.09,
"grad_norm": 0.63671875,
"learning_rate": 6e-05,
"loss": 3.8521,
"step": 3
},
{
"epoch": 0.12,
"grad_norm": 0.609375,
"learning_rate": 8e-05,
"loss": 3.8947,
"step": 4
},
{
"epoch": 0.16,
"grad_norm": 0.546875,
"learning_rate": 0.0001,
"loss": 3.6494,
"step": 5
},
{
"epoch": 0.19,
"grad_norm": 0.50390625,
"learning_rate": 0.00012,
"loss": 3.6457,
"step": 6
},
{
"epoch": 0.22,
"grad_norm": 0.640625,
"learning_rate": 0.00014,
"loss": 3.967,
"step": 7
},
{
"epoch": 0.25,
"grad_norm": 0.494140625,
"learning_rate": 0.00016,
"loss": 3.5364,
"step": 8
},
{
"epoch": 0.25,
"eval_loss": 3.6198840141296387,
"eval_runtime": 1.2681,
"eval_samples_per_second": 11.04,
"eval_steps_per_second": 5.52,
"step": 8
},
{
"epoch": 0.28,
"grad_norm": 0.51171875,
"learning_rate": 0.00018,
"loss": 3.5216,
"step": 9
},
{
"epoch": 0.31,
"grad_norm": 0.6171875,
"learning_rate": 0.0002,
"loss": 3.677,
"step": 10
},
{
"epoch": 0.34,
"grad_norm": 0.734375,
"learning_rate": 0.00019996456111234527,
"loss": 3.058,
"step": 11
},
{
"epoch": 0.38,
"grad_norm": 0.66796875,
"learning_rate": 0.0001998582695676762,
"loss": 3.1333,
"step": 12
},
{
"epoch": 0.41,
"grad_norm": 0.8359375,
"learning_rate": 0.000199681200703075,
"loss": 3.38,
"step": 13
},
{
"epoch": 0.44,
"grad_norm": 0.62109375,
"learning_rate": 0.00019943348002101371,
"loss": 3.1371,
"step": 14
},
{
"epoch": 0.47,
"grad_norm": 0.65625,
"learning_rate": 0.00019911528310040074,
"loss": 3.1479,
"step": 15
},
{
"epoch": 0.5,
"grad_norm": 0.6484375,
"learning_rate": 0.00019872683547213446,
"loss": 3.263,
"step": 16
},
{
"epoch": 0.5,
"eval_loss": 3.1820719242095947,
"eval_runtime": 1.2692,
"eval_samples_per_second": 11.031,
"eval_steps_per_second": 5.515,
"step": 16
},
{
"epoch": 0.53,
"grad_norm": 0.53125,
"learning_rate": 0.00019826841245925212,
"loss": 2.9833,
"step": 17
},
{
"epoch": 0.56,
"grad_norm": 0.5703125,
"learning_rate": 0.00019774033898178667,
"loss": 3.0787,
"step": 18
},
{
"epoch": 0.59,
"grad_norm": 0.71484375,
"learning_rate": 0.00019714298932647098,
"loss": 3.4132,
"step": 19
},
{
"epoch": 0.62,
"grad_norm": 0.73046875,
"learning_rate": 0.0001964767868814516,
"loss": 2.7304,
"step": 20
},
{
"epoch": 0.66,
"grad_norm": 0.55078125,
"learning_rate": 0.00019574220383620055,
"loss": 3.0116,
"step": 21
},
{
"epoch": 0.69,
"grad_norm": 0.6171875,
"learning_rate": 0.00019493976084683813,
"loss": 2.9474,
"step": 22
},
{
"epoch": 0.72,
"grad_norm": 0.61328125,
"learning_rate": 0.00019407002666710336,
"loss": 2.9415,
"step": 23
},
{
"epoch": 0.75,
"grad_norm": 0.6328125,
"learning_rate": 0.00019313361774523385,
"loss": 2.798,
"step": 24
},
{
"epoch": 0.75,
"eval_loss": 2.896176815032959,
"eval_runtime": 1.2765,
"eval_samples_per_second": 10.967,
"eval_steps_per_second": 5.484,
"step": 24
},
{
"epoch": 0.78,
"grad_norm": 0.8046875,
"learning_rate": 0.00019213119778704128,
"loss": 3.2157,
"step": 25
},
{
"epoch": 0.81,
"grad_norm": 0.7109375,
"learning_rate": 0.00019106347728549135,
"loss": 3.0666,
"step": 26
},
{
"epoch": 0.84,
"grad_norm": 0.59765625,
"learning_rate": 0.00018993121301712193,
"loss": 2.8219,
"step": 27
},
{
"epoch": 0.88,
"grad_norm": 0.75390625,
"learning_rate": 0.00018873520750565718,
"loss": 3.1164,
"step": 28
},
{
"epoch": 0.91,
"grad_norm": 0.67578125,
"learning_rate": 0.00018747630845319612,
"loss": 2.7154,
"step": 29
},
{
"epoch": 0.94,
"grad_norm": 0.625,
"learning_rate": 0.0001861554081393806,
"loss": 2.7395,
"step": 30
},
{
"epoch": 0.97,
"grad_norm": 0.8125,
"learning_rate": 0.0001847734427889671,
"loss": 2.8282,
"step": 31
},
{
"epoch": 1.0,
"grad_norm": 0.86328125,
"learning_rate": 0.0001833313919082515,
"loss": 2.7787,
"step": 32
},
{
"epoch": 1.0,
"eval_loss": 2.67726731300354,
"eval_runtime": 1.2769,
"eval_samples_per_second": 10.964,
"eval_steps_per_second": 5.482,
"step": 32
},
{
"epoch": 1.03,
"grad_norm": 0.703125,
"learning_rate": 0.0001818302775908169,
"loss": 2.5957,
"step": 33
},
{
"epoch": 1.06,
"grad_norm": 0.76953125,
"learning_rate": 0.00018027116379309638,
"loss": 2.7011,
"step": 34
},
{
"epoch": 1.09,
"grad_norm": 0.84765625,
"learning_rate": 0.00017865515558026428,
"loss": 2.6043,
"step": 35
},
{
"epoch": 1.12,
"grad_norm": 0.984375,
"learning_rate": 0.00017698339834299061,
"loss": 2.8607,
"step": 36
},
{
"epoch": 1.16,
"grad_norm": 0.7578125,
"learning_rate": 0.00017525707698561385,
"loss": 2.5949,
"step": 37
},
{
"epoch": 1.19,
"grad_norm": 0.94921875,
"learning_rate": 0.00017347741508630672,
"loss": 2.7476,
"step": 38
},
{
"epoch": 1.22,
"grad_norm": 0.88671875,
"learning_rate": 0.00017164567402983152,
"loss": 2.7991,
"step": 39
},
{
"epoch": 1.25,
"grad_norm": 0.8671875,
"learning_rate": 0.0001697631521134985,
"loss": 2.5959,
"step": 40
},
{
"epoch": 1.25,
"eval_loss": 2.5505764484405518,
"eval_runtime": 1.2761,
"eval_samples_per_second": 10.971,
"eval_steps_per_second": 5.486,
"step": 40
},
{
"epoch": 1.28,
"grad_norm": 0.984375,
"learning_rate": 0.00016783118362696163,
"loss": 2.5342,
"step": 41
},
{
"epoch": 1.31,
"grad_norm": 0.921875,
"learning_rate": 0.00016585113790650388,
"loss": 2.4969,
"step": 42
},
{
"epoch": 1.34,
"grad_norm": 1.125,
"learning_rate": 0.00016382441836448202,
"loss": 2.5723,
"step": 43
},
{
"epoch": 1.38,
"grad_norm": 1.015625,
"learning_rate": 0.0001617524614946192,
"loss": 2.6166,
"step": 44
},
{
"epoch": 1.41,
"grad_norm": 0.98828125,
"learning_rate": 0.00015963673585385016,
"loss": 2.4574,
"step": 45
},
{
"epoch": 1.44,
"grad_norm": 0.94921875,
"learning_rate": 0.0001574787410214407,
"loss": 2.5074,
"step": 46
},
{
"epoch": 1.47,
"grad_norm": 0.953125,
"learning_rate": 0.00015528000653611935,
"loss": 2.5394,
"step": 47
},
{
"epoch": 1.5,
"grad_norm": 0.828125,
"learning_rate": 0.00015304209081197425,
"loss": 2.4793,
"step": 48
},
{
"epoch": 1.5,
"eval_loss": 2.495466470718384,
"eval_runtime": 1.2814,
"eval_samples_per_second": 10.926,
"eval_steps_per_second": 5.463,
"step": 48
},
{
"epoch": 1.53,
"grad_norm": 0.921875,
"learning_rate": 0.000150766580033884,
"loss": 2.2909,
"step": 49
},
{
"epoch": 1.56,
"grad_norm": 0.74609375,
"learning_rate": 0.00014845508703326504,
"loss": 2.3424,
"step": 50
},
{
"epoch": 1.59,
"grad_norm": 0.87890625,
"learning_rate": 0.0001461092501449326,
"loss": 2.506,
"step": 51
},
{
"epoch": 1.62,
"grad_norm": 0.7890625,
"learning_rate": 0.00014373073204588556,
"loss": 2.4829,
"step": 52
},
{
"epoch": 1.66,
"grad_norm": 0.984375,
"learning_rate": 0.00014132121857683783,
"loss": 2.481,
"step": 53
},
{
"epoch": 1.69,
"grad_norm": 0.9296875,
"learning_rate": 0.00013888241754733208,
"loss": 2.5512,
"step": 54
},
{
"epoch": 1.72,
"grad_norm": 0.7890625,
"learning_rate": 0.00013641605752528224,
"loss": 2.5405,
"step": 55
},
{
"epoch": 1.75,
"grad_norm": 0.95703125,
"learning_rate": 0.00013392388661180303,
"loss": 2.5221,
"step": 56
},
{
"epoch": 1.75,
"eval_loss": 2.461298704147339,
"eval_runtime": 1.28,
"eval_samples_per_second": 10.937,
"eval_steps_per_second": 5.469,
"step": 56
},
{
"epoch": 1.78,
"grad_norm": 0.98046875,
"learning_rate": 0.0001314076712021949,
"loss": 2.5646,
"step": 57
},
{
"epoch": 1.81,
"grad_norm": 0.8125,
"learning_rate": 0.0001288691947339621,
"loss": 2.5079,
"step": 58
},
{
"epoch": 1.84,
"grad_norm": 0.8125,
"learning_rate": 0.00012631025642275212,
"loss": 2.4743,
"step": 59
},
{
"epoch": 1.88,
"grad_norm": 0.60546875,
"learning_rate": 0.0001237326699871115,
"loss": 2.3103,
"step": 60
},
{
"epoch": 1.91,
"grad_norm": 0.80859375,
"learning_rate": 0.00012113826236296244,
"loss": 2.4229,
"step": 61
},
{
"epoch": 1.94,
"grad_norm": 0.671875,
"learning_rate": 0.00011852887240871145,
"loss": 2.2709,
"step": 62
},
{
"epoch": 1.97,
"grad_norm": 0.8203125,
"learning_rate": 0.00011590634960190721,
"loss": 2.4868,
"step": 63
},
{
"epoch": 2.0,
"grad_norm": 0.62109375,
"learning_rate": 0.00011327255272837221,
"loss": 2.4384,
"step": 64
},
{
"epoch": 2.0,
"eval_loss": 2.4055111408233643,
"eval_runtime": 1.2798,
"eval_samples_per_second": 10.94,
"eval_steps_per_second": 5.47,
"step": 64
},
{
"epoch": 2.03,
"grad_norm": 0.6328125,
"learning_rate": 0.00011062934856473655,
"loss": 2.3671,
"step": 65
},
{
"epoch": 2.06,
"grad_norm": 0.69140625,
"learning_rate": 0.00010797861055530831,
"loss": 2.3381,
"step": 66
},
{
"epoch": 2.09,
"grad_norm": 0.671875,
"learning_rate": 0.00010532221748421787,
"loss": 2.2134,
"step": 67
},
{
"epoch": 2.12,
"grad_norm": 0.7890625,
"learning_rate": 0.00010266205214377748,
"loss": 2.2687,
"step": 68
},
{
"epoch": 2.16,
"grad_norm": 0.75,
"learning_rate": 0.0001,
"loss": 2.4273,
"step": 69
},
{
"epoch": 2.19,
"grad_norm": 0.8671875,
"learning_rate": 9.733794785622253e-05,
"loss": 2.4439,
"step": 70
},
{
"epoch": 2.22,
"grad_norm": 0.953125,
"learning_rate": 9.467778251578217e-05,
"loss": 2.6631,
"step": 71
},
{
"epoch": 2.25,
"grad_norm": 0.78125,
"learning_rate": 9.202138944469168e-05,
"loss": 2.295,
"step": 72
},
{
"epoch": 2.25,
"eval_loss": 2.3922784328460693,
"eval_runtime": 1.282,
"eval_samples_per_second": 10.921,
"eval_steps_per_second": 5.46,
"step": 72
},
{
"epoch": 2.28,
"grad_norm": 0.734375,
"learning_rate": 8.937065143526347e-05,
"loss": 2.4963,
"step": 73
},
{
"epoch": 2.31,
"grad_norm": 0.68359375,
"learning_rate": 8.672744727162781e-05,
"loss": 2.4274,
"step": 74
},
{
"epoch": 2.34,
"grad_norm": 0.9765625,
"learning_rate": 8.409365039809281e-05,
"loss": 2.4988,
"step": 75
},
{
"epoch": 2.38,
"grad_norm": 0.75390625,
"learning_rate": 8.147112759128859e-05,
"loss": 2.2886,
"step": 76
},
{
"epoch": 2.41,
"grad_norm": 0.80078125,
"learning_rate": 7.886173763703757e-05,
"loss": 2.1944,
"step": 77
},
{
"epoch": 2.44,
"grad_norm": 0.8828125,
"learning_rate": 7.626733001288851e-05,
"loss": 2.3283,
"step": 78
},
{
"epoch": 2.47,
"grad_norm": 0.82421875,
"learning_rate": 7.368974357724789e-05,
"loss": 2.3855,
"step": 79
},
{
"epoch": 2.5,
"grad_norm": 0.7890625,
"learning_rate": 7.113080526603792e-05,
"loss": 2.3943,
"step": 80
},
{
"epoch": 2.5,
"eval_loss": 2.386228084564209,
"eval_runtime": 1.2835,
"eval_samples_per_second": 10.908,
"eval_steps_per_second": 5.454,
"step": 80
},
{
"epoch": 2.53,
"grad_norm": 0.94921875,
"learning_rate": 6.859232879780515e-05,
"loss": 2.2725,
"step": 81
},
{
"epoch": 2.56,
"grad_norm": 0.75,
"learning_rate": 6.607611338819697e-05,
"loss": 2.1989,
"step": 82
},
{
"epoch": 2.59,
"grad_norm": 0.703125,
"learning_rate": 6.358394247471778e-05,
"loss": 2.1219,
"step": 83
},
{
"epoch": 2.62,
"grad_norm": 1.0390625,
"learning_rate": 6.111758245266794e-05,
"loss": 2.3478,
"step": 84
},
{
"epoch": 2.66,
"grad_norm": 0.78125,
"learning_rate": 5.867878142316221e-05,
"loss": 2.4196,
"step": 85
},
{
"epoch": 2.69,
"grad_norm": 0.6796875,
"learning_rate": 5.626926795411447e-05,
"loss": 2.1882,
"step": 86
},
{
"epoch": 2.72,
"grad_norm": 0.8203125,
"learning_rate": 5.38907498550674e-05,
"loss": 2.4703,
"step": 87
},
{
"epoch": 2.75,
"grad_norm": 0.8671875,
"learning_rate": 5.1544912966734994e-05,
"loss": 2.2398,
"step": 88
},
{
"epoch": 2.75,
"eval_loss": 2.3605105876922607,
"eval_runtime": 1.2887,
"eval_samples_per_second": 10.864,
"eval_steps_per_second": 5.432,
"step": 88
},
{
"epoch": 2.78,
"grad_norm": 0.81640625,
"learning_rate": 4.9233419966116036e-05,
"loss": 2.2939,
"step": 89
},
{
"epoch": 2.81,
"grad_norm": 1.0234375,
"learning_rate": 4.695790918802576e-05,
"loss": 2.2835,
"step": 90
},
{
"epoch": 2.84,
"grad_norm": 0.85546875,
"learning_rate": 4.47199934638807e-05,
"loss": 2.3145,
"step": 91
},
{
"epoch": 2.88,
"grad_norm": 0.5546875,
"learning_rate": 4.252125897855932e-05,
"loss": 2.2521,
"step": 92
},
{
"epoch": 2.91,
"grad_norm": 0.76953125,
"learning_rate": 4.036326414614985e-05,
"loss": 2.377,
"step": 93
},
{
"epoch": 2.94,
"grad_norm": 0.78125,
"learning_rate": 3.824753850538082e-05,
"loss": 2.4343,
"step": 94
},
{
"epoch": 2.97,
"grad_norm": 0.8046875,
"learning_rate": 3.617558163551802e-05,
"loss": 2.2051,
"step": 95
},
{
"epoch": 3.0,
"grad_norm": 0.88671875,
"learning_rate": 3.414886209349615e-05,
"loss": 2.2693,
"step": 96
},
{
"epoch": 3.0,
"eval_loss": 2.3525888919830322,
"eval_runtime": 1.2821,
"eval_samples_per_second": 10.919,
"eval_steps_per_second": 5.46,
"step": 96
},
{
"epoch": 3.03,
"grad_norm": 0.74609375,
"learning_rate": 3.216881637303839e-05,
"loss": 2.5048,
"step": 97
},
{
"epoch": 3.06,
"grad_norm": 0.95703125,
"learning_rate": 3.0236847886501542e-05,
"loss": 2.3404,
"step": 98
},
{
"epoch": 3.09,
"grad_norm": 0.69140625,
"learning_rate": 2.8354325970168484e-05,
"loss": 2.3147,
"step": 99
},
{
"epoch": 3.12,
"grad_norm": 0.765625,
"learning_rate": 2.6522584913693294e-05,
"loss": 2.448,
"step": 100
},
{
"epoch": 3.16,
"grad_norm": 0.671875,
"learning_rate": 2.4742923014386156e-05,
"loss": 2.2212,
"step": 101
},
{
"epoch": 3.19,
"grad_norm": 0.625,
"learning_rate": 2.301660165700936e-05,
"loss": 2.3427,
"step": 102
},
{
"epoch": 3.22,
"grad_norm": 0.6484375,
"learning_rate": 2.1344844419735755e-05,
"loss": 2.1,
"step": 103
},
{
"epoch": 3.25,
"grad_norm": 0.73046875,
"learning_rate": 1.9728836206903656e-05,
"loss": 2.425,
"step": 104
},
{
"epoch": 3.25,
"eval_loss": 2.3470816612243652,
"eval_runtime": 1.2874,
"eval_samples_per_second": 10.875,
"eval_steps_per_second": 5.437,
"step": 104
},
{
"epoch": 3.28,
"grad_norm": 0.734375,
"learning_rate": 1.8169722409183097e-05,
"loss": 2.2268,
"step": 105
},
{
"epoch": 3.31,
"grad_norm": 0.8359375,
"learning_rate": 1.6668608091748495e-05,
"loss": 2.3695,
"step": 106
},
{
"epoch": 3.34,
"grad_norm": 0.58203125,
"learning_rate": 1.522655721103291e-05,
"loss": 2.1864,
"step": 107
},
{
"epoch": 3.38,
"grad_norm": 0.5703125,
"learning_rate": 1.3844591860619383e-05,
"loss": 2.2613,
"step": 108
},
{
"epoch": 3.41,
"grad_norm": 0.7265625,
"learning_rate": 1.2523691546803873e-05,
"loss": 2.2771,
"step": 109
},
{
"epoch": 3.44,
"grad_norm": 0.59765625,
"learning_rate": 1.1264792494342857e-05,
"loss": 2.2056,
"step": 110
},
{
"epoch": 3.47,
"grad_norm": 0.6875,
"learning_rate": 1.0068786982878087e-05,
"loss": 2.4039,
"step": 111
},
{
"epoch": 3.5,
"grad_norm": 0.77734375,
"learning_rate": 8.936522714508678e-06,
"loss": 2.2857,
"step": 112
},
{
"epoch": 3.5,
"eval_loss": 2.3467748165130615,
"eval_runtime": 1.2808,
"eval_samples_per_second": 10.931,
"eval_steps_per_second": 5.466,
"step": 112
},
{
"epoch": 3.53,
"grad_norm": 0.671875,
"learning_rate": 7.868802212958703e-06,
"loss": 2.257,
"step": 113
},
{
"epoch": 3.56,
"grad_norm": 1.21875,
"learning_rate": 6.866382254766157e-06,
"loss": 2.4278,
"step": 114
},
{
"epoch": 3.59,
"grad_norm": 0.859375,
"learning_rate": 5.929973332896677e-06,
"loss": 2.2232,
"step": 115
},
{
"epoch": 3.62,
"grad_norm": 0.66015625,
"learning_rate": 5.060239153161872e-06,
"loss": 2.225,
"step": 116
},
{
"epoch": 3.66,
"grad_norm": 0.6875,
"learning_rate": 4.257796163799455e-06,
"loss": 2.2133,
"step": 117
},
{
"epoch": 3.69,
"grad_norm": 0.8671875,
"learning_rate": 3.5232131185484076e-06,
"loss": 2.2191,
"step": 118
},
{
"epoch": 3.72,
"grad_norm": 0.77734375,
"learning_rate": 2.857010673529015e-06,
"loss": 2.0715,
"step": 119
},
{
"epoch": 3.75,
"grad_norm": 0.72265625,
"learning_rate": 2.259661018213333e-06,
"loss": 2.2448,
"step": 120
},
{
"epoch": 3.75,
"eval_loss": 2.3450927734375,
"eval_runtime": 1.2817,
"eval_samples_per_second": 10.923,
"eval_steps_per_second": 5.461,
"step": 120
},
{
"epoch": 3.78,
"grad_norm": 0.62109375,
"learning_rate": 1.7315875407479032e-06,
"loss": 2.1337,
"step": 121
},
{
"epoch": 3.81,
"grad_norm": 0.765625,
"learning_rate": 1.2731645278655445e-06,
"loss": 2.4334,
"step": 122
},
{
"epoch": 3.84,
"grad_norm": 0.73828125,
"learning_rate": 8.847168995992916e-07,
"loss": 2.3989,
"step": 123
},
{
"epoch": 3.88,
"grad_norm": 0.71484375,
"learning_rate": 5.665199789862907e-07,
"loss": 2.3509,
"step": 124
},
{
"epoch": 3.91,
"grad_norm": 0.78125,
"learning_rate": 3.1879929692498757e-07,
"loss": 2.3366,
"step": 125
},
{
"epoch": 3.94,
"grad_norm": 0.625,
"learning_rate": 1.4173043232380557e-07,
"loss": 2.071,
"step": 126
},
{
"epoch": 3.97,
"grad_norm": 0.6015625,
"learning_rate": 3.5438887654737355e-08,
"loss": 2.2108,
"step": 127
},
{
"epoch": 4.0,
"grad_norm": 0.7578125,
"learning_rate": 0.0,
"loss": 2.1836,
"step": 128
},
{
"epoch": 4.0,
"eval_loss": 2.3465805053710938,
"eval_runtime": 1.2864,
"eval_samples_per_second": 10.883,
"eval_steps_per_second": 5.441,
"step": 128
}
],
"logging_steps": 1,
"max_steps": 128,
"num_input_tokens_seen": 0,
"num_train_epochs": 4,
"save_steps": 32,
"total_flos": 2.6669324546605056e+16,
"train_batch_size": 2,
"trial_name": null,
"trial_params": null
}