Transformers
PyTorch
English
gpt2
Generated from Trainer
Inference Endpoints
text-generation-inference
eloquent_keller / checkpoint-251 /trainer_state.json
tomekkorbak's picture
Training in progress, step 251
afc7f41
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.10626587637595258,
"global_step": 251,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0,
"learning_rate": 4.166666666666667e-06,
"loss": 3.0643,
"theoretical_loss": 3.321567680436603,
"tokens_seen": 2990538752
},
{
"epoch": 0.0,
"learning_rate": 8.333333333333334e-06,
"loss": 3.0798,
"theoretical_loss": 3.3215564803546,
"tokens_seen": 2990669824
},
{
"epoch": 0.0,
"learning_rate": 1.25e-05,
"loss": 2.9318,
"theoretical_loss": 3.321545280900887,
"tokens_seen": 2990800896
},
{
"epoch": 0.0,
"learning_rate": 1.6666666666666667e-05,
"loss": 2.8098,
"theoretical_loss": 3.3215340820754022,
"tokens_seen": 2990931968
},
{
"epoch": 0.0,
"learning_rate": 2.0833333333333336e-05,
"loss": 2.7055,
"theoretical_loss": 3.3215228838780817,
"tokens_seen": 2991063040
},
{
"epoch": 0.0,
"learning_rate": 2.5e-05,
"loss": 2.9762,
"theoretical_loss": 3.3215116863088636,
"tokens_seen": 2991194112
},
{
"epoch": 0.0,
"learning_rate": 2.916666666666667e-05,
"loss": 2.8724,
"theoretical_loss": 3.3215004893676854,
"tokens_seen": 2991325184
},
{
"epoch": 0.0,
"learning_rate": 3.3333333333333335e-05,
"loss": 3.0452,
"theoretical_loss": 3.321489293054483,
"tokens_seen": 2991456256
},
{
"epoch": 0.0,
"learning_rate": 3.7500000000000003e-05,
"loss": 2.8676,
"theoretical_loss": 3.321478097369195,
"tokens_seen": 2991587328
},
{
"epoch": 0.0,
"learning_rate": 4.166666666666667e-05,
"loss": 2.8343,
"theoretical_loss": 3.321466902311758,
"tokens_seen": 2991718400
},
{
"epoch": 0.0,
"learning_rate": 4.5833333333333334e-05,
"loss": 2.743,
"theoretical_loss": 3.3214557078821096,
"tokens_seen": 2991849472
},
{
"epoch": 0.01,
"learning_rate": 5e-05,
"loss": 2.5867,
"theoretical_loss": 3.321444514080187,
"tokens_seen": 2991980544
},
{
"epoch": 0.01,
"objective/train/docs_used": 1640856,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.7297048568725586,
"objective/train/theoretical_loss": 3.321438917414603,
"objective/train/tokens_used": 22097376,
"theoretical_loss": 3.321438917414603,
"tokens_seen": 2992046080
},
{
"epoch": 0.01,
"learning_rate": 5.4166666666666664e-05,
"loss": 2.7262,
"theoretical_loss": 3.321433320905927,
"tokens_seen": 2992111616
},
{
"epoch": 0.01,
"learning_rate": 5.833333333333334e-05,
"loss": 2.6517,
"theoretical_loss": 3.3214221283592678,
"tokens_seen": 2992242688
},
{
"epoch": 0.01,
"learning_rate": 6.25e-05,
"loss": 2.9399,
"theoretical_loss": 3.321410936440146,
"tokens_seen": 2992373760
},
{
"epoch": 0.01,
"learning_rate": 6.666666666666667e-05,
"loss": 2.7939,
"theoretical_loss": 3.3213997451485,
"tokens_seen": 2992504832
},
{
"epoch": 0.01,
"learning_rate": 7.083333333333334e-05,
"loss": 2.5715,
"theoretical_loss": 3.3213885544842654,
"tokens_seen": 2992635904
},
{
"epoch": 0.01,
"learning_rate": 7.500000000000001e-05,
"loss": 2.6047,
"theoretical_loss": 3.321377364447381,
"tokens_seen": 2992766976
},
{
"epoch": 0.01,
"learning_rate": 7.916666666666666e-05,
"loss": 2.6736,
"theoretical_loss": 3.3213661750377836,
"tokens_seen": 2992898048
},
{
"epoch": 0.01,
"learning_rate": 8.333333333333334e-05,
"loss": 2.6853,
"theoretical_loss": 3.3213549862554106,
"tokens_seen": 2993029120
},
{
"epoch": 0.01,
"learning_rate": 8.75e-05,
"loss": 2.3665,
"theoretical_loss": 3.3213437981001994,
"tokens_seen": 2993160192
},
{
"epoch": 0.01,
"learning_rate": 9.166666666666667e-05,
"loss": 2.618,
"theoretical_loss": 3.3213326105720875,
"tokens_seen": 2993291264
},
{
"epoch": 0.01,
"learning_rate": 9.583333333333334e-05,
"loss": 2.6614,
"theoretical_loss": 3.3213214236710122,
"tokens_seen": 2993422336
},
{
"epoch": 0.01,
"learning_rate": 0.0001,
"loss": 2.6163,
"theoretical_loss": 3.321310237396911,
"tokens_seen": 2993553408
},
{
"epoch": 0.01,
"objective/train/docs_used": 1641461,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.5642035007476807,
"objective/train/theoretical_loss": 3.3212990517497207,
"objective/train/tokens_used": 23735776,
"theoretical_loss": 3.3212990517497207,
"tokens_seen": 2993684480
},
{
"epoch": 0.01,
"learning_rate": 9.995722840034217e-05,
"loss": 2.7218,
"theoretical_loss": 3.3212990517497207,
"tokens_seen": 2993684480
},
{
"epoch": 0.01,
"learning_rate": 9.991445680068435e-05,
"loss": 2.5137,
"theoretical_loss": 3.3212878667293797,
"tokens_seen": 2993815552
},
{
"epoch": 0.01,
"learning_rate": 9.987168520102653e-05,
"loss": 2.5283,
"theoretical_loss": 3.321276682335825,
"tokens_seen": 2993946624
},
{
"epoch": 0.01,
"learning_rate": 9.98289136013687e-05,
"loss": 2.6367,
"theoretical_loss": 3.3212654985689936,
"tokens_seen": 2994077696
},
{
"epoch": 0.01,
"learning_rate": 9.978614200171087e-05,
"loss": 2.5822,
"theoretical_loss": 3.3212543154288237,
"tokens_seen": 2994208768
},
{
"epoch": 0.01,
"learning_rate": 9.974337040205303e-05,
"loss": 2.6296,
"theoretical_loss": 3.3212431329152525,
"tokens_seen": 2994339840
},
{
"epoch": 0.01,
"learning_rate": 9.970059880239521e-05,
"loss": 2.5596,
"theoretical_loss": 3.321231951028217,
"tokens_seen": 2994470912
},
{
"epoch": 0.01,
"learning_rate": 9.965782720273739e-05,
"loss": 2.5663,
"theoretical_loss": 3.3212207697676552,
"tokens_seen": 2994601984
},
{
"epoch": 0.01,
"learning_rate": 9.961505560307956e-05,
"loss": 2.5138,
"theoretical_loss": 3.3212095891335043,
"tokens_seen": 2994733056
},
{
"epoch": 0.01,
"learning_rate": 9.957228400342173e-05,
"loss": 2.5938,
"theoretical_loss": 3.321198409125702,
"tokens_seen": 2994864128
},
{
"epoch": 0.01,
"learning_rate": 9.95295124037639e-05,
"loss": 2.4583,
"theoretical_loss": 3.321187229744186,
"tokens_seen": 2994995200
},
{
"epoch": 0.02,
"learning_rate": 9.948674080410608e-05,
"loss": 2.5265,
"theoretical_loss": 3.321176050988893,
"tokens_seen": 2995126272
},
{
"epoch": 0.02,
"learning_rate": 9.944396920444825e-05,
"loss": 2.7376,
"theoretical_loss": 3.3211648728597614,
"tokens_seen": 2995257344
},
{
"epoch": 0.02,
"objective/train/docs_used": 1642666,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.7178521156311035,
"objective/train/theoretical_loss": 3.3211592840299864,
"objective/train/tokens_used": 25374176,
"theoretical_loss": 3.3211592840299864,
"tokens_seen": 2995322880
},
{
"epoch": 0.02,
"learning_rate": 9.940119760479042e-05,
"loss": 2.4464,
"theoretical_loss": 3.3211536953567284,
"tokens_seen": 2995388416
},
{
"epoch": 0.02,
"learning_rate": 9.93584260051326e-05,
"loss": 2.657,
"theoretical_loss": 3.321142518479731,
"tokens_seen": 2995519488
},
{
"epoch": 0.02,
"learning_rate": 9.931565440547476e-05,
"loss": 2.5525,
"theoretical_loss": 3.321131342228708,
"tokens_seen": 2995650560
},
{
"epoch": 0.02,
"learning_rate": 9.927288280581694e-05,
"loss": 2.6099,
"theoretical_loss": 3.321120166603596,
"tokens_seen": 2995781632
},
{
"epoch": 0.02,
"learning_rate": 9.923011120615912e-05,
"loss": 2.4712,
"theoretical_loss": 3.3211089916043326,
"tokens_seen": 2995912704
},
{
"epoch": 0.02,
"learning_rate": 9.918733960650128e-05,
"loss": 2.514,
"theoretical_loss": 3.3210978172308554,
"tokens_seen": 2996043776
},
{
"epoch": 0.02,
"learning_rate": 9.914456800684346e-05,
"loss": 2.4037,
"theoretical_loss": 3.3210866434831026,
"tokens_seen": 2996174848
},
{
"epoch": 0.02,
"learning_rate": 9.910179640718563e-05,
"loss": 2.4715,
"theoretical_loss": 3.3210754703610106,
"tokens_seen": 2996305920
},
{
"epoch": 0.02,
"learning_rate": 9.90590248075278e-05,
"loss": 2.4805,
"theoretical_loss": 3.321064297864518,
"tokens_seen": 2996436992
},
{
"epoch": 0.02,
"learning_rate": 9.901625320786998e-05,
"loss": 2.4755,
"theoretical_loss": 3.3210531259935627,
"tokens_seen": 2996568064
},
{
"epoch": 0.02,
"learning_rate": 9.897348160821215e-05,
"loss": 2.5174,
"theoretical_loss": 3.321041954748081,
"tokens_seen": 2996699136
},
{
"epoch": 0.02,
"learning_rate": 9.893071000855433e-05,
"loss": 2.5996,
"theoretical_loss": 3.321030784128012,
"tokens_seen": 2996830208
},
{
"epoch": 0.02,
"objective/train/docs_used": 1643300,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.879695415496826,
"objective/train/theoretical_loss": 3.321019614133292,
"objective/train/tokens_used": 27012576,
"theoretical_loss": 3.321019614133292,
"tokens_seen": 2996961280
},
{
"epoch": 0.02,
"learning_rate": 9.888793840889649e-05,
"loss": 2.5696,
"theoretical_loss": 3.321019614133292,
"tokens_seen": 2996961280
},
{
"epoch": 0.02,
"learning_rate": 9.884516680923867e-05,
"loss": 2.607,
"theoretical_loss": 3.3210084447638595,
"tokens_seen": 2997092352
},
{
"epoch": 0.02,
"learning_rate": 9.880239520958085e-05,
"loss": 2.4604,
"theoretical_loss": 3.320997276019652,
"tokens_seen": 2997223424
},
{
"epoch": 0.02,
"learning_rate": 9.875962360992301e-05,
"loss": 2.4603,
"theoretical_loss": 3.3209861079006067,
"tokens_seen": 2997354496
},
{
"epoch": 0.02,
"learning_rate": 9.871685201026519e-05,
"loss": 2.4374,
"theoretical_loss": 3.320974940406662,
"tokens_seen": 2997485568
},
{
"epoch": 0.02,
"learning_rate": 9.867408041060736e-05,
"loss": 2.5285,
"theoretical_loss": 3.320963773537755,
"tokens_seen": 2997616640
},
{
"epoch": 0.02,
"learning_rate": 9.863130881094953e-05,
"loss": 2.5895,
"theoretical_loss": 3.320952607293824,
"tokens_seen": 2997747712
},
{
"epoch": 0.02,
"learning_rate": 9.858853721129171e-05,
"loss": 2.5835,
"theoretical_loss": 3.320941441674806,
"tokens_seen": 2997878784
},
{
"epoch": 0.02,
"learning_rate": 9.854576561163388e-05,
"loss": 2.7362,
"theoretical_loss": 3.320930276680639,
"tokens_seen": 2998009856
},
{
"epoch": 0.02,
"learning_rate": 9.850299401197606e-05,
"loss": 2.5553,
"theoretical_loss": 3.3209191123112607,
"tokens_seen": 2998140928
},
{
"epoch": 0.03,
"learning_rate": 9.846022241231822e-05,
"loss": 2.6403,
"theoretical_loss": 3.320907948566609,
"tokens_seen": 2998272000
},
{
"epoch": 0.03,
"learning_rate": 9.84174508126604e-05,
"loss": 2.5129,
"theoretical_loss": 3.3208967854466214,
"tokens_seen": 2998403072
},
{
"epoch": 0.03,
"learning_rate": 9.837467921300258e-05,
"loss": 2.4238,
"theoretical_loss": 3.3208856229512356,
"tokens_seen": 2998534144
},
{
"epoch": 0.03,
"objective/train/docs_used": 1644380,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.4041128158569336,
"objective/train/theoretical_loss": 3.320880041937749,
"objective/train/tokens_used": 28650976,
"theoretical_loss": 3.320880041937749,
"tokens_seen": 2998599680
},
{
"epoch": 0.03,
"learning_rate": 9.833190761334474e-05,
"loss": 2.3705,
"theoretical_loss": 3.3208744610803898,
"tokens_seen": 2998665216
},
{
"epoch": 0.03,
"learning_rate": 9.828913601368692e-05,
"loss": 2.4602,
"theoretical_loss": 3.320863299834021,
"tokens_seen": 2998796288
},
{
"epoch": 0.03,
"learning_rate": 9.824636441402908e-05,
"loss": 2.5492,
"theoretical_loss": 3.320852139212068,
"tokens_seen": 2998927360
},
{
"epoch": 0.03,
"learning_rate": 9.820359281437126e-05,
"loss": 2.426,
"theoretical_loss": 3.3208409792144677,
"tokens_seen": 2999058432
},
{
"epoch": 0.03,
"learning_rate": 9.816082121471344e-05,
"loss": 2.4079,
"theoretical_loss": 3.320829819841158,
"tokens_seen": 2999189504
},
{
"epoch": 0.03,
"learning_rate": 9.81180496150556e-05,
"loss": 2.5189,
"theoretical_loss": 3.320818661092077,
"tokens_seen": 2999320576
},
{
"epoch": 0.03,
"learning_rate": 9.807527801539777e-05,
"loss": 2.4085,
"theoretical_loss": 3.3208075029671624,
"tokens_seen": 2999451648
},
{
"epoch": 0.03,
"learning_rate": 9.803250641573995e-05,
"loss": 2.4031,
"theoretical_loss": 3.320796345466352,
"tokens_seen": 2999582720
},
{
"epoch": 0.03,
"learning_rate": 9.798973481608213e-05,
"loss": 2.5067,
"theoretical_loss": 3.320785188589584,
"tokens_seen": 2999713792
},
{
"epoch": 0.03,
"learning_rate": 9.79469632164243e-05,
"loss": 2.4357,
"theoretical_loss": 3.3207740323367956,
"tokens_seen": 2999844864
},
{
"epoch": 0.03,
"learning_rate": 9.790419161676647e-05,
"loss": 2.4929,
"theoretical_loss": 3.3207628767079242,
"tokens_seen": 2999975936
},
{
"epoch": 0.03,
"learning_rate": 9.786142001710863e-05,
"loss": 2.4052,
"theoretical_loss": 3.3207517217029094,
"tokens_seen": 3000107008
},
{
"epoch": 0.03,
"objective/train/docs_used": 1645056,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.3271520137786865,
"objective/train/theoretical_loss": 3.3207405673216877,
"objective/train/tokens_used": 30289376,
"theoretical_loss": 3.3207405673216877,
"tokens_seen": 3000238080
},
{
"epoch": 0.03,
"learning_rate": 9.781864841745081e-05,
"loss": 2.5201,
"theoretical_loss": 3.3207405673216877,
"tokens_seen": 3000238080
},
{
"epoch": 0.03,
"learning_rate": 9.777587681779299e-05,
"loss": 2.5431,
"theoretical_loss": 3.320729413564197,
"tokens_seen": 3000369152
},
{
"epoch": 0.03,
"learning_rate": 9.773310521813517e-05,
"loss": 2.5359,
"theoretical_loss": 3.3207182604303753,
"tokens_seen": 3000500224
},
{
"epoch": 0.03,
"learning_rate": 9.769033361847733e-05,
"loss": 2.1929,
"theoretical_loss": 3.320707107920161,
"tokens_seen": 3000631296
},
{
"epoch": 0.03,
"learning_rate": 9.76475620188195e-05,
"loss": 2.5196,
"theoretical_loss": 3.3206959560334917,
"tokens_seen": 3000762368
},
{
"epoch": 0.03,
"learning_rate": 9.760479041916169e-05,
"loss": 2.3645,
"theoretical_loss": 3.320684804770305,
"tokens_seen": 3000893440
},
{
"epoch": 0.03,
"learning_rate": 9.756201881950386e-05,
"loss": 2.5329,
"theoretical_loss": 3.3206736541305393,
"tokens_seen": 3001024512
},
{
"epoch": 0.03,
"learning_rate": 9.751924721984602e-05,
"loss": 2.533,
"theoretical_loss": 3.3206625041141318,
"tokens_seen": 3001155584
},
{
"epoch": 0.04,
"learning_rate": 9.74764756201882e-05,
"loss": 2.4702,
"theoretical_loss": 3.3206513547210212,
"tokens_seen": 3001286656
},
{
"epoch": 0.04,
"learning_rate": 9.743370402053036e-05,
"loss": 2.6255,
"theoretical_loss": 3.320640205951145,
"tokens_seen": 3001417728
},
{
"epoch": 0.04,
"learning_rate": 9.739093242087256e-05,
"loss": 2.5139,
"theoretical_loss": 3.3206290578044415,
"tokens_seen": 3001548800
},
{
"epoch": 0.04,
"learning_rate": 9.734816082121472e-05,
"loss": 2.3796,
"theoretical_loss": 3.3206179102808484,
"tokens_seen": 3001679872
},
{
"epoch": 0.04,
"learning_rate": 9.730538922155689e-05,
"loss": 2.4997,
"theoretical_loss": 3.3206067633803036,
"tokens_seen": 3001810944
},
{
"epoch": 0.04,
"objective/train/docs_used": 1646327,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.398160219192505,
"objective/train/theoretical_loss": 3.320601190163655,
"objective/train/tokens_used": 31927776,
"theoretical_loss": 3.320601190163655,
"tokens_seen": 3001876480
},
{
"epoch": 0.04,
"learning_rate": 9.726261762189906e-05,
"loss": 2.3607,
"theoretical_loss": 3.320595617102745,
"tokens_seen": 3001942016
},
{
"epoch": 0.04,
"learning_rate": 9.721984602224123e-05,
"loss": 2.4803,
"theoretical_loss": 3.320584471448111,
"tokens_seen": 3002073088
},
{
"epoch": 0.04,
"learning_rate": 9.717707442258342e-05,
"loss": 2.348,
"theoretical_loss": 3.3205733264163393,
"tokens_seen": 3002204160
},
{
"epoch": 0.04,
"learning_rate": 9.713430282292558e-05,
"loss": 2.2828,
"theoretical_loss": 3.320562182007368,
"tokens_seen": 3002335232
},
{
"epoch": 0.04,
"learning_rate": 9.709153122326775e-05,
"loss": 2.4615,
"theoretical_loss": 3.320551038221135,
"tokens_seen": 3002466304
},
{
"epoch": 0.04,
"learning_rate": 9.704875962360993e-05,
"loss": 2.3988,
"theoretical_loss": 3.3205398950575784,
"tokens_seen": 3002597376
},
{
"epoch": 0.04,
"learning_rate": 9.700598802395209e-05,
"loss": 2.5582,
"theoretical_loss": 3.320528752516636,
"tokens_seen": 3002728448
},
{
"epoch": 0.04,
"learning_rate": 9.696321642429428e-05,
"loss": 2.3266,
"theoretical_loss": 3.3205176105982463,
"tokens_seen": 3002859520
},
{
"epoch": 0.04,
"learning_rate": 9.692044482463645e-05,
"loss": 2.5922,
"theoretical_loss": 3.320506469302347,
"tokens_seen": 3002990592
},
{
"epoch": 0.04,
"learning_rate": 9.687767322497861e-05,
"loss": 2.4959,
"theoretical_loss": 3.3204953286288763,
"tokens_seen": 3003121664
},
{
"epoch": 0.04,
"learning_rate": 9.683490162532079e-05,
"loss": 2.6068,
"theoretical_loss": 3.3204841885777725,
"tokens_seen": 3003252736
},
{
"epoch": 0.04,
"learning_rate": 9.679213002566297e-05,
"loss": 2.3996,
"theoretical_loss": 3.3204730491489727,
"tokens_seen": 3003383808
},
{
"epoch": 0.04,
"objective/train/docs_used": 1647543,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.3885486125946045,
"objective/train/theoretical_loss": 3.3204619103424164,
"objective/train/tokens_used": 33566176,
"theoretical_loss": 3.3204619103424164,
"tokens_seen": 3003514880
},
{
"epoch": 0.04,
"learning_rate": 9.674935842600514e-05,
"loss": 2.4692,
"theoretical_loss": 3.3204619103424164,
"tokens_seen": 3003514880
},
{
"epoch": 0.04,
"learning_rate": 9.670658682634731e-05,
"loss": 2.35,
"theoretical_loss": 3.3204507721580403,
"tokens_seen": 3003645952
},
{
"epoch": 0.04,
"learning_rate": 9.666381522668948e-05,
"loss": 2.3168,
"theoretical_loss": 3.3204396345957834,
"tokens_seen": 3003777024
},
{
"epoch": 0.04,
"learning_rate": 9.662104362703166e-05,
"loss": 2.4671,
"theoretical_loss": 3.320428497655584,
"tokens_seen": 3003908096
},
{
"epoch": 0.04,
"learning_rate": 9.657827202737383e-05,
"loss": 2.552,
"theoretical_loss": 3.320417361337379,
"tokens_seen": 3004039168
},
{
"epoch": 0.04,
"learning_rate": 9.6535500427716e-05,
"loss": 2.3902,
"theoretical_loss": 3.3204062256411078,
"tokens_seen": 3004170240
},
{
"epoch": 0.04,
"learning_rate": 9.649272882805818e-05,
"loss": 2.491,
"theoretical_loss": 3.320395090566708,
"tokens_seen": 3004301312
},
{
"epoch": 0.05,
"learning_rate": 9.644995722840034e-05,
"loss": 2.4628,
"theoretical_loss": 3.3203839561141173,
"tokens_seen": 3004432384
},
{
"epoch": 0.05,
"learning_rate": 9.640718562874252e-05,
"loss": 2.4858,
"theoretical_loss": 3.320372822283275,
"tokens_seen": 3004563456
},
{
"epoch": 0.05,
"learning_rate": 9.63644140290847e-05,
"loss": 2.5038,
"theoretical_loss": 3.3203616890741183,
"tokens_seen": 3004694528
},
{
"epoch": 0.05,
"learning_rate": 9.632164242942686e-05,
"loss": 2.2651,
"theoretical_loss": 3.3203505564865856,
"tokens_seen": 3004825600
},
{
"epoch": 0.05,
"learning_rate": 9.627887082976904e-05,
"loss": 2.4591,
"theoretical_loss": 3.3203394245206153,
"tokens_seen": 3004956672
},
{
"epoch": 0.05,
"learning_rate": 9.623609923011121e-05,
"loss": 2.4003,
"theoretical_loss": 3.320328293176145,
"tokens_seen": 3005087744
},
{
"epoch": 0.05,
"objective/train/docs_used": 1648109,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.3964030742645264,
"objective/train/theoretical_loss": 3.3203227277369534,
"objective/train/tokens_used": 35204576,
"theoretical_loss": 3.3203227277369534,
"tokens_seen": 3005153280
},
{
"epoch": 0.05,
"learning_rate": 9.619332763045337e-05,
"loss": 2.3952,
"theoretical_loss": 3.320317162453114,
"tokens_seen": 3005218816
},
{
"epoch": 0.05,
"learning_rate": 9.615055603079556e-05,
"loss": 2.4367,
"theoretical_loss": 3.3203060323514593,
"tokens_seen": 3005349888
},
{
"epoch": 0.05,
"learning_rate": 9.610778443113773e-05,
"loss": 2.4468,
"theoretical_loss": 3.3202949028711197,
"tokens_seen": 3005480960
},
{
"epoch": 0.05,
"learning_rate": 9.60650128314799e-05,
"loss": 2.3551,
"theoretical_loss": 3.3202837740120335,
"tokens_seen": 3005612032
},
{
"epoch": 0.05,
"learning_rate": 9.602224123182207e-05,
"loss": 2.3886,
"theoretical_loss": 3.3202726457741387,
"tokens_seen": 3005743104
},
{
"epoch": 0.05,
"learning_rate": 9.597946963216424e-05,
"loss": 2.4953,
"theoretical_loss": 3.320261518157374,
"tokens_seen": 3005874176
},
{
"epoch": 0.05,
"learning_rate": 9.593669803250643e-05,
"loss": 2.3074,
"theoretical_loss": 3.3202503911616765,
"tokens_seen": 3006005248
},
{
"epoch": 0.05,
"learning_rate": 9.589392643284859e-05,
"loss": 2.4135,
"theoretical_loss": 3.320239264786986,
"tokens_seen": 3006136320
},
{
"epoch": 0.05,
"learning_rate": 9.585115483319077e-05,
"loss": 2.431,
"theoretical_loss": 3.3202281390332393,
"tokens_seen": 3006267392
},
{
"epoch": 0.05,
"learning_rate": 9.580838323353294e-05,
"loss": 2.3277,
"theoretical_loss": 3.320217013900376,
"tokens_seen": 3006398464
},
{
"epoch": 0.05,
"learning_rate": 9.576561163387511e-05,
"loss": 2.5083,
"theoretical_loss": 3.3202058893883333,
"tokens_seen": 3006529536
},
{
"epoch": 0.05,
"learning_rate": 9.572284003421729e-05,
"loss": 2.4582,
"theoretical_loss": 3.3201947654970505,
"tokens_seen": 3006660608
},
{
"epoch": 0.05,
"objective/train/docs_used": 1649212,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.5212416648864746,
"objective/train/theoretical_loss": 3.320183642226465,
"objective/train/tokens_used": 36842976,
"theoretical_loss": 3.320183642226465,
"tokens_seen": 3006791680
},
{
"epoch": 0.05,
"learning_rate": 9.568006843455946e-05,
"loss": 2.3596,
"theoretical_loss": 3.320183642226465,
"tokens_seen": 3006791680
},
{
"epoch": 0.05,
"learning_rate": 9.563729683490164e-05,
"loss": 2.3923,
"theoretical_loss": 3.3201725195765155,
"tokens_seen": 3006922752
},
{
"epoch": 0.05,
"learning_rate": 9.55945252352438e-05,
"loss": 2.3563,
"theoretical_loss": 3.3201613975471402,
"tokens_seen": 3007053824
},
{
"epoch": 0.05,
"learning_rate": 9.555175363558598e-05,
"loss": 2.4285,
"theoretical_loss": 3.3201502761382775,
"tokens_seen": 3007184896
},
{
"epoch": 0.05,
"learning_rate": 9.550898203592816e-05,
"loss": 2.2928,
"theoretical_loss": 3.320139155349866,
"tokens_seen": 3007315968
},
{
"epoch": 0.06,
"learning_rate": 9.546621043627032e-05,
"loss": 2.4617,
"theoretical_loss": 3.3201280351818436,
"tokens_seen": 3007447040
},
{
"epoch": 0.06,
"learning_rate": 9.542343883661249e-05,
"loss": 2.4107,
"theoretical_loss": 3.320116915634149,
"tokens_seen": 3007578112
},
{
"epoch": 0.06,
"learning_rate": 9.538066723695466e-05,
"loss": 2.5715,
"theoretical_loss": 3.3201057967067205,
"tokens_seen": 3007709184
},
{
"epoch": 0.06,
"learning_rate": 9.533789563729684e-05,
"loss": 2.4217,
"theoretical_loss": 3.3200946783994962,
"tokens_seen": 3007840256
},
{
"epoch": 0.06,
"learning_rate": 9.529512403763902e-05,
"loss": 2.4315,
"theoretical_loss": 3.3200835607124146,
"tokens_seen": 3007971328
},
{
"epoch": 0.06,
"learning_rate": 9.525235243798119e-05,
"loss": 2.4307,
"theoretical_loss": 3.3200724436454143,
"tokens_seen": 3008102400
},
{
"epoch": 0.06,
"learning_rate": 9.520958083832335e-05,
"loss": 2.5032,
"theoretical_loss": 3.3200613271984336,
"tokens_seen": 3008233472
},
{
"epoch": 0.06,
"learning_rate": 9.516680923866553e-05,
"loss": 2.4379,
"theoretical_loss": 3.3200502113714108,
"tokens_seen": 3008364544
},
{
"epoch": 0.06,
"objective/train/docs_used": 1649940,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.399945020675659,
"objective/train/theoretical_loss": 3.3200446536903643,
"objective/train/tokens_used": 38481376,
"theoretical_loss": 3.3200446536903643,
"tokens_seen": 3008430080
},
{
"epoch": 0.06,
"learning_rate": 9.512403763900771e-05,
"loss": 2.5107,
"theoretical_loss": 3.3200390961642845,
"tokens_seen": 3008495616
},
{
"epoch": 0.06,
"learning_rate": 9.508126603934989e-05,
"loss": 2.4659,
"theoretical_loss": 3.3200279815769926,
"tokens_seen": 3008626688
},
{
"epoch": 0.06,
"learning_rate": 9.503849443969205e-05,
"loss": 2.4327,
"theoretical_loss": 3.3200168676094743,
"tokens_seen": 3008757760
},
{
"epoch": 0.06,
"learning_rate": 9.499572284003421e-05,
"loss": 2.2681,
"theoretical_loss": 3.320005754261668,
"tokens_seen": 3008888832
},
{
"epoch": 0.06,
"learning_rate": 9.49529512403764e-05,
"loss": 2.3802,
"theoretical_loss": 3.319994641533511,
"tokens_seen": 3009019904
},
{
"epoch": 0.06,
"learning_rate": 9.491017964071857e-05,
"loss": 2.5461,
"theoretical_loss": 3.319983529424943,
"tokens_seen": 3009150976
},
{
"epoch": 0.06,
"learning_rate": 9.486740804106075e-05,
"loss": 2.4021,
"theoretical_loss": 3.3199724179359027,
"tokens_seen": 3009282048
},
{
"epoch": 0.06,
"learning_rate": 9.482463644140291e-05,
"loss": 2.4038,
"theoretical_loss": 3.319961307066327,
"tokens_seen": 3009413120
},
{
"epoch": 0.06,
"learning_rate": 9.478186484174508e-05,
"loss": 2.4687,
"theoretical_loss": 3.3199501968161558,
"tokens_seen": 3009544192
},
{
"epoch": 0.06,
"learning_rate": 9.473909324208726e-05,
"loss": 2.4401,
"theoretical_loss": 3.319939087185327,
"tokens_seen": 3009675264
},
{
"epoch": 0.06,
"learning_rate": 9.469632164242944e-05,
"loss": 2.4778,
"theoretical_loss": 3.3199279781737796,
"tokens_seen": 3009806336
},
{
"epoch": 0.06,
"learning_rate": 9.46535500427716e-05,
"loss": 2.4166,
"theoretical_loss": 3.3199168697814514,
"tokens_seen": 3009937408
},
{
"epoch": 0.06,
"objective/train/docs_used": 1651249,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.412529230117798,
"objective/train/theoretical_loss": 3.3199057620082812,
"objective/train/tokens_used": 40119776,
"theoretical_loss": 3.3199057620082812,
"tokens_seen": 3010068480
},
{
"epoch": 0.06,
"learning_rate": 9.461077844311378e-05,
"loss": 2.3839,
"theoretical_loss": 3.3199057620082812,
"tokens_seen": 3010068480
},
{
"epoch": 0.06,
"learning_rate": 9.456800684345594e-05,
"loss": 2.5879,
"theoretical_loss": 3.319894654854208,
"tokens_seen": 3010199552
},
{
"epoch": 0.06,
"learning_rate": 9.452523524379812e-05,
"loss": 2.444,
"theoretical_loss": 3.3198835483191695,
"tokens_seen": 3010330624
},
{
"epoch": 0.06,
"learning_rate": 9.44824636441403e-05,
"loss": 2.2849,
"theoretical_loss": 3.319872442403105,
"tokens_seen": 3010461696
},
{
"epoch": 0.07,
"learning_rate": 9.443969204448247e-05,
"loss": 2.568,
"theoretical_loss": 3.3198613371059524,
"tokens_seen": 3010592768
},
{
"epoch": 0.07,
"learning_rate": 9.439692044482464e-05,
"loss": 2.4131,
"theoretical_loss": 3.319850232427651,
"tokens_seen": 3010723840
},
{
"epoch": 0.07,
"learning_rate": 9.435414884516681e-05,
"loss": 2.4222,
"theoretical_loss": 3.3198391283681383,
"tokens_seen": 3010854912
},
{
"epoch": 0.07,
"learning_rate": 9.431137724550899e-05,
"loss": 2.3993,
"theoretical_loss": 3.3198280249273546,
"tokens_seen": 3010985984
},
{
"epoch": 0.07,
"learning_rate": 9.426860564585116e-05,
"loss": 2.5294,
"theoretical_loss": 3.319816922105237,
"tokens_seen": 3011117056
},
{
"epoch": 0.07,
"learning_rate": 9.422583404619333e-05,
"loss": 2.4529,
"theoretical_loss": 3.319805819901724,
"tokens_seen": 3011248128
},
{
"epoch": 0.07,
"learning_rate": 9.418306244653551e-05,
"loss": 2.4924,
"theoretical_loss": 3.3197947183167553,
"tokens_seen": 3011379200
},
{
"epoch": 0.07,
"learning_rate": 9.414029084687767e-05,
"loss": 2.6054,
"theoretical_loss": 3.319783617350269,
"tokens_seen": 3011510272
},
{
"epoch": 0.07,
"learning_rate": 9.409751924721985e-05,
"loss": 2.388,
"theoretical_loss": 3.319772517002204,
"tokens_seen": 3011641344
},
{
"epoch": 0.07,
"objective/train/docs_used": 1651905,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.8164331912994385,
"objective/train/theoretical_loss": 3.31976696706006,
"objective/train/tokens_used": 41758176,
"theoretical_loss": 3.31976696706006,
"tokens_seen": 3011706880
},
{
"epoch": 0.07,
"learning_rate": 9.405474764756203e-05,
"loss": 2.4623,
"theoretical_loss": 3.319761417272498,
"tokens_seen": 3011772416
},
{
"epoch": 0.07,
"learning_rate": 9.40119760479042e-05,
"loss": 2.3955,
"theoretical_loss": 3.319750318161091,
"tokens_seen": 3011903488
},
{
"epoch": 0.07,
"learning_rate": 9.396920444824637e-05,
"loss": 2.2424,
"theoretical_loss": 3.3197392196679205,
"tokens_seen": 3012034560
},
{
"epoch": 0.07,
"learning_rate": 9.392643284858854e-05,
"loss": 2.3568,
"theoretical_loss": 3.3197281217929255,
"tokens_seen": 3012165632
},
{
"epoch": 0.07,
"learning_rate": 9.388366124893072e-05,
"loss": 2.3788,
"theoretical_loss": 3.319717024536045,
"tokens_seen": 3012296704
},
{
"epoch": 0.07,
"learning_rate": 9.38408896492729e-05,
"loss": 2.3081,
"theoretical_loss": 3.3197059278972176,
"tokens_seen": 3012427776
},
{
"epoch": 0.07,
"learning_rate": 9.379811804961506e-05,
"loss": 2.4277,
"theoretical_loss": 3.3196948318763817,
"tokens_seen": 3012558848
},
{
"epoch": 0.07,
"learning_rate": 9.375534644995724e-05,
"loss": 2.4135,
"theoretical_loss": 3.319683736473476,
"tokens_seen": 3012689920
},
{
"epoch": 0.07,
"learning_rate": 9.37125748502994e-05,
"loss": 2.3474,
"theoretical_loss": 3.3196726416884395,
"tokens_seen": 3012820992
},
{
"epoch": 0.07,
"learning_rate": 9.366980325064158e-05,
"loss": 2.5489,
"theoretical_loss": 3.3196615475212106,
"tokens_seen": 3012952064
},
{
"epoch": 0.07,
"learning_rate": 9.362703165098376e-05,
"loss": 2.4998,
"theoretical_loss": 3.3196504539717284,
"tokens_seen": 3013083136
},
{
"epoch": 0.07,
"learning_rate": 9.358426005132592e-05,
"loss": 2.5438,
"theoretical_loss": 3.3196393610399317,
"tokens_seen": 3013214208
},
{
"epoch": 0.07,
"objective/train/docs_used": 1652881,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.3386740684509277,
"objective/train/theoretical_loss": 3.3196282687257583,
"objective/train/tokens_used": 43396576,
"theoretical_loss": 3.3196282687257583,
"tokens_seen": 3013345280
},
{
"epoch": 0.07,
"learning_rate": 9.35414884516681e-05,
"loss": 2.5351,
"theoretical_loss": 3.3196282687257583,
"tokens_seen": 3013345280
},
{
"epoch": 0.07,
"learning_rate": 9.349871685201027e-05,
"loss": 2.3127,
"theoretical_loss": 3.3196171770291483,
"tokens_seen": 3013476352
},
{
"epoch": 0.07,
"learning_rate": 9.345594525235244e-05,
"loss": 2.435,
"theoretical_loss": 3.3196060859500394,
"tokens_seen": 3013607424
},
{
"epoch": 0.08,
"learning_rate": 9.341317365269462e-05,
"loss": 2.3466,
"theoretical_loss": 3.319594995488371,
"tokens_seen": 3013738496
},
{
"epoch": 0.08,
"learning_rate": 9.337040205303679e-05,
"loss": 2.5683,
"theoretical_loss": 3.3195839056440812,
"tokens_seen": 3013869568
},
{
"epoch": 0.08,
"learning_rate": 9.332763045337895e-05,
"loss": 2.4838,
"theoretical_loss": 3.3195728164171094,
"tokens_seen": 3014000640
},
{
"epoch": 0.08,
"learning_rate": 9.328485885372113e-05,
"loss": 2.5372,
"theoretical_loss": 3.319561727807394,
"tokens_seen": 3014131712
},
{
"epoch": 0.08,
"learning_rate": 9.324208725406331e-05,
"loss": 2.4055,
"theoretical_loss": 3.3195506398148744,
"tokens_seen": 3014262784
},
{
"epoch": 0.08,
"learning_rate": 9.319931565440549e-05,
"loss": 2.5566,
"theoretical_loss": 3.319539552439489,
"tokens_seen": 3014393856
},
{
"epoch": 0.08,
"learning_rate": 9.315654405474765e-05,
"loss": 2.4051,
"theoretical_loss": 3.3195284656811763,
"tokens_seen": 3014524928
},
{
"epoch": 0.08,
"learning_rate": 9.311377245508982e-05,
"loss": 2.5032,
"theoretical_loss": 3.319517379539876,
"tokens_seen": 3014656000
},
{
"epoch": 0.08,
"learning_rate": 9.3071000855432e-05,
"loss": 2.427,
"theoretical_loss": 3.3195062940155258,
"tokens_seen": 3014787072
},
{
"epoch": 0.08,
"learning_rate": 9.302822925577417e-05,
"loss": 2.4468,
"theoretical_loss": 3.3194952091080654,
"tokens_seen": 3014918144
},
{
"epoch": 0.08,
"objective/train/docs_used": 1653310,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.2789435386657715,
"objective/train/theoretical_loss": 3.3194896668856497,
"objective/train/tokens_used": 45034976,
"theoretical_loss": 3.3194896668856497,
"tokens_seen": 3014983680
},
{
"epoch": 0.08,
"learning_rate": 9.298545765611635e-05,
"loss": 2.3447,
"theoretical_loss": 3.3194841248174334,
"tokens_seen": 3015049216
},
{
"epoch": 0.08,
"learning_rate": 9.294268605645852e-05,
"loss": 2.3587,
"theoretical_loss": 3.3194730411435684,
"tokens_seen": 3015180288
},
{
"epoch": 0.08,
"learning_rate": 9.289991445680068e-05,
"loss": 2.6264,
"theoretical_loss": 3.3194619580864098,
"tokens_seen": 3015311360
},
{
"epoch": 0.08,
"learning_rate": 9.285714285714286e-05,
"loss": 2.57,
"theoretical_loss": 3.3194508756458965,
"tokens_seen": 3015442432
},
{
"epoch": 0.08,
"learning_rate": 9.281437125748504e-05,
"loss": 2.3972,
"theoretical_loss": 3.319439793821967,
"tokens_seen": 3015573504
},
{
"epoch": 0.08,
"learning_rate": 9.27715996578272e-05,
"loss": 2.4522,
"theoretical_loss": 3.3194287126145596,
"tokens_seen": 3015704576
},
{
"epoch": 0.08,
"learning_rate": 9.272882805816938e-05,
"loss": 2.4546,
"theoretical_loss": 3.3194176320236144,
"tokens_seen": 3015835648
},
{
"epoch": 0.08,
"learning_rate": 9.268605645851154e-05,
"loss": 2.6088,
"theoretical_loss": 3.31940655204907,
"tokens_seen": 3015966720
},
{
"epoch": 0.08,
"learning_rate": 9.264328485885372e-05,
"loss": 2.4454,
"theoretical_loss": 3.319395472690865,
"tokens_seen": 3016097792
},
{
"epoch": 0.08,
"learning_rate": 9.26005132591959e-05,
"loss": 2.3876,
"theoretical_loss": 3.3193843939489382,
"tokens_seen": 3016228864
},
{
"epoch": 0.08,
"learning_rate": 9.255774165953807e-05,
"loss": 2.4971,
"theoretical_loss": 3.319373315823229,
"tokens_seen": 3016359936
},
{
"epoch": 0.08,
"learning_rate": 9.251497005988024e-05,
"loss": 2.5668,
"theoretical_loss": 3.3193622383136763,
"tokens_seen": 3016491008
},
{
"epoch": 0.08,
"objective/train/docs_used": 1654644,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.148563861846924,
"objective/train/theoretical_loss": 3.3193511614202187,
"objective/train/tokens_used": 46673376,
"theoretical_loss": 3.3193511614202187,
"tokens_seen": 3016622080
},
{
"epoch": 0.08,
"learning_rate": 9.247219846022241e-05,
"loss": 2.3435,
"theoretical_loss": 3.3193511614202187,
"tokens_seen": 3016622080
},
{
"epoch": 0.09,
"learning_rate": 9.242942686056459e-05,
"loss": 2.5671,
"theoretical_loss": 3.319340085142796,
"tokens_seen": 3016753152
},
{
"epoch": 0.09,
"learning_rate": 9.238665526090677e-05,
"loss": 2.455,
"theoretical_loss": 3.319329009481346,
"tokens_seen": 3016884224
},
{
"epoch": 0.09,
"learning_rate": 9.234388366124893e-05,
"loss": 2.5706,
"theoretical_loss": 3.3193179344358086,
"tokens_seen": 3017015296
},
{
"epoch": 0.09,
"learning_rate": 9.230111206159111e-05,
"loss": 2.3164,
"theoretical_loss": 3.319306860006122,
"tokens_seen": 3017146368
},
{
"epoch": 0.09,
"learning_rate": 9.225834046193327e-05,
"loss": 2.317,
"theoretical_loss": 3.319295786192226,
"tokens_seen": 3017277440
},
{
"epoch": 0.09,
"learning_rate": 9.221556886227547e-05,
"loss": 2.3955,
"theoretical_loss": 3.319284712994059,
"tokens_seen": 3017408512
},
{
"epoch": 0.09,
"learning_rate": 9.217279726261763e-05,
"loss": 2.4648,
"theoretical_loss": 3.3192736404115606,
"tokens_seen": 3017539584
},
{
"epoch": 0.09,
"learning_rate": 9.21300256629598e-05,
"loss": 2.3474,
"theoretical_loss": 3.3192625684446693,
"tokens_seen": 3017670656
},
{
"epoch": 0.09,
"learning_rate": 9.208725406330197e-05,
"loss": 2.2225,
"theoretical_loss": 3.3192514970933242,
"tokens_seen": 3017801728
},
{
"epoch": 0.09,
"learning_rate": 9.204448246364414e-05,
"loss": 2.4497,
"theoretical_loss": 3.319240426357465,
"tokens_seen": 3017932800
},
{
"epoch": 0.09,
"learning_rate": 9.200171086398632e-05,
"loss": 2.4471,
"theoretical_loss": 3.31922935623703,
"tokens_seen": 3018063872
},
{
"epoch": 0.09,
"learning_rate": 9.19589392643285e-05,
"loss": 2.3816,
"theoretical_loss": 3.3192182867319584,
"tokens_seen": 3018194944
},
{
"epoch": 0.09,
"objective/train/docs_used": 1655335,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.1034493446350098,
"objective/train/theoretical_loss": 3.319212752210165,
"objective/train/tokens_used": 48311776,
"theoretical_loss": 3.319212752210165,
"tokens_seen": 3018260480
},
{
"epoch": 0.09,
"learning_rate": 9.191616766467066e-05,
"loss": 2.3561,
"theoretical_loss": 3.3192072178421896,
"tokens_seen": 3018326016
},
{
"epoch": 0.09,
"learning_rate": 9.187339606501284e-05,
"loss": 2.4573,
"theoretical_loss": 3.319196149567662,
"tokens_seen": 3018457088
},
{
"epoch": 0.09,
"learning_rate": 9.1830624465355e-05,
"loss": 2.3837,
"theoretical_loss": 3.3191850819083157,
"tokens_seen": 3018588160
},
{
"epoch": 0.09,
"learning_rate": 9.178785286569718e-05,
"loss": 2.4923,
"theoretical_loss": 3.319174014864089,
"tokens_seen": 3018719232
},
{
"epoch": 0.09,
"learning_rate": 9.174508126603936e-05,
"loss": 2.6121,
"theoretical_loss": 3.319162948434921,
"tokens_seen": 3018850304
},
{
"epoch": 0.09,
"learning_rate": 9.170230966638152e-05,
"loss": 2.5103,
"theoretical_loss": 3.319151882620752,
"tokens_seen": 3018981376
},
{
"epoch": 0.09,
"learning_rate": 9.16595380667237e-05,
"loss": 2.3644,
"theoretical_loss": 3.3191408174215193,
"tokens_seen": 3019112448
},
{
"epoch": 0.09,
"learning_rate": 9.161676646706587e-05,
"loss": 2.5582,
"theoretical_loss": 3.3191297528371635,
"tokens_seen": 3019243520
},
{
"epoch": 0.09,
"learning_rate": 9.157399486740804e-05,
"loss": 2.3977,
"theoretical_loss": 3.319118688867623,
"tokens_seen": 3019374592
},
{
"epoch": 0.09,
"learning_rate": 9.153122326775022e-05,
"loss": 2.3253,
"theoretical_loss": 3.319107625512837,
"tokens_seen": 3019505664
},
{
"epoch": 0.09,
"learning_rate": 9.148845166809239e-05,
"loss": 2.5582,
"theoretical_loss": 3.3190965627727445,
"tokens_seen": 3019636736
},
{
"epoch": 0.09,
"learning_rate": 9.144568006843457e-05,
"loss": 2.402,
"theoretical_loss": 3.3190855006472857,
"tokens_seen": 3019767808
},
{
"epoch": 0.09,
"objective/train/docs_used": 1656670,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.027528762817383,
"objective/train/theoretical_loss": 3.3190744391363984,
"objective/train/tokens_used": 49950176,
"theoretical_loss": 3.3190744391363984,
"tokens_seen": 3019898880
},
{
"epoch": 0.1,
"learning_rate": 9.140290846877674e-05,
"loss": 2.4387,
"theoretical_loss": 3.3190744391363984,
"tokens_seen": 3019898880
},
{
"epoch": 0.1,
"learning_rate": 9.136013686911891e-05,
"loss": 2.5059,
"theoretical_loss": 3.3190633782400223,
"tokens_seen": 3020029952
},
{
"epoch": 0.1,
"learning_rate": 9.131736526946109e-05,
"loss": 2.3856,
"theoretical_loss": 3.3190523179580973,
"tokens_seen": 3020161024
},
{
"epoch": 0.1,
"learning_rate": 9.127459366980325e-05,
"loss": 2.5568,
"theoretical_loss": 3.3190412582905617,
"tokens_seen": 3020292096
},
{
"epoch": 0.1,
"learning_rate": 9.123182207014542e-05,
"loss": 2.5093,
"theoretical_loss": 3.319030199237355,
"tokens_seen": 3020423168
},
{
"epoch": 0.1,
"learning_rate": 9.118905047048761e-05,
"loss": 2.3476,
"theoretical_loss": 3.3190191407984164,
"tokens_seen": 3020554240
},
{
"epoch": 0.1,
"learning_rate": 9.114627887082977e-05,
"loss": 2.4233,
"theoretical_loss": 3.3190080829736854,
"tokens_seen": 3020685312
},
{
"epoch": 0.1,
"learning_rate": 9.110350727117195e-05,
"loss": 2.4302,
"theoretical_loss": 3.318997025763101,
"tokens_seen": 3020816384
},
{
"epoch": 0.1,
"learning_rate": 9.106073567151412e-05,
"loss": 2.5779,
"theoretical_loss": 3.318985969166602,
"tokens_seen": 3020947456
},
{
"epoch": 0.1,
"learning_rate": 9.101796407185628e-05,
"loss": 2.4611,
"theoretical_loss": 3.3189749131841286,
"tokens_seen": 3021078528
},
{
"epoch": 0.1,
"learning_rate": 9.097519247219847e-05,
"loss": 2.3387,
"theoretical_loss": 3.3189638578156195,
"tokens_seen": 3021209600
},
{
"epoch": 0.1,
"learning_rate": 9.093242087254064e-05,
"loss": 2.6853,
"theoretical_loss": 3.3189528030610136,
"tokens_seen": 3021340672
},
{
"epoch": 0.1,
"learning_rate": 9.088964927288282e-05,
"loss": 2.4887,
"theoretical_loss": 3.318941748920251,
"tokens_seen": 3021471744
},
{
"epoch": 0.1,
"objective/train/docs_used": 1657192,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.7123944759368896,
"objective/train/theoretical_loss": 3.318936222080042,
"objective/train/tokens_used": 51588576,
"theoretical_loss": 3.318936222080042,
"tokens_seen": 3021537280
},
{
"epoch": 0.1,
"learning_rate": 9.084687767322498e-05,
"loss": 2.4619,
"theoretical_loss": 3.318930695393271,
"tokens_seen": 3021602816
},
{
"epoch": 0.1,
"learning_rate": 9.080410607356715e-05,
"loss": 2.4821,
"theoretical_loss": 3.3189196424800116,
"tokens_seen": 3021733888
},
{
"epoch": 0.1,
"learning_rate": 9.076133447390934e-05,
"loss": 2.4471,
"theoretical_loss": 3.3189085901804134,
"tokens_seen": 3021864960
},
{
"epoch": 0.1,
"learning_rate": 9.07185628742515e-05,
"loss": 2.3988,
"theoretical_loss": 3.3188975384944155,
"tokens_seen": 3021996032
},
{
"epoch": 0.1,
"learning_rate": 9.067579127459367e-05,
"loss": 2.469,
"theoretical_loss": 3.318886487421957,
"tokens_seen": 3022127104
},
{
"epoch": 0.1,
"learning_rate": 9.063301967493585e-05,
"loss": 2.5398,
"theoretical_loss": 3.318875436962977,
"tokens_seen": 3022258176
},
{
"epoch": 0.1,
"learning_rate": 9.059024807527801e-05,
"loss": 2.3125,
"theoretical_loss": 3.3188643871174155,
"tokens_seen": 3022389248
},
{
"epoch": 0.1,
"learning_rate": 9.05474764756202e-05,
"loss": 2.3502,
"theoretical_loss": 3.318853337885211,
"tokens_seen": 3022520320
},
{
"epoch": 0.1,
"learning_rate": 9.050470487596237e-05,
"loss": 2.4073,
"theoretical_loss": 3.318842289266304,
"tokens_seen": 3022651392
},
{
"epoch": 0.1,
"learning_rate": 9.046193327630453e-05,
"loss": 2.3899,
"theoretical_loss": 3.3188312412606327,
"tokens_seen": 3022782464
},
{
"epoch": 0.1,
"learning_rate": 9.041916167664671e-05,
"loss": 2.3516,
"theoretical_loss": 3.3188201938681368,
"tokens_seen": 3022913536
},
{
"epoch": 0.11,
"learning_rate": 9.037639007698889e-05,
"loss": 2.3968,
"theoretical_loss": 3.318809147088756,
"tokens_seen": 3023044608
},
{
"epoch": 0.11,
"objective/train/docs_used": 1658380,
"objective/train/instantaneous_batch_size": 16,
"objective/train/instantaneous_microbatch_size": 16384,
"objective/train/original_loss": 2.6505117416381836,
"objective/train/theoretical_loss": 3.3187981009224297,
"objective/train/tokens_used": 53226976,
"theoretical_loss": 3.3187981009224297,
"tokens_seen": 3023175680
},
{
"epoch": 0.11,
"learning_rate": 9.033361847733107e-05,
"loss": 2.4709,
"theoretical_loss": 3.3187981009224297,
"tokens_seen": 3023175680
},
{
"epoch": 0.11,
"learning_rate": 9.029084687767323e-05,
"loss": 2.3951,
"theoretical_loss": 3.3187870553690972,
"tokens_seen": 3023306752
}
],
"max_steps": 2362,
"num_train_epochs": 9223372036854775807,
"total_flos": 1.6789580808192e+16,
"trial_name": null,
"trial_params": null
}