| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.998272884283247, | |
| "eval_steps": 100, | |
| "global_step": 289, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 0.2451171875, | |
| "learning_rate": 1.7241379310344828e-07, | |
| "logits/chosen": -2.946108341217041, | |
| "logits/rejected": -2.9481201171875, | |
| "logps/chosen": -27.542381286621094, | |
| "logps/rejected": -28.998493194580078, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 0.2431640625, | |
| "learning_rate": 1.724137931034483e-06, | |
| "logits/chosen": -2.967113494873047, | |
| "logits/rejected": -2.9693994522094727, | |
| "logps/chosen": -27.679420471191406, | |
| "logps/rejected": -30.345211029052734, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.4166666567325592, | |
| "rewards/chosen": -0.00010507476690690964, | |
| "rewards/margins": 6.562391354236752e-05, | |
| "rewards/rejected": -0.0001706986950011924, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 0.259765625, | |
| "learning_rate": 3.448275862068966e-06, | |
| "logits/chosen": -2.9687092304229736, | |
| "logits/rejected": -2.9725170135498047, | |
| "logps/chosen": -27.916067123413086, | |
| "logps/rejected": -32.21611404418945, | |
| "loss": 0.6923, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.0016102585941553116, | |
| "rewards/margins": 0.0016238940879702568, | |
| "rewards/rejected": -1.3635644791065715e-05, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 0.263671875, | |
| "learning_rate": 4.999817502139027e-06, | |
| "logits/chosen": -2.991443157196045, | |
| "logits/rejected": -2.992274761199951, | |
| "logps/chosen": -27.992717742919922, | |
| "logps/rejected": -30.998449325561523, | |
| "loss": 0.6894, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": 0.010343936271965504, | |
| "rewards/margins": 0.007500241510570049, | |
| "rewards/rejected": 0.002843693597242236, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 0.359375, | |
| "learning_rate": 4.977949980164773e-06, | |
| "logits/chosen": -2.9539809226989746, | |
| "logits/rejected": -2.9519870281219482, | |
| "logps/chosen": -26.01015853881836, | |
| "logps/rejected": -30.401302337646484, | |
| "loss": 0.6829, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.03149240463972092, | |
| "rewards/margins": 0.02063887007534504, | |
| "rewards/rejected": 0.010853536427021027, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 0.34375, | |
| "learning_rate": 4.919948367622307e-06, | |
| "logits/chosen": -2.983638048171997, | |
| "logits/rejected": -2.989013195037842, | |
| "logps/chosen": -22.443767547607422, | |
| "logps/rejected": -31.806631088256836, | |
| "loss": 0.6664, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.06118880584836006, | |
| "rewards/margins": 0.05425649881362915, | |
| "rewards/rejected": 0.006932304706424475, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 0.484375, | |
| "learning_rate": 4.8266584586307555e-06, | |
| "logits/chosen": -2.988079786300659, | |
| "logits/rejected": -2.9914956092834473, | |
| "logps/chosen": -19.77129364013672, | |
| "logps/rejected": -29.180835723876953, | |
| "loss": 0.6535, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": 0.08603720366954803, | |
| "rewards/margins": 0.08121357858181, | |
| "rewards/rejected": 0.004823616705834866, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 0.8515625, | |
| "learning_rate": 4.699440630133794e-06, | |
| "logits/chosen": -2.995100736618042, | |
| "logits/rejected": -2.99615216255188, | |
| "logps/chosen": -17.9371395111084, | |
| "logps/rejected": -34.77935028076172, | |
| "loss": 0.6236, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.11037828773260117, | |
| "rewards/margins": 0.14534664154052734, | |
| "rewards/rejected": -0.03496835380792618, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "grad_norm": 0.84765625, | |
| "learning_rate": 4.5401500045405126e-06, | |
| "logits/chosen": -2.9611220359802246, | |
| "logits/rejected": -2.963106393814087, | |
| "logps/chosen": -11.03026008605957, | |
| "logps/rejected": -39.37724304199219, | |
| "loss": 0.5669, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.18076485395431519, | |
| "rewards/margins": 0.27246183156967163, | |
| "rewards/rejected": -0.09169697761535645, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "grad_norm": 0.59765625, | |
| "learning_rate": 4.351109397863309e-06, | |
| "logits/chosen": -2.970407009124756, | |
| "logits/rejected": -2.9756150245666504, | |
| "logps/chosen": -6.9094672203063965, | |
| "logps/rejected": -49.88671112060547, | |
| "loss": 0.5168, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.20805075764656067, | |
| "rewards/margins": 0.3929779827594757, | |
| "rewards/rejected": -0.18492721021175385, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "grad_norm": 0.828125, | |
| "learning_rate": 4.135075447829912e-06, | |
| "logits/chosen": -2.966463327407837, | |
| "logits/rejected": -2.973740816116333, | |
| "logps/chosen": -5.979574203491211, | |
| "logps/rejected": -56.75745391845703, | |
| "loss": 0.4825, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.23063524067401886, | |
| "rewards/margins": 0.4814692437648773, | |
| "rewards/rejected": -0.25083404779434204, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "eval_logits/chosen": -2.971499443054199, | |
| "eval_logits/rejected": -2.975092887878418, | |
| "eval_logps/chosen": -4.676602840423584, | |
| "eval_logps/rejected": -60.5069694519043, | |
| "eval_loss": 0.46322527527809143, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": 0.2303335815668106, | |
| "eval_rewards/margins": 0.5325379371643066, | |
| "eval_rewards/rejected": -0.3022043704986572, | |
| "eval_runtime": 59.0284, | |
| "eval_samples_per_second": 4.896, | |
| "eval_steps_per_second": 2.456, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "grad_norm": 2.15625, | |
| "learning_rate": 3.895198415897896e-06, | |
| "logits/chosen": -2.944152355194092, | |
| "logits/rejected": -2.948521614074707, | |
| "logps/chosen": -4.758464813232422, | |
| "logps/rejected": -65.92622375488281, | |
| "loss": 0.4433, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.23539750277996063, | |
| "rewards/margins": 0.5871593356132507, | |
| "rewards/rejected": -0.3517618179321289, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "grad_norm": 0.515625, | |
| "learning_rate": 3.634976249348867e-06, | |
| "logits/chosen": -2.956526279449463, | |
| "logits/rejected": -2.9596874713897705, | |
| "logps/chosen": -3.4839115142822266, | |
| "logps/rejected": -74.23634338378906, | |
| "loss": 0.4105, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2543177604675293, | |
| "rewards/margins": 0.6815550923347473, | |
| "rewards/rejected": -0.427237331867218, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "grad_norm": 0.65625, | |
| "learning_rate": 3.3582035733403963e-06, | |
| "logits/chosen": -2.9395928382873535, | |
| "logits/rejected": -2.9400253295898438, | |
| "logps/chosen": -3.3433098793029785, | |
| "logps/rejected": -81.93199920654297, | |
| "loss": 0.384, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.25131431221961975, | |
| "rewards/margins": 0.7623029947280884, | |
| "rewards/rejected": -0.5109885931015015, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 0.56640625, | |
| "learning_rate": 3.068916356726475e-06, | |
| "logits/chosen": -2.9392318725585938, | |
| "logits/rejected": -2.939736843109131, | |
| "logps/chosen": -2.7690482139587402, | |
| "logps/rejected": -91.77839660644531, | |
| "loss": 0.3548, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.24845953285694122, | |
| "rewards/margins": 0.8564491271972656, | |
| "rewards/rejected": -0.607989490032196, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "grad_norm": 0.52734375, | |
| "learning_rate": 2.771333058543416e-06, | |
| "logits/chosen": -2.9237265586853027, | |
| "logits/rejected": -2.9198527336120605, | |
| "logps/chosen": -3.30281138420105, | |
| "logps/rejected": -96.4010009765625, | |
| "loss": 0.3403, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2496950924396515, | |
| "rewards/margins": 0.9052786827087402, | |
| "rewards/rejected": -0.6555835604667664, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "grad_norm": 0.50390625, | |
| "learning_rate": 2.4697931133779566e-06, | |
| "logits/chosen": -2.9088094234466553, | |
| "logits/rejected": -2.9039511680603027, | |
| "logps/chosen": -3.719200849533081, | |
| "logps/rejected": -103.43408203125, | |
| "loss": 0.3218, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.24762046337127686, | |
| "rewards/margins": 0.9735357165336609, | |
| "rewards/rejected": -0.7259151935577393, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "grad_norm": 0.96875, | |
| "learning_rate": 2.168693652639432e-06, | |
| "logits/chosen": -2.9292285442352295, | |
| "logits/rejected": -2.9254345893859863, | |
| "logps/chosen": -1.5337612628936768, | |
| "logps/rejected": -105.90274810791016, | |
| "loss": 0.3082, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2660350203514099, | |
| "rewards/margins": 1.021663784980774, | |
| "rewards/rejected": -0.7556287050247192, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "grad_norm": 0.5546875, | |
| "learning_rate": 1.872425384482346e-06, | |
| "logits/chosen": -2.9200375080108643, | |
| "logits/rejected": -2.9096457958221436, | |
| "logps/chosen": -1.4461075067520142, | |
| "logps/rejected": -111.00732421875, | |
| "loss": 0.2964, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2637704610824585, | |
| "rewards/margins": 1.0691814422607422, | |
| "rewards/rejected": -0.8054108619689941, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "grad_norm": 2.6875, | |
| "learning_rate": 1.5853085673944695e-06, | |
| "logits/chosen": -2.9019381999969482, | |
| "logits/rejected": -2.889967441558838, | |
| "logps/chosen": -3.121753692626953, | |
| "logps/rejected": -114.66499328613281, | |
| "loss": 0.2975, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.24539904296398163, | |
| "rewards/margins": 1.0781965255737305, | |
| "rewards/rejected": -0.8327974081039429, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "grad_norm": 1.0859375, | |
| "learning_rate": 1.3115300110997097e-06, | |
| "logits/chosen": -2.867145538330078, | |
| "logits/rejected": -2.857390880584717, | |
| "logps/chosen": -4.095256805419922, | |
| "logps/rejected": -123.1200180053711, | |
| "loss": 0.2786, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.24890461564064026, | |
| "rewards/margins": 1.1587778329849243, | |
| "rewards/rejected": -0.9098731875419617, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "eval_logits/chosen": -2.8807153701782227, | |
| "eval_logits/rejected": -2.8679380416870117, | |
| "eval_logps/chosen": -6.8403167724609375, | |
| "eval_logps/rejected": -132.28236389160156, | |
| "eval_loss": 0.2587815523147583, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": 0.2086964100599289, | |
| "eval_rewards/margins": 1.2286547422409058, | |
| "eval_rewards/rejected": -1.019958257675171, | |
| "eval_runtime": 59.0007, | |
| "eval_samples_per_second": 4.898, | |
| "eval_steps_per_second": 2.458, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "grad_norm": 1.0390625, | |
| "learning_rate": 1.0550820234444627e-06, | |
| "logits/chosen": -2.8844265937805176, | |
| "logits/rejected": -2.8720266819000244, | |
| "logps/chosen": -7.328047752380371, | |
| "logps/rejected": -136.2236785888672, | |
| "loss": 0.2536, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2088628113269806, | |
| "rewards/margins": 1.2624285221099854, | |
| "rewards/rejected": -1.0535659790039062, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "grad_norm": 1.5, | |
| "learning_rate": 8.197041935593181e-07, | |
| "logits/chosen": -2.865748643875122, | |
| "logits/rejected": -2.8470582962036133, | |
| "logps/chosen": -3.5205326080322266, | |
| "logps/rejected": -143.3411865234375, | |
| "loss": 0.2291, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2387482225894928, | |
| "rewards/margins": 1.365382432937622, | |
| "rewards/rejected": -1.126634120941162, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "grad_norm": 0.97265625, | |
| "learning_rate": 6.088288602287159e-07, | |
| "logits/chosen": -2.8658416271209717, | |
| "logits/rejected": -2.8466126918792725, | |
| "logps/chosen": -1.298507809638977, | |
| "logps/rejected": -147.50143432617188, | |
| "loss": 0.2162, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2748534083366394, | |
| "rewards/margins": 1.4337130784988403, | |
| "rewards/rejected": -1.1588596105575562, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "grad_norm": 1.34375, | |
| "learning_rate": 4.255310606625124e-07, | |
| "logits/chosen": -2.8704545497894287, | |
| "logits/rejected": -2.8518197536468506, | |
| "logps/chosen": -2.5422964096069336, | |
| "logps/rejected": -151.9074249267578, | |
| "loss": 0.2081, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.25674667954444885, | |
| "rewards/margins": 1.4753159284591675, | |
| "rewards/rejected": -1.2185693979263306, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "grad_norm": 0.9921875, | |
| "learning_rate": 2.7248368952908055e-07, | |
| "logits/chosen": -2.861412525177002, | |
| "logits/rejected": -2.838165044784546, | |
| "logps/chosen": -2.5676398277282715, | |
| "logps/rejected": -152.24969482421875, | |
| "loss": 0.2105, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2601901888847351, | |
| "rewards/margins": 1.4616148471832275, | |
| "rewards/rejected": -1.2014245986938477, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "grad_norm": 0.88671875, | |
| "learning_rate": 1.5191852213221198e-07, | |
| "logits/chosen": -2.8463876247406006, | |
| "logits/rejected": -2.817927837371826, | |
| "logps/chosen": -3.504025936126709, | |
| "logps/rejected": -156.09738159179688, | |
| "loss": 0.2054, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2497003823518753, | |
| "rewards/margins": 1.4921144247055054, | |
| "rewards/rejected": -1.2424139976501465, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "grad_norm": 0.921875, | |
| "learning_rate": 6.559367010166629e-08, | |
| "logits/chosen": -2.8501267433166504, | |
| "logits/rejected": -2.8306031227111816, | |
| "logps/chosen": -2.110583782196045, | |
| "logps/rejected": -152.65463256835938, | |
| "loss": 0.2113, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.26196378469467163, | |
| "rewards/margins": 1.4719316959381104, | |
| "rewards/rejected": -1.209967851638794, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "grad_norm": 1.453125, | |
| "learning_rate": 1.4767944166687032e-08, | |
| "logits/chosen": -2.8537425994873047, | |
| "logits/rejected": -2.827571392059326, | |
| "logps/chosen": -2.493337869644165, | |
| "logps/rejected": -154.96142578125, | |
| "loss": 0.2011, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.2604244649410248, | |
| "rewards/margins": 1.5092506408691406, | |
| "rewards/rejected": -1.2488261461257935, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 289, | |
| "total_flos": 0.0, | |
| "train_loss": 0.4018066387275511, | |
| "train_runtime": 624.4981, | |
| "train_samples_per_second": 1.853, | |
| "train_steps_per_second": 0.463 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 289, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 200, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |