| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9945, |
| "eval_steps": 500, |
| "global_step": 153, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.01, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 8.605272045068777, |
| "learning_rate": 3.125e-08, |
| "logits/chosen": -2.8784992694854736, |
| "logits/rejected": -2.8769874572753906, |
| "logps/chosen": -263.9749755859375, |
| "logps/pi_response": -246.19029235839844, |
| "logps/ref_response": -246.19029235839844, |
| "logps/rejected": -308.2843322753906, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.07, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 8.688961504116353, |
| "learning_rate": 3.1249999999999997e-07, |
| "logits/chosen": -2.936194896697998, |
| "logits/rejected": -2.808932304382324, |
| "logps/chosen": -315.6687927246094, |
| "logps/pi_response": -209.20472717285156, |
| "logps/ref_response": -209.1347198486328, |
| "logps/rejected": -260.7985534667969, |
| "loss": 0.6928, |
| "rewards/accuracies": 0.4829059839248657, |
| "rewards/chosen": 0.00021380360703915358, |
| "rewards/margins": 0.0008379952632822096, |
| "rewards/rejected": -0.0006241916562430561, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.13, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 8.833821950128216, |
| "learning_rate": 4.989490450759331e-07, |
| "logits/chosen": -2.852677583694458, |
| "logits/rejected": -2.8317201137542725, |
| "logps/chosen": -274.80267333984375, |
| "logps/pi_response": -189.35801696777344, |
| "logps/ref_response": -187.89822387695312, |
| "logps/rejected": -261.1772766113281, |
| "loss": 0.6876, |
| "rewards/accuracies": 0.6846153736114502, |
| "rewards/chosen": -0.006662360858172178, |
| "rewards/margins": 0.012640128843486309, |
| "rewards/rejected": -0.0193024892359972, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.2, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 9.337463091325768, |
| "learning_rate": 4.872270441827174e-07, |
| "logits/chosen": -2.8088669776916504, |
| "logits/rejected": -2.791938304901123, |
| "logps/chosen": -275.8798828125, |
| "logps/pi_response": -215.20196533203125, |
| "logps/ref_response": -213.5146484375, |
| "logps/rejected": -255.837890625, |
| "loss": 0.6691, |
| "rewards/accuracies": 0.7153846025466919, |
| "rewards/chosen": 0.021489957347512245, |
| "rewards/margins": 0.061311714351177216, |
| "rewards/rejected": -0.03982176259160042, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.26, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 12.278601194362231, |
| "learning_rate": 4.6308512113530063e-07, |
| "logits/chosen": -2.7728219032287598, |
| "logits/rejected": -2.690376043319702, |
| "logps/chosen": -278.7479553222656, |
| "logps/pi_response": -190.5654296875, |
| "logps/ref_response": -177.33053588867188, |
| "logps/rejected": -246.11264038085938, |
| "loss": 0.6443, |
| "rewards/accuracies": 0.6730769276618958, |
| "rewards/chosen": -0.05301598832011223, |
| "rewards/margins": 0.10135015100240707, |
| "rewards/rejected": -0.154366135597229, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.33, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 15.472310357744927, |
| "learning_rate": 4.277872161641681e-07, |
| "logits/chosen": -2.792513608932495, |
| "logits/rejected": -2.7258100509643555, |
| "logps/chosen": -291.08642578125, |
| "logps/pi_response": -217.5293426513672, |
| "logps/ref_response": -194.07823181152344, |
| "logps/rejected": -272.1592102050781, |
| "loss": 0.619, |
| "rewards/accuracies": 0.6692307591438293, |
| "rewards/chosen": -0.05969160422682762, |
| "rewards/margins": 0.2206883430480957, |
| "rewards/rejected": -0.2803799510002136, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.39, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 12.018657879137255, |
| "learning_rate": 3.8318133624280046e-07, |
| "logits/chosen": -2.7461166381835938, |
| "logits/rejected": -2.6338207721710205, |
| "logps/chosen": -268.39324951171875, |
| "logps/pi_response": -218.18861389160156, |
| "logps/ref_response": -193.3256072998047, |
| "logps/rejected": -277.92572021484375, |
| "loss": 0.611, |
| "rewards/accuracies": 0.7153846025466919, |
| "rewards/chosen": -0.13407574594020844, |
| "rewards/margins": 0.21902315318584442, |
| "rewards/rejected": -0.35309889912605286, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.46, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 12.586943156361984, |
| "learning_rate": 3.316028034595861e-07, |
| "logits/chosen": -2.702092170715332, |
| "logits/rejected": -2.648845672607422, |
| "logps/chosen": -275.9683532714844, |
| "logps/pi_response": -199.25994873046875, |
| "logps/ref_response": -183.3825225830078, |
| "logps/rejected": -281.8118896484375, |
| "loss": 0.6125, |
| "rewards/accuracies": 0.6692307591438293, |
| "rewards/chosen": -0.07608187198638916, |
| "rewards/margins": 0.22089019417762756, |
| "rewards/rejected": -0.2969720661640167, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.52, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 16.37677218967305, |
| "learning_rate": 2.7575199021178855e-07, |
| "logits/chosen": -2.654991388320923, |
| "logits/rejected": -2.581737756729126, |
| "logps/chosen": -314.16900634765625, |
| "logps/pi_response": -250.20211791992188, |
| "logps/ref_response": -203.31488037109375, |
| "logps/rejected": -308.69287109375, |
| "loss": 0.5967, |
| "rewards/accuracies": 0.6730769276618958, |
| "rewards/chosen": -0.329426109790802, |
| "rewards/margins": 0.27185821533203125, |
| "rewards/rejected": -0.6012843251228333, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.58, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 22.869287847796812, |
| "learning_rate": 2.1855294234408068e-07, |
| "logits/chosen": -2.4663641452789307, |
| "logits/rejected": -2.1920886039733887, |
| "logps/chosen": -374.7882385253906, |
| "logps/pi_response": -300.48028564453125, |
| "logps/ref_response": -229.24087524414062, |
| "logps/rejected": -370.0035400390625, |
| "loss": 0.573, |
| "rewards/accuracies": 0.7153846025466919, |
| "rewards/chosen": -0.42087164521217346, |
| "rewards/margins": 0.46370625495910645, |
| "rewards/rejected": -0.8845779299736023, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.65, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 22.10929439369714, |
| "learning_rate": 1.6300029195778453e-07, |
| "logits/chosen": -2.2815823554992676, |
| "logits/rejected": -1.9420466423034668, |
| "logps/chosen": -328.23388671875, |
| "logps/pi_response": -285.6993408203125, |
| "logps/ref_response": -202.154541015625, |
| "logps/rejected": -346.80718994140625, |
| "loss": 0.5648, |
| "rewards/accuracies": 0.6653845906257629, |
| "rewards/chosen": -0.5251672863960266, |
| "rewards/margins": 0.47274622321128845, |
| "rewards/rejected": -0.9979135394096375, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.71, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 20.95321740793465, |
| "learning_rate": 1.1200247470632392e-07, |
| "logits/chosen": -2.211641311645508, |
| "logits/rejected": -1.855459451675415, |
| "logps/chosen": -360.8876953125, |
| "logps/pi_response": -303.0977783203125, |
| "logps/ref_response": -215.0885009765625, |
| "logps/rejected": -370.98193359375, |
| "loss": 0.563, |
| "rewards/accuracies": 0.7423076629638672, |
| "rewards/chosen": -0.47680747509002686, |
| "rewards/margins": 0.5583351850509644, |
| "rewards/rejected": -1.0351426601409912, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.78, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 21.53917118957897, |
| "learning_rate": 6.822945986946385e-08, |
| "logits/chosen": -1.8491864204406738, |
| "logits/rejected": -1.6956101655960083, |
| "logps/chosen": -344.0650939941406, |
| "logps/pi_response": -307.6352844238281, |
| "logps/ref_response": -204.07801818847656, |
| "logps/rejected": -390.5289001464844, |
| "loss": 0.5501, |
| "rewards/accuracies": 0.6884615421295166, |
| "rewards/chosen": -0.6317132711410522, |
| "rewards/margins": 0.5379453301429749, |
| "rewards/rejected": -1.1696586608886719, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.84, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 27.579199953433918, |
| "learning_rate": 3.397296523427806e-08, |
| "logits/chosen": -1.9508247375488281, |
| "logits/rejected": -1.6159588098526, |
| "logps/chosen": -333.64599609375, |
| "logps/pi_response": -301.04547119140625, |
| "logps/ref_response": -194.1094207763672, |
| "logps/rejected": -385.6200256347656, |
| "loss": 0.5332, |
| "rewards/accuracies": 0.7038461565971375, |
| "rewards/chosen": -0.6621810793876648, |
| "rewards/margins": 0.6021292805671692, |
| "rewards/rejected": -1.264310359954834, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.91, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 26.751074987142594, |
| "learning_rate": 1.1026475173977978e-08, |
| "logits/chosen": -1.9854283332824707, |
| "logits/rejected": -1.720418930053711, |
| "logps/chosen": -328.14459228515625, |
| "logps/pi_response": -306.83367919921875, |
| "logps/ref_response": -197.67745971679688, |
| "logps/rejected": -386.437255859375, |
| "loss": 0.5515, |
| "rewards/accuracies": 0.7038461565971375, |
| "rewards/chosen": -0.6694343686103821, |
| "rewards/margins": 0.5615480542182922, |
| "rewards/rejected": -1.2309825420379639, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.97, |
| "eta": 0.0010000000474974513, |
| "grad_norm": 27.667206944684594, |
| "learning_rate": 5.913435276374834e-10, |
| "logits/chosen": -1.9676626920700073, |
| "logits/rejected": -1.6368684768676758, |
| "logps/chosen": -339.4317321777344, |
| "logps/pi_response": -303.0911560058594, |
| "logps/ref_response": -192.59991455078125, |
| "logps/rejected": -375.9079895019531, |
| "loss": 0.5599, |
| "rewards/accuracies": 0.7192307710647583, |
| "rewards/chosen": -0.6414641737937927, |
| "rewards/margins": 0.5702866315841675, |
| "rewards/rejected": -1.2117507457733154, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.99, |
| "step": 153, |
| "total_flos": 0.0, |
| "train_loss": 0.5998621676482406, |
| "train_runtime": 41019.2972, |
| "train_samples_per_second": 0.488, |
| "train_steps_per_second": 0.004 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 153, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 100, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
