| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9984301412872841, | |
| "eval_steps": 500, | |
| "global_step": 159, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.006279434850863423, | |
| "grad_norm": 5.557445139643298, | |
| "learning_rate": 3.125e-08, | |
| "logits/chosen": 0.18015038967132568, | |
| "logits/rejected": 0.2519298493862152, | |
| "logps/chosen": -297.10906982421875, | |
| "logps/pi_response": -130.58929443359375, | |
| "logps/ref_response": -130.58929443359375, | |
| "logps/rejected": -316.44769287109375, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.06279434850863422, | |
| "grad_norm": 5.880023460230945, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logits/chosen": 0.16638809442520142, | |
| "logits/rejected": 0.3159521222114563, | |
| "logps/chosen": -243.77159118652344, | |
| "logps/pi_response": -120.18633270263672, | |
| "logps/ref_response": -120.15902709960938, | |
| "logps/rejected": -281.09716796875, | |
| "loss": 0.6928, | |
| "rewards/accuracies": 0.4444444477558136, | |
| "rewards/chosen": -0.0007160517852753401, | |
| "rewards/margins": 0.00040519109461456537, | |
| "rewards/rejected": -0.0011212429963052273, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.12558869701726844, | |
| "grad_norm": 6.011032264913819, | |
| "learning_rate": 4.990353313429303e-07, | |
| "logits/chosen": 0.1316775530576706, | |
| "logits/rejected": 0.32217010855674744, | |
| "logps/chosen": -244.0759735107422, | |
| "logps/pi_response": -121.6043701171875, | |
| "logps/ref_response": -121.85536193847656, | |
| "logps/rejected": -266.6847229003906, | |
| "loss": 0.6884, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.011644470505416393, | |
| "rewards/margins": 0.011915634386241436, | |
| "rewards/rejected": -0.02356010302901268, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.18838304552590268, | |
| "grad_norm": 6.296068063682766, | |
| "learning_rate": 4.882681251368548e-07, | |
| "logits/chosen": 0.17213600873947144, | |
| "logits/rejected": 0.3042981028556824, | |
| "logps/chosen": -244.4438018798828, | |
| "logps/pi_response": -109.73341369628906, | |
| "logps/ref_response": -110.8894271850586, | |
| "logps/rejected": -290.1441650390625, | |
| "loss": 0.6685, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.043184880167245865, | |
| "rewards/margins": 0.054663728922605515, | |
| "rewards/rejected": -0.09784860908985138, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.25117739403453687, | |
| "grad_norm": 5.762840753548806, | |
| "learning_rate": 4.6604720940421207e-07, | |
| "logits/chosen": 0.2107941210269928, | |
| "logits/rejected": 0.39838385581970215, | |
| "logps/chosen": -287.46002197265625, | |
| "logps/pi_response": -125.36665344238281, | |
| "logps/ref_response": -129.86325073242188, | |
| "logps/rejected": -316.40423583984375, | |
| "loss": 0.6349, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -0.08985555917024612, | |
| "rewards/margins": 0.1335289627313614, | |
| "rewards/rejected": -0.22338449954986572, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.3139717425431711, | |
| "grad_norm": 5.876442100853928, | |
| "learning_rate": 4.3344075855595097e-07, | |
| "logits/chosen": 0.3732234835624695, | |
| "logits/rejected": 0.5105798840522766, | |
| "logps/chosen": -247.15914916992188, | |
| "logps/pi_response": -109.07597351074219, | |
| "logps/ref_response": -116.5090560913086, | |
| "logps/rejected": -310.7102966308594, | |
| "loss": 0.606, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -0.11287806183099747, | |
| "rewards/margins": 0.24187734723091125, | |
| "rewards/rejected": -0.3547554314136505, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.37676609105180536, | |
| "grad_norm": 9.315356312481537, | |
| "learning_rate": 3.920161866827889e-07, | |
| "logits/chosen": 0.5162631273269653, | |
| "logits/rejected": 0.6798213124275208, | |
| "logps/chosen": -268.0201721191406, | |
| "logps/pi_response": -116.46971130371094, | |
| "logps/ref_response": -119.4989242553711, | |
| "logps/rejected": -347.75079345703125, | |
| "loss": 0.5814, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.20100387930870056, | |
| "rewards/margins": 0.41021671891212463, | |
| "rewards/rejected": -0.6112205386161804, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.43956043956043955, | |
| "grad_norm": 7.144146562974543, | |
| "learning_rate": 3.4376480090239047e-07, | |
| "logits/chosen": 0.6009117960929871, | |
| "logits/rejected": 0.767359733581543, | |
| "logps/chosen": -236.7908172607422, | |
| "logps/pi_response": -114.82955169677734, | |
| "logps/ref_response": -116.70068359375, | |
| "logps/rejected": -368.27850341796875, | |
| "loss": 0.5568, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.2643741965293884, | |
| "rewards/margins": 0.5534776449203491, | |
| "rewards/rejected": -0.8178518414497375, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.5023547880690737, | |
| "grad_norm": 7.105190568030611, | |
| "learning_rate": 2.910060778827554e-07, | |
| "logits/chosen": 0.47289925813674927, | |
| "logits/rejected": 0.8142975568771362, | |
| "logps/chosen": -325.69622802734375, | |
| "logps/pi_response": -129.21812438964844, | |
| "logps/ref_response": -127.53900146484375, | |
| "logps/rejected": -348.5531311035156, | |
| "loss": 0.5849, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.3693317174911499, | |
| "rewards/margins": 0.37722334265708923, | |
| "rewards/rejected": -0.7465550303459167, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.565149136577708, | |
| "grad_norm": 6.295924145315453, | |
| "learning_rate": 2.3627616503391812e-07, | |
| "logits/chosen": 0.43639689683914185, | |
| "logits/rejected": 0.8138043284416199, | |
| "logps/chosen": -301.4584045410156, | |
| "logps/pi_response": -135.66107177734375, | |
| "logps/ref_response": -129.38760375976562, | |
| "logps/rejected": -421.0269470214844, | |
| "loss": 0.5432, | |
| "rewards/accuracies": 0.7562500238418579, | |
| "rewards/chosen": -0.29649025201797485, | |
| "rewards/margins": 0.6172370910644531, | |
| "rewards/rejected": -0.9137271642684937, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.6279434850863422, | |
| "grad_norm": 7.528550263483296, | |
| "learning_rate": 1.8220596619089573e-07, | |
| "logits/chosen": 0.6732058525085449, | |
| "logits/rejected": 0.8459190130233765, | |
| "logps/chosen": -283.48883056640625, | |
| "logps/pi_response": -121.66845703125, | |
| "logps/ref_response": -114.0061264038086, | |
| "logps/rejected": -304.6356201171875, | |
| "loss": 0.5426, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -0.30830463767051697, | |
| "rewards/margins": 0.3349376320838928, | |
| "rewards/rejected": -0.6432422995567322, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.6907378335949764, | |
| "grad_norm": 7.109656663434607, | |
| "learning_rate": 1.3139467229135998e-07, | |
| "logits/chosen": 0.6394161581993103, | |
| "logits/rejected": 0.9435567855834961, | |
| "logps/chosen": -298.015869140625, | |
| "logps/pi_response": -136.70449829101562, | |
| "logps/ref_response": -125.8144760131836, | |
| "logps/rejected": -383.9209899902344, | |
| "loss": 0.5314, | |
| "rewards/accuracies": 0.7562500238418579, | |
| "rewards/chosen": -0.33838868141174316, | |
| "rewards/margins": 0.5101521015167236, | |
| "rewards/rejected": -0.8485407829284668, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.7535321821036107, | |
| "grad_norm": 7.158165163184529, | |
| "learning_rate": 8.628481651367875e-08, | |
| "logits/chosen": 0.5716279745101929, | |
| "logits/rejected": 0.8841035962104797, | |
| "logps/chosen": -320.6312255859375, | |
| "logps/pi_response": -131.89306640625, | |
| "logps/ref_response": -120.58707427978516, | |
| "logps/rejected": -369.5261535644531, | |
| "loss": 0.5541, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.4013099670410156, | |
| "rewards/margins": 0.5214625597000122, | |
| "rewards/rejected": -0.9227724075317383, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.8163265306122449, | |
| "grad_norm": 6.942114883986432, | |
| "learning_rate": 4.904486005914027e-08, | |
| "logits/chosen": 0.5996646881103516, | |
| "logits/rejected": 0.8270283937454224, | |
| "logps/chosen": -277.0096130371094, | |
| "logps/pi_response": -136.15554809570312, | |
| "logps/ref_response": -123.1449966430664, | |
| "logps/rejected": -372.5554504394531, | |
| "loss": 0.5271, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -0.3302404284477234, | |
| "rewards/margins": 0.5928428173065186, | |
| "rewards/rejected": -0.9230831861495972, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.8791208791208791, | |
| "grad_norm": 7.301927243964412, | |
| "learning_rate": 2.1464952759020856e-08, | |
| "logits/chosen": 0.5675501823425293, | |
| "logits/rejected": 0.8693594932556152, | |
| "logps/chosen": -284.0587158203125, | |
| "logps/pi_response": -131.0335235595703, | |
| "logps/ref_response": -121.63087463378906, | |
| "logps/rejected": -404.29278564453125, | |
| "loss": 0.53, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.3401293456554413, | |
| "rewards/margins": 0.748315155506134, | |
| "rewards/rejected": -1.088444471359253, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.9419152276295133, | |
| "grad_norm": 8.46328336009499, | |
| "learning_rate": 4.8708793644441086e-09, | |
| "logits/chosen": 0.5060345530509949, | |
| "logits/rejected": 0.8951950073242188, | |
| "logps/chosen": -317.99163818359375, | |
| "logps/pi_response": -145.62229919433594, | |
| "logps/ref_response": -132.86119079589844, | |
| "logps/rejected": -398.7874755859375, | |
| "loss": 0.5193, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -0.39088305830955505, | |
| "rewards/margins": 0.5848211050033569, | |
| "rewards/rejected": -0.9757040739059448, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.9984301412872841, | |
| "step": 159, | |
| "total_flos": 0.0, | |
| "train_loss": 0.5789602717513558, | |
| "train_runtime": 4365.2801, | |
| "train_samples_per_second": 4.668, | |
| "train_steps_per_second": 0.036 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 159, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |