|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 0.99581589958159, |
|
"eval_steps": 500, |
|
"global_step": 119, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 4.166666666666666e-08, |
|
"logits/chosen": -2.3755712509155273, |
|
"logits/rejected": -2.380011558532715, |
|
"logps/chosen": -396.8262023925781, |
|
"logps/pi_response": -147.47637939453125, |
|
"logps/ref_response": -147.47637939453125, |
|
"logps/rejected": -480.5025634765625, |
|
"loss": 0.6931, |
|
"rewards/accuracies": 0.0, |
|
"rewards/chosen": 0.0, |
|
"rewards/margins": 0.0, |
|
"rewards/rejected": 0.0, |
|
"step": 1 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 4.1666666666666667e-07, |
|
"logits/chosen": -2.306025266647339, |
|
"logits/rejected": -2.296987533569336, |
|
"logps/chosen": -434.2967529296875, |
|
"logps/pi_response": -168.16778564453125, |
|
"logps/ref_response": -168.15762329101562, |
|
"logps/rejected": -477.62066650390625, |
|
"loss": 0.6894, |
|
"rewards/accuracies": 0.5208333134651184, |
|
"rewards/chosen": -0.03276034817099571, |
|
"rewards/margins": 0.006516492459923029, |
|
"rewards/rejected": -0.0392768420279026, |
|
"step": 10 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 4.931352528237397e-07, |
|
"logits/chosen": -2.1829891204833984, |
|
"logits/rejected": -2.1276068687438965, |
|
"logps/chosen": -456.19000244140625, |
|
"logps/pi_response": -159.04637145996094, |
|
"logps/ref_response": -153.50245666503906, |
|
"logps/rejected": -582.487548828125, |
|
"loss": 0.6432, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -0.5500961542129517, |
|
"rewards/margins": 0.2686466574668884, |
|
"rewards/rejected": -0.8187428712844849, |
|
"step": 20 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 4.658920803689553e-07, |
|
"logits/chosen": -2.125579833984375, |
|
"logits/rejected": -2.0711355209350586, |
|
"logps/chosen": -478.7774353027344, |
|
"logps/pi_response": -221.3973846435547, |
|
"logps/ref_response": -205.2119903564453, |
|
"logps/rejected": -575.614990234375, |
|
"loss": 0.6322, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -0.5594707131385803, |
|
"rewards/margins": 0.3740437924861908, |
|
"rewards/rejected": -0.9335145950317383, |
|
"step": 30 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 4.201712553872657e-07, |
|
"logits/chosen": -2.1675801277160645, |
|
"logits/rejected": -2.1256229877471924, |
|
"logps/chosen": -491.529052734375, |
|
"logps/pi_response": -201.5789794921875, |
|
"logps/ref_response": -166.27740478515625, |
|
"logps/rejected": -564.7361450195312, |
|
"loss": 0.6135, |
|
"rewards/accuracies": 0.625, |
|
"rewards/chosen": -0.7727736830711365, |
|
"rewards/margins": 0.25983017683029175, |
|
"rewards/rejected": -1.0326038599014282, |
|
"step": 40 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 3.598859066780754e-07, |
|
"logits/chosen": -2.0738723278045654, |
|
"logits/rejected": -2.055086851119995, |
|
"logps/chosen": -498.4329528808594, |
|
"logps/pi_response": -215.8629913330078, |
|
"logps/ref_response": -181.7102508544922, |
|
"logps/rejected": -572.1842041015625, |
|
"loss": 0.5976, |
|
"rewards/accuracies": 0.6625000238418579, |
|
"rewards/chosen": -0.6373351812362671, |
|
"rewards/margins": 0.30541670322418213, |
|
"rewards/rejected": -0.9427520036697388, |
|
"step": 50 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 2.9019570347986706e-07, |
|
"logits/chosen": -2.0982589721679688, |
|
"logits/rejected": -2.0577781200408936, |
|
"logps/chosen": -498.26055908203125, |
|
"logps/pi_response": -217.1796875, |
|
"logps/ref_response": -177.24761962890625, |
|
"logps/rejected": -615.952880859375, |
|
"loss": 0.6182, |
|
"rewards/accuracies": 0.675000011920929, |
|
"rewards/chosen": -0.8855204582214355, |
|
"rewards/margins": 0.4653972089290619, |
|
"rewards/rejected": -1.3509178161621094, |
|
"step": 60 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 2.1706525253979534e-07, |
|
"logits/chosen": -2.0901355743408203, |
|
"logits/rejected": -2.017996072769165, |
|
"logps/chosen": -492.5106506347656, |
|
"logps/pi_response": -234.0646514892578, |
|
"logps/ref_response": -189.48513793945312, |
|
"logps/rejected": -597.0025024414062, |
|
"loss": 0.5864, |
|
"rewards/accuracies": 0.75, |
|
"rewards/chosen": -0.46928316354751587, |
|
"rewards/margins": 0.5200775265693665, |
|
"rewards/rejected": -0.9893606305122375, |
|
"step": 70 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 1.4675360263490295e-07, |
|
"logits/chosen": -1.9807932376861572, |
|
"logits/rejected": -1.936804175376892, |
|
"logps/chosen": -479.35595703125, |
|
"logps/pi_response": -225.93392944335938, |
|
"logps/ref_response": -176.03411865234375, |
|
"logps/rejected": -591.875244140625, |
|
"loss": 0.5799, |
|
"rewards/accuracies": 0.65625, |
|
"rewards/chosen": -0.7031043171882629, |
|
"rewards/margins": 0.40889111161231995, |
|
"rewards/rejected": -1.1119954586029053, |
|
"step": 80 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 8.527854855097224e-08, |
|
"logits/chosen": -1.8924801349639893, |
|
"logits/rejected": -1.8593276739120483, |
|
"logps/chosen": -575.15185546875, |
|
"logps/pi_response": -248.3000946044922, |
|
"logps/ref_response": -182.91275024414062, |
|
"logps/rejected": -703.632080078125, |
|
"loss": 0.5781, |
|
"rewards/accuracies": 0.699999988079071, |
|
"rewards/chosen": -1.2331480979919434, |
|
"rewards/margins": 0.5850399136543274, |
|
"rewards/rejected": -1.8181877136230469, |
|
"step": 90 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 3.790158337517127e-08, |
|
"logits/chosen": -1.9359309673309326, |
|
"logits/rejected": -1.9223928451538086, |
|
"logps/chosen": -529.174072265625, |
|
"logps/pi_response": -227.7006378173828, |
|
"logps/ref_response": -164.98428344726562, |
|
"logps/rejected": -648.9022216796875, |
|
"loss": 0.5683, |
|
"rewards/accuracies": 0.7250000238418579, |
|
"rewards/chosen": -1.045077919960022, |
|
"rewards/margins": 0.5439373850822449, |
|
"rewards/rejected": -1.589015245437622, |
|
"step": 100 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 8.677580722139671e-09, |
|
"logits/chosen": -1.8646703958511353, |
|
"logits/rejected": -1.8161742687225342, |
|
"logps/chosen": -512.7567138671875, |
|
"logps/pi_response": -230.37588500976562, |
|
"logps/ref_response": -167.1942138671875, |
|
"logps/rejected": -604.599609375, |
|
"loss": 0.5676, |
|
"rewards/accuracies": 0.706250011920929, |
|
"rewards/chosen": -1.034223198890686, |
|
"rewards/margins": 0.4625032842159271, |
|
"rewards/rejected": -1.496726632118225, |
|
"step": 110 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"step": 119, |
|
"total_flos": 0.0, |
|
"train_loss": 0.6084108673223928, |
|
"train_runtime": 3108.4145, |
|
"train_samples_per_second": 4.917, |
|
"train_steps_per_second": 0.038 |
|
} |
|
], |
|
"logging_steps": 10, |
|
"max_steps": 119, |
|
"num_input_tokens_seen": 0, |
|
"num_train_epochs": 1, |
|
"save_steps": 100, |
|
"total_flos": 0.0, |
|
"train_batch_size": 8, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|