|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 1.0, |
|
"eval_steps": 500, |
|
"global_step": 80, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 6.25e-08, |
|
"logits/chosen": -2.6214611530303955, |
|
"logits/rejected": -2.5660409927368164, |
|
"logps/chosen": -188.08230590820312, |
|
"logps/pi_response": -124.86982727050781, |
|
"logps/ref_response": -124.86982727050781, |
|
"logps/rejected": -255.6346435546875, |
|
"loss": 0.6931, |
|
"rewards/accuracies": 0.0, |
|
"rewards/chosen": 0.0, |
|
"rewards/margins": 0.0, |
|
"rewards/rejected": 0.0, |
|
"step": 1 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 4.990486745229364e-07, |
|
"logits/chosen": -2.51465106010437, |
|
"logits/rejected": -2.499199867248535, |
|
"logps/chosen": -234.35617065429688, |
|
"logps/pi_response": -144.1409912109375, |
|
"logps/ref_response": -142.20367431640625, |
|
"logps/rejected": -258.1360778808594, |
|
"loss": 0.6869, |
|
"rewards/accuracies": 0.4791666567325592, |
|
"rewards/chosen": -0.029688755050301552, |
|
"rewards/margins": 0.012110848911106586, |
|
"rewards/rejected": -0.04179960489273071, |
|
"step": 10 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 4.6650635094610966e-07, |
|
"logits/chosen": -2.4308013916015625, |
|
"logits/rejected": -2.438734769821167, |
|
"logps/chosen": -228.0776824951172, |
|
"logps/pi_response": -156.75161743164062, |
|
"logps/ref_response": -143.56219482421875, |
|
"logps/rejected": -305.380615234375, |
|
"loss": 0.6582, |
|
"rewards/accuracies": 0.59375, |
|
"rewards/chosen": -0.21737465262413025, |
|
"rewards/margins": 0.20173434913158417, |
|
"rewards/rejected": -0.4191090166568756, |
|
"step": 20 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 3.933941090877615e-07, |
|
"logits/chosen": -2.440542459487915, |
|
"logits/rejected": -2.3943440914154053, |
|
"logps/chosen": -224.8809051513672, |
|
"logps/pi_response": -155.57861328125, |
|
"logps/ref_response": -143.30210876464844, |
|
"logps/rejected": -315.95989990234375, |
|
"loss": 0.6526, |
|
"rewards/accuracies": 0.581250011920929, |
|
"rewards/chosen": -0.2337447851896286, |
|
"rewards/margins": 0.252723753452301, |
|
"rewards/rejected": -0.48646849393844604, |
|
"step": 30 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 2.934120444167326e-07, |
|
"logits/chosen": -2.3429083824157715, |
|
"logits/rejected": -2.3265538215637207, |
|
"logps/chosen": -307.50848388671875, |
|
"logps/pi_response": -218.9744110107422, |
|
"logps/ref_response": -160.5868377685547, |
|
"logps/rejected": -345.1627502441406, |
|
"loss": 0.6284, |
|
"rewards/accuracies": 0.6187499761581421, |
|
"rewards/chosen": -0.6049060821533203, |
|
"rewards/margins": 0.14748108386993408, |
|
"rewards/rejected": -0.7523871660232544, |
|
"step": 40 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 1.8529523872436977e-07, |
|
"logits/chosen": -2.3797526359558105, |
|
"logits/rejected": -2.336573839187622, |
|
"logps/chosen": -271.7038269042969, |
|
"logps/pi_response": -180.560302734375, |
|
"logps/ref_response": -147.65762329101562, |
|
"logps/rejected": -290.31854248046875, |
|
"loss": 0.6262, |
|
"rewards/accuracies": 0.65625, |
|
"rewards/chosen": -0.319719135761261, |
|
"rewards/margins": 0.23344525694847107, |
|
"rewards/rejected": -0.5531644225120544, |
|
"step": 50 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 8.930309757836516e-08, |
|
"logits/chosen": -2.3760523796081543, |
|
"logits/rejected": -2.32749605178833, |
|
"logps/chosen": -274.39361572265625, |
|
"logps/pi_response": -180.96006774902344, |
|
"logps/ref_response": -138.89596557617188, |
|
"logps/rejected": -319.18914794921875, |
|
"loss": 0.6005, |
|
"rewards/accuracies": 0.6625000238418579, |
|
"rewards/chosen": -0.49327319860458374, |
|
"rewards/margins": 0.29331183433532715, |
|
"rewards/rejected": -0.7865849733352661, |
|
"step": 60 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 2.3423053240837514e-08, |
|
"logits/chosen": -2.4027113914489746, |
|
"logits/rejected": -2.3286519050598145, |
|
"logps/chosen": -291.0926818847656, |
|
"logps/pi_response": -238.1649627685547, |
|
"logps/ref_response": -181.65982055664062, |
|
"logps/rejected": -385.4344177246094, |
|
"loss": 0.6027, |
|
"rewards/accuracies": 0.737500011920929, |
|
"rewards/chosen": -0.44906187057495117, |
|
"rewards/margins": 0.48221874237060547, |
|
"rewards/rejected": -0.9312806129455566, |
|
"step": 70 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"learning_rate": 0.0, |
|
"logits/chosen": -2.3703107833862305, |
|
"logits/rejected": -2.331961154937744, |
|
"logps/chosen": -283.42547607421875, |
|
"logps/pi_response": -191.7020263671875, |
|
"logps/ref_response": -152.6740264892578, |
|
"logps/rejected": -312.0522766113281, |
|
"loss": 0.6139, |
|
"rewards/accuracies": 0.637499988079071, |
|
"rewards/chosen": -0.4424005448818207, |
|
"rewards/margins": 0.24074189364910126, |
|
"rewards/rejected": -0.6831424832344055, |
|
"step": 80 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"step": 80, |
|
"total_flos": 0.0, |
|
"train_loss": 0.633746999502182, |
|
"train_runtime": 2276.5265, |
|
"train_samples_per_second": 4.476, |
|
"train_steps_per_second": 0.035 |
|
} |
|
], |
|
"logging_steps": 10, |
|
"max_steps": 80, |
|
"num_input_tokens_seen": 0, |
|
"num_train_epochs": 1, |
|
"save_steps": 100, |
|
"total_flos": 0.0, |
|
"train_batch_size": 8, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|