| { | |
| "best_metric": 0.004782966338098049, | |
| "best_model_checkpoint": "./mistral/22-04-24-Weni-WeniGPT-Agents-Mistral-1.0.0-SFT-1.0.25-DPO_Experiment on DPO with other hyperparameters and best SFT model of WeniGPT-2_max_steps-180_batch_8_2024-04-22_ppid_9/checkpoint-180", | |
| "epoch": 5.806451612903226, | |
| "eval_steps": 30, | |
| "global_step": 180, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.3225806451612903, | |
| "grad_norm": 24.6901912689209, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -1.8365566730499268, | |
| "logits/rejected": -1.8444175720214844, | |
| "logps/chosen": -195.56666564941406, | |
| "logps/rejected": -191.41812133789062, | |
| "loss": 0.653, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": 0.09110366553068161, | |
| "rewards/margins": 0.10920021682977676, | |
| "rewards/rejected": -0.018096545711159706, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.6451612903225806, | |
| "grad_norm": 13.191404342651367, | |
| "learning_rate": 4.71264367816092e-06, | |
| "logits/chosen": -1.7314943075180054, | |
| "logits/rejected": -1.766556978225708, | |
| "logps/chosen": -137.64279174804688, | |
| "logps/rejected": -220.5197296142578, | |
| "loss": 0.3701, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 1.1793702840805054, | |
| "rewards/margins": 1.4335505962371826, | |
| "rewards/rejected": -0.2541801631450653, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.967741935483871, | |
| "grad_norm": 20.181856155395508, | |
| "learning_rate": 4.42528735632184e-06, | |
| "logits/chosen": -1.8526036739349365, | |
| "logits/rejected": -1.86077880859375, | |
| "logps/chosen": -121.78877258300781, | |
| "logps/rejected": -151.2501678466797, | |
| "loss": 0.225, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.4271174669265747, | |
| "rewards/margins": 1.875509262084961, | |
| "rewards/rejected": -0.4483918249607086, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.967741935483871, | |
| "eval_logits/chosen": -1.9326385259628296, | |
| "eval_logits/rejected": -1.9565868377685547, | |
| "eval_logps/chosen": -123.71134185791016, | |
| "eval_logps/rejected": -215.0025634765625, | |
| "eval_loss": 0.22270560264587402, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": 2.6680667400360107, | |
| "eval_rewards/margins": 3.676541566848755, | |
| "eval_rewards/rejected": -1.008474588394165, | |
| "eval_runtime": 15.7165, | |
| "eval_samples_per_second": 1.782, | |
| "eval_steps_per_second": 0.445, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 1.2903225806451613, | |
| "grad_norm": 1.0404773950576782, | |
| "learning_rate": 4.137931034482759e-06, | |
| "logits/chosen": -1.9101307392120361, | |
| "logits/rejected": -1.9370834827423096, | |
| "logps/chosen": -146.01133728027344, | |
| "logps/rejected": -237.4972686767578, | |
| "loss": 0.0848, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.0706539154052734, | |
| "rewards/margins": 5.080114364624023, | |
| "rewards/rejected": -2.009460926055908, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 1.6129032258064515, | |
| "grad_norm": 17.344457626342773, | |
| "learning_rate": 3.850574712643678e-06, | |
| "logits/chosen": -1.8972657918930054, | |
| "logits/rejected": -1.9069604873657227, | |
| "logps/chosen": -195.85150146484375, | |
| "logps/rejected": -158.05677795410156, | |
| "loss": 0.0755, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 3.0010573863983154, | |
| "rewards/margins": 4.110104560852051, | |
| "rewards/rejected": -1.1090469360351562, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 1.935483870967742, | |
| "grad_norm": 0.7190752029418945, | |
| "learning_rate": 3.563218390804598e-06, | |
| "logits/chosen": -1.8297414779663086, | |
| "logits/rejected": -1.857456922531128, | |
| "logps/chosen": -144.957275390625, | |
| "logps/rejected": -232.25784301757812, | |
| "loss": 0.0853, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 3.4944889545440674, | |
| "rewards/margins": 5.610640525817871, | |
| "rewards/rejected": -2.116152286529541, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 1.935483870967742, | |
| "eval_logits/chosen": -1.9170362949371338, | |
| "eval_logits/rejected": -1.9400348663330078, | |
| "eval_logps/chosen": -118.40864562988281, | |
| "eval_logps/rejected": -221.06422424316406, | |
| "eval_loss": 0.10952840745449066, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": 3.7286033630371094, | |
| "eval_rewards/margins": 5.949405193328857, | |
| "eval_rewards/rejected": -2.220801830291748, | |
| "eval_runtime": 15.4022, | |
| "eval_samples_per_second": 1.818, | |
| "eval_steps_per_second": 0.454, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 2.258064516129032, | |
| "grad_norm": 6.895229339599609, | |
| "learning_rate": 3.2758620689655175e-06, | |
| "logits/chosen": -1.7604904174804688, | |
| "logits/rejected": -1.7763350009918213, | |
| "logps/chosen": -94.25442504882812, | |
| "logps/rejected": -200.06292724609375, | |
| "loss": 0.0336, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.892274856567383, | |
| "rewards/margins": 7.140706539154053, | |
| "rewards/rejected": -3.248431444168091, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 2.5806451612903225, | |
| "grad_norm": 3.1107871532440186, | |
| "learning_rate": 2.988505747126437e-06, | |
| "logits/chosen": -1.773847222328186, | |
| "logits/rejected": -1.7859563827514648, | |
| "logps/chosen": -134.7067108154297, | |
| "logps/rejected": -175.28102111816406, | |
| "loss": 0.0191, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 3.6734061241149902, | |
| "rewards/margins": 6.411807060241699, | |
| "rewards/rejected": -2.738400936126709, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 2.903225806451613, | |
| "grad_norm": 0.4654656946659088, | |
| "learning_rate": 2.7011494252873567e-06, | |
| "logits/chosen": -1.8632606267929077, | |
| "logits/rejected": -1.8870786428451538, | |
| "logps/chosen": -152.00164794921875, | |
| "logps/rejected": -194.2272186279297, | |
| "loss": 0.0285, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 3.7533652782440186, | |
| "rewards/margins": 6.942251682281494, | |
| "rewards/rejected": -3.1888861656188965, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 2.903225806451613, | |
| "eval_logits/chosen": -1.9063053131103516, | |
| "eval_logits/rejected": -1.930100679397583, | |
| "eval_logps/chosen": -116.32177734375, | |
| "eval_logps/rejected": -229.86549377441406, | |
| "eval_loss": 0.05446601286530495, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": 4.145977973937988, | |
| "eval_rewards/margins": 8.12703800201416, | |
| "eval_rewards/rejected": -3.9810593128204346, | |
| "eval_runtime": 15.401, | |
| "eval_samples_per_second": 1.818, | |
| "eval_steps_per_second": 0.455, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 3.225806451612903, | |
| "grad_norm": 0.023089677095413208, | |
| "learning_rate": 2.4137931034482762e-06, | |
| "logits/chosen": -1.8187129497528076, | |
| "logits/rejected": -1.8321533203125, | |
| "logps/chosen": -118.27799987792969, | |
| "logps/rejected": -238.4324493408203, | |
| "loss": 0.0031, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.572004318237305, | |
| "rewards/margins": 9.180875778198242, | |
| "rewards/rejected": -4.608872413635254, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 3.5483870967741935, | |
| "grad_norm": 0.9019374251365662, | |
| "learning_rate": 2.1264367816091954e-06, | |
| "logits/chosen": -1.903515100479126, | |
| "logits/rejected": -1.9107078313827515, | |
| "logps/chosen": -154.8245849609375, | |
| "logps/rejected": -195.23399353027344, | |
| "loss": 0.0127, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.906761646270752, | |
| "rewards/margins": 8.078901290893555, | |
| "rewards/rejected": -5.172140121459961, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 3.870967741935484, | |
| "grad_norm": 0.11791550368070602, | |
| "learning_rate": 1.839080459770115e-06, | |
| "logits/chosen": -1.7719767093658447, | |
| "logits/rejected": -1.7978557348251343, | |
| "logps/chosen": -135.73764038085938, | |
| "logps/rejected": -233.47396850585938, | |
| "loss": 0.001, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.7881596088409424, | |
| "rewards/margins": 10.058608055114746, | |
| "rewards/rejected": -6.270449161529541, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 3.870967741935484, | |
| "eval_logits/chosen": -1.8902080059051514, | |
| "eval_logits/rejected": -1.9141491651535034, | |
| "eval_logps/chosen": -116.14888763427734, | |
| "eval_logps/rejected": -235.0476531982422, | |
| "eval_loss": 0.04676542058587074, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": 4.18055534362793, | |
| "eval_rewards/margins": 9.198049545288086, | |
| "eval_rewards/rejected": -5.01749324798584, | |
| "eval_runtime": 15.4024, | |
| "eval_samples_per_second": 1.818, | |
| "eval_steps_per_second": 0.454, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 4.193548387096774, | |
| "grad_norm": 0.062105003744363785, | |
| "learning_rate": 1.5517241379310346e-06, | |
| "logits/chosen": -1.7840442657470703, | |
| "logits/rejected": -1.7999883890151978, | |
| "logps/chosen": -116.44026184082031, | |
| "logps/rejected": -252.85220336914062, | |
| "loss": 0.0192, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.862762451171875, | |
| "rewards/margins": 9.177450180053711, | |
| "rewards/rejected": -6.314688205718994, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 4.516129032258064, | |
| "grad_norm": 0.3371919095516205, | |
| "learning_rate": 1.2643678160919542e-06, | |
| "logits/chosen": -1.7637265920639038, | |
| "logits/rejected": -1.7790952920913696, | |
| "logps/chosen": -149.11972045898438, | |
| "logps/rejected": -200.7391357421875, | |
| "loss": 0.0156, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 1.933764100074768, | |
| "rewards/margins": 7.311190128326416, | |
| "rewards/rejected": -5.377426624298096, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 4.838709677419355, | |
| "grad_norm": 0.4724487066268921, | |
| "learning_rate": 9.770114942528738e-07, | |
| "logits/chosen": -1.7254791259765625, | |
| "logits/rejected": -1.754163384437561, | |
| "logps/chosen": -142.09568786621094, | |
| "logps/rejected": -264.6880798339844, | |
| "loss": 0.0021, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.6306958198547363, | |
| "rewards/margins": 9.091299057006836, | |
| "rewards/rejected": -5.460603713989258, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 4.838709677419355, | |
| "eval_logits/chosen": -1.881463885307312, | |
| "eval_logits/rejected": -1.9056099653244019, | |
| "eval_logps/chosen": -117.07276916503906, | |
| "eval_logps/rejected": -239.6072235107422, | |
| "eval_loss": 0.00868917815387249, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": 3.995779514312744, | |
| "eval_rewards/margins": 9.925186157226562, | |
| "eval_rewards/rejected": -5.929406642913818, | |
| "eval_runtime": 15.4035, | |
| "eval_samples_per_second": 1.818, | |
| "eval_steps_per_second": 0.454, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 5.161290322580645, | |
| "grad_norm": 0.4406607449054718, | |
| "learning_rate": 6.896551724137931e-07, | |
| "logits/chosen": -1.8259941339492798, | |
| "logits/rejected": -1.8320337533950806, | |
| "logps/chosen": -178.92930603027344, | |
| "logps/rejected": -274.9721984863281, | |
| "loss": 0.0028, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.2369651794433594, | |
| "rewards/margins": 9.267016410827637, | |
| "rewards/rejected": -7.030051231384277, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 5.483870967741936, | |
| "grad_norm": 0.02007269859313965, | |
| "learning_rate": 4.0229885057471266e-07, | |
| "logits/chosen": -1.7874090671539307, | |
| "logits/rejected": -1.8095813989639282, | |
| "logps/chosen": -149.6767120361328, | |
| "logps/rejected": -237.3863067626953, | |
| "loss": 0.0006, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.129965782165527, | |
| "rewards/margins": 11.051237106323242, | |
| "rewards/rejected": -6.921273708343506, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 5.806451612903226, | |
| "grad_norm": 0.5593944787979126, | |
| "learning_rate": 1.1494252873563219e-07, | |
| "logits/chosen": -1.8051351308822632, | |
| "logits/rejected": -1.8224855661392212, | |
| "logps/chosen": -168.2032928466797, | |
| "logps/rejected": -221.9829864501953, | |
| "loss": 0.0014, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.834609270095825, | |
| "rewards/margins": 9.291410446166992, | |
| "rewards/rejected": -6.456801414489746, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 5.806451612903226, | |
| "eval_logits/chosen": -1.8786267042160034, | |
| "eval_logits/rejected": -1.9027243852615356, | |
| "eval_logps/chosen": -117.46734619140625, | |
| "eval_logps/rejected": -240.9935760498047, | |
| "eval_loss": 0.004782966338098049, | |
| "eval_rewards/accuracies": 1.0, | |
| "eval_rewards/chosen": 3.9168648719787598, | |
| "eval_rewards/margins": 10.123543739318848, | |
| "eval_rewards/rejected": -6.206678867340088, | |
| "eval_runtime": 15.4023, | |
| "eval_samples_per_second": 1.818, | |
| "eval_steps_per_second": 0.454, | |
| "step": 180 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 180, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 6, | |
| "save_steps": 90, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |