File size: 10,914 Bytes
918c244 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 |
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 2.986666666666667,
"eval_steps": 500,
"global_step": 168,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.17777777777777778,
"grad_norm": 7.277172565460205,
"learning_rate": 2.9411764705882355e-06,
"logits/chosen": -0.4413936734199524,
"logits/rejected": -0.5268994569778442,
"logps/chosen": -1.6909767389297485,
"logps/rejected": -2.157106876373291,
"loss": 1.742,
"odds_ratio_loss": 15.103334426879883,
"rewards/accuracies": 0.78125,
"rewards/chosen": -0.1690976917743683,
"rewards/margins": 0.046613018959760666,
"rewards/rejected": -0.21571068465709686,
"sft_loss": 0.2316340208053589,
"step": 10
},
{
"epoch": 0.35555555555555557,
"grad_norm": 6.993782043457031,
"learning_rate": 4.995131923687488e-06,
"logits/chosen": -0.4119906425476074,
"logits/rejected": -0.48687878251075745,
"logps/chosen": -1.74609375,
"logps/rejected": -2.23624849319458,
"loss": 1.7977,
"odds_ratio_loss": 15.667881965637207,
"rewards/accuracies": 0.8125,
"rewards/chosen": -0.17460939288139343,
"rewards/margins": 0.04901544004678726,
"rewards/rejected": -0.22362485527992249,
"sft_loss": 0.23094086349010468,
"step": 20
},
{
"epoch": 0.5333333333333333,
"grad_norm": 7.126066207885742,
"learning_rate": 4.90911473983908e-06,
"logits/chosen": -0.3993254601955414,
"logits/rejected": -0.47226476669311523,
"logps/chosen": -1.6316206455230713,
"logps/rejected": -2.0731987953186035,
"loss": 1.685,
"odds_ratio_loss": 14.636944770812988,
"rewards/accuracies": 0.8062499761581421,
"rewards/chosen": -0.16316203773021698,
"rewards/margins": 0.044157832860946655,
"rewards/rejected": -0.20731988549232483,
"sft_loss": 0.22126968204975128,
"step": 30
},
{
"epoch": 0.7111111111111111,
"grad_norm": 6.876996040344238,
"learning_rate": 4.71919261421297e-06,
"logits/chosen": -0.4370671212673187,
"logits/rejected": -0.5137215852737427,
"logps/chosen": -1.3095628023147583,
"logps/rejected": -1.8373353481292725,
"loss": 1.3576,
"odds_ratio_loss": 11.971807479858398,
"rewards/accuracies": 0.831250011920929,
"rewards/chosen": -0.13095629215240479,
"rewards/margins": 0.052777260541915894,
"rewards/rejected": -0.18373355269432068,
"sft_loss": 0.16043850779533386,
"step": 40
},
{
"epoch": 0.8888888888888888,
"grad_norm": 3.4186394214630127,
"learning_rate": 4.43355687413747e-06,
"logits/chosen": -0.4473581910133362,
"logits/rejected": -0.5121863484382629,
"logps/chosen": -1.047701120376587,
"logps/rejected": -1.479405164718628,
"loss": 1.0996,
"odds_ratio_loss": 9.729310989379883,
"rewards/accuracies": 0.7562500238418579,
"rewards/chosen": -0.10477012395858765,
"rewards/margins": 0.04317039996385574,
"rewards/rejected": -0.1479405164718628,
"sft_loss": 0.12663736939430237,
"step": 50
},
{
"epoch": 1.0666666666666667,
"grad_norm": 4.834543704986572,
"learning_rate": 4.064526968101844e-06,
"logits/chosen": -0.39888542890548706,
"logits/rejected": -0.4785118103027344,
"logps/chosen": -0.7870318293571472,
"logps/rejected": -1.2147241830825806,
"loss": 0.8362,
"odds_ratio_loss": 7.4531731605529785,
"rewards/accuracies": 0.768750011920929,
"rewards/chosen": -0.07870318740606308,
"rewards/margins": 0.04276924580335617,
"rewards/rejected": -0.12147243320941925,
"sft_loss": 0.09090302884578705,
"step": 60
},
{
"epoch": 1.2444444444444445,
"grad_norm": 2.8773512840270996,
"learning_rate": 3.6280191288478437e-06,
"logits/chosen": -0.4217619299888611,
"logits/rejected": -0.49851852655410767,
"logps/chosen": -0.5581613779067993,
"logps/rejected": -1.076547384262085,
"loss": 0.5998,
"odds_ratio_loss": 5.396755695343018,
"rewards/accuracies": 0.8374999761581421,
"rewards/chosen": -0.05581614375114441,
"rewards/margins": 0.051838599145412445,
"rewards/rejected": -0.10765473544597626,
"sft_loss": 0.06012673303484917,
"step": 70
},
{
"epoch": 1.4222222222222223,
"grad_norm": 2.8275744915008545,
"learning_rate": 3.142859907420615e-06,
"logits/chosen": -0.39267396926879883,
"logits/rejected": -0.46761494874954224,
"logps/chosen": -0.43946489691734314,
"logps/rejected": -0.9307826161384583,
"loss": 0.4834,
"odds_ratio_loss": 4.3501482009887695,
"rewards/accuracies": 0.831250011920929,
"rewards/chosen": -0.04394649341702461,
"rewards/margins": 0.049131762236356735,
"rewards/rejected": -0.09307824075222015,
"sft_loss": 0.04834695905447006,
"step": 80
},
{
"epoch": 1.6,
"grad_norm": 2.467008590698242,
"learning_rate": 2.629974185404951e-06,
"logits/chosen": -0.3878730535507202,
"logits/rejected": -0.4690762460231781,
"logps/chosen": -0.3482344150543213,
"logps/rejected": -0.9180346727371216,
"loss": 0.3918,
"odds_ratio_loss": 3.5334713459014893,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.03482344001531601,
"rewards/margins": 0.05698002502322197,
"rewards/rejected": -0.09180346131324768,
"sft_loss": 0.03849966078996658,
"step": 90
},
{
"epoch": 1.7777777777777777,
"grad_norm": 2.4158785343170166,
"learning_rate": 2.1114826863194882e-06,
"logits/chosen": -0.3763123154640198,
"logits/rejected": -0.47139158844947815,
"logps/chosen": -0.27210018038749695,
"logps/rejected": -0.8651291728019714,
"loss": 0.3089,
"odds_ratio_loss": 2.8031647205352783,
"rewards/accuracies": 0.875,
"rewards/chosen": -0.027210021391510963,
"rewards/margins": 0.05930290371179581,
"rewards/rejected": -0.08651291579008102,
"sft_loss": 0.02853994630277157,
"step": 100
},
{
"epoch": 1.9555555555555557,
"grad_norm": 2.5559449195861816,
"learning_rate": 1.6097479104361328e-06,
"logits/chosen": -0.4010067880153656,
"logits/rejected": -0.4844478666782379,
"logps/chosen": -0.2540341019630432,
"logps/rejected": -0.6811193823814392,
"loss": 0.3004,
"odds_ratio_loss": 2.749537706375122,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.02540341392159462,
"rewards/margins": 0.04270853474736214,
"rewards/rejected": -0.06811194121837616,
"sft_loss": 0.025406217202544212,
"step": 110
},
{
"epoch": 2.1333333333333333,
"grad_norm": 2.1673660278320312,
"learning_rate": 1.1464096417858821e-06,
"logits/chosen": -0.3880887031555176,
"logits/rejected": -0.47383826971054077,
"logps/chosen": -0.17626990377902985,
"logps/rejected": -0.8207994699478149,
"loss": 0.2064,
"odds_ratio_loss": 1.8402626514434814,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.017626991495490074,
"rewards/margins": 0.06445295363664627,
"rewards/rejected": -0.0820799320936203,
"sft_loss": 0.022340241819620132,
"step": 120
},
{
"epoch": 2.311111111111111,
"grad_norm": 1.6694434881210327,
"learning_rate": 7.414516258630245e-07,
"logits/chosen": -0.4004104733467102,
"logits/rejected": -0.48559218645095825,
"logps/chosen": -0.18084414303302765,
"logps/rejected": -0.7277659177780151,
"loss": 0.2208,
"odds_ratio_loss": 1.993011474609375,
"rewards/accuracies": 0.8125,
"rewards/chosen": -0.018084412440657616,
"rewards/margins": 0.054692186415195465,
"rewards/rejected": -0.07277660071849823,
"sft_loss": 0.021496199071407318,
"step": 130
},
{
"epoch": 2.488888888888889,
"grad_norm": 2.6840009689331055,
"learning_rate": 4.123396721497977e-07,
"logits/chosen": -0.38309675455093384,
"logits/rejected": -0.4632611870765686,
"logps/chosen": -0.17778636515140533,
"logps/rejected": -0.7036616802215576,
"loss": 0.2182,
"odds_ratio_loss": 1.9652601480484009,
"rewards/accuracies": 0.831250011920929,
"rewards/chosen": -0.017778638750314713,
"rewards/margins": 0.052587538957595825,
"rewards/rejected": -0.07036618143320084,
"sft_loss": 0.021626269444823265,
"step": 140
},
{
"epoch": 2.6666666666666665,
"grad_norm": 2.5022149085998535,
"learning_rate": 1.7326835503629542e-07,
"logits/chosen": -0.41912952065467834,
"logits/rejected": -0.48157674074172974,
"logps/chosen": -0.1846485584974289,
"logps/rejected": -0.6781209707260132,
"loss": 0.2302,
"odds_ratio_loss": 2.0842204093933105,
"rewards/accuracies": 0.7875000238418579,
"rewards/chosen": -0.01846485398709774,
"rewards/margins": 0.04934724420309067,
"rewards/rejected": -0.06781210005283356,
"sft_loss": 0.02174842543900013,
"step": 150
},
{
"epoch": 2.8444444444444446,
"grad_norm": 2.2835490703582764,
"learning_rate": 3.4548802869627806e-08,
"logits/chosen": -0.3882659673690796,
"logits/rejected": -0.46682611107826233,
"logps/chosen": -0.15616735816001892,
"logps/rejected": -0.6447629332542419,
"loss": 0.193,
"odds_ratio_loss": 1.7490888833999634,
"rewards/accuracies": 0.8374999761581421,
"rewards/chosen": -0.015616737306118011,
"rewards/margins": 0.04885955527424812,
"rewards/rejected": -0.06447629630565643,
"sft_loss": 0.018099095672369003,
"step": 160
},
{
"epoch": 2.986666666666667,
"step": 168,
"total_flos": 2.94774302048256e+16,
"train_loss": 0.703327341448693,
"train_runtime": 380.828,
"train_samples_per_second": 7.09,
"train_steps_per_second": 0.441
}
],
"logging_steps": 10,
"max_steps": 168,
"num_input_tokens_seen": 0,
"num_train_epochs": 3,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 2.94774302048256e+16,
"train_batch_size": 2,
"trial_name": null,
"trial_params": null
}
|