{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.986666666666667,
  "eval_steps": 500,
  "global_step": 168,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.17777777777777778,
      "grad_norm": 7.277172565460205,
      "learning_rate": 2.9411764705882355e-06,
      "logits/chosen": -0.4413936734199524,
      "logits/rejected": -0.5268994569778442,
      "logps/chosen": -1.6909767389297485,
      "logps/rejected": -2.157106876373291,
      "loss": 1.742,
      "odds_ratio_loss": 15.103334426879883,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.1690976917743683,
      "rewards/margins": 0.046613018959760666,
      "rewards/rejected": -0.21571068465709686,
      "sft_loss": 0.2316340208053589,
      "step": 10
    },
    {
      "epoch": 0.35555555555555557,
      "grad_norm": 6.993782043457031,
      "learning_rate": 4.995131923687488e-06,
      "logits/chosen": -0.4119906425476074,
      "logits/rejected": -0.48687878251075745,
      "logps/chosen": -1.74609375,
      "logps/rejected": -2.23624849319458,
      "loss": 1.7977,
      "odds_ratio_loss": 15.667881965637207,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.17460939288139343,
      "rewards/margins": 0.04901544004678726,
      "rewards/rejected": -0.22362485527992249,
      "sft_loss": 0.23094086349010468,
      "step": 20
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 7.126066207885742,
      "learning_rate": 4.90911473983908e-06,
      "logits/chosen": -0.3993254601955414,
      "logits/rejected": -0.47226476669311523,
      "logps/chosen": -1.6316206455230713,
      "logps/rejected": -2.0731987953186035,
      "loss": 1.685,
      "odds_ratio_loss": 14.636944770812988,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.16316203773021698,
      "rewards/margins": 0.044157832860946655,
      "rewards/rejected": -0.20731988549232483,
      "sft_loss": 0.22126968204975128,
      "step": 30
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 6.876996040344238,
      "learning_rate": 4.71919261421297e-06,
      "logits/chosen": -0.4370671212673187,
      "logits/rejected": -0.5137215852737427,
      "logps/chosen": -1.3095628023147583,
      "logps/rejected": -1.8373353481292725,
      "loss": 1.3576,
      "odds_ratio_loss": 11.971807479858398,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.13095629215240479,
      "rewards/margins": 0.052777260541915894,
      "rewards/rejected": -0.18373355269432068,
      "sft_loss": 0.16043850779533386,
      "step": 40
    },
    {
      "epoch": 0.8888888888888888,
      "grad_norm": 3.4186394214630127,
      "learning_rate": 4.43355687413747e-06,
      "logits/chosen": -0.4473581910133362,
      "logits/rejected": -0.5121863484382629,
      "logps/chosen": -1.047701120376587,
      "logps/rejected": -1.479405164718628,
      "loss": 1.0996,
      "odds_ratio_loss": 9.729310989379883,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.10477012395858765,
      "rewards/margins": 0.04317039996385574,
      "rewards/rejected": -0.1479405164718628,
      "sft_loss": 0.12663736939430237,
      "step": 50
    },
    {
      "epoch": 1.0666666666666667,
      "grad_norm": 4.834543704986572,
      "learning_rate": 4.064526968101844e-06,
      "logits/chosen": -0.39888542890548706,
      "logits/rejected": -0.4785118103027344,
      "logps/chosen": -0.7870318293571472,
      "logps/rejected": -1.2147241830825806,
      "loss": 0.8362,
      "odds_ratio_loss": 7.4531731605529785,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.07870318740606308,
      "rewards/margins": 0.04276924580335617,
      "rewards/rejected": -0.12147243320941925,
      "sft_loss": 0.09090302884578705,
      "step": 60
    },
    {
      "epoch": 1.2444444444444445,
      "grad_norm": 2.8773512840270996,
      "learning_rate": 3.6280191288478437e-06,
      "logits/chosen": -0.4217619299888611,
      "logits/rejected": -0.49851852655410767,
      "logps/chosen": -0.5581613779067993,
      "logps/rejected": -1.076547384262085,
      "loss": 0.5998,
      "odds_ratio_loss": 5.396755695343018,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.05581614375114441,
      "rewards/margins": 0.051838599145412445,
      "rewards/rejected": -0.10765473544597626,
      "sft_loss": 0.06012673303484917,
      "step": 70
    },
    {
      "epoch": 1.4222222222222223,
      "grad_norm": 2.8275744915008545,
      "learning_rate": 3.142859907420615e-06,
      "logits/chosen": -0.39267396926879883,
      "logits/rejected": -0.46761494874954224,
      "logps/chosen": -0.43946489691734314,
      "logps/rejected": -0.9307826161384583,
      "loss": 0.4834,
      "odds_ratio_loss": 4.3501482009887695,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.04394649341702461,
      "rewards/margins": 0.049131762236356735,
      "rewards/rejected": -0.09307824075222015,
      "sft_loss": 0.04834695905447006,
      "step": 80
    },
    {
      "epoch": 1.6,
      "grad_norm": 2.467008590698242,
      "learning_rate": 2.629974185404951e-06,
      "logits/chosen": -0.3878730535507202,
      "logits/rejected": -0.4690762460231781,
      "logps/chosen": -0.3482344150543213,
      "logps/rejected": -0.9180346727371216,
      "loss": 0.3918,
      "odds_ratio_loss": 3.5334713459014893,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03482344001531601,
      "rewards/margins": 0.05698002502322197,
      "rewards/rejected": -0.09180346131324768,
      "sft_loss": 0.03849966078996658,
      "step": 90
    },
    {
      "epoch": 1.7777777777777777,
      "grad_norm": 2.4158785343170166,
      "learning_rate": 2.1114826863194882e-06,
      "logits/chosen": -0.3763123154640198,
      "logits/rejected": -0.47139158844947815,
      "logps/chosen": -0.27210018038749695,
      "logps/rejected": -0.8651291728019714,
      "loss": 0.3089,
      "odds_ratio_loss": 2.8031647205352783,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.027210021391510963,
      "rewards/margins": 0.05930290371179581,
      "rewards/rejected": -0.08651291579008102,
      "sft_loss": 0.02853994630277157,
      "step": 100
    },
    {
      "epoch": 1.9555555555555557,
      "grad_norm": 2.5559449195861816,
      "learning_rate": 1.6097479104361328e-06,
      "logits/chosen": -0.4010067880153656,
      "logits/rejected": -0.4844478666782379,
      "logps/chosen": -0.2540341019630432,
      "logps/rejected": -0.6811193823814392,
      "loss": 0.3004,
      "odds_ratio_loss": 2.749537706375122,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.02540341392159462,
      "rewards/margins": 0.04270853474736214,
      "rewards/rejected": -0.06811194121837616,
      "sft_loss": 0.025406217202544212,
      "step": 110
    },
    {
      "epoch": 2.1333333333333333,
      "grad_norm": 2.1673660278320312,
      "learning_rate": 1.1464096417858821e-06,
      "logits/chosen": -0.3880887031555176,
      "logits/rejected": -0.47383826971054077,
      "logps/chosen": -0.17626990377902985,
      "logps/rejected": -0.8207994699478149,
      "loss": 0.2064,
      "odds_ratio_loss": 1.8402626514434814,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.017626991495490074,
      "rewards/margins": 0.06445295363664627,
      "rewards/rejected": -0.0820799320936203,
      "sft_loss": 0.022340241819620132,
      "step": 120
    },
    {
      "epoch": 2.311111111111111,
      "grad_norm": 1.6694434881210327,
      "learning_rate": 7.414516258630245e-07,
      "logits/chosen": -0.4004104733467102,
      "logits/rejected": -0.48559218645095825,
      "logps/chosen": -0.18084414303302765,
      "logps/rejected": -0.7277659177780151,
      "loss": 0.2208,
      "odds_ratio_loss": 1.993011474609375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.018084412440657616,
      "rewards/margins": 0.054692186415195465,
      "rewards/rejected": -0.07277660071849823,
      "sft_loss": 0.021496199071407318,
      "step": 130
    },
    {
      "epoch": 2.488888888888889,
      "grad_norm": 2.6840009689331055,
      "learning_rate": 4.123396721497977e-07,
      "logits/chosen": -0.38309675455093384,
      "logits/rejected": -0.4632611870765686,
      "logps/chosen": -0.17778636515140533,
      "logps/rejected": -0.7036616802215576,
      "loss": 0.2182,
      "odds_ratio_loss": 1.9652601480484009,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.017778638750314713,
      "rewards/margins": 0.052587538957595825,
      "rewards/rejected": -0.07036618143320084,
      "sft_loss": 0.021626269444823265,
      "step": 140
    },
    {
      "epoch": 2.6666666666666665,
      "grad_norm": 2.5022149085998535,
      "learning_rate": 1.7326835503629542e-07,
      "logits/chosen": -0.41912952065467834,
      "logits/rejected": -0.48157674074172974,
      "logps/chosen": -0.1846485584974289,
      "logps/rejected": -0.6781209707260132,
      "loss": 0.2302,
      "odds_ratio_loss": 2.0842204093933105,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.01846485398709774,
      "rewards/margins": 0.04934724420309067,
      "rewards/rejected": -0.06781210005283356,
      "sft_loss": 0.02174842543900013,
      "step": 150
    },
    {
      "epoch": 2.8444444444444446,
      "grad_norm": 2.2835490703582764,
      "learning_rate": 3.4548802869627806e-08,
      "logits/chosen": -0.3882659673690796,
      "logits/rejected": -0.46682611107826233,
      "logps/chosen": -0.15616735816001892,
      "logps/rejected": -0.6447629332542419,
      "loss": 0.193,
      "odds_ratio_loss": 1.7490888833999634,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.015616737306118011,
      "rewards/margins": 0.04885955527424812,
      "rewards/rejected": -0.06447629630565643,
      "sft_loss": 0.018099095672369003,
      "step": 160
    },
    {
      "epoch": 2.986666666666667,
      "step": 168,
      "total_flos": 2.94774302048256e+16,
      "train_loss": 0.703327341448693,
      "train_runtime": 380.828,
      "train_samples_per_second": 7.09,
      "train_steps_per_second": 0.441
    }
  ],
  "logging_steps": 10,
  "max_steps": 168,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 2.94774302048256e+16,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}