{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 37,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "debug/policy_chosen_logits": -3.41206693649292,
      "debug/policy_chosen_logps": -143.21307373046875,
      "debug/policy_rejected_logits": -3.2551214694976807,
      "debug/policy_rejected_logps": -150.2328338623047,
      "debug/reference_chosen_logps": -143.21307373046875,
      "debug/reference_rejected_logps": -150.2328338623047,
      "epoch": 0.02702702702702703,
      "grad_norm": 3.5140838495338116,
      "learning_rate": 1e-06,
      "logits/chosen": -3.41206693649292,
      "logits/rejected": -3.2551214694976807,
      "logps/chosen": -143.21307373046875,
      "logps/rejected": -150.2328338623047,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "debug/policy_chosen_logits": -3.4049649238586426,
      "debug/policy_chosen_logps": -143.7670135498047,
      "debug/policy_rejected_logits": -3.4008069038391113,
      "debug/policy_rejected_logps": -138.0754852294922,
      "debug/reference_chosen_logps": -143.82278442382812,
      "debug/reference_rejected_logps": -138.1428985595703,
      "epoch": 0.05405405405405406,
      "grad_norm": 3.526231895119079,
      "learning_rate": 1e-06,
      "logits/chosen": -3.4049649238586426,
      "logits/rejected": -3.4008069038391113,
      "logps/chosen": -143.7670135498047,
      "logps/rejected": -138.0754852294922,
      "loss": 0.4994,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0005577945848926902,
      "rewards/margins": -0.00011638639261946082,
      "rewards/rejected": 0.000674180977512151,
      "step": 2
    },
    {
      "debug/policy_chosen_logits": -3.2883780002593994,
      "debug/policy_chosen_logps": -143.35385131835938,
      "debug/policy_rejected_logits": -3.2103564739227295,
      "debug/policy_rejected_logps": -139.85372924804688,
      "debug/reference_chosen_logps": -143.38311767578125,
      "debug/reference_rejected_logps": -139.98326110839844,
      "epoch": 0.08108108108108109,
      "grad_norm": 3.618148106626628,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2883780002593994,
      "logits/rejected": -3.2103564739227295,
      "logps/chosen": -143.35385131835938,
      "logps/rejected": -139.85372924804688,
      "loss": 0.5006,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.00029258732683956623,
      "rewards/margins": -0.001002683537080884,
      "rewards/rejected": 0.0012952708639204502,
      "step": 3
    },
    {
      "debug/policy_chosen_logits": -3.294776678085327,
      "debug/policy_chosen_logps": -132.79519653320312,
      "debug/policy_rejected_logits": -3.4150912761688232,
      "debug/policy_rejected_logps": -138.08180236816406,
      "debug/reference_chosen_logps": -132.62167358398438,
      "debug/reference_rejected_logps": -137.927978515625,
      "epoch": 0.10810810810810811,
      "grad_norm": 3.5093502617713077,
      "learning_rate": 1e-06,
      "logits/chosen": -3.294776678085327,
      "logits/rejected": -3.4150912761688232,
      "logps/chosen": -132.79519653320312,
      "logps/rejected": -138.08180236816406,
      "loss": 0.4994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0017352199647575617,
      "rewards/margins": -0.0001968955621123314,
      "rewards/rejected": -0.0015383241698145866,
      "step": 4
    },
    {
      "debug/policy_chosen_logits": -3.343012809753418,
      "debug/policy_chosen_logps": -126.95188903808594,
      "debug/policy_rejected_logits": -3.3124771118164062,
      "debug/policy_rejected_logps": -129.9374237060547,
      "debug/reference_chosen_logps": -126.82910919189453,
      "debug/reference_rejected_logps": -129.91493225097656,
      "epoch": 0.13513513513513514,
      "grad_norm": 3.250245901043351,
      "learning_rate": 1e-06,
      "logits/chosen": -3.343012809753418,
      "logits/rejected": -3.3124771118164062,
      "logps/chosen": -126.95188903808594,
      "logps/rejected": -129.9374237060547,
      "loss": 0.5009,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.001227750675752759,
      "rewards/margins": -0.001002778997644782,
      "rewards/rejected": -0.00022497176541946828,
      "step": 5
    },
    {
      "debug/policy_chosen_logits": -3.2374277114868164,
      "debug/policy_chosen_logps": -154.26406860351562,
      "debug/policy_rejected_logits": -3.2978382110595703,
      "debug/policy_rejected_logps": -143.40101623535156,
      "debug/reference_chosen_logps": -153.9961395263672,
      "debug/reference_rejected_logps": -143.3165283203125,
      "epoch": 0.16216216216216217,
      "grad_norm": 3.439711062354016,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2374277114868164,
      "logits/rejected": -3.2978382110595703,
      "logps/chosen": -154.26406860351562,
      "logps/rejected": -143.40101623535156,
      "loss": 0.5009,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.002679176162928343,
      "rewards/margins": -0.0018340873066335917,
      "rewards/rejected": -0.000845088972710073,
      "step": 6
    },
    {
      "debug/policy_chosen_logits": -3.2252252101898193,
      "debug/policy_chosen_logps": -162.3474578857422,
      "debug/policy_rejected_logits": -3.230537176132202,
      "debug/policy_rejected_logps": -147.3297576904297,
      "debug/reference_chosen_logps": -162.26889038085938,
      "debug/reference_rejected_logps": -147.57199096679688,
      "epoch": 0.1891891891891892,
      "grad_norm": 3.5128248367150468,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2252252101898193,
      "logits/rejected": -3.230537176132202,
      "logps/chosen": -162.3474578857422,
      "logps/rejected": -147.3297576904297,
      "loss": 0.5005,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0007855510921217501,
      "rewards/margins": -0.003207969479262829,
      "rewards/rejected": 0.0024224184453487396,
      "step": 7
    },
    {
      "debug/policy_chosen_logits": -3.3006181716918945,
      "debug/policy_chosen_logps": -149.18630981445312,
      "debug/policy_rejected_logits": -3.3146488666534424,
      "debug/policy_rejected_logps": -147.72348022460938,
      "debug/reference_chosen_logps": -149.20960998535156,
      "debug/reference_rejected_logps": -147.54893493652344,
      "epoch": 0.21621621621621623,
      "grad_norm": 3.499360817762659,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3006181716918945,
      "logits/rejected": -3.3146488666534424,
      "logps/chosen": -149.18630981445312,
      "logps/rejected": -147.72348022460938,
      "loss": 0.4992,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.000233030179515481,
      "rewards/margins": 0.001978444866836071,
      "rewards/rejected": -0.00174541468732059,
      "step": 8
    },
    {
      "debug/policy_chosen_logits": -3.3503048419952393,
      "debug/policy_chosen_logps": -150.99197387695312,
      "debug/policy_rejected_logits": -3.2822837829589844,
      "debug/policy_rejected_logps": -145.26345825195312,
      "debug/reference_chosen_logps": -151.06939697265625,
      "debug/reference_rejected_logps": -145.52816772460938,
      "epoch": 0.24324324324324326,
      "grad_norm": 3.4264003876223215,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3503048419952393,
      "logits/rejected": -3.2822837829589844,
      "logps/chosen": -150.99197387695312,
      "logps/rejected": -145.26345825195312,
      "loss": 0.4987,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0007742022862657905,
      "rewards/margins": -0.0018729494186118245,
      "rewards/rejected": 0.002647151704877615,
      "step": 9
    },
    {
      "debug/policy_chosen_logits": -3.304420232772827,
      "debug/policy_chosen_logps": -133.21044921875,
      "debug/policy_rejected_logits": -3.412870168685913,
      "debug/policy_rejected_logps": -125.90888977050781,
      "debug/reference_chosen_logps": -133.26504516601562,
      "debug/reference_rejected_logps": -125.95523834228516,
      "epoch": 0.2702702702702703,
      "grad_norm": 3.3820605783151074,
      "learning_rate": 1e-06,
      "logits/chosen": -3.304420232772827,
      "logits/rejected": -3.412870168685913,
      "logps/chosen": -133.21044921875,
      "logps/rejected": -125.90888977050781,
      "loss": 0.5005,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0005457591032609344,
      "rewards/margins": 8.22543297545053e-05,
      "rewards/rejected": 0.0004635047516785562,
      "step": 10
    },
    {
      "debug/policy_chosen_logits": -3.3585329055786133,
      "debug/policy_chosen_logps": -139.380859375,
      "debug/policy_rejected_logits": -3.3599538803100586,
      "debug/policy_rejected_logps": -132.3596649169922,
      "debug/reference_chosen_logps": -139.71542358398438,
      "debug/reference_rejected_logps": -132.61981201171875,
      "epoch": 0.2972972972972973,
      "grad_norm": 3.288218575224273,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3585329055786133,
      "logits/rejected": -3.3599538803100586,
      "logps/chosen": -139.380859375,
      "logps/rejected": -132.3596649169922,
      "loss": 0.4999,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0033456420060247183,
      "rewards/margins": 0.0007441901252605021,
      "rewards/rejected": 0.0026014517061412334,
      "step": 11
    },
    {
      "debug/policy_chosen_logits": -3.2882096767425537,
      "debug/policy_chosen_logps": -156.53334045410156,
      "debug/policy_rejected_logits": -3.2960500717163086,
      "debug/policy_rejected_logps": -143.13995361328125,
      "debug/reference_chosen_logps": -156.6325225830078,
      "debug/reference_rejected_logps": -142.83905029296875,
      "epoch": 0.32432432432432434,
      "grad_norm": 3.813316903837083,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2882096767425537,
      "logits/rejected": -3.2960500717163086,
      "logps/chosen": -156.53334045410156,
      "logps/rejected": -143.13995361328125,
      "loss": 0.4989,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0009917640127241611,
      "rewards/margins": 0.004000749439001083,
      "rewards/rejected": -0.0030089854262769222,
      "step": 12
    },
    {
      "debug/policy_chosen_logits": -3.2771499156951904,
      "debug/policy_chosen_logps": -133.85403442382812,
      "debug/policy_rejected_logits": -3.3577096462249756,
      "debug/policy_rejected_logps": -145.81051635742188,
      "debug/reference_chosen_logps": -134.21636962890625,
      "debug/reference_rejected_logps": -145.71588134765625,
      "epoch": 0.35135135135135137,
      "grad_norm": 3.5238547026932254,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2771499156951904,
      "logits/rejected": -3.3577096462249756,
      "logps/chosen": -133.85403442382812,
      "logps/rejected": -145.81051635742188,
      "loss": 0.5004,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.003623266238719225,
      "rewards/margins": 0.004569797310978174,
      "rewards/rejected": -0.0009465311886742711,
      "step": 13
    },
    {
      "debug/policy_chosen_logits": -3.4804084300994873,
      "debug/policy_chosen_logps": -165.59515380859375,
      "debug/policy_rejected_logits": -3.5207834243774414,
      "debug/policy_rejected_logps": -139.4119873046875,
      "debug/reference_chosen_logps": -165.61618041992188,
      "debug/reference_rejected_logps": -139.546142578125,
      "epoch": 0.3783783783783784,
      "grad_norm": 3.271576173715924,
      "learning_rate": 1e-06,
      "logits/chosen": -3.4804084300994873,
      "logits/rejected": -3.5207834243774414,
      "logps/chosen": -165.59515380859375,
      "logps/rejected": -139.4119873046875,
      "loss": 0.4997,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.00021032325457781553,
      "rewards/margins": -0.0011312393471598625,
      "rewards/rejected": 0.0013415624853223562,
      "step": 14
    },
    {
      "debug/policy_chosen_logits": -3.371809244155884,
      "debug/policy_chosen_logps": -138.2847900390625,
      "debug/policy_rejected_logits": -3.2725796699523926,
      "debug/policy_rejected_logps": -146.83644104003906,
      "debug/reference_chosen_logps": -138.04966735839844,
      "debug/reference_rejected_logps": -146.4534454345703,
      "epoch": 0.40540540540540543,
      "grad_norm": 3.5549880331732084,
      "learning_rate": 1e-06,
      "logits/chosen": -3.371809244155884,
      "logits/rejected": -3.2725796699523926,
      "logps/chosen": -138.2847900390625,
      "logps/rejected": -146.83644104003906,
      "loss": 0.4978,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.002351150382310152,
      "rewards/margins": 0.0014788818079978228,
      "rewards/rejected": -0.0038300324231386185,
      "step": 15
    },
    {
      "debug/policy_chosen_logits": -3.2811360359191895,
      "debug/policy_chosen_logps": -145.2406005859375,
      "debug/policy_rejected_logits": -3.2666900157928467,
      "debug/policy_rejected_logps": -136.08499145507812,
      "debug/reference_chosen_logps": -145.38519287109375,
      "debug/reference_rejected_logps": -136.1790313720703,
      "epoch": 0.43243243243243246,
      "grad_norm": 3.390241199656903,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2811360359191895,
      "logits/rejected": -3.2666900157928467,
      "logps/chosen": -145.2406005859375,
      "logps/rejected": -136.08499145507812,
      "loss": 0.4992,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0014458559453487396,
      "rewards/margins": 0.0005054186331108212,
      "rewards/rejected": 0.0009404371376149356,
      "step": 16
    },
    {
      "debug/policy_chosen_logits": -3.4209792613983154,
      "debug/policy_chosen_logps": -154.63943481445312,
      "debug/policy_rejected_logits": -3.3486948013305664,
      "debug/policy_rejected_logps": -133.35781860351562,
      "debug/reference_chosen_logps": -154.70895385742188,
      "debug/reference_rejected_logps": -133.4116668701172,
      "epoch": 0.4594594594594595,
      "grad_norm": 3.6742049248327437,
      "learning_rate": 1e-06,
      "logits/chosen": -3.4209792613983154,
      "logits/rejected": -3.3486948013305664,
      "logps/chosen": -154.63943481445312,
      "logps/rejected": -133.35781860351562,
      "loss": 0.4989,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0006950664683245122,
      "rewards/margins": 0.00015656478353776038,
      "rewards/rejected": 0.0005385016556829214,
      "step": 17
    },
    {
      "debug/policy_chosen_logits": -3.298342704772949,
      "debug/policy_chosen_logps": -146.52044677734375,
      "debug/policy_rejected_logits": -3.3201894760131836,
      "debug/policy_rejected_logps": -156.97808837890625,
      "debug/reference_chosen_logps": -146.34840393066406,
      "debug/reference_rejected_logps": -156.84378051757812,
      "epoch": 0.4864864864864865,
      "grad_norm": 3.5194962148324103,
      "learning_rate": 1e-06,
      "logits/chosen": -3.298342704772949,
      "logits/rejected": -3.3201894760131836,
      "logps/chosen": -146.52044677734375,
      "logps/rejected": -156.97808837890625,
      "loss": 0.4973,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0017202282324433327,
      "rewards/margins": -0.0003771304036490619,
      "rewards/rejected": -0.001343097654171288,
      "step": 18
    },
    {
      "debug/policy_chosen_logits": -3.3400371074676514,
      "debug/policy_chosen_logps": -132.01437377929688,
      "debug/policy_rejected_logits": -3.36023211479187,
      "debug/policy_rejected_logps": -148.31887817382812,
      "debug/reference_chosen_logps": -131.88931274414062,
      "debug/reference_rejected_logps": -148.13345336914062,
      "epoch": 0.5135135135135135,
      "grad_norm": 3.61313418435521,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3400371074676514,
      "logits/rejected": -3.36023211479187,
      "logps/chosen": -132.01437377929688,
      "logps/rejected": -148.31887817382812,
      "loss": 0.498,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00125067715998739,
      "rewards/margins": 0.0006034375401213765,
      "rewards/rejected": -0.001854114467278123,
      "step": 19
    },
    {
      "debug/policy_chosen_logits": -3.2641985416412354,
      "debug/policy_chosen_logps": -134.37350463867188,
      "debug/policy_rejected_logits": -3.255669355392456,
      "debug/policy_rejected_logps": -124.11454772949219,
      "debug/reference_chosen_logps": -134.53549194335938,
      "debug/reference_rejected_logps": -123.25413513183594,
      "epoch": 0.5405405405405406,
      "grad_norm": 3.4240940627433663,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2641985416412354,
      "logits/rejected": -3.255669355392456,
      "logps/chosen": -134.37350463867188,
      "logps/rejected": -124.11454772949219,
      "loss": 0.4969,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0016198731027543545,
      "rewards/margins": 0.010224045254290104,
      "rewards/rejected": -0.008604173548519611,
      "step": 20
    },
    {
      "debug/policy_chosen_logits": -3.389690637588501,
      "debug/policy_chosen_logps": -144.95230102539062,
      "debug/policy_rejected_logits": -3.3237528800964355,
      "debug/policy_rejected_logps": -140.4608917236328,
      "debug/reference_chosen_logps": -145.19155883789062,
      "debug/reference_rejected_logps": -139.9624481201172,
      "epoch": 0.5675675675675675,
      "grad_norm": 3.5636895297483804,
      "learning_rate": 1e-06,
      "logits/chosen": -3.389690637588501,
      "logits/rejected": -3.3237528800964355,
      "logps/chosen": -144.95230102539062,
      "logps/rejected": -140.4608917236328,
      "loss": 0.4972,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0023925015702843666,
      "rewards/margins": 0.007377022877335548,
      "rewards/rejected": -0.004984522238373756,
      "step": 21
    },
    {
      "debug/policy_chosen_logits": -3.2673821449279785,
      "debug/policy_chosen_logps": -159.7883758544922,
      "debug/policy_rejected_logits": -3.2593047618865967,
      "debug/policy_rejected_logps": -141.72947692871094,
      "debug/reference_chosen_logps": -159.80300903320312,
      "debug/reference_rejected_logps": -142.28695678710938,
      "epoch": 0.5945945945945946,
      "grad_norm": 3.8014985351330988,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2673821449279785,
      "logits/rejected": -3.2593047618865967,
      "logps/chosen": -159.7883758544922,
      "logps/rejected": -141.72947692871094,
      "loss": 0.499,
      "rewards/accuracies": 0.125,
      "rewards/chosen": 0.00014627468772232533,
      "rewards/margins": -0.005428638309240341,
      "rewards/rejected": 0.00557491322979331,
      "step": 22
    },
    {
      "debug/policy_chosen_logits": -3.3076963424682617,
      "debug/policy_chosen_logps": -147.4810791015625,
      "debug/policy_rejected_logits": -3.3063511848449707,
      "debug/policy_rejected_logps": -155.9129638671875,
      "debug/reference_chosen_logps": -147.6890869140625,
      "debug/reference_rejected_logps": -155.93359375,
      "epoch": 0.6216216216216216,
      "grad_norm": 3.5943677168048143,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3076963424682617,
      "logits/rejected": -3.3063511848449707,
      "logps/chosen": -147.4810791015625,
      "logps/rejected": -155.9129638671875,
      "loss": 0.499,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0020801923237740993,
      "rewards/margins": 0.001873721950687468,
      "rewards/rejected": 0.00020647048950195312,
      "step": 23
    },
    {
      "debug/policy_chosen_logits": -3.3162076473236084,
      "debug/policy_chosen_logps": -146.12327575683594,
      "debug/policy_rejected_logits": -3.2808167934417725,
      "debug/policy_rejected_logps": -150.74459838867188,
      "debug/reference_chosen_logps": -146.20852661132812,
      "debug/reference_rejected_logps": -151.0212860107422,
      "epoch": 0.6486486486486487,
      "grad_norm": 3.490255067904084,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3162076473236084,
      "logits/rejected": -3.2808167934417725,
      "logps/chosen": -146.12327575683594,
      "logps/rejected": -150.74459838867188,
      "loss": 0.4979,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0008525657467544079,
      "rewards/margins": -0.0019143482204526663,
      "rewards/rejected": 0.002766914200037718,
      "step": 24
    },
    {
      "debug/policy_chosen_logits": -3.341583490371704,
      "debug/policy_chosen_logps": -145.74295043945312,
      "debug/policy_rejected_logits": -3.2622642517089844,
      "debug/policy_rejected_logps": -148.95095825195312,
      "debug/reference_chosen_logps": -145.28700256347656,
      "debug/reference_rejected_logps": -148.38320922851562,
      "epoch": 0.6756756756756757,
      "grad_norm": 3.4930826080691433,
      "learning_rate": 1e-06,
      "logits/chosen": -3.341583490371704,
      "logits/rejected": -3.2622642517089844,
      "logps/chosen": -145.74295043945312,
      "logps/rejected": -148.95095825195312,
      "loss": 0.4992,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0045594689436256886,
      "rewards/margins": 0.001117925625294447,
      "rewards/rejected": -0.0056773945689201355,
      "step": 25
    },
    {
      "debug/policy_chosen_logits": -3.3710758686065674,
      "debug/policy_chosen_logps": -134.76580810546875,
      "debug/policy_rejected_logits": -3.268131971359253,
      "debug/policy_rejected_logps": -164.87716674804688,
      "debug/reference_chosen_logps": -134.9357452392578,
      "debug/reference_rejected_logps": -164.06057739257812,
      "epoch": 0.7027027027027027,
      "grad_norm": 3.5324806343379795,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3710758686065674,
      "logits/rejected": -3.268131971359253,
      "logps/chosen": -134.76580810546875,
      "logps/rejected": -164.87716674804688,
      "loss": 0.4961,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0016993426252156496,
      "rewards/margins": 0.009865359403192997,
      "rewards/rejected": -0.008166017010807991,
      "step": 26
    },
    {
      "debug/policy_chosen_logits": -3.276531934738159,
      "debug/policy_chosen_logps": -128.13336181640625,
      "debug/policy_rejected_logits": -3.2201032638549805,
      "debug/policy_rejected_logps": -148.01954650878906,
      "debug/reference_chosen_logps": -127.79987335205078,
      "debug/reference_rejected_logps": -148.602783203125,
      "epoch": 0.7297297297297297,
      "grad_norm": 3.5504923092772165,
      "learning_rate": 1e-06,
      "logits/chosen": -3.276531934738159,
      "logits/rejected": -3.2201032638549805,
      "logps/chosen": -128.13336181640625,
      "logps/rejected": -148.01954650878906,
      "loss": 0.4978,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0033348274882882833,
      "rewards/margins": -0.009167090058326721,
      "rewards/rejected": 0.005832261871546507,
      "step": 27
    },
    {
      "debug/policy_chosen_logits": -3.3670849800109863,
      "debug/policy_chosen_logps": -142.864013671875,
      "debug/policy_rejected_logits": -3.314009428024292,
      "debug/policy_rejected_logps": -159.29888916015625,
      "debug/reference_chosen_logps": -143.54417419433594,
      "debug/reference_rejected_logps": -159.19894409179688,
      "epoch": 0.7567567567567568,
      "grad_norm": 3.6333481085660533,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3670849800109863,
      "logits/rejected": -3.314009428024292,
      "logps/chosen": -142.864013671875,
      "logps/rejected": -159.29888916015625,
      "loss": 0.4946,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.006801585666835308,
      "rewards/margins": 0.007801036350429058,
      "rewards/rejected": -0.0009994508000090718,
      "step": 28
    },
    {
      "debug/policy_chosen_logits": -3.4510252475738525,
      "debug/policy_chosen_logps": -144.7528839111328,
      "debug/policy_rejected_logits": -3.4992246627807617,
      "debug/policy_rejected_logps": -129.56163024902344,
      "debug/reference_chosen_logps": -144.27435302734375,
      "debug/reference_rejected_logps": -128.7982177734375,
      "epoch": 0.7837837837837838,
      "grad_norm": 3.519707420288352,
      "learning_rate": 1e-06,
      "logits/chosen": -3.4510252475738525,
      "logits/rejected": -3.4992246627807617,
      "logps/chosen": -144.7528839111328,
      "logps/rejected": -129.56163024902344,
      "loss": 0.4917,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0047853561118245125,
      "rewards/margins": 0.0028487201780080795,
      "rewards/rejected": -0.007634077221155167,
      "step": 29
    },
    {
      "debug/policy_chosen_logits": -3.442349910736084,
      "debug/policy_chosen_logps": -129.01779174804688,
      "debug/policy_rejected_logits": -3.355781078338623,
      "debug/policy_rejected_logps": -125.97911071777344,
      "debug/reference_chosen_logps": -129.1351318359375,
      "debug/reference_rejected_logps": -125.61131286621094,
      "epoch": 0.8108108108108109,
      "grad_norm": 3.436282122029211,
      "learning_rate": 1e-06,
      "logits/chosen": -3.442349910736084,
      "logits/rejected": -3.355781078338623,
      "logps/chosen": -129.01779174804688,
      "logps/rejected": -125.97911071777344,
      "loss": 0.4957,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0011734389699995518,
      "rewards/margins": 0.004851359874010086,
      "rewards/rejected": -0.003677921136841178,
      "step": 30
    },
    {
      "debug/policy_chosen_logits": -3.46421480178833,
      "debug/policy_chosen_logps": -148.1017608642578,
      "debug/policy_rejected_logits": -3.4284045696258545,
      "debug/policy_rejected_logps": -155.14401245117188,
      "debug/reference_chosen_logps": -147.857666015625,
      "debug/reference_rejected_logps": -153.7999267578125,
      "epoch": 0.8378378378378378,
      "grad_norm": 3.573380219675295,
      "learning_rate": 1e-06,
      "logits/chosen": -3.46421480178833,
      "logits/rejected": -3.4284045696258545,
      "logps/chosen": -148.1017608642578,
      "logps/rejected": -155.14401245117188,
      "loss": 0.4994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.002440805546939373,
      "rewards/margins": 0.010999973863363266,
      "rewards/rejected": -0.013440780341625214,
      "step": 31
    },
    {
      "debug/policy_chosen_logits": -3.296452522277832,
      "debug/policy_chosen_logps": -146.0049285888672,
      "debug/policy_rejected_logits": -3.3844947814941406,
      "debug/policy_rejected_logps": -145.01739501953125,
      "debug/reference_chosen_logps": -146.03688049316406,
      "debug/reference_rejected_logps": -144.96478271484375,
      "epoch": 0.8648648648648649,
      "grad_norm": 3.7844677501283552,
      "learning_rate": 1e-06,
      "logits/chosen": -3.296452522277832,
      "logits/rejected": -3.3844947814941406,
      "logps/chosen": -146.0049285888672,
      "logps/rejected": -145.01739501953125,
      "loss": 0.4983,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0003195476019755006,
      "rewards/margins": 0.0008457758231088519,
      "rewards/rejected": -0.000526227755472064,
      "step": 32
    },
    {
      "debug/policy_chosen_logits": -3.393280506134033,
      "debug/policy_chosen_logps": -135.49090576171875,
      "debug/policy_rejected_logits": -3.4107859134674072,
      "debug/policy_rejected_logps": -126.17267608642578,
      "debug/reference_chosen_logps": -135.9659423828125,
      "debug/reference_rejected_logps": -123.26311492919922,
      "epoch": 0.8918918918918919,
      "grad_norm": 3.6987210522943323,
      "learning_rate": 1e-06,
      "logits/chosen": -3.393280506134033,
      "logits/rejected": -3.4107859134674072,
      "logps/chosen": -135.49090576171875,
      "logps/rejected": -126.17267608642578,
      "loss": 0.4982,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00475044222548604,
      "rewards/margins": 0.03384600579738617,
      "rewards/rejected": -0.029095562174916267,
      "step": 33
    },
    {
      "debug/policy_chosen_logits": -3.3232836723327637,
      "debug/policy_chosen_logps": -159.23611450195312,
      "debug/policy_rejected_logits": -3.3686249256134033,
      "debug/policy_rejected_logps": -165.2159423828125,
      "debug/reference_chosen_logps": -158.15670776367188,
      "debug/reference_rejected_logps": -164.94976806640625,
      "epoch": 0.918918918918919,
      "grad_norm": 3.553957133112115,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3232836723327637,
      "logits/rejected": -3.3686249256134033,
      "logps/chosen": -159.23611450195312,
      "logps/rejected": -165.2159423828125,
      "loss": 0.4998,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.010794105939567089,
      "rewards/margins": -0.008132400922477245,
      "rewards/rejected": -0.0026617045514285564,
      "step": 34
    },
    {
      "debug/policy_chosen_logits": -3.3675332069396973,
      "debug/policy_chosen_logps": -129.2323455810547,
      "debug/policy_rejected_logits": -3.39815092086792,
      "debug/policy_rejected_logps": -157.40768432617188,
      "debug/reference_chosen_logps": -129.46514892578125,
      "debug/reference_rejected_logps": -157.58755493164062,
      "epoch": 0.9459459459459459,
      "grad_norm": 3.6687608874400897,
      "learning_rate": 1e-06,
      "logits/chosen": -3.3675332069396973,
      "logits/rejected": -3.39815092086792,
      "logps/chosen": -129.2323455810547,
      "logps/rejected": -157.40768432617188,
      "loss": 0.4972,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0023280431050807238,
      "rewards/margins": 0.0005296231247484684,
      "rewards/rejected": 0.0017984198639169335,
      "step": 35
    },
    {
      "debug/policy_chosen_logits": -3.2530903816223145,
      "debug/policy_chosen_logps": -139.84768676757812,
      "debug/policy_rejected_logits": -3.380523204803467,
      "debug/policy_rejected_logps": -127.18656921386719,
      "debug/reference_chosen_logps": -140.35733032226562,
      "debug/reference_rejected_logps": -127.09129333496094,
      "epoch": 0.972972972972973,
      "grad_norm": 3.747493235351115,
      "learning_rate": 1e-06,
      "logits/chosen": -3.2530903816223145,
      "logits/rejected": -3.380523204803467,
      "logps/chosen": -139.84768676757812,
      "logps/rejected": -127.18656921386719,
      "loss": 0.4949,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.005096264183521271,
      "rewards/margins": 0.006049060728400946,
      "rewards/rejected": -0.0009527970105409622,
      "step": 36
    },
    {
      "debug/policy_chosen_logits": -3.1672980785369873,
      "debug/policy_chosen_logps": -165.81317138671875,
      "debug/policy_rejected_logits": -3.1891133785247803,
      "debug/policy_rejected_logps": -135.89328002929688,
      "debug/reference_chosen_logps": -164.97256469726562,
      "debug/reference_rejected_logps": -135.76358032226562,
      "epoch": 1.0,
      "grad_norm": 3.5111470282705084,
      "learning_rate": 1e-06,
      "logits/chosen": -3.1672980785369873,
      "logits/rejected": -3.1891133785247803,
      "logps/chosen": -165.81317138671875,
      "logps/rejected": -135.89328002929688,
      "loss": 0.4809,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.00840616226196289,
      "rewards/margins": -0.007109222002327442,
      "rewards/rejected": -0.0012969397939741611,
      "step": 37
    },
    {
      "epoch": 1.0,
      "step": 37,
      "total_flos": 0.0,
      "train_loss": 0.4979530750094233,
      "train_runtime": 139.1789,
      "train_samples_per_second": 16.849,
      "train_steps_per_second": 0.266
    }
  ],
  "logging_steps": 1,
  "max_steps": 37,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}