{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9984301412872841,
  "eval_steps": 500,
  "global_step": 159,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.006279434850863423,
      "grad_norm": 10.430633053065636,
      "learning_rate": 1.875e-08,
      "logits/chosen": 1.9384945631027222,
      "logits/rejected": 1.9276118278503418,
      "logps/chosen": -271.40283203125,
      "logps/pi_response": -164.53562927246094,
      "logps/ref_response": -164.53562927246094,
      "logps/rejected": -331.295166015625,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.06279434850863422,
      "grad_norm": 12.398250917152115,
      "learning_rate": 1.875e-07,
      "logits/chosen": 1.375727891921997,
      "logits/rejected": 1.5644251108169556,
      "logps/chosen": -274.7219543457031,
      "logps/pi_response": -147.36016845703125,
      "logps/ref_response": -147.31085205078125,
      "logps/rejected": -435.1158142089844,
      "loss": 0.6938,
      "rewards/accuracies": 0.4444444477558136,
      "rewards/chosen": -0.0004548828292172402,
      "rewards/margins": -0.00019649550085887313,
      "rewards/rejected": -0.0002583875320851803,
      "step": 10
    },
    {
      "epoch": 0.12558869701726844,
      "grad_norm": 8.66500301618874,
      "learning_rate": 2.9942119880575817e-07,
      "logits/chosen": 1.4651533365249634,
      "logits/rejected": 1.655652403831482,
      "logps/chosen": -306.84954833984375,
      "logps/pi_response": -169.71615600585938,
      "logps/ref_response": -169.712646484375,
      "logps/rejected": -432.33612060546875,
      "loss": 0.6897,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.007200081832706928,
      "rewards/margins": 0.005233117379248142,
      "rewards/rejected": -0.01243319921195507,
      "step": 20
    },
    {
      "epoch": 0.18838304552590268,
      "grad_norm": 9.072183489589987,
      "learning_rate": 2.929608750821129e-07,
      "logits/chosen": 1.4678703546524048,
      "logits/rejected": 1.6495119333267212,
      "logps/chosen": -303.84332275390625,
      "logps/pi_response": -163.2515106201172,
      "logps/ref_response": -162.7336883544922,
      "logps/rejected": -418.14385986328125,
      "loss": 0.6725,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04476435109972954,
      "rewards/margins": 0.027997547760605812,
      "rewards/rejected": -0.0727619007229805,
      "step": 30
    },
    {
      "epoch": 0.25117739403453687,
      "grad_norm": 7.9558717739641445,
      "learning_rate": 2.7962832564252725e-07,
      "logits/chosen": 1.2683569192886353,
      "logits/rejected": 1.6301815509796143,
      "logps/chosen": -296.2575988769531,
      "logps/pi_response": -164.4340362548828,
      "logps/ref_response": -164.83663940429688,
      "logps/rejected": -465.8512268066406,
      "loss": 0.6494,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.08111439645290375,
      "rewards/margins": 0.10455696284770966,
      "rewards/rejected": -0.1856713443994522,
      "step": 40
    },
    {
      "epoch": 0.3139717425431711,
      "grad_norm": 9.575559868471615,
      "learning_rate": 2.6006445513357056e-07,
      "logits/chosen": 1.1666990518569946,
      "logits/rejected": 1.442095398902893,
      "logps/chosen": -279.1966857910156,
      "logps/pi_response": -159.5543670654297,
      "logps/ref_response": -159.3308563232422,
      "logps/rejected": -451.89306640625,
      "loss": 0.6245,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1434028297662735,
      "rewards/margins": 0.18818414211273193,
      "rewards/rejected": -0.33158695697784424,
      "step": 50
    },
    {
      "epoch": 0.37676609105180536,
      "grad_norm": 7.235393720583013,
      "learning_rate": 2.3520971200967334e-07,
      "logits/chosen": 1.2313123941421509,
      "logits/rejected": 1.3668440580368042,
      "logps/chosen": -317.02679443359375,
      "logps/pi_response": -161.31985473632812,
      "logps/ref_response": -161.15023803710938,
      "logps/rejected": -474.43475341796875,
      "loss": 0.6077,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.21095111966133118,
      "rewards/margins": 0.2169308215379715,
      "rewards/rejected": -0.4278818964958191,
      "step": 60
    },
    {
      "epoch": 0.43956043956043955,
      "grad_norm": 7.493835961341088,
      "learning_rate": 2.0625888054143427e-07,
      "logits/chosen": 1.0015811920166016,
      "logits/rejected": 1.1100066900253296,
      "logps/chosen": -305.7568359375,
      "logps/pi_response": -157.75637817382812,
      "logps/ref_response": -160.76815795898438,
      "logps/rejected": -470.54522705078125,
      "loss": 0.6033,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.1870805323123932,
      "rewards/margins": 0.23715010285377502,
      "rewards/rejected": -0.42423057556152344,
      "step": 70
    },
    {
      "epoch": 0.5023547880690737,
      "grad_norm": 6.931068106744734,
      "learning_rate": 1.7460364672965327e-07,
      "logits/chosen": 0.8727623224258423,
      "logits/rejected": 1.101215124130249,
      "logps/chosen": -268.4558410644531,
      "logps/pi_response": -142.99215698242188,
      "logps/ref_response": -145.04220581054688,
      "logps/rejected": -471.4241638183594,
      "loss": 0.5909,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.18771064281463623,
      "rewards/margins": 0.33435240387916565,
      "rewards/rejected": -0.5220630764961243,
      "step": 80
    },
    {
      "epoch": 0.565149136577708,
      "grad_norm": 7.075043680291521,
      "learning_rate": 1.4176569902035086e-07,
      "logits/chosen": 0.7811240553855896,
      "logits/rejected": 1.0503191947937012,
      "logps/chosen": -308.6497497558594,
      "logps/pi_response": -152.72732543945312,
      "logps/ref_response": -156.76785278320312,
      "logps/rejected": -511.48516845703125,
      "loss": 0.5858,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2674116790294647,
      "rewards/margins": 0.39314374327659607,
      "rewards/rejected": -0.6605554819107056,
      "step": 90
    },
    {
      "epoch": 0.6279434850863422,
      "grad_norm": 7.416871526087097,
      "learning_rate": 1.0932357971453743e-07,
      "logits/chosen": 0.6527765393257141,
      "logits/rejected": 0.881365180015564,
      "logps/chosen": -290.30291748046875,
      "logps/pi_response": -137.30520629882812,
      "logps/ref_response": -141.33499145507812,
      "logps/rejected": -528.6287231445312,
      "loss": 0.5723,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2469521462917328,
      "rewards/margins": 0.4689961075782776,
      "rewards/rejected": -0.7159483432769775,
      "step": 100
    },
    {
      "epoch": 0.6907378335949764,
      "grad_norm": 6.868702818175434,
      "learning_rate": 7.883680337481599e-08,
      "logits/chosen": 0.5740480422973633,
      "logits/rejected": 0.858615517616272,
      "logps/chosen": -303.41119384765625,
      "logps/pi_response": -136.2499542236328,
      "logps/ref_response": -139.31204223632812,
      "logps/rejected": -475.9146423339844,
      "loss": 0.5688,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.3227171003818512,
      "rewards/margins": 0.34727370738983154,
      "rewards/rejected": -0.6699907183647156,
      "step": 110
    },
    {
      "epoch": 0.7535321821036107,
      "grad_norm": 7.117305529201249,
      "learning_rate": 5.177088990820725e-08,
      "logits/chosen": 0.8573010563850403,
      "logits/rejected": 1.0616093873977661,
      "logps/chosen": -343.8194885253906,
      "logps/pi_response": -163.02926635742188,
      "logps/ref_response": -168.28973388671875,
      "logps/rejected": -572.6094970703125,
      "loss": 0.5672,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.34128373861312866,
      "rewards/margins": 0.48513007164001465,
      "rewards/rejected": -0.8264138102531433,
      "step": 120
    },
    {
      "epoch": 0.8163265306122449,
      "grad_norm": 7.6342489936056275,
      "learning_rate": 2.942691603548416e-08,
      "logits/chosen": 0.6449292302131653,
      "logits/rejected": 0.8830472826957703,
      "logps/chosen": -324.7782287597656,
      "logps/pi_response": -156.79000854492188,
      "logps/ref_response": -161.82154846191406,
      "logps/rejected": -524.2703857421875,
      "loss": 0.5844,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.3194042146205902,
      "rewards/margins": 0.4711576998233795,
      "rewards/rejected": -0.7905619740486145,
      "step": 130
    },
    {
      "epoch": 0.8791208791208791,
      "grad_norm": 6.752884100895449,
      "learning_rate": 1.2878971655412513e-08,
      "logits/chosen": 0.8525398373603821,
      "logits/rejected": 1.0120213031768799,
      "logps/chosen": -330.0039978027344,
      "logps/pi_response": -156.17630004882812,
      "logps/ref_response": -161.65866088867188,
      "logps/rejected": -559.6900634765625,
      "loss": 0.5577,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.2980651557445526,
      "rewards/margins": 0.5507915616035461,
      "rewards/rejected": -0.8488567471504211,
      "step": 140
    },
    {
      "epoch": 0.9419152276295133,
      "grad_norm": 7.563586487187802,
      "learning_rate": 2.922527618666465e-09,
      "logits/chosen": 0.5885689854621887,
      "logits/rejected": 0.828190803527832,
      "logps/chosen": -297.84844970703125,
      "logps/pi_response": -152.97872924804688,
      "logps/ref_response": -157.01953125,
      "logps/rejected": -508.4051818847656,
      "loss": 0.5623,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.2831505537033081,
      "rewards/margins": 0.48666033148765564,
      "rewards/rejected": -0.7698109149932861,
      "step": 150
    },
    {
      "epoch": 0.9984301412872841,
      "step": 159,
      "total_flos": 0.0,
      "train_loss": 0.6040852474716475,
      "train_runtime": 3000.2234,
      "train_samples_per_second": 6.792,
      "train_steps_per_second": 0.053
    }
  ],
  "logging_steps": 10,
  "max_steps": 159,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}