{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.968,
  "eval_steps": 100,
  "global_step": 248,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.02,
      "learning_rate": 2.0000000000000002e-07,
      "logits/chosen": 0.9600980877876282,
      "logits/rejected": 1.094868540763855,
      "logps/chosen": -119.03445434570312,
      "logps/rejected": -112.18000030517578,
      "loss": 0.0003,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.16,
      "learning_rate": 2.0000000000000003e-06,
      "logits/chosen": 0.8413593769073486,
      "logits/rejected": 0.813139796257019,
      "logps/chosen": -142.51600646972656,
      "logps/rejected": -130.04800415039062,
      "loss": 0.0004,
      "rewards/accuracies": 0.3194444477558136,
      "rewards/chosen": 0.0009210502030327916,
      "rewards/margins": -8.903745765564963e-05,
      "rewards/rejected": 0.0010100876679643989,
      "step": 10
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.000000000000001e-06,
      "logits/chosen": 0.8089853525161743,
      "logits/rejected": 0.8187138438224792,
      "logps/chosen": -155.18431091308594,
      "logps/rejected": -143.44085693359375,
      "loss": 0.0004,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 0.0007359608425758779,
      "rewards/margins": -0.0004602448898367584,
      "rewards/rejected": 0.001196205848827958,
      "step": 20
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.993800445762451e-06,
      "logits/chosen": 0.8762779235839844,
      "logits/rejected": 0.9211899638175964,
      "logps/chosen": -154.8330535888672,
      "logps/rejected": -141.16299438476562,
      "loss": 0.0004,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -1.3224826943769585e-05,
      "rewards/margins": -0.00042751478031277657,
      "rewards/rejected": 0.0004142899124417454,
      "step": 30
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.944388344834205e-06,
      "logits/chosen": 0.8115983009338379,
      "logits/rejected": 0.8376752734184265,
      "logps/chosen": -172.55001831054688,
      "logps/rejected": -157.711669921875,
      "loss": 0.0004,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": 0.00026369161787442863,
      "rewards/margins": 4.611946133081801e-05,
      "rewards/rejected": 0.0002175722038373351,
      "step": 40
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.8465431931347904e-06,
      "logits/chosen": 0.8291877508163452,
      "logits/rejected": 0.8113203048706055,
      "logps/chosen": -160.0177764892578,
      "logps/rejected": -146.55140686035156,
      "loss": 0.0004,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -5.383044481277466e-07,
      "rewards/margins": -8.223028999054804e-05,
      "rewards/rejected": 8.169189095497131e-05,
      "step": 50
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.702203692102539e-06,
      "logits/chosen": 0.7359780073165894,
      "logits/rejected": 0.7216537594795227,
      "logps/chosen": -146.64401245117188,
      "logps/rejected": -134.4973907470703,
      "loss": 0.0003,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": 0.0005743899382650852,
      "rewards/margins": 0.00039931651554070413,
      "rewards/rejected": 0.00017507340817246586,
      "step": 60
    },
    {
      "epoch": 1.12,
      "learning_rate": 4.514229781074239e-06,
      "logits/chosen": 0.8711406588554382,
      "logits/rejected": 0.8447128534317017,
      "logps/chosen": -146.93917846679688,
      "logps/rejected": -137.2763214111328,
      "loss": 0.0003,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -9.175995364785194e-05,
      "rewards/margins": 0.00013136237976141274,
      "rewards/rejected": -0.00022312237706501037,
      "step": 70
    },
    {
      "epoch": 1.28,
      "learning_rate": 4.286345970517195e-06,
      "logits/chosen": 0.8513940572738647,
      "logits/rejected": 0.8943384289741516,
      "logps/chosen": -156.4779510498047,
      "logps/rejected": -141.64044189453125,
      "loss": 0.0003,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.0007812186959199607,
      "rewards/margins": 0.0002858097432181239,
      "rewards/rejected": -0.0010670283809304237,
      "step": 80
    },
    {
      "epoch": 1.44,
      "learning_rate": 4.023067544670082e-06,
      "logits/chosen": 0.8046412467956543,
      "logits/rejected": 0.8172038793563843,
      "logps/chosen": -159.82638549804688,
      "logps/rejected": -148.19137573242188,
      "loss": 0.0003,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.0005336708854883909,
      "rewards/margins": 0.000577736587729305,
      "rewards/rejected": -0.0011114075314253569,
      "step": 90
    },
    {
      "epoch": 1.6,
      "learning_rate": 3.7296110958116845e-06,
      "logits/chosen": 0.7820717096328735,
      "logits/rejected": 0.8074045181274414,
      "logps/chosen": -149.77316284179688,
      "logps/rejected": -137.23370361328125,
      "loss": 0.0003,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.00042833946645259857,
      "rewards/margins": 0.0002858256921172142,
      "rewards/rejected": -0.0007141652167774737,
      "step": 100
    },
    {
      "epoch": 1.6,
      "eval_logits/chosen": 0.8386867046356201,
      "eval_logits/rejected": 0.8959885835647583,
      "eval_logps/chosen": -256.5564880371094,
      "eval_logps/rejected": -233.50169372558594,
      "eval_loss": 0.0003867279156111181,
      "eval_rewards/accuracies": 0.4855000078678131,
      "eval_rewards/chosen": 0.0005771049181930721,
      "eval_rewards/margins": 0.00021130235109012574,
      "eval_rewards/rejected": 0.00036580252344720066,
      "eval_runtime": 895.2516,
      "eval_samples_per_second": 2.234,
      "eval_steps_per_second": 0.559,
      "step": 100
    },
    {
      "epoch": 1.76,
      "learning_rate": 3.4117911628292944e-06,
      "logits/chosen": 0.8274463415145874,
      "logits/rejected": 0.7994173765182495,
      "logps/chosen": -160.32693481445312,
      "logps/rejected": -144.13430786132812,
      "loss": 0.0004,
      "rewards/accuracies": 0.3687500059604645,
      "rewards/chosen": -0.0019955369643867016,
      "rewards/margins": -0.0002371647860854864,
      "rewards/rejected": -0.0017583720618858933,
      "step": 110
    },
    {
      "epoch": 1.92,
      "learning_rate": 3.075905022087675e-06,
      "logits/chosen": 0.8538025617599487,
      "logits/rejected": 0.8362048268318176,
      "logps/chosen": -144.97744750976562,
      "logps/rejected": -132.49935913085938,
      "loss": 0.0003,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0011052186600863934,
      "rewards/margins": 0.00014151711366139352,
      "rewards/rejected": -0.0012467356864362955,
      "step": 120
    },
    {
      "epoch": 2.08,
      "learning_rate": 2.728607913349464e-06,
      "logits/chosen": 0.8091080784797668,
      "logits/rejected": 0.7883086800575256,
      "logps/chosen": -168.44651794433594,
      "logps/rejected": -154.5172882080078,
      "loss": 0.0003,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.001501282094977796,
      "rewards/margins": 0.00047290659858845174,
      "rewards/rejected": -0.001974188955500722,
      "step": 130
    },
    {
      "epoch": 2.24,
      "learning_rate": 2.376781173017589e-06,
      "logits/chosen": 0.8410237431526184,
      "logits/rejected": 0.8125711679458618,
      "logps/chosen": -135.33348083496094,
      "logps/rejected": -125.95147705078125,
      "loss": 0.0003,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.0011694638524204493,
      "rewards/margins": 0.0012481597950682044,
      "rewards/rejected": -0.002417623531073332,
      "step": 140
    },
    {
      "epoch": 2.4,
      "learning_rate": 2.0273958875043877e-06,
      "logits/chosen": 0.8500027656555176,
      "logits/rejected": 0.8616348505020142,
      "logps/chosen": -163.2034454345703,
      "logps/rejected": -147.33352661132812,
      "loss": 0.0003,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.0014132513897493482,
      "rewards/margins": 0.0005158716812729836,
      "rewards/rejected": -0.0019291229546070099,
      "step": 150
    },
    {
      "epoch": 2.56,
      "learning_rate": 1.6873747682962393e-06,
      "logits/chosen": 0.7654728293418884,
      "logits/rejected": 0.7762084007263184,
      "logps/chosen": -164.34304809570312,
      "logps/rejected": -151.01991271972656,
      "loss": 0.0003,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.002124720485880971,
      "rewards/margins": 0.0006703810067847371,
      "rewards/rejected": -0.00279510160908103,
      "step": 160
    },
    {
      "epoch": 2.72,
      "learning_rate": 1.363454985517803e-06,
      "logits/chosen": 0.8323400616645813,
      "logits/rejected": 0.8276194334030151,
      "logps/chosen": -139.4966278076172,
      "logps/rejected": -126.7676010131836,
      "loss": 0.0003,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.002051023533567786,
      "rewards/margins": 0.0008618654683232307,
      "rewards/rejected": -0.002912889001891017,
      "step": 170
    },
    {
      "epoch": 2.88,
      "learning_rate": 1.062054677808238e-06,
      "logits/chosen": 0.8200883865356445,
      "logits/rejected": 0.8247148394584656,
      "logps/chosen": -147.3248291015625,
      "logps/rejected": -131.50941467285156,
      "loss": 0.0004,
      "rewards/accuracies": 0.39375001192092896,
      "rewards/chosen": -0.002400606404989958,
      "rewards/margins": -0.0002633151598274708,
      "rewards/rejected": -0.0021372910123318434,
      "step": 180
    },
    {
      "epoch": 3.04,
      "learning_rate": 7.891457834794711e-07,
      "logits/chosen": 0.7988116145133972,
      "logits/rejected": 0.7790786027908325,
      "logps/chosen": -157.28366088867188,
      "logps/rejected": -148.06373596191406,
      "loss": 0.0003,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.0013300395803526044,
      "rewards/margins": 0.0013687892351299524,
      "rewards/rejected": -0.002698828699067235,
      "step": 190
    },
    {
      "epoch": 3.2,
      "learning_rate": 5.501357126768117e-07,
      "logits/chosen": 0.8291190266609192,
      "logits/rejected": 0.8441616296768188,
      "logps/chosen": -179.22891235351562,
      "logps/rejected": -165.08668518066406,
      "loss": 0.0003,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.0026950170285999775,
      "rewards/margins": 0.0003695812192745507,
      "rewards/rejected": -0.0030645981896668673,
      "step": 200
    },
    {
      "epoch": 3.2,
      "eval_logits/chosen": 0.8411728143692017,
      "eval_logits/rejected": 0.8983786106109619,
      "eval_logps/chosen": -256.48114013671875,
      "eval_logps/rejected": -233.4491729736328,
      "eval_loss": 0.0003846254840027541,
      "eval_rewards/accuracies": 0.5099999904632568,
      "eval_rewards/chosen": 0.0013305767206475139,
      "eval_rewards/margins": 0.00043944927165284753,
      "eval_rewards/rejected": 0.0008911274489946663,
      "eval_runtime": 662.5153,
      "eval_samples_per_second": 3.019,
      "eval_steps_per_second": 0.755,
      "step": 200
    },
    {
      "epoch": 3.36,
      "learning_rate": 3.4976020508682345e-07,
      "logits/chosen": 0.7520900964736938,
      "logits/rejected": 0.7521581053733826,
      "logps/chosen": -142.36843872070312,
      "logps/rejected": -131.4374542236328,
      "loss": 0.0003,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.0021503984462469816,
      "rewards/margins": 0.0008300153422169387,
      "rewards/rejected": -0.0029804136138409376,
      "step": 210
    },
    {
      "epoch": 3.52,
      "learning_rate": 1.9198949610721273e-07,
      "logits/chosen": 0.8309981226921082,
      "logits/rejected": 0.811602771282196,
      "logps/chosen": -150.01573181152344,
      "logps/rejected": -136.91067504882812,
      "loss": 0.0003,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.00250506354495883,
      "rewards/margins": 0.0008244045311585069,
      "rewards/rejected": -0.0033294681925326586,
      "step": 220
    },
    {
      "epoch": 3.68,
      "learning_rate": 7.994965069994143e-08,
      "logits/chosen": 0.8669425249099731,
      "logits/rejected": 0.880296528339386,
      "logps/chosen": -159.811767578125,
      "logps/rejected": -145.57418823242188,
      "loss": 0.0003,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.0020190321374684572,
      "rewards/margins": 0.0008607100462540984,
      "rewards/rejected": -0.002879742067307234,
      "step": 230
    },
    {
      "epoch": 3.84,
      "learning_rate": 1.5860623616664183e-08,
      "logits/chosen": 0.7746256589889526,
      "logits/rejected": 0.7747436761856079,
      "logps/chosen": -154.1355743408203,
      "logps/rejected": -141.05926513671875,
      "loss": 0.0003,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.001991902943700552,
      "rewards/margins": 0.000743478478398174,
      "rewards/rejected": -0.0027353812474757433,
      "step": 240
    },
    {
      "epoch": 3.97,
      "step": 248,
      "total_flos": 0.0,
      "train_loss": 0.0003399105843263603,
      "train_runtime": 4613.339,
      "train_samples_per_second": 0.867,
      "train_steps_per_second": 0.054
    }
  ],
  "logging_steps": 10,
  "max_steps": 248,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 4,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}