File size: 37,543 Bytes
f799945 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 |
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 100,
"global_step": 43,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"debug/policy_chosen_logits": -1.5398600101470947,
"debug/policy_chosen_logps": -263.839111328125,
"debug/policy_rejected_logits": -1.3813989162445068,
"debug/policy_rejected_logps": -320.2253112792969,
"debug/reference_chosen_logps": -263.839111328125,
"debug/reference_rejected_logps": -320.2253112792969,
"epoch": 0.023255813953488372,
"grad_norm": 17.674977894001827,
"learning_rate": 1e-06,
"logits/chosen": -1.5398600101470947,
"logits/rejected": -1.3813989162445068,
"logps/chosen": -263.839111328125,
"logps/rejected": -320.2253112792969,
"loss": 0.5,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"debug/policy_chosen_logits": -1.61044442653656,
"debug/policy_chosen_logps": -222.2188720703125,
"debug/policy_rejected_logits": -1.3342773914337158,
"debug/policy_rejected_logps": -305.922607421875,
"debug/reference_chosen_logps": -221.88748168945312,
"debug/reference_rejected_logps": -306.1925048828125,
"epoch": 0.046511627906976744,
"grad_norm": 40.70183376910324,
"learning_rate": 1e-06,
"logits/chosen": -1.61044442653656,
"logits/rejected": -1.3342773914337158,
"logps/chosen": -222.2188720703125,
"logps/rejected": -305.922607421875,
"loss": 0.5035,
"rewards/accuracies": 0.375,
"rewards/chosen": -0.003313712775707245,
"rewards/margins": -0.006012477912008762,
"rewards/rejected": 0.00269876467064023,
"step": 2
},
{
"debug/policy_chosen_logits": -1.4612830877304077,
"debug/policy_chosen_logps": -211.158935546875,
"debug/policy_rejected_logits": -1.4117305278778076,
"debug/policy_rejected_logps": -295.76220703125,
"debug/reference_chosen_logps": -210.71981811523438,
"debug/reference_rejected_logps": -295.57293701171875,
"epoch": 0.06976744186046512,
"grad_norm": 18.75915861796485,
"learning_rate": 1e-06,
"logits/chosen": -1.4612830877304077,
"logits/rejected": -1.4117305278778076,
"logps/chosen": -211.158935546875,
"logps/rejected": -295.76220703125,
"loss": 0.5071,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.004391231574118137,
"rewards/margins": -0.0024983598850667477,
"rewards/rejected": -0.001892871456220746,
"step": 3
},
{
"debug/policy_chosen_logits": -1.613046646118164,
"debug/policy_chosen_logps": -302.75689697265625,
"debug/policy_rejected_logits": -1.5550684928894043,
"debug/policy_rejected_logps": -374.60919189453125,
"debug/reference_chosen_logps": -300.4857177734375,
"debug/reference_rejected_logps": -372.85357666015625,
"epoch": 0.09302325581395349,
"grad_norm": 16.079388960546417,
"learning_rate": 1e-06,
"logits/chosen": -1.613046646118164,
"logits/rejected": -1.5550684928894043,
"logps/chosen": -302.75689697265625,
"logps/rejected": -374.60919189453125,
"loss": 0.503,
"rewards/accuracies": 0.25,
"rewards/chosen": -0.02271179109811783,
"rewards/margins": -0.005155391059815884,
"rewards/rejected": -0.01755640096962452,
"step": 4
},
{
"debug/policy_chosen_logits": -1.453744649887085,
"debug/policy_chosen_logps": -324.67041015625,
"debug/policy_rejected_logits": -1.337669849395752,
"debug/policy_rejected_logps": -347.67181396484375,
"debug/reference_chosen_logps": -320.1375732421875,
"debug/reference_rejected_logps": -345.66351318359375,
"epoch": 0.11627906976744186,
"grad_norm": 41.72011905946601,
"learning_rate": 1e-06,
"logits/chosen": -1.453744649887085,
"logits/rejected": -1.337669849395752,
"logps/chosen": -324.67041015625,
"logps/rejected": -347.67181396484375,
"loss": 0.5138,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.04532808065414429,
"rewards/margins": -0.02524520829319954,
"rewards/rejected": -0.020082874223589897,
"step": 5
},
{
"debug/policy_chosen_logits": -1.6281760931015015,
"debug/policy_chosen_logps": -235.80917358398438,
"debug/policy_rejected_logits": -1.4969021081924438,
"debug/policy_rejected_logps": -356.2055358886719,
"debug/reference_chosen_logps": -230.56964111328125,
"debug/reference_rejected_logps": -353.9713134765625,
"epoch": 0.13953488372093023,
"grad_norm": 51.19218432335038,
"learning_rate": 1e-06,
"logits/chosen": -1.6281760931015015,
"logits/rejected": -1.4969021081924438,
"logps/chosen": -235.80917358398438,
"logps/rejected": -356.2055358886719,
"loss": 0.5101,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.0523955337703228,
"rewards/margins": -0.030053365975618362,
"rewards/rejected": -0.022342165932059288,
"step": 6
},
{
"debug/policy_chosen_logits": -1.3837248086929321,
"debug/policy_chosen_logps": -246.68148803710938,
"debug/policy_rejected_logits": -1.1755256652832031,
"debug/policy_rejected_logps": -349.03863525390625,
"debug/reference_chosen_logps": -243.39242553710938,
"debug/reference_rejected_logps": -341.8122863769531,
"epoch": 0.16279069767441862,
"grad_norm": 87.4581502140832,
"learning_rate": 1e-06,
"logits/chosen": -1.3837248086929321,
"logits/rejected": -1.1755256652832031,
"logps/chosen": -246.68148803710938,
"logps/rejected": -349.03863525390625,
"loss": 0.5205,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.03289069980382919,
"rewards/margins": 0.03937286138534546,
"rewards/rejected": -0.07226356118917465,
"step": 7
},
{
"debug/policy_chosen_logits": -1.4155532121658325,
"debug/policy_chosen_logps": -323.04852294921875,
"debug/policy_rejected_logits": -1.4017454385757446,
"debug/policy_rejected_logps": -340.23590087890625,
"debug/reference_chosen_logps": -320.93182373046875,
"debug/reference_rejected_logps": -338.517578125,
"epoch": 0.18604651162790697,
"grad_norm": 16.872050626378687,
"learning_rate": 1e-06,
"logits/chosen": -1.4155532121658325,
"logits/rejected": -1.4017454385757446,
"logps/chosen": -323.04852294921875,
"logps/rejected": -340.23590087890625,
"loss": 0.4939,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.021166954189538956,
"rewards/margins": -0.003983440343290567,
"rewards/rejected": -0.017183512449264526,
"step": 8
},
{
"debug/policy_chosen_logits": -1.5506556034088135,
"debug/policy_chosen_logps": -292.11676025390625,
"debug/policy_rejected_logits": -1.4309040307998657,
"debug/policy_rejected_logps": -307.30072021484375,
"debug/reference_chosen_logps": -288.0973205566406,
"debug/reference_rejected_logps": -306.0505676269531,
"epoch": 0.20930232558139536,
"grad_norm": 12.819676343754237,
"learning_rate": 1e-06,
"logits/chosen": -1.5506556034088135,
"logits/rejected": -1.4309040307998657,
"logps/chosen": -292.11676025390625,
"logps/rejected": -307.30072021484375,
"loss": 0.509,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.04019462317228317,
"rewards/margins": -0.027693118900060654,
"rewards/rejected": -0.012501506134867668,
"step": 9
},
{
"debug/policy_chosen_logits": -1.5686500072479248,
"debug/policy_chosen_logps": -248.2123260498047,
"debug/policy_rejected_logits": -1.543015480041504,
"debug/policy_rejected_logps": -311.9925537109375,
"debug/reference_chosen_logps": -249.5205078125,
"debug/reference_rejected_logps": -309.8684997558594,
"epoch": 0.23255813953488372,
"grad_norm": 15.183229742103123,
"learning_rate": 1e-06,
"logits/chosen": -1.5686500072479248,
"logits/rejected": -1.543015480041504,
"logps/chosen": -248.2123260498047,
"logps/rejected": -311.9925537109375,
"loss": 0.479,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.013081644661724567,
"rewards/margins": 0.0343221053481102,
"rewards/rejected": -0.021240463480353355,
"step": 10
},
{
"debug/policy_chosen_logits": -1.6759617328643799,
"debug/policy_chosen_logps": -241.01116943359375,
"debug/policy_rejected_logits": -1.577236533164978,
"debug/policy_rejected_logps": -277.329833984375,
"debug/reference_chosen_logps": -243.44976806640625,
"debug/reference_rejected_logps": -281.47589111328125,
"epoch": 0.2558139534883721,
"grad_norm": 31.90728084918248,
"learning_rate": 1e-06,
"logits/chosen": -1.6759617328643799,
"logits/rejected": -1.577236533164978,
"logps/chosen": -241.01116943359375,
"logps/rejected": -277.329833984375,
"loss": 0.4928,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.024385996162891388,
"rewards/margins": -0.01707470789551735,
"rewards/rejected": 0.04146070405840874,
"step": 11
},
{
"debug/policy_chosen_logits": -1.5532420873641968,
"debug/policy_chosen_logps": -223.29437255859375,
"debug/policy_rejected_logits": -1.4593762159347534,
"debug/policy_rejected_logps": -401.5757141113281,
"debug/reference_chosen_logps": -226.59278869628906,
"debug/reference_rejected_logps": -400.6639099121094,
"epoch": 0.27906976744186046,
"grad_norm": 19.326544561851396,
"learning_rate": 1e-06,
"logits/chosen": -1.5532420873641968,
"logits/rejected": -1.4593762159347534,
"logps/chosen": -223.29437255859375,
"logps/rejected": -401.5757141113281,
"loss": 0.506,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.03298404812812805,
"rewards/margins": 0.042102016508579254,
"rewards/rejected": -0.009117968380451202,
"step": 12
},
{
"debug/policy_chosen_logits": -1.4542545080184937,
"debug/policy_chosen_logps": -257.8163146972656,
"debug/policy_rejected_logits": -1.385650634765625,
"debug/policy_rejected_logps": -263.74664306640625,
"debug/reference_chosen_logps": -256.629638671875,
"debug/reference_rejected_logps": -271.56353759765625,
"epoch": 0.3023255813953488,
"grad_norm": 68.62278258007824,
"learning_rate": 1e-06,
"logits/chosen": -1.4542545080184937,
"logits/rejected": -1.385650634765625,
"logps/chosen": -257.8163146972656,
"logps/rejected": -263.74664306640625,
"loss": 0.5367,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.011866549961268902,
"rewards/margins": -0.09003522992134094,
"rewards/rejected": 0.07816867530345917,
"step": 13
},
{
"debug/policy_chosen_logits": -1.6388806104660034,
"debug/policy_chosen_logps": -191.90548706054688,
"debug/policy_rejected_logits": -1.3299458026885986,
"debug/policy_rejected_logps": -283.2551574707031,
"debug/reference_chosen_logps": -193.77301025390625,
"debug/reference_rejected_logps": -284.05615234375,
"epoch": 0.32558139534883723,
"grad_norm": 54.79498866692366,
"learning_rate": 1e-06,
"logits/chosen": -1.6388806104660034,
"logits/rejected": -1.3299458026885986,
"logps/chosen": -191.90548706054688,
"logps/rejected": -283.2551574707031,
"loss": 0.5391,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.018675191327929497,
"rewards/margins": 0.010665085166692734,
"rewards/rejected": 0.008010102435946465,
"step": 14
},
{
"debug/policy_chosen_logits": -1.4526491165161133,
"debug/policy_chosen_logps": -284.34832763671875,
"debug/policy_rejected_logits": -1.4107521772384644,
"debug/policy_rejected_logps": -363.9824523925781,
"debug/reference_chosen_logps": -282.1776123046875,
"debug/reference_rejected_logps": -362.1690368652344,
"epoch": 0.3488372093023256,
"grad_norm": 55.58252467843115,
"learning_rate": 1e-06,
"logits/chosen": -1.4526491165161133,
"logits/rejected": -1.4107521772384644,
"logps/chosen": -284.34832763671875,
"logps/rejected": -363.9824523925781,
"loss": 0.5002,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.021707000210881233,
"rewards/margins": -0.0035727117210626602,
"rewards/rejected": -0.018134288489818573,
"step": 15
},
{
"debug/policy_chosen_logits": -1.5884922742843628,
"debug/policy_chosen_logps": -250.37115478515625,
"debug/policy_rejected_logits": -1.519769549369812,
"debug/policy_rejected_logps": -297.04339599609375,
"debug/reference_chosen_logps": -246.65272521972656,
"debug/reference_rejected_logps": -291.1093444824219,
"epoch": 0.37209302325581395,
"grad_norm": 20.97275013484062,
"learning_rate": 1e-06,
"logits/chosen": -1.5884922742843628,
"logits/rejected": -1.519769549369812,
"logps/chosen": -250.37115478515625,
"logps/rejected": -297.04339599609375,
"loss": 0.4843,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.037184178829193115,
"rewards/margins": 0.022156618535518646,
"rewards/rejected": -0.05934080481529236,
"step": 16
},
{
"debug/policy_chosen_logits": -1.5344979763031006,
"debug/policy_chosen_logps": -214.46148681640625,
"debug/policy_rejected_logits": -1.4357935190200806,
"debug/policy_rejected_logps": -316.449951171875,
"debug/reference_chosen_logps": -210.27589416503906,
"debug/reference_rejected_logps": -310.38165283203125,
"epoch": 0.3953488372093023,
"grad_norm": 43.215377561087685,
"learning_rate": 1e-06,
"logits/chosen": -1.5344979763031006,
"logits/rejected": -1.4357935190200806,
"logps/chosen": -214.46148681640625,
"logps/rejected": -316.449951171875,
"loss": 0.5047,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.04185575246810913,
"rewards/margins": 0.01882733218371868,
"rewards/rejected": -0.06068308651447296,
"step": 17
},
{
"debug/policy_chosen_logits": -1.5920026302337646,
"debug/policy_chosen_logps": -313.2801513671875,
"debug/policy_rejected_logits": -1.4743777513504028,
"debug/policy_rejected_logps": -344.209716796875,
"debug/reference_chosen_logps": -304.38165283203125,
"debug/reference_rejected_logps": -334.8297424316406,
"epoch": 0.4186046511627907,
"grad_norm": 46.01762275116522,
"learning_rate": 1e-06,
"logits/chosen": -1.5920026302337646,
"logits/rejected": -1.4743777513504028,
"logps/chosen": -313.2801513671875,
"logps/rejected": -344.209716796875,
"loss": 0.5061,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.0889848843216896,
"rewards/margins": 0.004814721643924713,
"rewards/rejected": -0.09379960596561432,
"step": 18
},
{
"debug/policy_chosen_logits": -1.5471267700195312,
"debug/policy_chosen_logps": -275.2354736328125,
"debug/policy_rejected_logits": -1.5307285785675049,
"debug/policy_rejected_logps": -272.6752624511719,
"debug/reference_chosen_logps": -266.9699401855469,
"debug/reference_rejected_logps": -267.95025634765625,
"epoch": 0.4418604651162791,
"grad_norm": 78.66006276541071,
"learning_rate": 1e-06,
"logits/chosen": -1.5471267700195312,
"logits/rejected": -1.5307285785675049,
"logps/chosen": -275.2354736328125,
"logps/rejected": -272.6752624511719,
"loss": 0.5021,
"rewards/accuracies": 0.25,
"rewards/chosen": -0.08265500515699387,
"rewards/margins": -0.03540489077568054,
"rewards/rejected": -0.047250114381313324,
"step": 19
},
{
"debug/policy_chosen_logits": -1.5639888048171997,
"debug/policy_chosen_logps": -260.4048767089844,
"debug/policy_rejected_logits": -1.4268858432769775,
"debug/policy_rejected_logps": -320.64324951171875,
"debug/reference_chosen_logps": -257.83966064453125,
"debug/reference_rejected_logps": -314.41796875,
"epoch": 0.46511627906976744,
"grad_norm": 76.10262014579365,
"learning_rate": 1e-06,
"logits/chosen": -1.5639888048171997,
"logits/rejected": -1.4268858432769775,
"logps/chosen": -260.4048767089844,
"logps/rejected": -320.64324951171875,
"loss": 0.4966,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.025652198120951653,
"rewards/margins": 0.036600261926651,
"rewards/rejected": -0.0622524619102478,
"step": 20
},
{
"debug/policy_chosen_logits": -1.5689480304718018,
"debug/policy_chosen_logps": -301.9723205566406,
"debug/policy_rejected_logits": -1.5015813112258911,
"debug/policy_rejected_logps": -363.80145263671875,
"debug/reference_chosen_logps": -296.4005432128906,
"debug/reference_rejected_logps": -355.6829833984375,
"epoch": 0.4883720930232558,
"grad_norm": 57.20135841653825,
"learning_rate": 1e-06,
"logits/chosen": -1.5689480304718018,
"logits/rejected": -1.5015813112258911,
"logps/chosen": -301.9723205566406,
"logps/rejected": -363.80145263671875,
"loss": 0.5027,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.055717699229717255,
"rewards/margins": 0.025466954335570335,
"rewards/rejected": -0.08118465542793274,
"step": 21
},
{
"debug/policy_chosen_logits": -1.6258084774017334,
"debug/policy_chosen_logps": -271.4725341796875,
"debug/policy_rejected_logits": -1.5413790941238403,
"debug/policy_rejected_logps": -298.4366760253906,
"debug/reference_chosen_logps": -269.2203369140625,
"debug/reference_rejected_logps": -294.7615966796875,
"epoch": 0.5116279069767442,
"grad_norm": 24.247441456009533,
"learning_rate": 1e-06,
"logits/chosen": -1.6258084774017334,
"logits/rejected": -1.5413790941238403,
"logps/chosen": -271.4725341796875,
"logps/rejected": -298.4366760253906,
"loss": 0.4965,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.022522106766700745,
"rewards/margins": 0.014228800311684608,
"rewards/rejected": -0.0367509089410305,
"step": 22
},
{
"debug/policy_chosen_logits": -1.4780645370483398,
"debug/policy_chosen_logps": -238.43905639648438,
"debug/policy_rejected_logits": -1.4756673574447632,
"debug/policy_rejected_logps": -300.5503845214844,
"debug/reference_chosen_logps": -234.42031860351562,
"debug/reference_rejected_logps": -297.0867919921875,
"epoch": 0.5348837209302325,
"grad_norm": 40.36957484697572,
"learning_rate": 1e-06,
"logits/chosen": -1.4780645370483398,
"logits/rejected": -1.4756673574447632,
"logps/chosen": -238.43905639648438,
"logps/rejected": -300.5503845214844,
"loss": 0.5159,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.040187377482652664,
"rewards/margins": -0.005551624111831188,
"rewards/rejected": -0.0346357524394989,
"step": 23
},
{
"debug/policy_chosen_logits": -1.653166651725769,
"debug/policy_chosen_logps": -278.77362060546875,
"debug/policy_rejected_logits": -1.4196488857269287,
"debug/policy_rejected_logps": -267.0096435546875,
"debug/reference_chosen_logps": -280.239013671875,
"debug/reference_rejected_logps": -267.35540771484375,
"epoch": 0.5581395348837209,
"grad_norm": 43.31573267916226,
"learning_rate": 1e-06,
"logits/chosen": -1.653166651725769,
"logits/rejected": -1.4196488857269287,
"logps/chosen": -278.77362060546875,
"logps/rejected": -267.0096435546875,
"loss": 0.5133,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.014654004946351051,
"rewards/margins": 0.011196250095963478,
"rewards/rejected": 0.0034577548503875732,
"step": 24
},
{
"debug/policy_chosen_logits": -1.5763293504714966,
"debug/policy_chosen_logps": -231.9792938232422,
"debug/policy_rejected_logits": -1.5205391645431519,
"debug/policy_rejected_logps": -304.4539794921875,
"debug/reference_chosen_logps": -230.9835662841797,
"debug/reference_rejected_logps": -301.9881896972656,
"epoch": 0.5813953488372093,
"grad_norm": 26.712778912083667,
"learning_rate": 1e-06,
"logits/chosen": -1.5763293504714966,
"logits/rejected": -1.5205391645431519,
"logps/chosen": -231.9792938232422,
"logps/rejected": -304.4539794921875,
"loss": 0.4975,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.009957179427146912,
"rewards/margins": 0.014700757339596748,
"rewards/rejected": -0.02465793862938881,
"step": 25
},
{
"debug/policy_chosen_logits": -1.4429453611373901,
"debug/policy_chosen_logps": -223.239501953125,
"debug/policy_rejected_logits": -1.4664965867996216,
"debug/policy_rejected_logps": -303.63232421875,
"debug/reference_chosen_logps": -226.43624877929688,
"debug/reference_rejected_logps": -305.3199157714844,
"epoch": 0.6046511627906976,
"grad_norm": 19.046211934497915,
"learning_rate": 1e-06,
"logits/chosen": -1.4429453611373901,
"logits/rejected": -1.4664965867996216,
"logps/chosen": -223.239501953125,
"logps/rejected": -303.63232421875,
"loss": 0.4694,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.03196748346090317,
"rewards/margins": 0.015091512352228165,
"rewards/rejected": 0.016875972971320152,
"step": 26
},
{
"debug/policy_chosen_logits": -1.565663456916809,
"debug/policy_chosen_logps": -309.9948425292969,
"debug/policy_rejected_logits": -1.5396883487701416,
"debug/policy_rejected_logps": -343.0301818847656,
"debug/reference_chosen_logps": -308.83489990234375,
"debug/reference_rejected_logps": -338.6658935546875,
"epoch": 0.627906976744186,
"grad_norm": 37.980256590319236,
"learning_rate": 1e-06,
"logits/chosen": -1.565663456916809,
"logits/rejected": -1.5396883487701416,
"logps/chosen": -309.9948425292969,
"logps/rejected": -343.0301818847656,
"loss": 0.4978,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.011599387973546982,
"rewards/margins": 0.032043568789958954,
"rewards/rejected": -0.043642960488796234,
"step": 27
},
{
"debug/policy_chosen_logits": -1.5224864482879639,
"debug/policy_chosen_logps": -277.25262451171875,
"debug/policy_rejected_logits": -1.2660479545593262,
"debug/policy_rejected_logps": -272.9588623046875,
"debug/reference_chosen_logps": -273.28631591796875,
"debug/reference_rejected_logps": -273.3193359375,
"epoch": 0.6511627906976745,
"grad_norm": 26.186511827710103,
"learning_rate": 1e-06,
"logits/chosen": -1.5224864482879639,
"logits/rejected": -1.2660479545593262,
"logps/chosen": -277.25262451171875,
"logps/rejected": -272.9588623046875,
"loss": 0.4751,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.0396631583571434,
"rewards/margins": -0.04326799511909485,
"rewards/rejected": 0.003604833036661148,
"step": 28
},
{
"debug/policy_chosen_logits": -1.366220235824585,
"debug/policy_chosen_logps": -253.40408325195312,
"debug/policy_rejected_logits": -1.3099398612976074,
"debug/policy_rejected_logps": -305.3914489746094,
"debug/reference_chosen_logps": -254.5312957763672,
"debug/reference_rejected_logps": -306.05023193359375,
"epoch": 0.6744186046511628,
"grad_norm": 17.36440597445389,
"learning_rate": 1e-06,
"logits/chosen": -1.366220235824585,
"logits/rejected": -1.3099398612976074,
"logps/chosen": -253.40408325195312,
"logps/rejected": -305.3914489746094,
"loss": 0.4956,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.011272145435214043,
"rewards/margins": 0.004684487357735634,
"rewards/rejected": 0.00658765435218811,
"step": 29
},
{
"debug/policy_chosen_logits": -1.464882493019104,
"debug/policy_chosen_logps": -207.21820068359375,
"debug/policy_rejected_logits": -1.0828033685684204,
"debug/policy_rejected_logps": -319.6998291015625,
"debug/reference_chosen_logps": -202.86053466796875,
"debug/reference_rejected_logps": -312.14776611328125,
"epoch": 0.6976744186046512,
"grad_norm": 35.44932621138554,
"learning_rate": 1e-06,
"logits/chosen": -1.464882493019104,
"logits/rejected": -1.0828033685684204,
"logps/chosen": -207.21820068359375,
"logps/rejected": -319.6998291015625,
"loss": 0.5006,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.043576523661613464,
"rewards/margins": 0.03194410353899002,
"rewards/rejected": -0.07552062720060349,
"step": 30
},
{
"debug/policy_chosen_logits": -1.4816325902938843,
"debug/policy_chosen_logps": -232.97183227539062,
"debug/policy_rejected_logits": -1.4852979183197021,
"debug/policy_rejected_logps": -330.938232421875,
"debug/reference_chosen_logps": -228.60247802734375,
"debug/reference_rejected_logps": -322.2069091796875,
"epoch": 0.7209302325581395,
"grad_norm": 69.51972685873595,
"learning_rate": 1e-06,
"logits/chosen": -1.4816325902938843,
"logits/rejected": -1.4852979183197021,
"logps/chosen": -232.97183227539062,
"logps/rejected": -330.938232421875,
"loss": 0.5002,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.04369337856769562,
"rewards/margins": 0.04361977428197861,
"rewards/rejected": -0.08731315284967422,
"step": 31
},
{
"debug/policy_chosen_logits": -1.5637061595916748,
"debug/policy_chosen_logps": -243.05120849609375,
"debug/policy_rejected_logits": -1.4639126062393188,
"debug/policy_rejected_logps": -335.36761474609375,
"debug/reference_chosen_logps": -240.65737915039062,
"debug/reference_rejected_logps": -330.8468322753906,
"epoch": 0.7441860465116279,
"grad_norm": 27.689750506820502,
"learning_rate": 1e-06,
"logits/chosen": -1.5637061595916748,
"logits/rejected": -1.4639126062393188,
"logps/chosen": -243.05120849609375,
"logps/rejected": -335.36761474609375,
"loss": 0.4746,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.023938149213790894,
"rewards/margins": 0.021269541233778,
"rewards/rejected": -0.04520769417285919,
"step": 32
},
{
"debug/policy_chosen_logits": -1.5378698110580444,
"debug/policy_chosen_logps": -269.176513671875,
"debug/policy_rejected_logits": -1.480587363243103,
"debug/policy_rejected_logps": -268.183349609375,
"debug/reference_chosen_logps": -267.6761474609375,
"debug/reference_rejected_logps": -264.76300048828125,
"epoch": 0.7674418604651163,
"grad_norm": 16.782807931183452,
"learning_rate": 1e-06,
"logits/chosen": -1.5378698110580444,
"logits/rejected": -1.480587363243103,
"logps/chosen": -269.176513671875,
"logps/rejected": -268.183349609375,
"loss": 0.491,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.015003662556409836,
"rewards/margins": 0.01920013502240181,
"rewards/rejected": -0.034203797578811646,
"step": 33
},
{
"debug/policy_chosen_logits": -1.509446620941162,
"debug/policy_chosen_logps": -289.96466064453125,
"debug/policy_rejected_logits": -1.3191570043563843,
"debug/policy_rejected_logps": -325.3876953125,
"debug/reference_chosen_logps": -288.51171875,
"debug/reference_rejected_logps": -320.84259033203125,
"epoch": 0.7906976744186046,
"grad_norm": 43.72267267692569,
"learning_rate": 1e-06,
"logits/chosen": -1.509446620941162,
"logits/rejected": -1.3191570043563843,
"logps/chosen": -289.96466064453125,
"logps/rejected": -325.3876953125,
"loss": 0.4688,
"rewards/accuracies": 0.875,
"rewards/chosen": -0.014529475010931492,
"rewards/margins": 0.030921556055545807,
"rewards/rejected": -0.045451030135154724,
"step": 34
},
{
"debug/policy_chosen_logits": -1.5837349891662598,
"debug/policy_chosen_logps": -256.12457275390625,
"debug/policy_rejected_logits": -1.4979993104934692,
"debug/policy_rejected_logps": -346.24853515625,
"debug/reference_chosen_logps": -261.654052734375,
"debug/reference_rejected_logps": -343.11456298828125,
"epoch": 0.813953488372093,
"grad_norm": 20.72530257477532,
"learning_rate": 1e-06,
"logits/chosen": -1.5837349891662598,
"logits/rejected": -1.4979993104934692,
"logps/chosen": -256.12457275390625,
"logps/rejected": -346.24853515625,
"loss": 0.4555,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.05529472604393959,
"rewards/margins": 0.08663440495729446,
"rewards/rejected": -0.03133968263864517,
"step": 35
},
{
"debug/policy_chosen_logits": -1.5132783651351929,
"debug/policy_chosen_logps": -203.21551513671875,
"debug/policy_rejected_logits": -1.5186952352523804,
"debug/policy_rejected_logps": -217.06405639648438,
"debug/reference_chosen_logps": -207.17909240722656,
"debug/reference_rejected_logps": -222.4234161376953,
"epoch": 0.8372093023255814,
"grad_norm": 21.776316541560245,
"learning_rate": 1e-06,
"logits/chosen": -1.5132783651351929,
"logits/rejected": -1.5186952352523804,
"logps/chosen": -203.21551513671875,
"logps/rejected": -217.06405639648438,
"loss": 0.4668,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.03963588550686836,
"rewards/margins": -0.013957731425762177,
"rewards/rejected": 0.05359361693263054,
"step": 36
},
{
"debug/policy_chosen_logits": -1.588102102279663,
"debug/policy_chosen_logps": -200.3746795654297,
"debug/policy_rejected_logits": -1.5428119897842407,
"debug/policy_rejected_logps": -309.95166015625,
"debug/reference_chosen_logps": -209.82540893554688,
"debug/reference_rejected_logps": -305.77777099609375,
"epoch": 0.8604651162790697,
"grad_norm": 19.97759398330279,
"learning_rate": 1e-06,
"logits/chosen": -1.588102102279663,
"logits/rejected": -1.5428119897842407,
"logps/chosen": -200.3746795654297,
"logps/rejected": -309.95166015625,
"loss": 0.4543,
"rewards/accuracies": 1.0,
"rewards/chosen": 0.09450740367174149,
"rewards/margins": 0.13624611496925354,
"rewards/rejected": -0.04173869639635086,
"step": 37
},
{
"debug/policy_chosen_logits": -1.6548794507980347,
"debug/policy_chosen_logps": -257.4342956542969,
"debug/policy_rejected_logits": -1.496774673461914,
"debug/policy_rejected_logps": -306.6890563964844,
"debug/reference_chosen_logps": -263.0815734863281,
"debug/reference_rejected_logps": -307.71331787109375,
"epoch": 0.8837209302325582,
"grad_norm": 35.00520893739856,
"learning_rate": 1e-06,
"logits/chosen": -1.6548794507980347,
"logits/rejected": -1.496774673461914,
"logps/chosen": -257.4342956542969,
"logps/rejected": -306.6890563964844,
"loss": 0.476,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.05647262558341026,
"rewards/margins": 0.046230219304561615,
"rewards/rejected": 0.01024240255355835,
"step": 38
},
{
"debug/policy_chosen_logits": -1.449007272720337,
"debug/policy_chosen_logps": -267.24298095703125,
"debug/policy_rejected_logits": -1.4096888303756714,
"debug/policy_rejected_logps": -289.27166748046875,
"debug/reference_chosen_logps": -269.7801818847656,
"debug/reference_rejected_logps": -288.45440673828125,
"epoch": 0.9069767441860465,
"grad_norm": 18.02813938430424,
"learning_rate": 1e-06,
"logits/chosen": -1.449007272720337,
"logits/rejected": -1.4096888303756714,
"logps/chosen": -267.24298095703125,
"logps/rejected": -289.27166748046875,
"loss": 0.4743,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.025372076779603958,
"rewards/margins": 0.033544398844242096,
"rewards/rejected": -0.008172323927283287,
"step": 39
},
{
"debug/policy_chosen_logits": -1.5711109638214111,
"debug/policy_chosen_logps": -211.0213623046875,
"debug/policy_rejected_logits": -1.473678708076477,
"debug/policy_rejected_logps": -334.5294189453125,
"debug/reference_chosen_logps": -212.76715087890625,
"debug/reference_rejected_logps": -332.358154296875,
"epoch": 0.9302325581395349,
"grad_norm": 13.548944902515293,
"learning_rate": 1e-06,
"logits/chosen": -1.5711109638214111,
"logits/rejected": -1.473678708076477,
"logps/chosen": -211.0213623046875,
"logps/rejected": -334.5294189453125,
"loss": 0.4847,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.01745777204632759,
"rewards/margins": 0.039170339703559875,
"rewards/rejected": -0.021712567657232285,
"step": 40
},
{
"debug/policy_chosen_logits": -1.5397162437438965,
"debug/policy_chosen_logps": -238.81680297851562,
"debug/policy_rejected_logits": -1.4631048440933228,
"debug/policy_rejected_logps": -299.21343994140625,
"debug/reference_chosen_logps": -237.25958251953125,
"debug/reference_rejected_logps": -294.679931640625,
"epoch": 0.9534883720930233,
"grad_norm": 13.60929652129651,
"learning_rate": 1e-06,
"logits/chosen": -1.5397162437438965,
"logits/rejected": -1.4631048440933228,
"logps/chosen": -238.81680297851562,
"logps/rejected": -299.21343994140625,
"loss": 0.4666,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.01557239517569542,
"rewards/margins": 0.029763124883174896,
"rewards/rejected": -0.045335520058870316,
"step": 41
},
{
"debug/policy_chosen_logits": -1.4046356678009033,
"debug/policy_chosen_logps": -228.942138671875,
"debug/policy_rejected_logits": -1.301234245300293,
"debug/policy_rejected_logps": -319.590087890625,
"debug/reference_chosen_logps": -227.15533447265625,
"debug/reference_rejected_logps": -310.7023620605469,
"epoch": 0.9767441860465116,
"grad_norm": 20.029756661206523,
"learning_rate": 1e-06,
"logits/chosen": -1.4046356678009033,
"logits/rejected": -1.301234245300293,
"logps/chosen": -228.942138671875,
"logps/rejected": -319.590087890625,
"loss": 0.4742,
"rewards/accuracies": 0.875,
"rewards/chosen": -0.017868060618638992,
"rewards/margins": 0.07100919634103775,
"rewards/rejected": -0.08887725323438644,
"step": 42
},
{
"debug/policy_chosen_logits": -1.4242496490478516,
"debug/policy_chosen_logps": -248.90008544921875,
"debug/policy_rejected_logits": -1.344144582748413,
"debug/policy_rejected_logps": -312.7857360839844,
"debug/reference_chosen_logps": -248.36306762695312,
"debug/reference_rejected_logps": -306.93707275390625,
"epoch": 1.0,
"grad_norm": 16.354244880023245,
"learning_rate": 1e-06,
"logits/chosen": -1.4242496490478516,
"logits/rejected": -1.344144582748413,
"logps/chosen": -248.90008544921875,
"logps/rejected": -312.7857360839844,
"loss": 0.4466,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.005370025523006916,
"rewards/margins": 0.053116798400878906,
"rewards/rejected": -0.05848682299256325,
"step": 43
},
{
"epoch": 1.0,
"step": 43,
"total_flos": 0.0,
"train_loss": 0.49317165862682255,
"train_runtime": 155.8617,
"train_samples_per_second": 17.445,
"train_steps_per_second": 0.276
}
],
"logging_steps": 1,
"max_steps": 43,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
|