File size: 45,097 Bytes
56d50a9 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 |
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 100,
"global_step": 52,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"debug/policy_chosen_logits": -2.977583885192871,
"debug/policy_chosen_logps": -214.68138122558594,
"debug/policy_rejected_logits": -3.0865864753723145,
"debug/policy_rejected_logps": -221.65213012695312,
"debug/reference_chosen_logps": -214.68138122558594,
"debug/reference_rejected_logps": -221.65213012695312,
"epoch": 0.019230769230769232,
"grad_norm": 5.166761446238812,
"learning_rate": 1e-06,
"logits/chosen": -2.977583885192871,
"logits/rejected": -3.0865864753723145,
"logps/chosen": -214.68138122558594,
"logps/rejected": -221.65213012695312,
"loss": 0.5,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"debug/policy_chosen_logits": -3.110696792602539,
"debug/policy_chosen_logps": -208.40696716308594,
"debug/policy_rejected_logits": -3.141209602355957,
"debug/policy_rejected_logps": -220.74032592773438,
"debug/reference_chosen_logps": -208.67347717285156,
"debug/reference_rejected_logps": -220.81326293945312,
"epoch": 0.038461538461538464,
"grad_norm": 4.748582416027732,
"learning_rate": 1e-06,
"logits/chosen": -3.110696792602539,
"logits/rejected": -3.141209602355957,
"logps/chosen": -208.40696716308594,
"logps/rejected": -220.74032592773438,
"loss": 0.4986,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.0026651951484382153,
"rewards/margins": 0.001935672713443637,
"rewards/rejected": 0.000729522667825222,
"step": 2
},
{
"debug/policy_chosen_logits": -3.122938394546509,
"debug/policy_chosen_logps": -191.87767028808594,
"debug/policy_rejected_logits": -3.2007791996002197,
"debug/policy_rejected_logps": -196.9674530029297,
"debug/reference_chosen_logps": -192.09266662597656,
"debug/reference_rejected_logps": -196.9289093017578,
"epoch": 0.057692307692307696,
"grad_norm": 4.85937471576945,
"learning_rate": 1e-06,
"logits/chosen": -3.122938394546509,
"logits/rejected": -3.2007791996002197,
"logps/chosen": -191.87767028808594,
"logps/rejected": -196.9674530029297,
"loss": 0.4979,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.0021500205621123314,
"rewards/margins": 0.0025354004465043545,
"rewards/rejected": -0.0003853798261843622,
"step": 3
},
{
"debug/policy_chosen_logits": -2.9445621967315674,
"debug/policy_chosen_logps": -202.26788330078125,
"debug/policy_rejected_logits": -2.958071708679199,
"debug/policy_rejected_logps": -236.9288330078125,
"debug/reference_chosen_logps": -202.63815307617188,
"debug/reference_rejected_logps": -236.94044494628906,
"epoch": 0.07692307692307693,
"grad_norm": 4.68477948866532,
"learning_rate": 1e-06,
"logits/chosen": -2.9445621967315674,
"logits/rejected": -2.958071708679199,
"logps/chosen": -202.26788330078125,
"logps/rejected": -236.9288330078125,
"loss": 0.4975,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.0037027548532932997,
"rewards/margins": 0.0035865209065377712,
"rewards/rejected": 0.00011623383034020662,
"step": 4
},
{
"debug/policy_chosen_logits": -3.0143117904663086,
"debug/policy_chosen_logps": -206.9991455078125,
"debug/policy_rejected_logits": -3.0474984645843506,
"debug/policy_rejected_logps": -184.9208984375,
"debug/reference_chosen_logps": -207.888671875,
"debug/reference_rejected_logps": -185.94973754882812,
"epoch": 0.09615384615384616,
"grad_norm": 5.103004709570949,
"learning_rate": 1e-06,
"logits/chosen": -3.0143117904663086,
"logits/rejected": -3.0474984645843506,
"logps/chosen": -206.9991455078125,
"logps/rejected": -184.9208984375,
"loss": 0.4956,
"rewards/accuracies": 0.375,
"rewards/chosen": 0.008895359002053738,
"rewards/margins": -0.0013930893037468195,
"rewards/rejected": 0.010288448072969913,
"step": 5
},
{
"debug/policy_chosen_logits": -3.1157450675964355,
"debug/policy_chosen_logps": -191.06285095214844,
"debug/policy_rejected_logits": -3.1180226802825928,
"debug/policy_rejected_logps": -217.71697998046875,
"debug/reference_chosen_logps": -192.79592895507812,
"debug/reference_rejected_logps": -218.74090576171875,
"epoch": 0.11538461538461539,
"grad_norm": 4.434924732032029,
"learning_rate": 1e-06,
"logits/chosen": -3.1157450675964355,
"logits/rejected": -3.1180226802825928,
"logps/chosen": -191.06285095214844,
"logps/rejected": -217.71697998046875,
"loss": 0.4926,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.017330702394247055,
"rewards/margins": 0.00709140719845891,
"rewards/rejected": 0.010239295661449432,
"step": 6
},
{
"debug/policy_chosen_logits": -3.011009454727173,
"debug/policy_chosen_logps": -214.3972625732422,
"debug/policy_rejected_logits": -3.1553845405578613,
"debug/policy_rejected_logps": -205.53399658203125,
"debug/reference_chosen_logps": -215.999267578125,
"debug/reference_rejected_logps": -204.93153381347656,
"epoch": 0.1346153846153846,
"grad_norm": 5.27755315826341,
"learning_rate": 1e-06,
"logits/chosen": -3.011009454727173,
"logits/rejected": -3.1553845405578613,
"logps/chosen": -214.3972625732422,
"logps/rejected": -205.53399658203125,
"loss": 0.4874,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.016020087525248528,
"rewards/margins": 0.02204471454024315,
"rewards/rejected": -0.006024627946317196,
"step": 7
},
{
"debug/policy_chosen_logits": -3.0167155265808105,
"debug/policy_chosen_logps": -179.38259887695312,
"debug/policy_rejected_logits": -3.06713604927063,
"debug/policy_rejected_logps": -190.42263793945312,
"debug/reference_chosen_logps": -180.68841552734375,
"debug/reference_rejected_logps": -191.515869140625,
"epoch": 0.15384615384615385,
"grad_norm": 4.472209218113578,
"learning_rate": 1e-06,
"logits/chosen": -3.0167155265808105,
"logits/rejected": -3.06713604927063,
"logps/chosen": -179.38259887695312,
"logps/rejected": -190.42263793945312,
"loss": 0.4889,
"rewards/accuracies": 0.25,
"rewards/chosen": 0.013058356940746307,
"rewards/margins": 0.002126025967299938,
"rewards/rejected": 0.01093233097344637,
"step": 8
},
{
"debug/policy_chosen_logits": -3.1482882499694824,
"debug/policy_chosen_logps": -191.95626831054688,
"debug/policy_rejected_logits": -3.1080360412597656,
"debug/policy_rejected_logps": -211.66036987304688,
"debug/reference_chosen_logps": -192.33502197265625,
"debug/reference_rejected_logps": -211.17388916015625,
"epoch": 0.17307692307692307,
"grad_norm": 4.711300291586481,
"learning_rate": 1e-06,
"logits/chosen": -3.1482882499694824,
"logits/rejected": -3.1080360412597656,
"logps/chosen": -191.95626831054688,
"logps/rejected": -211.66036987304688,
"loss": 0.4863,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.0037874598056077957,
"rewards/margins": 0.008652305230498314,
"rewards/rejected": -0.004864844959229231,
"step": 9
},
{
"debug/policy_chosen_logits": -3.047020196914673,
"debug/policy_chosen_logps": -200.92193603515625,
"debug/policy_rejected_logits": -3.1241281032562256,
"debug/policy_rejected_logps": -228.44821166992188,
"debug/reference_chosen_logps": -201.93951416015625,
"debug/reference_rejected_logps": -226.07154846191406,
"epoch": 0.19230769230769232,
"grad_norm": 4.555391857522008,
"learning_rate": 1e-06,
"logits/chosen": -3.047020196914673,
"logits/rejected": -3.1241281032562256,
"logps/chosen": -200.92193603515625,
"logps/rejected": -228.44821166992188,
"loss": 0.4822,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.010175762698054314,
"rewards/margins": 0.033942315727472305,
"rewards/rejected": -0.02376655675470829,
"step": 10
},
{
"debug/policy_chosen_logits": -3.047590494155884,
"debug/policy_chosen_logps": -193.70095825195312,
"debug/policy_rejected_logits": -2.9977893829345703,
"debug/policy_rejected_logps": -240.172607421875,
"debug/reference_chosen_logps": -194.73997497558594,
"debug/reference_rejected_logps": -239.21884155273438,
"epoch": 0.21153846153846154,
"grad_norm": 5.679651909554435,
"learning_rate": 1e-06,
"logits/chosen": -3.047590494155884,
"logits/rejected": -2.9977893829345703,
"logps/chosen": -193.70095825195312,
"logps/rejected": -240.172607421875,
"loss": 0.4719,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.010390243493020535,
"rewards/margins": 0.019927941262722015,
"rewards/rejected": -0.009537696838378906,
"step": 11
},
{
"debug/policy_chosen_logits": -3.063314199447632,
"debug/policy_chosen_logps": -193.97750854492188,
"debug/policy_rejected_logits": -2.8785159587860107,
"debug/policy_rejected_logps": -206.611083984375,
"debug/reference_chosen_logps": -195.43072509765625,
"debug/reference_rejected_logps": -203.80999755859375,
"epoch": 0.23076923076923078,
"grad_norm": 5.02375537373724,
"learning_rate": 1e-06,
"logits/chosen": -3.063314199447632,
"logits/rejected": -2.8785159587860107,
"logps/chosen": -193.97750854492188,
"logps/rejected": -206.611083984375,
"loss": 0.4596,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.014532221481204033,
"rewards/margins": 0.0425429493188858,
"rewards/rejected": -0.02801072970032692,
"step": 12
},
{
"debug/policy_chosen_logits": -3.060885190963745,
"debug/policy_chosen_logps": -198.6640625,
"debug/policy_rejected_logits": -3.0962471961975098,
"debug/policy_rejected_logps": -247.57131958007812,
"debug/reference_chosen_logps": -202.70645141601562,
"debug/reference_rejected_logps": -239.8416290283203,
"epoch": 0.25,
"grad_norm": 4.916536953774363,
"learning_rate": 1e-06,
"logits/chosen": -3.060885190963745,
"logits/rejected": -3.0962471961975098,
"logps/chosen": -198.6640625,
"logps/rejected": -247.57131958007812,
"loss": 0.4602,
"rewards/accuracies": 1.0,
"rewards/chosen": 0.04042387008666992,
"rewards/margins": 0.11772066354751587,
"rewards/rejected": -0.07729679346084595,
"step": 13
},
{
"debug/policy_chosen_logits": -3.025531768798828,
"debug/policy_chosen_logps": -200.4529266357422,
"debug/policy_rejected_logits": -2.992382287979126,
"debug/policy_rejected_logps": -220.69435119628906,
"debug/reference_chosen_logps": -204.46011352539062,
"debug/reference_rejected_logps": -221.68063354492188,
"epoch": 0.2692307692307692,
"grad_norm": 4.5758304591077765,
"learning_rate": 1e-06,
"logits/chosen": -3.025531768798828,
"logits/rejected": -2.992382287979126,
"logps/chosen": -200.4529266357422,
"logps/rejected": -220.69435119628906,
"loss": 0.4713,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.040071770548820496,
"rewards/margins": 0.030209042131900787,
"rewards/rejected": 0.009862728416919708,
"step": 14
},
{
"debug/policy_chosen_logits": -3.1965172290802,
"debug/policy_chosen_logps": -186.35000610351562,
"debug/policy_rejected_logits": -3.1802818775177,
"debug/policy_rejected_logps": -210.35556030273438,
"debug/reference_chosen_logps": -189.31112670898438,
"debug/reference_rejected_logps": -211.51010131835938,
"epoch": 0.28846153846153844,
"grad_norm": 5.776855357402379,
"learning_rate": 1e-06,
"logits/chosen": -3.1965172290802,
"logits/rejected": -3.1802818775177,
"logps/chosen": -186.35000610351562,
"logps/rejected": -210.35556030273438,
"loss": 0.4747,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.02961129881441593,
"rewards/margins": 0.018065985292196274,
"rewards/rejected": 0.011545314453542233,
"step": 15
},
{
"debug/policy_chosen_logits": -3.004584550857544,
"debug/policy_chosen_logps": -219.46209716796875,
"debug/policy_rejected_logits": -2.997267007827759,
"debug/policy_rejected_logps": -239.15829467773438,
"debug/reference_chosen_logps": -221.38467407226562,
"debug/reference_rejected_logps": -233.982177734375,
"epoch": 0.3076923076923077,
"grad_norm": 5.1995499627013,
"learning_rate": 1e-06,
"logits/chosen": -3.004584550857544,
"logits/rejected": -2.997267007827759,
"logps/chosen": -219.46209716796875,
"logps/rejected": -239.15829467773438,
"loss": 0.4659,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.019225770607590675,
"rewards/margins": 0.07098689675331116,
"rewards/rejected": -0.05176112800836563,
"step": 16
},
{
"debug/policy_chosen_logits": -3.0550475120544434,
"debug/policy_chosen_logps": -196.09921264648438,
"debug/policy_rejected_logits": -3.0315375328063965,
"debug/policy_rejected_logps": -228.4376983642578,
"debug/reference_chosen_logps": -196.76771545410156,
"debug/reference_rejected_logps": -219.703857421875,
"epoch": 0.3269230769230769,
"grad_norm": 4.905311259261036,
"learning_rate": 1e-06,
"logits/chosen": -3.0550475120544434,
"logits/rejected": -3.0315375328063965,
"logps/chosen": -196.09921264648438,
"logps/rejected": -228.4376983642578,
"loss": 0.4668,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.006685066036880016,
"rewards/margins": 0.09402336925268173,
"rewards/rejected": -0.08733831346035004,
"step": 17
},
{
"debug/policy_chosen_logits": -3.109316110610962,
"debug/policy_chosen_logps": -192.22500610351562,
"debug/policy_rejected_logits": -3.070436954498291,
"debug/policy_rejected_logps": -225.37353515625,
"debug/reference_chosen_logps": -193.43701171875,
"debug/reference_rejected_logps": -215.90811157226562,
"epoch": 0.34615384615384615,
"grad_norm": 5.438987049949224,
"learning_rate": 1e-06,
"logits/chosen": -3.109316110610962,
"logits/rejected": -3.070436954498291,
"logps/chosen": -192.22500610351562,
"logps/rejected": -225.37353515625,
"loss": 0.4604,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.012120017781853676,
"rewards/margins": 0.10677413642406464,
"rewards/rejected": -0.09465412050485611,
"step": 18
},
{
"debug/policy_chosen_logits": -3.156306505203247,
"debug/policy_chosen_logps": -198.84149169921875,
"debug/policy_rejected_logits": -3.163816213607788,
"debug/policy_rejected_logps": -207.2037811279297,
"debug/reference_chosen_logps": -199.2224884033203,
"debug/reference_rejected_logps": -204.29345703125,
"epoch": 0.36538461538461536,
"grad_norm": 5.3459094259339395,
"learning_rate": 1e-06,
"logits/chosen": -3.156306505203247,
"logits/rejected": -3.163816213607788,
"logps/chosen": -198.84149169921875,
"logps/rejected": -207.2037811279297,
"loss": 0.4623,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.003809986636042595,
"rewards/margins": 0.032913051545619965,
"rewards/rejected": -0.02910306677222252,
"step": 19
},
{
"debug/policy_chosen_logits": -3.09970760345459,
"debug/policy_chosen_logps": -195.97071838378906,
"debug/policy_rejected_logits": -3.0445327758789062,
"debug/policy_rejected_logps": -233.5975341796875,
"debug/reference_chosen_logps": -196.68661499023438,
"debug/reference_rejected_logps": -226.482666015625,
"epoch": 0.38461538461538464,
"grad_norm": 5.574869499523263,
"learning_rate": 1e-06,
"logits/chosen": -3.09970760345459,
"logits/rejected": -3.0445327758789062,
"logps/chosen": -195.97071838378906,
"logps/rejected": -233.5975341796875,
"loss": 0.4437,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.007159043103456497,
"rewards/margins": 0.07830756902694702,
"rewards/rejected": -0.07114852219820023,
"step": 20
},
{
"debug/policy_chosen_logits": -3.048816680908203,
"debug/policy_chosen_logps": -187.65283203125,
"debug/policy_rejected_logits": -3.0968563556671143,
"debug/policy_rejected_logps": -222.02886962890625,
"debug/reference_chosen_logps": -191.67349243164062,
"debug/reference_rejected_logps": -218.13577270507812,
"epoch": 0.40384615384615385,
"grad_norm": 4.472533940520768,
"learning_rate": 1e-06,
"logits/chosen": -3.048816680908203,
"logits/rejected": -3.0968563556671143,
"logps/chosen": -187.65283203125,
"logps/rejected": -222.02886962890625,
"loss": 0.4355,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.04020654410123825,
"rewards/margins": 0.07913752645254135,
"rewards/rejected": -0.0389309898018837,
"step": 21
},
{
"debug/policy_chosen_logits": -3.1425256729125977,
"debug/policy_chosen_logps": -201.33770751953125,
"debug/policy_rejected_logits": -3.146122932434082,
"debug/policy_rejected_logps": -221.27914428710938,
"debug/reference_chosen_logps": -205.18829345703125,
"debug/reference_rejected_logps": -218.09457397460938,
"epoch": 0.4230769230769231,
"grad_norm": 6.018529206237279,
"learning_rate": 1e-06,
"logits/chosen": -3.1425256729125977,
"logits/rejected": -3.146122932434082,
"logps/chosen": -201.33770751953125,
"logps/rejected": -221.27914428710938,
"loss": 0.4552,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.038505878299474716,
"rewards/margins": 0.07035169750452042,
"rewards/rejected": -0.0318458154797554,
"step": 22
},
{
"debug/policy_chosen_logits": -3.187493324279785,
"debug/policy_chosen_logps": -197.76002502441406,
"debug/policy_rejected_logits": -3.1831955909729004,
"debug/policy_rejected_logps": -201.7500762939453,
"debug/reference_chosen_logps": -200.38540649414062,
"debug/reference_rejected_logps": -205.30821228027344,
"epoch": 0.4423076923076923,
"grad_norm": 4.661142809825874,
"learning_rate": 1e-06,
"logits/chosen": -3.187493324279785,
"logits/rejected": -3.1831955909729004,
"logps/chosen": -197.76002502441406,
"logps/rejected": -201.7500762939453,
"loss": 0.4572,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.02625381574034691,
"rewards/margins": -0.009327447973191738,
"rewards/rejected": 0.03558126464486122,
"step": 23
},
{
"debug/policy_chosen_logits": -3.20326566696167,
"debug/policy_chosen_logps": -191.74195861816406,
"debug/policy_rejected_logits": -3.187241792678833,
"debug/policy_rejected_logps": -233.16586303710938,
"debug/reference_chosen_logps": -196.749267578125,
"debug/reference_rejected_logps": -227.8116455078125,
"epoch": 0.46153846153846156,
"grad_norm": 4.44342516621906,
"learning_rate": 1e-06,
"logits/chosen": -3.20326566696167,
"logits/rejected": -3.187241792678833,
"logps/chosen": -191.74195861816406,
"logps/rejected": -233.16586303710938,
"loss": 0.4244,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.05007295310497284,
"rewards/margins": 0.10361497104167938,
"rewards/rejected": -0.05354202166199684,
"step": 24
},
{
"debug/policy_chosen_logits": -3.142024517059326,
"debug/policy_chosen_logps": -203.33738708496094,
"debug/policy_rejected_logits": -3.017565965652466,
"debug/policy_rejected_logps": -238.41554260253906,
"debug/reference_chosen_logps": -205.4508056640625,
"debug/reference_rejected_logps": -228.70516967773438,
"epoch": 0.4807692307692308,
"grad_norm": 4.368252676849126,
"learning_rate": 1e-06,
"logits/chosen": -3.142024517059326,
"logits/rejected": -3.017565965652466,
"logps/chosen": -203.33738708496094,
"logps/rejected": -238.41554260253906,
"loss": 0.4321,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.021134089678525925,
"rewards/margins": 0.11823777854442596,
"rewards/rejected": -0.09710369259119034,
"step": 25
},
{
"debug/policy_chosen_logits": -3.0380425453186035,
"debug/policy_chosen_logps": -200.80526733398438,
"debug/policy_rejected_logits": -3.081782102584839,
"debug/policy_rejected_logps": -218.62149047851562,
"debug/reference_chosen_logps": -203.88671875,
"debug/reference_rejected_logps": -216.6439208984375,
"epoch": 0.5,
"grad_norm": 4.4745876509482025,
"learning_rate": 1e-06,
"logits/chosen": -3.0380425453186035,
"logits/rejected": -3.081782102584839,
"logps/chosen": -200.80526733398438,
"logps/rejected": -218.62149047851562,
"loss": 0.4491,
"rewards/accuracies": 0.375,
"rewards/chosen": 0.030814513564109802,
"rewards/margins": 0.05059013515710831,
"rewards/rejected": -0.019775621592998505,
"step": 26
},
{
"debug/policy_chosen_logits": -3.1514594554901123,
"debug/policy_chosen_logps": -195.95147705078125,
"debug/policy_rejected_logits": -2.9963762760162354,
"debug/policy_rejected_logps": -275.3726806640625,
"debug/reference_chosen_logps": -193.73902893066406,
"debug/reference_rejected_logps": -263.09857177734375,
"epoch": 0.5192307692307693,
"grad_norm": 4.481499913956976,
"learning_rate": 1e-06,
"logits/chosen": -3.1514594554901123,
"logits/rejected": -2.9963762760162354,
"logps/chosen": -195.95147705078125,
"logps/rejected": -275.3726806640625,
"loss": 0.4362,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.02212451957166195,
"rewards/margins": 0.10061690211296082,
"rewards/rejected": -0.12274143099784851,
"step": 27
},
{
"debug/policy_chosen_logits": -3.1708903312683105,
"debug/policy_chosen_logps": -188.44619750976562,
"debug/policy_rejected_logits": -3.2108025550842285,
"debug/policy_rejected_logps": -220.68988037109375,
"debug/reference_chosen_logps": -194.43800354003906,
"debug/reference_rejected_logps": -220.9051513671875,
"epoch": 0.5384615384615384,
"grad_norm": 4.764561262542065,
"learning_rate": 1e-06,
"logits/chosen": -3.1708903312683105,
"logits/rejected": -3.2108025550842285,
"logps/chosen": -188.44619750976562,
"logps/rejected": -220.68988037109375,
"loss": 0.4431,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.05991803854703903,
"rewards/margins": 0.05776527523994446,
"rewards/rejected": 0.002152767963707447,
"step": 28
},
{
"debug/policy_chosen_logits": -3.1714835166931152,
"debug/policy_chosen_logps": -180.14614868164062,
"debug/policy_rejected_logits": -3.08394718170166,
"debug/policy_rejected_logps": -239.15164184570312,
"debug/reference_chosen_logps": -184.656494140625,
"debug/reference_rejected_logps": -224.29171752929688,
"epoch": 0.5576923076923077,
"grad_norm": 4.849507609873306,
"learning_rate": 1e-06,
"logits/chosen": -3.1714835166931152,
"logits/rejected": -3.08394718170166,
"logps/chosen": -180.14614868164062,
"logps/rejected": -239.15164184570312,
"loss": 0.3837,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.045103415846824646,
"rewards/margins": 0.1937025487422943,
"rewards/rejected": -0.14859913289546967,
"step": 29
},
{
"debug/policy_chosen_logits": -3.178560972213745,
"debug/policy_chosen_logps": -210.00115966796875,
"debug/policy_rejected_logits": -3.1779253482818604,
"debug/policy_rejected_logps": -235.44644165039062,
"debug/reference_chosen_logps": -213.55612182617188,
"debug/reference_rejected_logps": -221.76431274414062,
"epoch": 0.5769230769230769,
"grad_norm": 4.895017276343638,
"learning_rate": 1e-06,
"logits/chosen": -3.178560972213745,
"logits/rejected": -3.1779253482818604,
"logps/chosen": -210.00115966796875,
"logps/rejected": -235.44644165039062,
"loss": 0.3956,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.03554954379796982,
"rewards/margins": 0.1723707914352417,
"rewards/rejected": -0.13682125508785248,
"step": 30
},
{
"debug/policy_chosen_logits": -3.108640670776367,
"debug/policy_chosen_logps": -200.23190307617188,
"debug/policy_rejected_logits": -3.1070878505706787,
"debug/policy_rejected_logps": -206.5513153076172,
"debug/reference_chosen_logps": -209.71009826660156,
"debug/reference_rejected_logps": -205.9014892578125,
"epoch": 0.5961538461538461,
"grad_norm": 5.522641702393358,
"learning_rate": 1e-06,
"logits/chosen": -3.108640670776367,
"logits/rejected": -3.1070878505706787,
"logps/chosen": -200.23190307617188,
"logps/rejected": -206.5513153076172,
"loss": 0.4094,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.09478198736906052,
"rewards/margins": 0.10128023475408554,
"rewards/rejected": -0.0064982399344444275,
"step": 31
},
{
"debug/policy_chosen_logits": -3.165921211242676,
"debug/policy_chosen_logps": -176.6615753173828,
"debug/policy_rejected_logits": -3.197509288787842,
"debug/policy_rejected_logps": -212.1288299560547,
"debug/reference_chosen_logps": -191.31610107421875,
"debug/reference_rejected_logps": -211.03121948242188,
"epoch": 0.6153846153846154,
"grad_norm": 5.252195941070867,
"learning_rate": 1e-06,
"logits/chosen": -3.165921211242676,
"logits/rejected": -3.197509288787842,
"logps/chosen": -176.6615753173828,
"logps/rejected": -212.1288299560547,
"loss": 0.3729,
"rewards/accuracies": 1.0,
"rewards/chosen": 0.14654535055160522,
"rewards/margins": 0.15752162039279938,
"rewards/rejected": -0.010976276360452175,
"step": 32
},
{
"debug/policy_chosen_logits": -3.020163059234619,
"debug/policy_chosen_logps": -203.67031860351562,
"debug/policy_rejected_logits": -3.0926342010498047,
"debug/policy_rejected_logps": -210.38372802734375,
"debug/reference_chosen_logps": -209.38458251953125,
"debug/reference_rejected_logps": -205.83929443359375,
"epoch": 0.6346153846153846,
"grad_norm": 4.8112163863395665,
"learning_rate": 1e-06,
"logits/chosen": -3.020163059234619,
"logits/rejected": -3.0926342010498047,
"logps/chosen": -203.67031860351562,
"logps/rejected": -210.38372802734375,
"loss": 0.3845,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.05714261904358864,
"rewards/margins": 0.10258688032627106,
"rewards/rejected": -0.04544425755739212,
"step": 33
},
{
"debug/policy_chosen_logits": -3.226917028427124,
"debug/policy_chosen_logps": -197.40658569335938,
"debug/policy_rejected_logits": -3.1886653900146484,
"debug/policy_rejected_logps": -249.4960174560547,
"debug/reference_chosen_logps": -208.50588989257812,
"debug/reference_rejected_logps": -234.88482666015625,
"epoch": 0.6538461538461539,
"grad_norm": 4.70294242121678,
"learning_rate": 1e-06,
"logits/chosen": -3.226917028427124,
"logits/rejected": -3.1886653900146484,
"logps/chosen": -197.40658569335938,
"logps/rejected": -249.4960174560547,
"loss": 0.3647,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.11099302023649216,
"rewards/margins": 0.25710487365722656,
"rewards/rejected": -0.146111860871315,
"step": 34
},
{
"debug/policy_chosen_logits": -3.1596603393554688,
"debug/policy_chosen_logps": -196.2579345703125,
"debug/policy_rejected_logits": -3.1261677742004395,
"debug/policy_rejected_logps": -231.90347290039062,
"debug/reference_chosen_logps": -205.25323486328125,
"debug/reference_rejected_logps": -227.40982055664062,
"epoch": 0.6730769230769231,
"grad_norm": 7.012486637898167,
"learning_rate": 1e-06,
"logits/chosen": -3.1596603393554688,
"logits/rejected": -3.1261677742004395,
"logps/chosen": -196.2579345703125,
"logps/rejected": -231.90347290039062,
"loss": 0.407,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.08995288610458374,
"rewards/margins": 0.13488951325416565,
"rewards/rejected": -0.044936634600162506,
"step": 35
},
{
"debug/policy_chosen_logits": -3.1420416831970215,
"debug/policy_chosen_logps": -187.47885131835938,
"debug/policy_rejected_logits": -3.067408800125122,
"debug/policy_rejected_logps": -227.22024536132812,
"debug/reference_chosen_logps": -199.94757080078125,
"debug/reference_rejected_logps": -219.46438598632812,
"epoch": 0.6923076923076923,
"grad_norm": 5.480923899428052,
"learning_rate": 1e-06,
"logits/chosen": -3.1420416831970215,
"logits/rejected": -3.067408800125122,
"logps/chosen": -187.47885131835938,
"logps/rejected": -227.22024536132812,
"loss": 0.3983,
"rewards/accuracies": 1.0,
"rewards/chosen": 0.12468719482421875,
"rewards/margins": 0.2022458016872406,
"rewards/rejected": -0.07755860686302185,
"step": 36
},
{
"debug/policy_chosen_logits": -3.134657621383667,
"debug/policy_chosen_logps": -198.46621704101562,
"debug/policy_rejected_logits": -3.191622495651245,
"debug/policy_rejected_logps": -238.76580810546875,
"debug/reference_chosen_logps": -210.65228271484375,
"debug/reference_rejected_logps": -226.888427734375,
"epoch": 0.7115384615384616,
"grad_norm": 6.49950029807871,
"learning_rate": 1e-06,
"logits/chosen": -3.134657621383667,
"logits/rejected": -3.191622495651245,
"logps/chosen": -198.46621704101562,
"logps/rejected": -238.76580810546875,
"loss": 0.4017,
"rewards/accuracies": 1.0,
"rewards/chosen": 0.12186044454574585,
"rewards/margins": 0.24063441157341003,
"rewards/rejected": -0.11877395212650299,
"step": 37
},
{
"debug/policy_chosen_logits": -3.2385828495025635,
"debug/policy_chosen_logps": -170.713623046875,
"debug/policy_rejected_logits": -3.207604169845581,
"debug/policy_rejected_logps": -235.9530029296875,
"debug/reference_chosen_logps": -183.66558837890625,
"debug/reference_rejected_logps": -223.42767333984375,
"epoch": 0.7307692307692307,
"grad_norm": 5.516561491681134,
"learning_rate": 1e-06,
"logits/chosen": -3.2385828495025635,
"logits/rejected": -3.207604169845581,
"logps/chosen": -170.713623046875,
"logps/rejected": -235.9530029296875,
"loss": 0.4081,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.12951962649822235,
"rewards/margins": 0.2547728717327118,
"rewards/rejected": -0.12525323033332825,
"step": 38
},
{
"debug/policy_chosen_logits": -3.0979700088500977,
"debug/policy_chosen_logps": -162.31280517578125,
"debug/policy_rejected_logits": -3.186298131942749,
"debug/policy_rejected_logps": -235.3626708984375,
"debug/reference_chosen_logps": -172.52468872070312,
"debug/reference_rejected_logps": -221.17556762695312,
"epoch": 0.75,
"grad_norm": 6.774526914117382,
"learning_rate": 1e-06,
"logits/chosen": -3.0979700088500977,
"logits/rejected": -3.186298131942749,
"logps/chosen": -162.31280517578125,
"logps/rejected": -235.3626708984375,
"loss": 0.3939,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.10211898386478424,
"rewards/margins": 0.2439899444580078,
"rewards/rejected": -0.14187094569206238,
"step": 39
},
{
"debug/policy_chosen_logits": -3.089684009552002,
"debug/policy_chosen_logps": -196.1321563720703,
"debug/policy_rejected_logits": -3.04327392578125,
"debug/policy_rejected_logps": -216.24703979492188,
"debug/reference_chosen_logps": -200.5460205078125,
"debug/reference_rejected_logps": -207.2269287109375,
"epoch": 0.7692307692307693,
"grad_norm": 5.675881779895136,
"learning_rate": 1e-06,
"logits/chosen": -3.089684009552002,
"logits/rejected": -3.04327392578125,
"logps/chosen": -196.1321563720703,
"logps/rejected": -216.24703979492188,
"loss": 0.4025,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.04413875192403793,
"rewards/margins": 0.1343398243188858,
"rewards/rejected": -0.09020107239484787,
"step": 40
},
{
"debug/policy_chosen_logits": -3.1140072345733643,
"debug/policy_chosen_logps": -187.277099609375,
"debug/policy_rejected_logits": -3.2128348350524902,
"debug/policy_rejected_logps": -254.4541015625,
"debug/reference_chosen_logps": -192.44972229003906,
"debug/reference_rejected_logps": -245.08372497558594,
"epoch": 0.7884615384615384,
"grad_norm": 6.653395887552183,
"learning_rate": 1e-06,
"logits/chosen": -3.1140072345733643,
"logits/rejected": -3.2128348350524902,
"logps/chosen": -187.277099609375,
"logps/rejected": -254.4541015625,
"loss": 0.4269,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.051726073026657104,
"rewards/margins": 0.14542999863624573,
"rewards/rejected": -0.09370393306016922,
"step": 41
},
{
"debug/policy_chosen_logits": -3.3676302433013916,
"debug/policy_chosen_logps": -159.673095703125,
"debug/policy_rejected_logits": -3.310539484024048,
"debug/policy_rejected_logps": -204.17922973632812,
"debug/reference_chosen_logps": -172.7714385986328,
"debug/reference_rejected_logps": -196.74644470214844,
"epoch": 0.8076923076923077,
"grad_norm": 5.688046713439665,
"learning_rate": 1e-06,
"logits/chosen": -3.3676302433013916,
"logits/rejected": -3.310539484024048,
"logps/chosen": -159.673095703125,
"logps/rejected": -204.17922973632812,
"loss": 0.3881,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.1309833526611328,
"rewards/margins": 0.20531128346920013,
"rewards/rejected": -0.07432794570922852,
"step": 42
},
{
"debug/policy_chosen_logits": -3.048048496246338,
"debug/policy_chosen_logps": -195.83262634277344,
"debug/policy_rejected_logits": -3.1337571144104004,
"debug/policy_rejected_logps": -220.1021728515625,
"debug/reference_chosen_logps": -204.2503662109375,
"debug/reference_rejected_logps": -214.80859375,
"epoch": 0.8269230769230769,
"grad_norm": 6.260667799966591,
"learning_rate": 1e-06,
"logits/chosen": -3.048048496246338,
"logits/rejected": -3.1337571144104004,
"logps/chosen": -195.83262634277344,
"logps/rejected": -220.1021728515625,
"loss": 0.4081,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.08417723327875137,
"rewards/margins": 0.13711285591125488,
"rewards/rejected": -0.05293561890721321,
"step": 43
},
{
"debug/policy_chosen_logits": -3.2857210636138916,
"debug/policy_chosen_logps": -190.01461791992188,
"debug/policy_rejected_logits": -3.210080146789551,
"debug/policy_rejected_logps": -202.32200622558594,
"debug/reference_chosen_logps": -192.96429443359375,
"debug/reference_rejected_logps": -204.96524047851562,
"epoch": 0.8461538461538461,
"grad_norm": 5.41978019999578,
"learning_rate": 1e-06,
"logits/chosen": -3.2857210636138916,
"logits/rejected": -3.210080146789551,
"logps/chosen": -190.01461791992188,
"logps/rejected": -202.32200622558594,
"loss": 0.3812,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.02949686348438263,
"rewards/margins": 0.0030643679201602936,
"rewards/rejected": 0.026432491838932037,
"step": 44
},
{
"debug/policy_chosen_logits": -3.212117910385132,
"debug/policy_chosen_logps": -168.555419921875,
"debug/policy_rejected_logits": -3.191093683242798,
"debug/policy_rejected_logps": -196.78350830078125,
"debug/reference_chosen_logps": -184.03561401367188,
"debug/reference_rejected_logps": -194.875,
"epoch": 0.8653846153846154,
"grad_norm": 8.557722940045432,
"learning_rate": 1e-06,
"logits/chosen": -3.212117910385132,
"logits/rejected": -3.191093683242798,
"logps/chosen": -168.555419921875,
"logps/rejected": -196.78350830078125,
"loss": 0.4219,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.1548018455505371,
"rewards/margins": 0.1738869845867157,
"rewards/rejected": -0.01908515766263008,
"step": 45
},
{
"debug/policy_chosen_logits": -3.4355483055114746,
"debug/policy_chosen_logps": -174.82119750976562,
"debug/policy_rejected_logits": -3.4658639430999756,
"debug/policy_rejected_logps": -179.5398712158203,
"debug/reference_chosen_logps": -183.99972534179688,
"debug/reference_rejected_logps": -182.81251525878906,
"epoch": 0.8846153846153846,
"grad_norm": 5.9566531465155,
"learning_rate": 1e-06,
"logits/chosen": -3.4355483055114746,
"logits/rejected": -3.4658639430999756,
"logps/chosen": -174.82119750976562,
"logps/rejected": -179.5398712158203,
"loss": 0.4262,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.09178514778614044,
"rewards/margins": 0.059058789163827896,
"rewards/rejected": 0.032726362347602844,
"step": 46
},
{
"debug/policy_chosen_logits": -3.0792293548583984,
"debug/policy_chosen_logps": -181.77552795410156,
"debug/policy_rejected_logits": -3.082854986190796,
"debug/policy_rejected_logps": -220.6517791748047,
"debug/reference_chosen_logps": -191.10177612304688,
"debug/reference_rejected_logps": -199.66552734375,
"epoch": 0.9038461538461539,
"grad_norm": 9.698103359515986,
"learning_rate": 1e-06,
"logits/chosen": -3.0792293548583984,
"logits/rejected": -3.082854986190796,
"logps/chosen": -181.77552795410156,
"logps/rejected": -220.6517791748047,
"loss": 0.3918,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.09326266497373581,
"rewards/margins": 0.3031250536441803,
"rewards/rejected": -0.20986239612102509,
"step": 47
},
{
"debug/policy_chosen_logits": -3.1799800395965576,
"debug/policy_chosen_logps": -182.957763671875,
"debug/policy_rejected_logits": -3.1335575580596924,
"debug/policy_rejected_logps": -217.8682403564453,
"debug/reference_chosen_logps": -194.94949340820312,
"debug/reference_rejected_logps": -203.79934692382812,
"epoch": 0.9230769230769231,
"grad_norm": 6.73392677471291,
"learning_rate": 1e-06,
"logits/chosen": -3.1799800395965576,
"logits/rejected": -3.1335575580596924,
"logps/chosen": -182.957763671875,
"logps/rejected": -217.8682403564453,
"loss": 0.34,
"rewards/accuracies": 1.0,
"rewards/chosen": 0.11991731822490692,
"rewards/margins": 0.26060622930526733,
"rewards/rejected": -0.1406889110803604,
"step": 48
},
{
"debug/policy_chosen_logits": -3.0402865409851074,
"debug/policy_chosen_logps": -204.64993286132812,
"debug/policy_rejected_logits": -3.1806182861328125,
"debug/policy_rejected_logps": -223.6003875732422,
"debug/reference_chosen_logps": -214.6299285888672,
"debug/reference_rejected_logps": -218.46998596191406,
"epoch": 0.9423076923076923,
"grad_norm": 5.290743823323874,
"learning_rate": 1e-06,
"logits/chosen": -3.0402865409851074,
"logits/rejected": -3.1806182861328125,
"logps/chosen": -204.64993286132812,
"logps/rejected": -223.6003875732422,
"loss": 0.3762,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.09979984909296036,
"rewards/margins": 0.1511038839817047,
"rewards/rejected": -0.05130405351519585,
"step": 49
},
{
"debug/policy_chosen_logits": -3.2792398929595947,
"debug/policy_chosen_logps": -172.821533203125,
"debug/policy_rejected_logits": -3.199223756790161,
"debug/policy_rejected_logps": -226.37725830078125,
"debug/reference_chosen_logps": -185.41844177246094,
"debug/reference_rejected_logps": -212.13818359375,
"epoch": 0.9615384615384616,
"grad_norm": 12.334225732861476,
"learning_rate": 1e-06,
"logits/chosen": -3.2792398929595947,
"logits/rejected": -3.199223756790161,
"logps/chosen": -172.821533203125,
"logps/rejected": -226.37725830078125,
"loss": 0.3587,
"rewards/accuracies": 1.0,
"rewards/chosen": 0.12596909701824188,
"rewards/margins": 0.268359899520874,
"rewards/rejected": -0.14239083230495453,
"step": 50
},
{
"debug/policy_chosen_logits": -3.1161139011383057,
"debug/policy_chosen_logps": -183.79429626464844,
"debug/policy_rejected_logits": -3.1612720489501953,
"debug/policy_rejected_logps": -238.53965759277344,
"debug/reference_chosen_logps": -192.13421630859375,
"debug/reference_rejected_logps": -220.17738342285156,
"epoch": 0.9807692307692307,
"grad_norm": 6.871974967844878,
"learning_rate": 1e-06,
"logits/chosen": -3.1161139011383057,
"logits/rejected": -3.1612720489501953,
"logps/chosen": -183.79429626464844,
"logps/rejected": -238.53965759277344,
"loss": 0.3778,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.0833992063999176,
"rewards/margins": 0.26702186465263367,
"rewards/rejected": -0.18362264335155487,
"step": 51
},
{
"debug/policy_chosen_logits": -3.1640007495880127,
"debug/policy_chosen_logps": -184.35934448242188,
"debug/policy_rejected_logits": -3.1118485927581787,
"debug/policy_rejected_logps": -235.7113037109375,
"debug/reference_chosen_logps": -200.53074645996094,
"debug/reference_rejected_logps": -225.58468627929688,
"epoch": 1.0,
"grad_norm": 6.270849179735478,
"learning_rate": 1e-06,
"logits/chosen": -3.1640007495880127,
"logits/rejected": -3.1118485927581787,
"logps/chosen": -184.35934448242188,
"logps/rejected": -235.7113037109375,
"loss": 0.3221,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.1617138683795929,
"rewards/margins": 0.2629801034927368,
"rewards/rejected": -0.10126623511314392,
"step": 52
},
{
"epoch": 1.0,
"step": 52,
"total_flos": 0.0,
"train_loss": 0.43149175896094394,
"train_runtime": 170.8012,
"train_samples_per_second": 19.385,
"train_steps_per_second": 0.304
}
],
"logging_steps": 1,
"max_steps": 52,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
|