Hugging Face's logo

HuggingFaceH4
/
zephyr-orpo-141b-A35b-v0.1

Text Generation
Transformers
TensorBoard
Safetensors
mixtral
trl
orpo
Generated from Trainer
conversational
text-generation-inference
Inference Endpoints
Model card Files Metrics Community
15
lewtun HF staff commited on
Commit
b08d449
•
1 Parent(s): 3f3b467

Model save

Browse files
Files changed (13) hide show
  1. README.md +59 -0
  2. all_results.json +8 -0
  3. runs/Apr10_15-55-51_ip-26-0-161-78/events.out.tfevents.1712764896.ip-26-0-161-78.3320890.0 +3 -0
  4. runs/Apr10_16-59-50_ip-26-0-163-226/events.out.tfevents.1712768751.ip-26-0-163-226.3229494.0 +3 -0
  5. runs/Apr10_17-24-41_ip-26-0-171-230/events.out.tfevents.1712770137.ip-26-0-171-230.599304.0 +3 -0
  6. runs/Apr10_17-51-26_ip-26-0-160-100/events.out.tfevents.1712771849.ip-26-0-160-100.3551619.0 +3 -0
  7. runs/Apr10_18-13-54_ip-26-0-165-38/events.out.tfevents.1712773202.ip-26-0-165-38.2582152.0 +3 -0
  8. runs/Apr10_18-24-34_ip-26-0-165-38/events.out.tfevents.1712773732.ip-26-0-165-38.2584151.0 +3 -0
  9. runs/Apr10_19-03-29_ip-26-0-169-247/events.out.tfevents.1712776302.ip-26-0-169-247.3774585.0 +3 -0
  10. runs/Apr10_19-21-46_ip-26-0-167-245/events.out.tfevents.1712777451.ip-26-0-167-245.2282940.0 +3 -0
  11. runs/Apr10_19-36-14_ip-26-0-162-79/events.out.tfevents.1712778153.ip-26-0-162-79.1673127.0 +3 -0
  12. train_results.json +8 -0
  13. trainer_state.json +1200 -0
README.md ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ base_model: mistral-community/Mixtral-8x22B-v0.1
4
+ tags:
5
+ - trl
6
+ - orpo
7
+ - generated_from_trainer
8
+ model-index:
9
+ - name: Mixtral-8x22B-capybara-v0.1
10
+ results: []
11
+ ---
12
+
13
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
14
+ should probably proofread and complete it, then remove this comment. -->
15
+
16
+ # Mixtral-8x22B-capybara-v0.1
17
+
18
+ This model is a fine-tuned version of [mistral-community/Mixtral-8x22B-v0.1](https://huggingface.co/mistral-community/Mixtral-8x22B-v0.1) on an unknown dataset.
19
+
20
+ ## Model description
21
+
22
+ More information needed
23
+
24
+ ## Intended uses & limitations
25
+
26
+ More information needed
27
+
28
+ ## Training and evaluation data
29
+
30
+ More information needed
31
+
32
+ ## Training procedure
33
+
34
+ ### Training hyperparameters
35
+
36
+ The following hyperparameters were used during training:
37
+ - learning_rate: 5e-06
38
+ - train_batch_size: 1
39
+ - eval_batch_size: 8
40
+ - seed: 42
41
+ - distributed_type: multi-GPU
42
+ - num_devices: 32
43
+ - total_train_batch_size: 32
44
+ - total_eval_batch_size: 256
45
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
46
+ - lr_scheduler_type: inverse_sqrt
47
+ - lr_scheduler_warmup_steps: 100
48
+ - num_epochs: 3
49
+
50
+ ### Training results
51
+
52
+
53
+
54
+ ### Framework versions
55
+
56
+ - Transformers 4.39.3
57
+ - Pytorch 2.1.2+cu121
58
+ - Datasets 2.18.0
59
+ - Tokenizers 0.15.1
all_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 3.0,
3
+ "train_loss": 1.0000138937419827,
4
+ "train_runtime": 4770.0198,
5
+ "train_samples": 6932,
6
+ "train_samples_per_second": 4.36,
7
+ "train_steps_per_second": 0.136
8
+ }
runs/Apr10_15-55-51_ip-26-0-161-78/events.out.tfevents.1712764896.ip-26-0-161-78.3320890.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4d6f27682d27ab10b89eb8d334a4db1501d631b01daf35e43de26321efa03675
3
+ size 5178
runs/Apr10_16-59-50_ip-26-0-163-226/events.out.tfevents.1712768751.ip-26-0-163-226.3229494.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12a83f13bba8a1fe7471ddbbaf698377bd67cbf1e1f0c4259ff3b5eeb03f97ec
3
+ size 5179
runs/Apr10_17-24-41_ip-26-0-171-230/events.out.tfevents.1712770137.ip-26-0-171-230.599304.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:89ff2d19a08dff7a1b8a098a3814a3c03acbcb69b49e716fe51bbdba567fdd8a
3
+ size 5179
runs/Apr10_17-51-26_ip-26-0-160-100/events.out.tfevents.1712771849.ip-26-0-160-100.3551619.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b1702b8347406a65415bf47d6b036b0c220950122661fb858b3d2f0708a37a96
3
+ size 5179
runs/Apr10_18-13-54_ip-26-0-165-38/events.out.tfevents.1712773202.ip-26-0-165-38.2582152.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3da4e8336a313842eff8a04781fef94986289902b8f7fb7b366d42575edf94d1
3
+ size 5175
runs/Apr10_18-24-34_ip-26-0-165-38/events.out.tfevents.1712773732.ip-26-0-165-38.2584151.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:262824914b5e33e6160bad18eb2ce847b1944117514789bb95ba0e3f1daca571
3
+ size 5181
runs/Apr10_19-03-29_ip-26-0-169-247/events.out.tfevents.1712776302.ip-26-0-169-247.3774585.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:515e0a7d57e9a7661e7a137ec64b870fedb8c27d21526b63c823104638295bd6
3
+ size 5220
runs/Apr10_19-21-46_ip-26-0-167-245/events.out.tfevents.1712777451.ip-26-0-167-245.2282940.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e8a20f8b1d96cb0321c058b88c1026a5ff27541eaeb08b2025d445c7c2c07d4
3
+ size 5220
runs/Apr10_19-36-14_ip-26-0-162-79/events.out.tfevents.1712778153.ip-26-0-162-79.1673127.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:73d238630f88a0982f923d1ea7df0ff7de1e40a90211673b7434feb64f0993ec
3
+ size 61291
train_results.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 3.0,
3
+ "train_loss": 1.0000138937419827,
4
+ "train_runtime": 4770.0198,
5
+ "train_samples": 6932,
6
+ "train_samples_per_second": 4.36,
7
+ "train_steps_per_second": 0.136
8
+ }
trainer_state.json ADDED
@@ -0,0 +1,1200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 3.0,
5
+ "eval_steps": 500,
6
+ "global_step": 651,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.05,
13
+ "grad_norm": 2496.0,
14
+ "learning_rate": 5.000000000000001e-07,
15
+ "log_odds_chosen": 0.36155954003334045,
16
+ "log_odds_ratio": -0.6397812962532043,
17
+ "logits/chosen": 4.0045647621154785,
18
+ "logits/rejected": 5.319853782653809,
19
+ "logps/chosen": -0.9873844981193542,
20
+ "logps/rejected": -1.2530113458633423,
21
+ "loss": 1.8618,
22
+ "nll_loss": 3.376544952392578,
23
+ "rewards/accuracies": 0.6000000238418579,
24
+ "rewards/chosen": -0.04936922341585159,
25
+ "rewards/margins": 0.01328133512288332,
26
+ "rewards/rejected": -0.06265056133270264,
27
+ "step": 10
28
+ },
29
+ {
30
+ "epoch": 0.09,
31
+ "grad_norm": 4112384.0,
32
+ "learning_rate": 1.0000000000000002e-06,
33
+ "log_odds_chosen": 0.17483997344970703,
34
+ "log_odds_ratio": -0.6283982992172241,
35
+ "logits/chosen": 4.776286602020264,
36
+ "logits/rejected": 5.276551246643066,
37
+ "logps/chosen": -1.1025744676589966,
38
+ "logps/rejected": -1.2446129322052002,
39
+ "loss": 1.935,
40
+ "nll_loss": 2.290414571762085,
41
+ "rewards/accuracies": 0.6000000238418579,
42
+ "rewards/chosen": -0.05512872338294983,
43
+ "rewards/margins": 0.007101929280906916,
44
+ "rewards/rejected": -0.06223064661026001,
45
+ "step": 20
46
+ },
47
+ {
48
+ "epoch": 0.14,
49
+ "grad_norm": 23680.0,
50
+ "learning_rate": 1.5e-06,
51
+ "log_odds_chosen": 0.47547435760498047,
52
+ "log_odds_ratio": -0.5693398118019104,
53
+ "logits/chosen": 4.594313144683838,
54
+ "logits/rejected": 5.171442031860352,
55
+ "logps/chosen": -0.9896373748779297,
56
+ "logps/rejected": -1.2604167461395264,
57
+ "loss": 1.9598,
58
+ "nll_loss": 1.6909611225128174,
59
+ "rewards/accuracies": 0.699999988079071,
60
+ "rewards/chosen": -0.049481868743896484,
61
+ "rewards/margins": 0.013538961298763752,
62
+ "rewards/rejected": -0.06302083283662796,
63
+ "step": 30
64
+ },
65
+ {
66
+ "epoch": 0.18,
67
+ "grad_norm": 222208.0,
68
+ "learning_rate": 2.0000000000000003e-06,
69
+ "log_odds_chosen": 0.2728760838508606,
70
+ "log_odds_ratio": -0.6436117887496948,
71
+ "logits/chosen": 5.256069660186768,
72
+ "logits/rejected": 5.317893981933594,
73
+ "logps/chosen": -0.9026492834091187,
74
+ "logps/rejected": -1.0579217672348022,
75
+ "loss": 1.7292,
76
+ "nll_loss": 1.168599009513855,
77
+ "rewards/accuracies": 0.6000000238418579,
78
+ "rewards/chosen": -0.04513246938586235,
79
+ "rewards/margins": 0.007763628847897053,
80
+ "rewards/rejected": -0.05289609357714653,
81
+ "step": 40
82
+ },
83
+ {
84
+ "epoch": 0.23,
85
+ "grad_norm": 41216.0,
86
+ "learning_rate": 2.5e-06,
87
+ "log_odds_chosen": -0.07172749936580658,
88
+ "log_odds_ratio": -0.9093732833862305,
89
+ "logits/chosen": 4.6413750648498535,
90
+ "logits/rejected": 5.3312811851501465,
91
+ "logps/chosen": -1.0293734073638916,
92
+ "logps/rejected": -0.9991890788078308,
93
+ "loss": 1.9033,
94
+ "nll_loss": 2.1190409660339355,
95
+ "rewards/accuracies": 0.699999988079071,
96
+ "rewards/chosen": -0.05146867036819458,
97
+ "rewards/margins": -0.0015092171961441636,
98
+ "rewards/rejected": -0.0499594546854496,
99
+ "step": 50
100
+ },
101
+ {
102
+ "epoch": 0.28,
103
+ "grad_norm": 7962624.0,
104
+ "learning_rate": 3e-06,
105
+ "log_odds_chosen": 0.6953669190406799,
106
+ "log_odds_ratio": -0.4808632731437683,
107
+ "logits/chosen": 5.134734153747559,
108
+ "logits/rejected": 5.177048683166504,
109
+ "logps/chosen": -0.74892258644104,
110
+ "logps/rejected": -1.1702710390090942,
111
+ "loss": 1.8265,
112
+ "nll_loss": 1.2245725393295288,
113
+ "rewards/accuracies": 0.800000011920929,
114
+ "rewards/chosen": -0.03744612634181976,
115
+ "rewards/margins": 0.02106742560863495,
116
+ "rewards/rejected": -0.05851355195045471,
117
+ "step": 60
118
+ },
119
+ {
120
+ "epoch": 0.32,
121
+ "grad_norm": 244736.0,
122
+ "learning_rate": 3.5e-06,
123
+ "log_odds_chosen": -0.30761510133743286,
124
+ "log_odds_ratio": -1.0599069595336914,
125
+ "logits/chosen": 4.717621326446533,
126
+ "logits/rejected": 5.133732318878174,
127
+ "logps/chosen": -1.1152620315551758,
128
+ "logps/rejected": -0.8680723309516907,
129
+ "loss": 1.8814,
130
+ "nll_loss": 1.776172399520874,
131
+ "rewards/accuracies": 0.6000000238418579,
132
+ "rewards/chosen": -0.05576310306787491,
133
+ "rewards/margins": -0.012359483167529106,
134
+ "rewards/rejected": -0.04340361803770065,
135
+ "step": 70
136
+ },
137
+ {
138
+ "epoch": 0.37,
139
+ "grad_norm": 12096.0,
140
+ "learning_rate": 4.000000000000001e-06,
141
+ "log_odds_chosen": 0.38896164298057556,
142
+ "log_odds_ratio": -0.5763465166091919,
143
+ "logits/chosen": 4.805164337158203,
144
+ "logits/rejected": 5.310858249664307,
145
+ "logps/chosen": -0.9377196431159973,
146
+ "logps/rejected": -1.1725047826766968,
147
+ "loss": 1.7939,
148
+ "nll_loss": 1.305657148361206,
149
+ "rewards/accuracies": 0.6000000238418579,
150
+ "rewards/chosen": -0.046885985881090164,
151
+ "rewards/margins": 0.011739259585738182,
152
+ "rewards/rejected": -0.0586252398788929,
153
+ "step": 80
154
+ },
155
+ {
156
+ "epoch": 0.41,
157
+ "grad_norm": 17280.0,
158
+ "learning_rate": 4.5e-06,
159
+ "log_odds_chosen": 0.5031664371490479,
160
+ "log_odds_ratio": -0.5130521655082703,
161
+ "logits/chosen": 4.390740394592285,
162
+ "logits/rejected": 4.711813926696777,
163
+ "logps/chosen": -0.8132585287094116,
164
+ "logps/rejected": -1.101137399673462,
165
+ "loss": 1.8672,
166
+ "nll_loss": 1.8442224264144897,
167
+ "rewards/accuracies": 0.8999999761581421,
168
+ "rewards/chosen": -0.04066292941570282,
169
+ "rewards/margins": 0.014393949881196022,
170
+ "rewards/rejected": -0.05505687743425369,
171
+ "step": 90
172
+ },
173
+ {
174
+ "epoch": 0.46,
175
+ "grad_norm": 9920.0,
176
+ "learning_rate": 5e-06,
177
+ "log_odds_chosen": 0.9018317461013794,
178
+ "log_odds_ratio": -0.3739783763885498,
179
+ "logits/chosen": 3.827307939529419,
180
+ "logits/rejected": 5.207882404327393,
181
+ "logps/chosen": -0.7615492939949036,
182
+ "logps/rejected": -1.3325573205947876,
183
+ "loss": 1.5324,
184
+ "nll_loss": 1.6830791234970093,
185
+ "rewards/accuracies": 0.8999999761581421,
186
+ "rewards/chosen": -0.0380774661898613,
187
+ "rewards/margins": 0.028550397604703903,
188
+ "rewards/rejected": -0.0666278749704361,
189
+ "step": 100
190
+ },
191
+ {
192
+ "epoch": 0.51,
193
+ "grad_norm": 17792.0,
194
+ "learning_rate": 4.767312946227961e-06,
195
+ "log_odds_chosen": 0.6007639169692993,
196
+ "log_odds_ratio": -0.5869214534759521,
197
+ "logits/chosen": 4.3159074783325195,
198
+ "logits/rejected": 4.817818641662598,
199
+ "logps/chosen": -0.9503151178359985,
200
+ "logps/rejected": -1.3361114263534546,
201
+ "loss": 1.4121,
202
+ "nll_loss": 1.579045295715332,
203
+ "rewards/accuracies": 0.699999988079071,
204
+ "rewards/chosen": -0.047515757381916046,
205
+ "rewards/margins": 0.019289815798401833,
206
+ "rewards/rejected": -0.06680557131767273,
207
+ "step": 110
208
+ },
209
+ {
210
+ "epoch": 0.55,
211
+ "grad_norm": 56832.0,
212
+ "learning_rate": 4.564354645876385e-06,
213
+ "log_odds_chosen": 0.5472079515457153,
214
+ "log_odds_ratio": -0.5241577625274658,
215
+ "logits/chosen": 5.075867652893066,
216
+ "logits/rejected": 5.5994157791137695,
217
+ "logps/chosen": -0.9527261853218079,
218
+ "logps/rejected": -1.2945894002914429,
219
+ "loss": 1.51,
220
+ "nll_loss": 1.2633002996444702,
221
+ "rewards/accuracies": 0.699999988079071,
222
+ "rewards/chosen": -0.047636307775974274,
223
+ "rewards/margins": 0.01709316112101078,
224
+ "rewards/rejected": -0.0647294670343399,
225
+ "step": 120
226
+ },
227
+ {
228
+ "epoch": 0.6,
229
+ "grad_norm": 364544.0,
230
+ "learning_rate": 4.385290096535147e-06,
231
+ "log_odds_chosen": -0.02357400581240654,
232
+ "log_odds_ratio": -0.7699296474456787,
233
+ "logits/chosen": 3.9909451007843018,
234
+ "logits/rejected": 5.117685794830322,
235
+ "logps/chosen": -1.079882264137268,
236
+ "logps/rejected": -1.0565487146377563,
237
+ "loss": 1.5702,
238
+ "nll_loss": 1.8465524911880493,
239
+ "rewards/accuracies": 0.5,
240
+ "rewards/chosen": -0.053994111716747284,
241
+ "rewards/margins": -0.0011666819918900728,
242
+ "rewards/rejected": -0.05282743647694588,
243
+ "step": 130
244
+ },
245
+ {
246
+ "epoch": 0.65,
247
+ "grad_norm": 24448.0,
248
+ "learning_rate": 4.2257712736425835e-06,
249
+ "log_odds_chosen": 0.37071943283081055,
250
+ "log_odds_ratio": -0.5912718176841736,
251
+ "logits/chosen": 4.673558235168457,
252
+ "logits/rejected": 4.776795387268066,
253
+ "logps/chosen": -0.8323698043823242,
254
+ "logps/rejected": -1.0506634712219238,
255
+ "loss": 1.3833,
256
+ "nll_loss": 1.5265856981277466,
257
+ "rewards/accuracies": 0.800000011920929,
258
+ "rewards/chosen": -0.04161848872900009,
259
+ "rewards/margins": 0.010914680548012257,
260
+ "rewards/rejected": -0.052533168345689774,
261
+ "step": 140
262
+ },
263
+ {
264
+ "epoch": 0.69,
265
+ "grad_norm": 667648.0,
266
+ "learning_rate": 4.082482904638631e-06,
267
+ "log_odds_chosen": 0.38299956917762756,
268
+ "log_odds_ratio": -0.6216747164726257,
269
+ "logits/chosen": 5.035830974578857,
270
+ "logits/rejected": 4.980803966522217,
271
+ "logps/chosen": -0.9842357635498047,
272
+ "logps/rejected": -1.176418662071228,
273
+ "loss": 1.3014,
274
+ "nll_loss": 1.2037413120269775,
275
+ "rewards/accuracies": 0.699999988079071,
276
+ "rewards/chosen": -0.04921179264783859,
277
+ "rewards/margins": 0.009609140455722809,
278
+ "rewards/rejected": -0.0588209331035614,
279
+ "step": 150
280
+ },
281
+ {
282
+ "epoch": 0.74,
283
+ "grad_norm": 15296.0,
284
+ "learning_rate": 3.952847075210474e-06,
285
+ "log_odds_chosen": 0.5589774250984192,
286
+ "log_odds_ratio": -0.5005263090133667,
287
+ "logits/chosen": 4.990937232971191,
288
+ "logits/rejected": 5.509407043457031,
289
+ "logps/chosen": -1.038329839706421,
290
+ "logps/rejected": -1.4047755002975464,
291
+ "loss": 1.4222,
292
+ "nll_loss": 1.3109643459320068,
293
+ "rewards/accuracies": 0.8999999761581421,
294
+ "rewards/chosen": -0.05191649869084358,
295
+ "rewards/margins": 0.018322281539440155,
296
+ "rewards/rejected": -0.07023878395557404,
297
+ "step": 160
298
+ },
299
+ {
300
+ "epoch": 0.78,
301
+ "grad_norm": 12416.0,
302
+ "learning_rate": 3.834824944236852e-06,
303
+ "log_odds_chosen": 0.4185755252838135,
304
+ "log_odds_ratio": -0.5574899315834045,
305
+ "logits/chosen": 4.170102119445801,
306
+ "logits/rejected": 5.193001747131348,
307
+ "logps/chosen": -0.8057904243469238,
308
+ "logps/rejected": -1.1126911640167236,
309
+ "loss": 1.2946,
310
+ "nll_loss": 1.3742918968200684,
311
+ "rewards/accuracies": 0.800000011920929,
312
+ "rewards/chosen": -0.04028952494263649,
313
+ "rewards/margins": 0.015345032326877117,
314
+ "rewards/rejected": -0.05563455820083618,
315
+ "step": 170
316
+ },
317
+ {
318
+ "epoch": 0.83,
319
+ "grad_norm": 4928.0,
320
+ "learning_rate": 3.72677996249965e-06,
321
+ "log_odds_chosen": 0.23164959251880646,
322
+ "log_odds_ratio": -0.6397107243537903,
323
+ "logits/chosen": 4.471838474273682,
324
+ "logits/rejected": 5.085393905639648,
325
+ "logps/chosen": -0.8749902844429016,
326
+ "logps/rejected": -1.0519354343414307,
327
+ "loss": 1.2939,
328
+ "nll_loss": 1.6231539249420166,
329
+ "rewards/accuracies": 0.6000000238418579,
330
+ "rewards/chosen": -0.04374951496720314,
331
+ "rewards/margins": 0.00884725246578455,
332
+ "rewards/rejected": -0.052596766501665115,
333
+ "step": 180
334
+ },
335
+ {
336
+ "epoch": 0.88,
337
+ "grad_norm": 16768.0,
338
+ "learning_rate": 3.6273812505500587e-06,
339
+ "log_odds_chosen": 0.812569260597229,
340
+ "log_odds_ratio": -0.43427157402038574,
341
+ "logits/chosen": 5.2673540115356445,
342
+ "logits/rejected": 5.124138355255127,
343
+ "logps/chosen": -0.8323872685432434,
344
+ "logps/rejected": -1.265616536140442,
345
+ "loss": 1.1174,
346
+ "nll_loss": 0.9819033741950989,
347
+ "rewards/accuracies": 0.8999999761581421,
348
+ "rewards/chosen": -0.04161936417222023,
349
+ "rewards/margins": 0.021661465987563133,
350
+ "rewards/rejected": -0.06328083574771881,
351
+ "step": 190
352
+ },
353
+ {
354
+ "epoch": 0.92,
355
+ "grad_norm": 38400.0,
356
+ "learning_rate": 3.5355339059327378e-06,
357
+ "log_odds_chosen": 0.28463655710220337,
358
+ "log_odds_ratio": -0.6241921186447144,
359
+ "logits/chosen": 5.328371524810791,
360
+ "logits/rejected": 5.427257061004639,
361
+ "logps/chosen": -1.1006910800933838,
362
+ "logps/rejected": -1.294104814529419,
363
+ "loss": 1.1771,
364
+ "nll_loss": 1.1568324565887451,
365
+ "rewards/accuracies": 0.800000011920929,
366
+ "rewards/chosen": -0.05503455549478531,
367
+ "rewards/margins": 0.009670679457485676,
368
+ "rewards/rejected": -0.06470523774623871,
369
+ "step": 200
370
+ },
371
+ {
372
+ "epoch": 0.97,
373
+ "grad_norm": 99840.0,
374
+ "learning_rate": 3.450327796711771e-06,
375
+ "log_odds_chosen": 0.387382835149765,
376
+ "log_odds_ratio": -0.6240562796592712,
377
+ "logits/chosen": 4.951775550842285,
378
+ "logits/rejected": 5.1796345710754395,
379
+ "logps/chosen": -0.9047640562057495,
380
+ "logps/rejected": -1.1922653913497925,
381
+ "loss": 1.2614,
382
+ "nll_loss": 1.1042182445526123,
383
+ "rewards/accuracies": 0.6000000238418579,
384
+ "rewards/chosen": -0.04523820802569389,
385
+ "rewards/margins": 0.014375070109963417,
386
+ "rewards/rejected": -0.05961327999830246,
387
+ "step": 210
388
+ },
389
+ {
390
+ "epoch": 1.01,
391
+ "grad_norm": 503808.0,
392
+ "learning_rate": 3.3709993123162106e-06,
393
+ "log_odds_chosen": 0.9024753570556641,
394
+ "log_odds_ratio": -0.4256620407104492,
395
+ "logits/chosen": 4.911884307861328,
396
+ "logits/rejected": 5.383030891418457,
397
+ "logps/chosen": -0.6001554131507874,
398
+ "logps/rejected": -1.0823005437850952,
399
+ "loss": 1.4667,
400
+ "nll_loss": 1.102745771408081,
401
+ "rewards/accuracies": 0.8999999761581421,
402
+ "rewards/chosen": -0.030007774010300636,
403
+ "rewards/margins": 0.024107253178954124,
404
+ "rewards/rejected": -0.054115019738674164,
405
+ "step": 220
406
+ },
407
+ {
408
+ "epoch": 1.06,
409
+ "grad_norm": 2384.0,
410
+ "learning_rate": 3.296902366978936e-06,
411
+ "log_odds_chosen": 0.30322012305259705,
412
+ "log_odds_ratio": -0.6324062943458557,
413
+ "logits/chosen": 5.081802845001221,
414
+ "logits/rejected": 5.494980812072754,
415
+ "logps/chosen": -0.9133793711662292,
416
+ "logps/rejected": -1.0385487079620361,
417
+ "loss": 1.124,
418
+ "nll_loss": 0.9831510782241821,
419
+ "rewards/accuracies": 0.699999988079071,
420
+ "rewards/chosen": -0.04566896706819534,
421
+ "rewards/margins": 0.00625845929607749,
422
+ "rewards/rejected": -0.05192742869257927,
423
+ "step": 230
424
+ },
425
+ {
426
+ "epoch": 1.11,
427
+ "grad_norm": 31232.0,
428
+ "learning_rate": 3.2274861218395142e-06,
429
+ "log_odds_chosen": 0.30336421728134155,
430
+ "log_odds_ratio": -0.6088391542434692,
431
+ "logits/chosen": 4.869197845458984,
432
+ "logits/rejected": 5.251407623291016,
433
+ "logps/chosen": -0.8927870988845825,
434
+ "logps/rejected": -1.0718307495117188,
435
+ "loss": 1.0758,
436
+ "nll_loss": 1.5037825107574463,
437
+ "rewards/accuracies": 0.6000000238418579,
438
+ "rewards/chosen": -0.04463934525847435,
439
+ "rewards/margins": 0.008952188305556774,
440
+ "rewards/rejected": -0.053591541945934296,
441
+ "step": 240
442
+ },
443
+ {
444
+ "epoch": 1.15,
445
+ "grad_norm": 2864.0,
446
+ "learning_rate": 3.1622776601683796e-06,
447
+ "log_odds_chosen": -0.1395704448223114,
448
+ "log_odds_ratio": -0.8436253666877747,
449
+ "logits/chosen": 5.441056251525879,
450
+ "logits/rejected": 5.426650047302246,
451
+ "logps/chosen": -1.041529893875122,
452
+ "logps/rejected": -0.9539015889167786,
453
+ "loss": 1.042,
454
+ "nll_loss": 0.900956928730011,
455
+ "rewards/accuracies": 0.4000000059604645,
456
+ "rewards/chosen": -0.05207649990916252,
457
+ "rewards/margins": -0.004381421022117138,
458
+ "rewards/rejected": -0.04769507795572281,
459
+ "step": 250
460
+ },
461
+ {
462
+ "epoch": 1.2,
463
+ "grad_norm": 964.0,
464
+ "learning_rate": 3.1008683647302113e-06,
465
+ "log_odds_chosen": 0.44115790724754333,
466
+ "log_odds_ratio": -0.5350314378738403,
467
+ "logits/chosen": 5.245479106903076,
468
+ "logits/rejected": 4.632310390472412,
469
+ "logps/chosen": -0.8721857070922852,
470
+ "logps/rejected": -1.108418583869934,
471
+ "loss": 0.9244,
472
+ "nll_loss": 0.8430414199829102,
473
+ "rewards/accuracies": 0.800000011920929,
474
+ "rewards/chosen": -0.04360928758978844,
475
+ "rewards/margins": 0.01181163638830185,
476
+ "rewards/rejected": -0.055420927703380585,
477
+ "step": 260
478
+ },
479
+ {
480
+ "epoch": 1.24,
481
+ "grad_norm": 6.78125,
482
+ "learning_rate": 3.0429030972509227e-06,
483
+ "log_odds_chosen": 0.04378192871809006,
484
+ "log_odds_ratio": -0.8174707293510437,
485
+ "logits/chosen": 4.69882869720459,
486
+ "logits/rejected": 5.443737983703613,
487
+ "logps/chosen": -0.7305840253829956,
488
+ "logps/rejected": -0.7943456172943115,
489
+ "loss": 0.8501,
490
+ "nll_loss": 0.6877334117889404,
491
+ "rewards/accuracies": 0.6000000238418579,
492
+ "rewards/chosen": -0.03652920573949814,
493
+ "rewards/margins": 0.0031880741007626057,
494
+ "rewards/rejected": -0.03971727937459946,
495
+ "step": 270
496
+ },
497
+ {
498
+ "epoch": 1.29,
499
+ "grad_norm": 2.140625,
500
+ "learning_rate": 2.988071523335984e-06,
501
+ "log_odds_chosen": 0.5857657790184021,
502
+ "log_odds_ratio": -0.5777389407157898,
503
+ "logits/chosen": 4.741288661956787,
504
+ "logits/rejected": 5.175159454345703,
505
+ "logps/chosen": -0.8554542660713196,
506
+ "logps/rejected": -1.0246210098266602,
507
+ "loss": 0.7456,
508
+ "nll_loss": 0.7096272706985474,
509
+ "rewards/accuracies": 0.6000000238418579,
510
+ "rewards/chosen": -0.04277271404862404,
511
+ "rewards/margins": 0.008458340540528297,
512
+ "rewards/rejected": -0.051231056451797485,
513
+ "step": 280
514
+ },
515
+ {
516
+ "epoch": 1.34,
517
+ "grad_norm": 2.859375,
518
+ "learning_rate": 2.9361010975735177e-06,
519
+ "log_odds_chosen": 0.40030980110168457,
520
+ "log_odds_ratio": -0.5996511578559875,
521
+ "logits/chosen": 4.385786533355713,
522
+ "logits/rejected": 5.33310604095459,
523
+ "logps/chosen": -0.9153855443000793,
524
+ "logps/rejected": -1.1298904418945312,
525
+ "loss": 0.7218,
526
+ "nll_loss": 0.6553577184677124,
527
+ "rewards/accuracies": 0.6000000238418579,
528
+ "rewards/chosen": -0.04576927423477173,
529
+ "rewards/margins": 0.010725250467658043,
530
+ "rewards/rejected": -0.05649452656507492,
531
+ "step": 290
532
+ },
533
+ {
534
+ "epoch": 1.38,
535
+ "grad_norm": 2.15625,
536
+ "learning_rate": 2.8867513459481293e-06,
537
+ "log_odds_chosen": 0.20400409400463104,
538
+ "log_odds_ratio": -0.807258129119873,
539
+ "logits/chosen": 4.453027725219727,
540
+ "logits/rejected": 5.463714599609375,
541
+ "logps/chosen": -0.8655136227607727,
542
+ "logps/rejected": -1.121850848197937,
543
+ "loss": 0.7357,
544
+ "nll_loss": 0.7287333607673645,
545
+ "rewards/accuracies": 0.699999988079071,
546
+ "rewards/chosen": -0.04327567666769028,
547
+ "rewards/margins": 0.012816866859793663,
548
+ "rewards/rejected": -0.05609254911541939,
549
+ "step": 300
550
+ },
551
+ {
552
+ "epoch": 1.43,
553
+ "grad_norm": 2.75,
554
+ "learning_rate": 2.839809171235324e-06,
555
+ "log_odds_chosen": 1.1084682941436768,
556
+ "log_odds_ratio": -0.36698225140571594,
557
+ "logits/chosen": 3.3886070251464844,
558
+ "logits/rejected": 5.912220478057861,
559
+ "logps/chosen": -0.6694556474685669,
560
+ "logps/rejected": -1.2311700582504272,
561
+ "loss": 0.7327,
562
+ "nll_loss": 0.4876071810722351,
563
+ "rewards/accuracies": 1.0,
564
+ "rewards/chosen": -0.033472783863544464,
565
+ "rewards/margins": 0.028085719794034958,
566
+ "rewards/rejected": -0.06155850365757942,
567
+ "step": 310
568
+ },
569
+ {
570
+ "epoch": 1.47,
571
+ "grad_norm": 2.171875,
572
+ "learning_rate": 2.7950849718747376e-06,
573
+ "log_odds_chosen": 0.13975438475608826,
574
+ "log_odds_ratio": -0.663262128829956,
575
+ "logits/chosen": 5.9078688621521,
576
+ "logits/rejected": 4.301927089691162,
577
+ "logps/chosen": -0.9628068804740906,
578
+ "logps/rejected": -1.071995735168457,
579
+ "loss": 0.76,
580
+ "nll_loss": 0.7910596132278442,
581
+ "rewards/accuracies": 0.5,
582
+ "rewards/chosen": -0.04814033955335617,
583
+ "rewards/margins": 0.0054594483226537704,
584
+ "rewards/rejected": -0.05359978601336479,
585
+ "step": 320
586
+ },
587
+ {
588
+ "epoch": 1.52,
589
+ "grad_norm": 2.0625,
590
+ "learning_rate": 2.752409412815902e-06,
591
+ "log_odds_chosen": 1.0402824878692627,
592
+ "log_odds_ratio": -0.41337329149246216,
593
+ "logits/chosen": 4.443751811981201,
594
+ "logits/rejected": 5.312446594238281,
595
+ "logps/chosen": -0.5369973182678223,
596
+ "logps/rejected": -0.9893845319747925,
597
+ "loss": 0.7277,
598
+ "nll_loss": 0.5271695852279663,
599
+ "rewards/accuracies": 0.800000011920929,
600
+ "rewards/chosen": -0.026849865913391113,
601
+ "rewards/margins": 0.02261935919523239,
602
+ "rewards/rejected": -0.0494692288339138,
603
+ "step": 330
604
+ },
605
+ {
606
+ "epoch": 1.57,
607
+ "grad_norm": 2.25,
608
+ "learning_rate": 2.711630722733202e-06,
609
+ "log_odds_chosen": 0.3988404870033264,
610
+ "log_odds_ratio": -0.5601134896278381,
611
+ "logits/chosen": 4.886635780334473,
612
+ "logits/rejected": 5.539236068725586,
613
+ "logps/chosen": -1.0322339534759521,
614
+ "logps/rejected": -1.3051972389221191,
615
+ "loss": 0.7194,
616
+ "nll_loss": 0.7682756185531616,
617
+ "rewards/accuracies": 0.699999988079071,
618
+ "rewards/chosen": -0.05161169916391373,
619
+ "rewards/margins": 0.01364816538989544,
620
+ "rewards/rejected": -0.06525985896587372,
621
+ "step": 340
622
+ },
623
+ {
624
+ "epoch": 1.61,
625
+ "grad_norm": 2.1875,
626
+ "learning_rate": 2.6726124191242444e-06,
627
+ "log_odds_chosen": 0.3155137598514557,
628
+ "log_odds_ratio": -0.6714231967926025,
629
+ "logits/chosen": 5.050114631652832,
630
+ "logits/rejected": 5.338369846343994,
631
+ "logps/chosen": -1.0256156921386719,
632
+ "logps/rejected": -1.1557905673980713,
633
+ "loss": 0.7013,
634
+ "nll_loss": 0.7552909851074219,
635
+ "rewards/accuracies": 0.5,
636
+ "rewards/chosen": -0.05128078535199165,
637
+ "rewards/margins": 0.006508743856102228,
638
+ "rewards/rejected": -0.05778952315449715,
639
+ "step": 350
640
+ },
641
+ {
642
+ "epoch": 1.66,
643
+ "grad_norm": 2.09375,
644
+ "learning_rate": 2.6352313834736496e-06,
645
+ "log_odds_chosen": 0.33062583208084106,
646
+ "log_odds_ratio": -0.605568528175354,
647
+ "logits/chosen": 4.567168235778809,
648
+ "logits/rejected": 5.823721408843994,
649
+ "logps/chosen": -0.844105064868927,
650
+ "logps/rejected": -1.0698764324188232,
651
+ "loss": 0.698,
652
+ "nll_loss": 0.6597741842269897,
653
+ "rewards/accuracies": 0.699999988079071,
654
+ "rewards/chosen": -0.04220524802803993,
655
+ "rewards/margins": 0.011288573034107685,
656
+ "rewards/rejected": -0.05349382013082504,
657
+ "step": 360
658
+ },
659
+ {
660
+ "epoch": 1.71,
661
+ "grad_norm": 2.0,
662
+ "learning_rate": 2.599376224550182e-06,
663
+ "log_odds_chosen": 0.46411484479904175,
664
+ "log_odds_ratio": -0.5467833280563354,
665
+ "logits/chosen": 4.998323917388916,
666
+ "logits/rejected": 5.422191619873047,
667
+ "logps/chosen": -0.8868681788444519,
668
+ "logps/rejected": -1.1595653295516968,
669
+ "loss": 0.7081,
670
+ "nll_loss": 0.7420350313186646,
671
+ "rewards/accuracies": 0.800000011920929,
672
+ "rewards/chosen": -0.04434341564774513,
673
+ "rewards/margins": 0.013634858652949333,
674
+ "rewards/rejected": -0.057978272438049316,
675
+ "step": 370
676
+ },
677
+ {
678
+ "epoch": 1.75,
679
+ "grad_norm": 2.421875,
680
+ "learning_rate": 2.564945880212886e-06,
681
+ "log_odds_chosen": 0.3259461522102356,
682
+ "log_odds_ratio": -0.5724526643753052,
683
+ "logits/chosen": 5.517279148101807,
684
+ "logits/rejected": 4.9622650146484375,
685
+ "logps/chosen": -0.9444965124130249,
686
+ "logps/rejected": -1.102052092552185,
687
+ "loss": 0.7063,
688
+ "nll_loss": 0.7701026797294617,
689
+ "rewards/accuracies": 0.699999988079071,
690
+ "rewards/chosen": -0.047224827110767365,
691
+ "rewards/margins": 0.007877781055867672,
692
+ "rewards/rejected": -0.05510260909795761,
693
+ "step": 380
694
+ },
695
+ {
696
+ "epoch": 1.8,
697
+ "grad_norm": 3.859375,
698
+ "learning_rate": 2.5318484177091667e-06,
699
+ "log_odds_chosen": 0.6275227665901184,
700
+ "log_odds_ratio": -0.5797664523124695,
701
+ "logits/chosen": 4.297969341278076,
702
+ "logits/rejected": 6.322115421295166,
703
+ "logps/chosen": -0.7682819962501526,
704
+ "logps/rejected": -1.2000246047973633,
705
+ "loss": 0.7015,
706
+ "nll_loss": 0.6223348379135132,
707
+ "rewards/accuracies": 0.800000011920929,
708
+ "rewards/chosen": -0.03841409832239151,
709
+ "rewards/margins": 0.021587131544947624,
710
+ "rewards/rejected": -0.06000123172998428,
711
+ "step": 390
712
+ },
713
+ {
714
+ "epoch": 1.84,
715
+ "grad_norm": 4.125,
716
+ "learning_rate": 2.5e-06,
717
+ "log_odds_chosen": 0.31113579869270325,
718
+ "log_odds_ratio": -0.584196925163269,
719
+ "logits/chosen": 5.111819267272949,
720
+ "logits/rejected": 5.423724174499512,
721
+ "logps/chosen": -0.7397966980934143,
722
+ "logps/rejected": -0.9515731930732727,
723
+ "loss": 0.7028,
724
+ "nll_loss": 0.623633861541748,
725
+ "rewards/accuracies": 0.699999988079071,
726
+ "rewards/chosen": -0.036989834159612656,
727
+ "rewards/margins": 0.01058882288634777,
728
+ "rewards/rejected": -0.04757865145802498,
729
+ "step": 400
730
+ },
731
+ {
732
+ "epoch": 1.89,
733
+ "grad_norm": 2.140625,
734
+ "learning_rate": 2.4693239916239746e-06,
735
+ "log_odds_chosen": 0.2728329598903656,
736
+ "log_odds_ratio": -0.6246954798698425,
737
+ "logits/chosen": 4.5603532791137695,
738
+ "logits/rejected": 5.455700874328613,
739
+ "logps/chosen": -0.8029536008834839,
740
+ "logps/rejected": -0.9662898778915405,
741
+ "loss": 0.6951,
742
+ "nll_loss": 0.6521136164665222,
743
+ "rewards/accuracies": 0.6000000238418579,
744
+ "rewards/chosen": -0.040147680789232254,
745
+ "rewards/margins": 0.008166818879544735,
746
+ "rewards/rejected": -0.048314500600099564,
747
+ "step": 410
748
+ },
749
+ {
750
+ "epoch": 1.94,
751
+ "grad_norm": 1.984375,
752
+ "learning_rate": 2.4397501823713327e-06,
753
+ "log_odds_chosen": 1.0010753870010376,
754
+ "log_odds_ratio": -0.35868290066719055,
755
+ "logits/chosen": 5.044535160064697,
756
+ "logits/rejected": 5.757393836975098,
757
+ "logps/chosen": -0.7278403043746948,
758
+ "logps/rejected": -1.1986501216888428,
759
+ "loss": 0.7033,
760
+ "nll_loss": 0.628697395324707,
761
+ "rewards/accuracies": 0.8999999761581421,
762
+ "rewards/chosen": -0.03639201447367668,
763
+ "rewards/margins": 0.023540491238236427,
764
+ "rewards/rejected": -0.05993250757455826,
765
+ "step": 420
766
+ },
767
+ {
768
+ "epoch": 1.98,
769
+ "grad_norm": 2.28125,
770
+ "learning_rate": 2.411214110852061e-06,
771
+ "log_odds_chosen": 0.22137129306793213,
772
+ "log_odds_ratio": -0.6573923826217651,
773
+ "logits/chosen": 4.241551399230957,
774
+ "logits/rejected": 5.332159519195557,
775
+ "logps/chosen": -0.769220769405365,
776
+ "logps/rejected": -0.8231722116470337,
777
+ "loss": 0.7093,
778
+ "nll_loss": 0.6223791241645813,
779
+ "rewards/accuracies": 0.4000000059604645,
780
+ "rewards/chosen": -0.03846103698015213,
781
+ "rewards/margins": 0.0026975716464221478,
782
+ "rewards/rejected": -0.04115860536694527,
783
+ "step": 430
784
+ },
785
+ {
786
+ "epoch": 2.03,
787
+ "grad_norm": 2.15625,
788
+ "learning_rate": 2.3836564731139807e-06,
789
+ "log_odds_chosen": 0.37268373370170593,
790
+ "log_odds_ratio": -0.570591390132904,
791
+ "logits/chosen": 4.9182658195495605,
792
+ "logits/rejected": 5.131478309631348,
793
+ "logps/chosen": -0.8495089411735535,
794
+ "logps/rejected": -1.0931196212768555,
795
+ "loss": 0.6606,
796
+ "nll_loss": 0.6880626082420349,
797
+ "rewards/accuracies": 0.6000000238418579,
798
+ "rewards/chosen": -0.04247545078396797,
799
+ "rewards/margins": 0.012180537916719913,
800
+ "rewards/rejected": -0.05465598776936531,
801
+ "step": 440
802
+ },
803
+ {
804
+ "epoch": 2.07,
805
+ "grad_norm": 2.03125,
806
+ "learning_rate": 2.357022603955159e-06,
807
+ "log_odds_chosen": 0.9277127385139465,
808
+ "log_odds_ratio": -0.5885446071624756,
809
+ "logits/chosen": 3.5330519676208496,
810
+ "logits/rejected": 4.548266410827637,
811
+ "logps/chosen": -0.5304622650146484,
812
+ "logps/rejected": -1.0346746444702148,
813
+ "loss": 0.661,
814
+ "nll_loss": 0.5373905301094055,
815
+ "rewards/accuracies": 0.800000011920929,
816
+ "rewards/chosen": -0.026523113250732422,
817
+ "rewards/margins": 0.02521061897277832,
818
+ "rewards/rejected": -0.05173373222351074,
819
+ "step": 450
820
+ },
821
+ {
822
+ "epoch": 2.12,
823
+ "grad_norm": 2.125,
824
+ "learning_rate": 2.3312620206007847e-06,
825
+ "log_odds_chosen": 0.7354871034622192,
826
+ "log_odds_ratio": -0.4610438346862793,
827
+ "logits/chosen": 4.341551780700684,
828
+ "logits/rejected": 5.085806846618652,
829
+ "logps/chosen": -0.7264617085456848,
830
+ "logps/rejected": -1.1319414377212524,
831
+ "loss": 0.6391,
832
+ "nll_loss": 0.5838049054145813,
833
+ "rewards/accuracies": 0.800000011920929,
834
+ "rewards/chosen": -0.03632308170199394,
835
+ "rewards/margins": 0.020273985341191292,
836
+ "rewards/rejected": -0.05659707263112068,
837
+ "step": 460
838
+ },
839
+ {
840
+ "epoch": 2.17,
841
+ "grad_norm": 2.5,
842
+ "learning_rate": 2.3063280200722128e-06,
843
+ "log_odds_chosen": 1.372476577758789,
844
+ "log_odds_ratio": -0.32559940218925476,
845
+ "logits/chosen": 3.357041835784912,
846
+ "logits/rejected": 4.5395894050598145,
847
+ "logps/chosen": -0.5446572303771973,
848
+ "logps/rejected": -1.0288522243499756,
849
+ "loss": 0.63,
850
+ "nll_loss": 0.5199865102767944,
851
+ "rewards/accuracies": 0.8999999761581421,
852
+ "rewards/chosen": -0.027232861146330833,
853
+ "rewards/margins": 0.024209752678871155,
854
+ "rewards/rejected": -0.05144261568784714,
855
+ "step": 470
856
+ },
857
+ {
858
+ "epoch": 2.21,
859
+ "grad_norm": 2.109375,
860
+ "learning_rate": 2.2821773229381924e-06,
861
+ "log_odds_chosen": 0.516548752784729,
862
+ "log_odds_ratio": -0.7071049213409424,
863
+ "logits/chosen": 4.261706829071045,
864
+ "logits/rejected": 4.245719909667969,
865
+ "logps/chosen": -0.7887482047080994,
866
+ "logps/rejected": -1.266065001487732,
867
+ "loss": 0.6282,
868
+ "nll_loss": 0.7050595879554749,
869
+ "rewards/accuracies": 0.6000000238418579,
870
+ "rewards/chosen": -0.039437416940927505,
871
+ "rewards/margins": 0.023865830153226852,
872
+ "rewards/rejected": -0.06330324709415436,
873
+ "step": 480
874
+ },
875
+ {
876
+ "epoch": 2.26,
877
+ "grad_norm": 2.15625,
878
+ "learning_rate": 2.2587697572631284e-06,
879
+ "log_odds_chosen": 0.1166841983795166,
880
+ "log_odds_ratio": -0.7867618799209595,
881
+ "logits/chosen": 5.030650615692139,
882
+ "logits/rejected": 4.7974395751953125,
883
+ "logps/chosen": -1.0410363674163818,
884
+ "logps/rejected": -1.0709666013717651,
885
+ "loss": 0.6494,
886
+ "nll_loss": 0.7816016674041748,
887
+ "rewards/accuracies": 0.6000000238418579,
888
+ "rewards/chosen": -0.05205183103680611,
889
+ "rewards/margins": 0.0014965005684643984,
890
+ "rewards/rejected": -0.05354832857847214,
891
+ "step": 490
892
+ },
893
+ {
894
+ "epoch": 2.3,
895
+ "grad_norm": 2.15625,
896
+ "learning_rate": 2.23606797749979e-06,
897
+ "log_odds_chosen": 0.26380103826522827,
898
+ "log_odds_ratio": -0.68238365650177,
899
+ "logits/chosen": 4.641995429992676,
900
+ "logits/rejected": 5.189269542694092,
901
+ "logps/chosen": -0.9602524042129517,
902
+ "logps/rejected": -1.0924670696258545,
903
+ "loss": 0.6551,
904
+ "nll_loss": 0.8767480850219727,
905
+ "rewards/accuracies": 0.699999988079071,
906
+ "rewards/chosen": -0.048012617975473404,
907
+ "rewards/margins": 0.006610742304474115,
908
+ "rewards/rejected": -0.05462335795164108,
909
+ "step": 500
910
+ },
911
+ {
912
+ "epoch": 2.35,
913
+ "grad_norm": 2.265625,
914
+ "learning_rate": 2.2140372138502386e-06,
915
+ "log_odds_chosen": 0.7083855867385864,
916
+ "log_odds_ratio": -0.4410790801048279,
917
+ "logits/chosen": 4.488713264465332,
918
+ "logits/rejected": 4.330811977386475,
919
+ "logps/chosen": -0.7253609299659729,
920
+ "logps/rejected": -1.1262476444244385,
921
+ "loss": 0.6693,
922
+ "nll_loss": 0.6643433570861816,
923
+ "rewards/accuracies": 0.8999999761581421,
924
+ "rewards/chosen": -0.036268047988414764,
925
+ "rewards/margins": 0.02004433050751686,
926
+ "rewards/rejected": -0.056312382221221924,
927
+ "step": 510
928
+ },
929
+ {
930
+ "epoch": 2.4,
931
+ "grad_norm": 2.1875,
932
+ "learning_rate": 2.1926450482675734e-06,
933
+ "log_odds_chosen": 0.30893248319625854,
934
+ "log_odds_ratio": -0.6462469696998596,
935
+ "logits/chosen": 3.3600478172302246,
936
+ "logits/rejected": 4.821731090545654,
937
+ "logps/chosen": -0.8601899147033691,
938
+ "logps/rejected": -1.0197805166244507,
939
+ "loss": 0.6303,
940
+ "nll_loss": 0.6425656080245972,
941
+ "rewards/accuracies": 0.699999988079071,
942
+ "rewards/chosen": -0.043009497225284576,
943
+ "rewards/margins": 0.007979532703757286,
944
+ "rewards/rejected": -0.05098903179168701,
945
+ "step": 520
946
+ },
947
+ {
948
+ "epoch": 2.44,
949
+ "grad_norm": 2.140625,
950
+ "learning_rate": 2.1718612138153473e-06,
951
+ "log_odds_chosen": 0.5819677710533142,
952
+ "log_odds_ratio": -0.7259879112243652,
953
+ "logits/chosen": 3.3069472312927246,
954
+ "logits/rejected": 4.239446640014648,
955
+ "logps/chosen": -0.7454169988632202,
956
+ "logps/rejected": -0.7617279291152954,
957
+ "loss": 0.6362,
958
+ "nll_loss": 0.6466267108917236,
959
+ "rewards/accuracies": 0.4000000059604645,
960
+ "rewards/chosen": -0.03727085143327713,
961
+ "rewards/margins": 0.0008155457908287644,
962
+ "rewards/rejected": -0.03808639571070671,
963
+ "step": 530
964
+ },
965
+ {
966
+ "epoch": 2.49,
967
+ "grad_norm": 1.859375,
968
+ "learning_rate": 2.151657414559676e-06,
969
+ "log_odds_chosen": 0.232131689786911,
970
+ "log_odds_ratio": -0.7130674123764038,
971
+ "logits/chosen": 4.628894329071045,
972
+ "logits/rejected": 4.495694637298584,
973
+ "logps/chosen": -0.9905622601509094,
974
+ "logps/rejected": -1.0986087322235107,
975
+ "loss": 0.6571,
976
+ "nll_loss": 0.7514830231666565,
977
+ "rewards/accuracies": 0.30000001192092896,
978
+ "rewards/chosen": -0.04952811077237129,
979
+ "rewards/margins": 0.00540232565253973,
980
+ "rewards/rejected": -0.0549304373562336,
981
+ "step": 540
982
+ },
983
+ {
984
+ "epoch": 2.53,
985
+ "grad_norm": 1.8984375,
986
+ "learning_rate": 2.132007163556104e-06,
987
+ "log_odds_chosen": 1.0160751342773438,
988
+ "log_odds_ratio": -0.6234133243560791,
989
+ "logits/chosen": 3.5942962169647217,
990
+ "logits/rejected": 3.701565980911255,
991
+ "logps/chosen": -0.6263929009437561,
992
+ "logps/rejected": -0.9312576055526733,
993
+ "loss": 0.6479,
994
+ "nll_loss": 0.5540992021560669,
995
+ "rewards/accuracies": 0.6000000238418579,
996
+ "rewards/chosen": -0.03131964057683945,
997
+ "rewards/margins": 0.01524323783814907,
998
+ "rewards/rejected": -0.04656287655234337,
999
+ "step": 550
1000
+ },
1001
+ {
1002
+ "epoch": 2.58,
1003
+ "grad_norm": 1.828125,
1004
+ "learning_rate": 2.1128856368212917e-06,
1005
+ "log_odds_chosen": 0.5333415865898132,
1006
+ "log_odds_ratio": -0.5421962738037109,
1007
+ "logits/chosen": 3.2995147705078125,
1008
+ "logits/rejected": 3.9002597332000732,
1009
+ "logps/chosen": -0.7015010714530945,
1010
+ "logps/rejected": -0.9539710283279419,
1011
+ "loss": 0.6152,
1012
+ "nll_loss": 0.4980201721191406,
1013
+ "rewards/accuracies": 0.699999988079071,
1014
+ "rewards/chosen": -0.03507505729794502,
1015
+ "rewards/margins": 0.012623499147593975,
1016
+ "rewards/rejected": -0.047698549926280975,
1017
+ "step": 560
1018
+ },
1019
+ {
1020
+ "epoch": 2.63,
1021
+ "grad_norm": 2.140625,
1022
+ "learning_rate": 2.0942695414584777e-06,
1023
+ "log_odds_chosen": 1.1786051988601685,
1024
+ "log_odds_ratio": -0.35151195526123047,
1025
+ "logits/chosen": 3.9222683906555176,
1026
+ "logits/rejected": 5.216652870178223,
1027
+ "logps/chosen": -0.7344393730163574,
1028
+ "logps/rejected": -1.5133073329925537,
1029
+ "loss": 0.673,
1030
+ "nll_loss": 0.6333657503128052,
1031
+ "rewards/accuracies": 0.8999999761581421,
1032
+ "rewards/chosen": -0.03672196716070175,
1033
+ "rewards/margins": 0.038943398743867874,
1034
+ "rewards/rejected": -0.07566536217927933,
1035
+ "step": 570
1036
+ },
1037
+ {
1038
+ "epoch": 2.67,
1039
+ "grad_norm": 2.0,
1040
+ "learning_rate": 2.0761369963434992e-06,
1041
+ "log_odds_chosen": 1.2405891418457031,
1042
+ "log_odds_ratio": -0.35236161947250366,
1043
+ "logits/chosen": 2.746840476989746,
1044
+ "logits/rejected": 4.9477057456970215,
1045
+ "logps/chosen": -0.5109530687332153,
1046
+ "logps/rejected": -1.0042306184768677,
1047
+ "loss": 0.6259,
1048
+ "nll_loss": 0.4207887649536133,
1049
+ "rewards/accuracies": 0.800000011920929,
1050
+ "rewards/chosen": -0.025547657161951065,
1051
+ "rewards/margins": 0.024663874879479408,
1052
+ "rewards/rejected": -0.050211526453495026,
1053
+ "step": 580
1054
+ },
1055
+ {
1056
+ "epoch": 2.72,
1057
+ "grad_norm": 2.078125,
1058
+ "learning_rate": 2.058467423981546e-06,
1059
+ "log_odds_chosen": 0.8945177793502808,
1060
+ "log_odds_ratio": -0.573771059513092,
1061
+ "logits/chosen": 3.4927985668182373,
1062
+ "logits/rejected": 4.610069274902344,
1063
+ "logps/chosen": -0.7422350645065308,
1064
+ "logps/rejected": -0.9318183660507202,
1065
+ "loss": 0.6196,
1066
+ "nll_loss": 0.5690957903862,
1067
+ "rewards/accuracies": 0.6000000238418579,
1068
+ "rewards/chosen": -0.03711175173521042,
1069
+ "rewards/margins": 0.00947917066514492,
1070
+ "rewards/rejected": -0.04659091681241989,
1071
+ "step": 590
1072
+ },
1073
+ {
1074
+ "epoch": 2.76,
1075
+ "grad_norm": 2.484375,
1076
+ "learning_rate": 2.0412414523193154e-06,
1077
+ "log_odds_chosen": 0.9618481397628784,
1078
+ "log_odds_ratio": -0.41677170991897583,
1079
+ "logits/chosen": 3.545571804046631,
1080
+ "logits/rejected": 4.716848850250244,
1081
+ "logps/chosen": -0.7041240334510803,
1082
+ "logps/rejected": -1.2747822999954224,
1083
+ "loss": 0.6273,
1084
+ "nll_loss": 0.608167290687561,
1085
+ "rewards/accuracies": 0.800000011920929,
1086
+ "rewards/chosen": -0.035206202417612076,
1087
+ "rewards/margins": 0.028532912954688072,
1088
+ "rewards/rejected": -0.063739113509655,
1089
+ "step": 600
1090
+ },
1091
+ {
1092
+ "epoch": 2.81,
1093
+ "grad_norm": 2.203125,
1094
+ "learning_rate": 2.0244408254472904e-06,
1095
+ "log_odds_chosen": 0.5268529653549194,
1096
+ "log_odds_ratio": -0.5601459741592407,
1097
+ "logits/chosen": 3.7574939727783203,
1098
+ "logits/rejected": 4.722257137298584,
1099
+ "logps/chosen": -0.7759762406349182,
1100
+ "logps/rejected": -1.0274509191513062,
1101
+ "loss": 0.6365,
1102
+ "nll_loss": 0.5871884822845459,
1103
+ "rewards/accuracies": 0.8999999761581421,
1104
+ "rewards/chosen": -0.03879881277680397,
1105
+ "rewards/margins": 0.01257373671978712,
1106
+ "rewards/rejected": -0.05137254670262337,
1107
+ "step": 610
1108
+ },
1109
+ {
1110
+ "epoch": 2.86,
1111
+ "grad_norm": 1.859375,
1112
+ "learning_rate": 2.0080483222562476e-06,
1113
+ "log_odds_chosen": 0.8587082028388977,
1114
+ "log_odds_ratio": -0.42638325691223145,
1115
+ "logits/chosen": 4.095767021179199,
1116
+ "logits/rejected": 4.707412242889404,
1117
+ "logps/chosen": -0.5631445050239563,
1118
+ "logps/rejected": -0.8936156034469604,
1119
+ "loss": 0.6224,
1120
+ "nll_loss": 0.6269885897636414,
1121
+ "rewards/accuracies": 0.800000011920929,
1122
+ "rewards/chosen": -0.028157228603959084,
1123
+ "rewards/margins": 0.016523560509085655,
1124
+ "rewards/rejected": -0.04468078538775444,
1125
+ "step": 620
1126
+ },
1127
+ {
1128
+ "epoch": 2.9,
1129
+ "grad_norm": 2.546875,
1130
+ "learning_rate": 1.9920476822239895e-06,
1131
+ "log_odds_chosen": 0.2341475933790207,
1132
+ "log_odds_ratio": -0.647922158241272,
1133
+ "logits/chosen": 4.118049144744873,
1134
+ "logits/rejected": 4.044008731842041,
1135
+ "logps/chosen": -0.9274168014526367,
1136
+ "logps/rejected": -1.0269124507904053,
1137
+ "loss": 0.6281,
1138
+ "nll_loss": 0.7285808324813843,
1139
+ "rewards/accuracies": 0.6000000238418579,
1140
+ "rewards/chosen": -0.046370841562747955,
1141
+ "rewards/margins": 0.004974778275936842,
1142
+ "rewards/rejected": -0.051345616579055786,
1143
+ "step": 630
1144
+ },
1145
+ {
1146
+ "epoch": 2.95,
1147
+ "grad_norm": 2.09375,
1148
+ "learning_rate": 1.976423537605237e-06,
1149
+ "log_odds_chosen": 0.8695243000984192,
1150
+ "log_odds_ratio": -0.3998942971229553,
1151
+ "logits/chosen": 3.509381055831909,
1152
+ "logits/rejected": 4.522087574005127,
1153
+ "logps/chosen": -0.6447582244873047,
1154
+ "logps/rejected": -1.073907494544983,
1155
+ "loss": 0.6332,
1156
+ "nll_loss": 0.5656287670135498,
1157
+ "rewards/accuracies": 0.800000011920929,
1158
+ "rewards/chosen": -0.03223791345953941,
1159
+ "rewards/margins": 0.02145746536552906,
1160
+ "rewards/rejected": -0.05369538068771362,
1161
+ "step": 640
1162
+ },
1163
+ {
1164
+ "epoch": 3.0,
1165
+ "grad_norm": 2.328125,
1166
+ "learning_rate": 1.961161351381841e-06,
1167
+ "log_odds_chosen": 0.9357713460922241,
1168
+ "log_odds_ratio": -0.4956935942173004,
1169
+ "logits/chosen": 3.176776170730591,
1170
+ "logits/rejected": 4.386256217956543,
1171
+ "logps/chosen": -0.6535663604736328,
1172
+ "logps/rejected": -0.9676573872566223,
1173
+ "loss": 0.6171,
1174
+ "nll_loss": 0.5301100015640259,
1175
+ "rewards/accuracies": 0.699999988079071,
1176
+ "rewards/chosen": -0.03267831727862358,
1177
+ "rewards/margins": 0.015704555436968803,
1178
+ "rewards/rejected": -0.04838287830352783,
1179
+ "step": 650
1180
+ },
1181
+ {
1182
+ "epoch": 3.0,
1183
+ "step": 651,
1184
+ "total_flos": 0.0,
1185
+ "train_loss": 1.0000138937419827,
1186
+ "train_runtime": 4770.0198,
1187
+ "train_samples_per_second": 4.36,
1188
+ "train_steps_per_second": 0.136
1189
+ }
1190
+ ],
1191
+ "logging_steps": 10,
1192
+ "max_steps": 651,
1193
+ "num_input_tokens_seen": 0,
1194
+ "num_train_epochs": 3,
1195
+ "save_steps": 500,
1196
+ "total_flos": 0.0,
1197
+ "train_batch_size": 1,
1198
+ "trial_name": null,
1199
+ "trial_params": null
1200
+ }