sanqiang commited on
Commit
d36a35d
1 Parent(s): d82e274

Model save

Browse files
README.md CHANGED
@@ -2,15 +2,9 @@
2
  license: other
3
  base_model: HuggingFaceH4/zephyr-7b-gemma-sft-v0.1
4
  tags:
5
- - alignment-handbook
6
  - trl
7
  - dpo
8
  - generated_from_trainer
9
- - trl
10
- - dpo
11
- - generated_from_trainer
12
- datasets:
13
- - argilla/dpo-mix-7k
14
  model-index:
15
  - name: zephyr-7b-gemma-dpo
16
  results: []
@@ -19,20 +13,9 @@ model-index:
19
  <!-- This model card has been generated automatically according to the information the Trainer had access to. You
20
  should probably proofread and complete it, then remove this comment. -->
21
 
22
- [<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/sanqiang/wdpo/runs/cbwapxr3)
23
  # zephyr-7b-gemma-dpo
24
 
25
- This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-gemma-sft-v0.1](https://huggingface.co/HuggingFaceH4/zephyr-7b-gemma-sft-v0.1) on the argilla/dpo-mix-7k dataset.
26
- It achieves the following results on the evaluation set:
27
- - Loss: 0.4694
28
- - Rewards/chosen: -3.9430
29
- - Rewards/rejected: -5.5568
30
- - Rewards/accuracies: 0.7292
31
- - Rewards/margins: 1.6137
32
- - Logps/rejected: -473.0876
33
- - Logps/chosen: -441.5945
34
- - Logits/rejected: 88.3937
35
- - Logits/chosen: 94.2780
36
 
37
  ## Model description
38
 
@@ -52,14 +35,14 @@ More information needed
52
 
53
  The following hyperparameters were used during training:
54
  - learning_rate: 5e-07
55
- - train_batch_size: 2
56
- - eval_batch_size: 4
57
  - seed: 42
58
  - distributed_type: multi-GPU
59
  - num_devices: 8
60
- - gradient_accumulation_steps: 8
61
  - total_train_batch_size: 128
62
- - total_eval_batch_size: 32
63
  - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
64
  - lr_scheduler_type: cosine
65
  - lr_scheduler_warmup_ratio: 0.1
@@ -67,14 +50,11 @@ The following hyperparameters were used during training:
67
 
68
  ### Training results
69
 
70
- | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
71
- |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
72
- | 0.1534 | 1.8957 | 100 | 0.4631 | -3.9815 | -5.5353 | 0.7083 | 1.5538 | -472.6590 | -442.3637 | 88.4677 | 94.3385 |
73
 
74
 
75
  ### Framework versions
76
 
77
- - Transformers 4.41.0
78
  - Pytorch 2.1.2+cu121
79
  - Datasets 2.14.6
80
  - Tokenizers 0.19.1
 
2
  license: other
3
  base_model: HuggingFaceH4/zephyr-7b-gemma-sft-v0.1
4
  tags:
 
5
  - trl
6
  - dpo
7
  - generated_from_trainer
 
 
 
 
 
8
  model-index:
9
  - name: zephyr-7b-gemma-dpo
10
  results: []
 
13
  <!-- This model card has been generated automatically according to the information the Trainer had access to. You
14
  should probably proofread and complete it, then remove this comment. -->
15
 
 
16
  # zephyr-7b-gemma-dpo
17
 
18
+ This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-gemma-sft-v0.1](https://huggingface.co/HuggingFaceH4/zephyr-7b-gemma-sft-v0.1) on the None dataset.
 
 
 
 
 
 
 
 
 
 
19
 
20
  ## Model description
21
 
 
35
 
36
  The following hyperparameters were used during training:
37
  - learning_rate: 5e-07
38
+ - train_batch_size: 1
39
+ - eval_batch_size: 1
40
  - seed: 42
41
  - distributed_type: multi-GPU
42
  - num_devices: 8
43
+ - gradient_accumulation_steps: 16
44
  - total_train_batch_size: 128
45
+ - total_eval_batch_size: 8
46
  - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
47
  - lr_scheduler_type: cosine
48
  - lr_scheduler_warmup_ratio: 0.1
 
50
 
51
  ### Training results
52
 
 
 
 
53
 
54
 
55
  ### Framework versions
56
 
57
+ - Transformers 4.41.1
58
  - Pytorch 2.1.2+cu121
59
  - Datasets 2.14.6
60
  - Tokenizers 0.19.1
all_results.json CHANGED
@@ -1,22 +1,9 @@
1
  {
2
  "epoch": 1.971563981042654,
3
- "eval_logits/chosen": 94.27800750732422,
4
- "eval_logits/rejected": 88.3936767578125,
5
- "eval_logps/chosen": -441.594482421875,
6
- "eval_logps/rejected": -473.087646484375,
7
- "eval_loss": 0.4694259464740753,
8
- "eval_rewards/accuracies": 0.7291666865348816,
9
- "eval_rewards/chosen": -3.94303035736084,
10
- "eval_rewards/margins": 1.613731026649475,
11
- "eval_rewards/rejected": -5.556761264801025,
12
- "eval_runtime": 20.9081,
13
- "eval_samples": 750,
14
- "eval_samples_per_second": 35.871,
15
- "eval_steps_per_second": 1.148,
16
  "total_flos": 0.0,
17
- "train_loss": 0.38818228932527393,
18
- "train_runtime": 1555.646,
19
  "train_samples": 6750,
20
- "train_samples_per_second": 8.678,
21
- "train_steps_per_second": 0.067
22
  }
 
1
  {
2
  "epoch": 1.971563981042654,
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  "total_flos": 0.0,
4
+ "train_loss": 0.39618923515081406,
5
+ "train_runtime": 6217.6844,
6
  "train_samples": 6750,
7
+ "train_samples_per_second": 2.171,
8
+ "train_steps_per_second": 0.017
9
  }
config.json CHANGED
@@ -23,7 +23,7 @@
23
  "rope_scaling": null,
24
  "rope_theta": 10000.0,
25
  "torch_dtype": "bfloat16",
26
- "transformers_version": "4.41.0",
27
- "use_cache": true,
28
  "vocab_size": 256000
29
  }
 
23
  "rope_scaling": null,
24
  "rope_theta": 10000.0,
25
  "torch_dtype": "bfloat16",
26
+ "transformers_version": "4.41.1",
27
+ "use_cache": false,
28
  "vocab_size": 256000
29
  }
generation_config.json CHANGED
@@ -3,5 +3,5 @@
3
  "bos_token_id": 2,
4
  "eos_token_id": 1,
5
  "pad_token_id": 0,
6
- "transformers_version": "4.41.0"
7
  }
 
3
  "bos_token_id": 2,
4
  "eos_token_id": 1,
5
  "pad_token_id": 0,
6
+ "transformers_version": "4.41.1"
7
  }
model-00001-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:907718dc7abf77c674dfe5561577edaa20f6fa1f02bf06b9de7a03e0c15e2485
3
  size 4995496656
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8173c05d1efe08c4065544d793175322eb2200be1887726dea6644d5277d73ac
3
  size 4995496656
model-00002-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4670809bdabbdfeeaa959616f541ed35c881f72174d92315b90b89b7c7ed884a
3
  size 4982953168
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:02256ff0bcb27472feccfb9450965b57ad05a4ab4aa4bec3b416265f8261aaf8
3
  size 4982953168
model-00003-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d9782ec18e737bd2641fe5faa1dfc877b1e190168f373747e733b300ba25c56e
3
  size 4982953200
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f4eeebdfd0c17424180ff75420235e53fc228ffae6dac9af48c4fd07b803d870
3
  size 4982953200
model-00004-of-00004.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:15dd432be6f7df2743b1bef22516b2f769aa7e7804086c1875c9abd5f2f2e0b5
3
  size 2113988336
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f8ea7640a19dbd05fd2186e383d4a64ec43af0f5b7299d629a37ff306fc7e9f8
3
  size 2113988336
runs/May24_04-18-29_pytorchjob-summarization-debugkd-ifxb-sanqiang-zhao-master-0/events.out.tfevents.1716524387.pytorchjob-summarization-debugkd-ifxb-sanqiang-zhao-master-0.8891.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac4ecd006f7c6c3cd9fc1ab0bdf028b9f97a21b144976016c57eade392f96f90
3
+ size 12870
train_results.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "epoch": 1.971563981042654,
3
  "total_flos": 0.0,
4
- "train_loss": 0.38818228932527393,
5
- "train_runtime": 1555.646,
6
  "train_samples": 6750,
7
- "train_samples_per_second": 8.678,
8
- "train_steps_per_second": 0.067
9
  }
 
1
  {
2
  "epoch": 1.971563981042654,
3
  "total_flos": 0.0,
4
+ "train_loss": 0.39618923515081406,
5
+ "train_runtime": 6217.6844,
6
  "train_samples": 6750,
7
+ "train_samples_per_second": 2.171,
8
+ "train_steps_per_second": 0.017
9
  }
trainer_state.json CHANGED
@@ -2,7 +2,7 @@
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
  "epoch": 1.971563981042654,
5
- "eval_steps": 100,
6
  "global_step": 104,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
@@ -10,12 +10,12 @@
10
  "log_history": [
11
  {
12
  "epoch": 0.018957345971563982,
13
- "grad_norm": 134.93724401851352,
14
  "learning_rate": 4.545454545454545e-08,
15
- "logits/chosen": 117.67350769042969,
16
- "logits/rejected": 126.90988159179688,
17
- "logps/chosen": -336.5020751953125,
18
- "logps/rejected": -438.0943298339844,
19
  "loss": 0.6931,
20
  "rewards/accuracies": 0.0,
21
  "rewards/chosen": 0.0,
@@ -25,178 +25,162 @@
25
  },
26
  {
27
  "epoch": 0.1895734597156398,
28
- "grad_norm": 136.34386152980758,
29
  "learning_rate": 4.545454545454545e-07,
30
- "logits/chosen": 134.7221221923828,
31
- "logits/rejected": 138.1984100341797,
32
- "logps/chosen": -395.924560546875,
33
- "logps/rejected": -439.0712890625,
34
- "loss": 0.7055,
35
- "rewards/accuracies": 0.4097222089767456,
36
- "rewards/chosen": -0.003234411356970668,
37
- "rewards/margins": -0.03947298228740692,
38
- "rewards/rejected": 0.036238569766283035,
39
  "step": 10
40
  },
41
  {
42
  "epoch": 0.3791469194312796,
43
- "grad_norm": 135.65125923828373,
44
  "learning_rate": 4.885348141000122e-07,
45
- "logits/chosen": 121.62422180175781,
46
- "logits/rejected": 125.15272521972656,
47
- "logps/chosen": -369.32891845703125,
48
- "logps/rejected": -422.75689697265625,
49
- "loss": 0.6228,
50
- "rewards/accuracies": 0.581250011920929,
51
- "rewards/chosen": 0.170469269156456,
52
- "rewards/margins": 0.30520111322402954,
53
- "rewards/rejected": -0.13473184406757355,
54
  "step": 20
55
  },
56
  {
57
  "epoch": 0.5687203791469194,
58
- "grad_norm": 114.07857539812969,
59
  "learning_rate": 4.5025027361734613e-07,
60
- "logits/chosen": 141.53656005859375,
61
- "logits/rejected": 135.02110290527344,
62
- "logps/chosen": -418.7715759277344,
63
- "logps/rejected": -464.8955078125,
64
- "loss": 0.56,
65
- "rewards/accuracies": 0.675000011920929,
66
- "rewards/chosen": -1.2944904565811157,
67
- "rewards/margins": 0.8948480486869812,
68
- "rewards/rejected": -2.189338445663452,
69
  "step": 30
70
  },
71
  {
72
  "epoch": 0.7582938388625592,
73
- "grad_norm": 103.49873893671008,
74
  "learning_rate": 3.893311157806091e-07,
75
- "logits/chosen": 123.42630767822266,
76
- "logits/rejected": 112.1140365600586,
77
- "logps/chosen": -401.87945556640625,
78
- "logps/rejected": -429.3699645996094,
79
- "loss": 0.5472,
80
  "rewards/accuracies": 0.7250000238418579,
81
- "rewards/chosen": -2.4228084087371826,
82
- "rewards/margins": 1.1271207332611084,
83
- "rewards/rejected": -3.54992938041687,
84
  "step": 40
85
  },
86
  {
87
  "epoch": 0.9478672985781991,
88
- "grad_norm": 120.25232045055446,
89
  "learning_rate": 3.126631330646801e-07,
90
- "logits/chosen": 138.51483154296875,
91
- "logits/rejected": 142.50746154785156,
92
- "logps/chosen": -464.8746643066406,
93
- "logps/rejected": -547.7520141601562,
94
- "loss": 0.5034,
95
- "rewards/accuracies": 0.737500011920929,
96
- "rewards/chosen": -2.2957112789154053,
97
- "rewards/margins": 1.2795140743255615,
98
- "rewards/rejected": -3.575225353240967,
99
  "step": 50
100
  },
101
  {
102
  "epoch": 1.1374407582938388,
103
- "grad_norm": 60.01877502993502,
104
  "learning_rate": 2.2891223348923882e-07,
105
- "logits/chosen": 131.36839294433594,
106
- "logits/rejected": 134.95950317382812,
107
- "logps/chosen": -444.547607421875,
108
- "logps/rejected": -531.6339721679688,
109
- "loss": 0.3025,
110
- "rewards/accuracies": 0.9125000238418579,
111
- "rewards/chosen": -2.315918207168579,
112
- "rewards/margins": 2.439805269241333,
113
- "rewards/rejected": -4.755723476409912,
114
  "step": 60
115
  },
116
  {
117
  "epoch": 1.3270142180094786,
118
- "grad_norm": 47.79216068377628,
119
  "learning_rate": 1.4754491880085317e-07,
120
- "logits/chosen": 124.45259094238281,
121
- "logits/rejected": 126.58074951171875,
122
- "logps/chosen": -414.2548828125,
123
- "logps/rejected": -518.2446899414062,
124
- "loss": 0.1949,
125
- "rewards/accuracies": 0.9375,
126
- "rewards/chosen": -2.3579883575439453,
127
- "rewards/margins": 2.8935599327087402,
128
- "rewards/rejected": -5.251548767089844,
129
  "step": 70
130
  },
131
  {
132
  "epoch": 1.5165876777251186,
133
- "grad_norm": 55.81661982856631,
134
  "learning_rate": 7.775827023107834e-08,
135
- "logits/chosen": 110.53846740722656,
136
- "logits/rejected": 127.3997573852539,
137
- "logps/chosen": -416.94775390625,
138
- "logps/rejected": -536.0177001953125,
139
- "loss": 0.1738,
140
  "rewards/accuracies": 0.9312499761581421,
141
- "rewards/chosen": -3.1685705184936523,
142
- "rewards/margins": 2.967393159866333,
143
- "rewards/rejected": -6.135963439941406,
144
  "step": 80
145
  },
146
  {
147
  "epoch": 1.7061611374407581,
148
- "grad_norm": 46.35415773419707,
149
  "learning_rate": 2.7440387297912122e-08,
150
- "logits/chosen": 108.8740234375,
151
- "logits/rejected": 121.63133239746094,
152
- "logps/chosen": -452.612060546875,
153
- "logps/rejected": -570.0894775390625,
154
- "loss": 0.1585,
155
  "rewards/accuracies": 0.96875,
156
- "rewards/chosen": -3.32800030708313,
157
- "rewards/margins": 3.267381191253662,
158
- "rewards/rejected": -6.595381259918213,
159
  "step": 90
160
  },
161
  {
162
  "epoch": 1.8957345971563981,
163
- "grad_norm": 43.55253667946353,
164
  "learning_rate": 2.27878296044029e-09,
165
- "logits/chosen": 115.157958984375,
166
- "logits/rejected": 114.7303695678711,
167
- "logps/chosen": -443.5245056152344,
168
- "logps/rejected": -538.7801513671875,
169
- "loss": 0.1534,
170
- "rewards/accuracies": 0.956250011920929,
171
- "rewards/chosen": -3.0830078125,
172
- "rewards/margins": 3.0285675525665283,
173
- "rewards/rejected": -6.111575126647949,
174
- "step": 100
175
- },
176
- {
177
- "epoch": 1.8957345971563981,
178
- "eval_logits/chosen": 94.33853912353516,
179
- "eval_logits/rejected": 88.46772003173828,
180
- "eval_logps/chosen": -442.3636779785156,
181
- "eval_logps/rejected": -472.6589660644531,
182
- "eval_loss": 0.4630958139896393,
183
- "eval_rewards/accuracies": 0.7083333134651184,
184
- "eval_rewards/chosen": -3.9814913272857666,
185
- "eval_rewards/margins": 1.5538396835327148,
186
- "eval_rewards/rejected": -5.5353312492370605,
187
- "eval_runtime": 21.3945,
188
- "eval_samples_per_second": 35.056,
189
- "eval_steps_per_second": 1.122,
190
  "step": 100
191
  },
192
  {
193
  "epoch": 1.971563981042654,
194
  "step": 104,
195
  "total_flos": 0.0,
196
- "train_loss": 0.38818228932527393,
197
- "train_runtime": 1555.646,
198
- "train_samples_per_second": 8.678,
199
- "train_steps_per_second": 0.067
200
  }
201
  ],
202
  "logging_steps": 10,
@@ -217,7 +201,7 @@
217
  }
218
  },
219
  "total_flos": 0.0,
220
- "train_batch_size": 2,
221
  "trial_name": null,
222
  "trial_params": null
223
  }
 
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
  "epoch": 1.971563981042654,
5
+ "eval_steps": 500,
6
  "global_step": 104,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
 
10
  "log_history": [
11
  {
12
  "epoch": 0.018957345971563982,
13
+ "grad_norm": 135.46242416026297,
14
  "learning_rate": 4.545454545454545e-08,
15
+ "logits/chosen": 203.78909301757812,
16
+ "logits/rejected": 182.07696533203125,
17
+ "logps/chosen": -434.955322265625,
18
+ "logps/rejected": -429.50384521484375,
19
  "loss": 0.6931,
20
  "rewards/accuracies": 0.0,
21
  "rewards/chosen": 0.0,
 
25
  },
26
  {
27
  "epoch": 0.1895734597156398,
28
+ "grad_norm": 153.21007203142685,
29
  "learning_rate": 4.545454545454545e-07,
30
+ "logits/chosen": 172.51792907714844,
31
+ "logits/rejected": 181.91905212402344,
32
+ "logps/chosen": -379.5159912109375,
33
+ "logps/rejected": -448.1414794921875,
34
+ "loss": 0.7124,
35
+ "rewards/accuracies": 0.4861111044883728,
36
+ "rewards/chosen": 0.027371780946850777,
37
+ "rewards/margins": 0.008838895708322525,
38
+ "rewards/rejected": 0.01853288896381855,
39
  "step": 10
40
  },
41
  {
42
  "epoch": 0.3791469194312796,
43
+ "grad_norm": 142.08069477633126,
44
  "learning_rate": 4.885348141000122e-07,
45
+ "logits/chosen": 173.6682891845703,
46
+ "logits/rejected": 175.75106811523438,
47
+ "logps/chosen": -381.7967834472656,
48
+ "logps/rejected": -430.2057189941406,
49
+ "loss": 0.6304,
50
+ "rewards/accuracies": 0.637499988079071,
51
+ "rewards/chosen": 0.19567528367042542,
52
+ "rewards/margins": 0.36299628019332886,
53
+ "rewards/rejected": -0.16732101142406464,
54
  "step": 20
55
  },
56
  {
57
  "epoch": 0.5687203791469194,
58
+ "grad_norm": 120.91669313906786,
59
  "learning_rate": 4.5025027361734613e-07,
60
+ "logits/chosen": 164.82431030273438,
61
+ "logits/rejected": 173.40679931640625,
62
+ "logps/chosen": -365.9583740234375,
63
+ "logps/rejected": -445.6947326660156,
64
+ "loss": 0.5761,
65
+ "rewards/accuracies": 0.75,
66
+ "rewards/chosen": -1.0274367332458496,
67
+ "rewards/margins": 1.1871185302734375,
68
+ "rewards/rejected": -2.214555263519287,
69
  "step": 30
70
  },
71
  {
72
  "epoch": 0.7582938388625592,
73
+ "grad_norm": 99.23172159300925,
74
  "learning_rate": 3.893311157806091e-07,
75
+ "logits/chosen": 170.08607482910156,
76
+ "logits/rejected": 162.03958129882812,
77
+ "logps/chosen": -414.80340576171875,
78
+ "logps/rejected": -454.820556640625,
79
+ "loss": 0.5641,
80
  "rewards/accuracies": 0.7250000238418579,
81
+ "rewards/chosen": -2.624403715133667,
82
+ "rewards/margins": 0.9876155853271484,
83
+ "rewards/rejected": -3.6120193004608154,
84
  "step": 40
85
  },
86
  {
87
  "epoch": 0.9478672985781991,
88
+ "grad_norm": 107.79387973262907,
89
  "learning_rate": 3.126631330646801e-07,
90
+ "logits/chosen": 177.14224243164062,
91
+ "logits/rejected": 174.7544708251953,
92
+ "logps/chosen": -459.03759765625,
93
+ "logps/rejected": -512.6439208984375,
94
+ "loss": 0.5158,
95
+ "rewards/accuracies": 0.78125,
96
+ "rewards/chosen": -3.4753570556640625,
97
+ "rewards/margins": 1.3291194438934326,
98
+ "rewards/rejected": -4.804476737976074,
99
  "step": 50
100
  },
101
  {
102
  "epoch": 1.1374407582938388,
103
+ "grad_norm": 53.02406081573369,
104
  "learning_rate": 2.2891223348923882e-07,
105
+ "logits/chosen": 167.0361328125,
106
+ "logits/rejected": 171.70101928710938,
107
+ "logps/chosen": -414.3773498535156,
108
+ "logps/rejected": -524.4793701171875,
109
+ "loss": 0.3097,
110
+ "rewards/accuracies": 0.8687499761581421,
111
+ "rewards/chosen": -3.200485944747925,
112
+ "rewards/margins": 2.440056085586548,
113
+ "rewards/rejected": -5.6405415534973145,
114
  "step": 60
115
  },
116
  {
117
  "epoch": 1.3270142180094786,
118
+ "grad_norm": 62.10149124492704,
119
  "learning_rate": 1.4754491880085317e-07,
120
+ "logits/chosen": 157.87100219726562,
121
+ "logits/rejected": 165.49331665039062,
122
+ "logps/chosen": -453.851318359375,
123
+ "logps/rejected": -509.77960205078125,
124
+ "loss": 0.2085,
125
+ "rewards/accuracies": 0.956250011920929,
126
+ "rewards/chosen": -3.4425768852233887,
127
+ "rewards/margins": 2.912635326385498,
128
+ "rewards/rejected": -6.355212211608887,
129
  "step": 70
130
  },
131
  {
132
  "epoch": 1.5165876777251186,
133
+ "grad_norm": 92.48557214337542,
134
  "learning_rate": 7.775827023107834e-08,
135
+ "logits/chosen": 155.17611694335938,
136
+ "logits/rejected": 170.6410369873047,
137
+ "logps/chosen": -473.2515563964844,
138
+ "logps/rejected": -549.7650146484375,
139
+ "loss": 0.187,
140
  "rewards/accuracies": 0.9312499761581421,
141
+ "rewards/chosen": -3.7577788829803467,
142
+ "rewards/margins": 2.7696285247802734,
143
+ "rewards/rejected": -6.527407646179199,
144
  "step": 80
145
  },
146
  {
147
  "epoch": 1.7061611374407581,
148
+ "grad_norm": 50.507688635772325,
149
  "learning_rate": 2.7440387297912122e-08,
150
+ "logits/chosen": 160.5852508544922,
151
+ "logits/rejected": 169.7069854736328,
152
+ "logps/chosen": -444.99725341796875,
153
+ "logps/rejected": -585.0924072265625,
154
+ "loss": 0.1675,
155
  "rewards/accuracies": 0.96875,
156
+ "rewards/chosen": -3.7263991832733154,
157
+ "rewards/margins": 3.2523887157440186,
158
+ "rewards/rejected": -6.97878885269165,
159
  "step": 90
160
  },
161
  {
162
  "epoch": 1.8957345971563981,
163
+ "grad_norm": 44.901315947621455,
164
  "learning_rate": 2.27878296044029e-09,
165
+ "logits/chosen": 160.45559692382812,
166
+ "logits/rejected": 161.6788330078125,
167
+ "logps/chosen": -444.6620178222656,
168
+ "logps/rejected": -523.9886474609375,
169
+ "loss": 0.1723,
170
+ "rewards/accuracies": 0.9437500238418579,
171
+ "rewards/chosen": -3.935753583908081,
172
+ "rewards/margins": 3.035583734512329,
173
+ "rewards/rejected": -6.971337795257568,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
174
  "step": 100
175
  },
176
  {
177
  "epoch": 1.971563981042654,
178
  "step": 104,
179
  "total_flos": 0.0,
180
+ "train_loss": 0.39618923515081406,
181
+ "train_runtime": 6217.6844,
182
+ "train_samples_per_second": 2.171,
183
+ "train_steps_per_second": 0.017
184
  }
185
  ],
186
  "logging_steps": 10,
 
201
  }
202
  },
203
  "total_flos": 0.0,
204
+ "train_batch_size": 1,
205
  "trial_name": null,
206
  "trial_params": null
207
  }
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:36c830e7f828745f5f4993639dca5d4e9350a3782d777acf9e26bd26d79229bd
3
  size 6456
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:72c54496520c00ad2acc4b9f3014bb5fb90f7cc0675061cc92b9c15fbde068ae
3
  size 6456